Skip to content

T
tool-code
Commit afe6589d authored by rico.liu's avatar rico.liu
Browse files
Options

update

parent 16f76ec8
Show whitespace changes
Inline Side-by-side
Showing with 881 additions and 60 deletions
数据治理平台线下处理/ESCore/ES.py
View file @ afe6589d
...@@ -929,9 +929,9 @@ class ES_Client: ...@@ -929,9 +929,9 @@ class ES_Client:
} }
} }
res = self.es.update_by_query(index=self.index_name,body=body)
print(self.es.update_by_query(index=self.index_name,body=body)) print(res)
return res
def UpdateByMultipleFiveCondition(self,search_field_list,search_field_value_list,update_field,update_field_value): def UpdateByMultipleFiveCondition(self,search_field_list,search_field_value_list,update_field,update_field_value):
inline = "ctx._source['" + str(update_field) + "']='" + str(update_field_value) + "'" inline = "ctx._source['" + str(update_field) + "']='" + str(update_field_value) + "'"
......
数据治理平台线下处理/线下处理/参数补充线下处理脚本.py → 数据治理平台线下处理/线下处理/参数补充线下处理导出导入脚本.py
View file @ afe6589d
...@@ -15,7 +15,27 @@ import pymysql ...@@ -15,7 +15,27 @@ import pymysql
from public import zgc_api,Index from public import zgc_api,Index
from SimilarCharactor.string_similarity import similarity_cn,similarity_en from SimilarCharactor.string_similarity import similarity_cn,similarity_en
def export_params_data(es,filter_col,filter_value,category_name,params_list):
#获取指定类别的参数项
def get_category_params_list(category_name):
conn_zi_new = pymssql.connect(host='123.56.115.207', user='zgcprice3311',password='zgcprice20200628',database= 'ZI_NEW',autocommit=True)
cursor_zi_new = conn_zi_new.cursor()
cursor_zi_new.execute(f"select subtitle,identy from vw_property where name = '{category_name}' and subtitle != '产品型号'")
db_params = pd.DataFrame(cursor_zi_new.fetchall(), columns=[tuple[0] for tuple in cursor_zi_new.description])
db_params['require_param'] = db_params['identy'].apply(lambda x: x[0])
db_params['standard_param'] = db_params['identy'].apply(lambda x: x[2])
db_params = db_params[(db_params['standard_param'] != '0') & (db_params['require_param'] != '0')]
db_params['subtitle'] = db_params['subtitle'].apply(lambda x: x.strip())
params_list = db_params['subtitle'].tolist()
return params_list
def export_params_data(es,filter_col,filter_value,category_name):
#导出指定项目数据 #导出指定项目数据
if filter_col == 'DATA_BATCH': if filter_col == 'DATA_BATCH':
...@@ -34,6 +54,8 @@ def export_params_data(es,filter_col,filter_value,category_name,params_list): ...@@ -34,6 +54,8 @@ def export_params_data(es,filter_col,filter_value,category_name,params_list):
cat_df = df[(df['FINISH_P_LASTCATEGORY_NAME'] == category_name) & (df['FLOW_NODE_STATUS'] == FLOW_NODE_STATUS) & (df['FLOW_FINISH_STATUS'] == 'N')] cat_df = df[(df['FINISH_P_LASTCATEGORY_NAME'] == category_name) & (df['FLOW_NODE_STATUS'] == FLOW_NODE_STATUS) & (df['FLOW_FINISH_STATUS'] == 'N')]
params_list = get_category_params_list(category_name)
for param in params_list: for param in params_list:
param_var = '_' + ''.join(param.split()).replace('（','').replace('）','').replace('/','').replace('(','').replace(')','').replace('+','').replace('-','').replace('*','').replace('.','') param_var = '_' + ''.join(param.split()).replace('（','').replace('）','').replace('/','').replace('(','').replace(')','').replace('+','').replace('-','').replace('*','').replace('.','')
exec('%s_list=[]'%param_var) exec('%s_list=[]'%param_var)
...@@ -58,14 +80,13 @@ def export_params_data(es,filter_col,filter_value,category_name,params_list): ...@@ -58,14 +80,13 @@ def export_params_data(es,filter_col,filter_value,category_name,params_list):
category_name = category_name.replace("/","_") category_name = category_name.replace("/","_")
cat_df.to_excel(f"{category_name}参数补充{date}.xlsx") cat_df.to_excel(f"{category_name}参数补充{date}.xlsx")
def export_params_finish_data(es,filter_col,filter_value,category_name):
def export_params_finish_data(es,filter_col,filter_value,category_name,params_list):
#导出指定项目数据 #导出指定项目数据
df = es.SearchByCondition(filter_col,filter_value) df = es.SearchByCondition(filter_col,filter_value)
#df = df[df['DATA_BATCH'] == "202111160940430001"] #df = df[df['DATA_BATCH'] == "202111160940430001"]
#导出补参环节数据 #导出补参环节数据
params_list = get_category_params_list(category_name)
cat_df = df[df['FINISH_P_LASTCATEGORY_NAME'] == category_name] cat_df = df[df['FINISH_P_LASTCATEGORY_NAME'] == category_name]
cat_df['FINISH_P_PARAMS'] = cat_df['FINISH_P_PARAMS'].apply(lambda x : str(x)) cat_df['FINISH_P_PARAMS'] = cat_df['FINISH_P_PARAMS'].apply(lambda x : str(x))
cat_df['flag'] = cat_df['FINISH_P_PARAMS'].apply(lambda x :0 if x == 'None' or x == 'nan' else 1) cat_df['flag'] = cat_df['FINISH_P_PARAMS'].apply(lambda x :0 if x == 'None' or x == 'nan' else 1)
...@@ -103,7 +124,7 @@ def get_zgc_param_values(zgc_categoryname,id_list,param_value_dict_list): ...@@ -103,7 +124,7 @@ def get_zgc_param_values(zgc_categoryname,id_list,param_value_dict_list):
cursor_zi_new = conn_zi_new.cursor() cursor_zi_new = conn_zi_new.cursor()
cursor_zi_new.execute(f"select * from ShuJuZiDian_Cfg where categoryname = '{zgc_categoryname}'") cursor_zi_new.execute(f"select * from ShuJuZiDian_Cfg where categoryname = '{zgc_categoryname}' and stdvalue is not Null")
data = cursor_zi_new.fetchall() data = cursor_zi_new.fetchall()
cat_data_dict = pd.DataFrame(data, columns=[tuple[0] for tuple in cursor_zi_new.description]) cat_data_dict = pd.DataFrame(data, columns=[tuple[0] for tuple in cursor_zi_new.description])
...@@ -181,7 +202,7 @@ def get_zgc_param_values(zgc_categoryname,id_list,param_value_dict_list): ...@@ -181,7 +202,7 @@ def get_zgc_param_values(zgc_categoryname,id_list,param_value_dict_list):
res = {'code':code,'msg':msg,'paramsValue_dict':{}} res = {'code':code,'msg':msg,'paramsValue_dict':{}}
return res return res
def check_data(category_name,path): def check_data(category_name,path,assign_params_list):
df = pd.read_excel(path,converters = {'产品编码':str}) df = pd.read_excel(path,converters = {'产品编码':str})
try: try:
...@@ -204,17 +225,23 @@ def check_data(category_name,path): ...@@ -204,17 +225,23 @@ def check_data(category_name,path):
#进行非标转标 #进行非标转标
##加载库中参数项数据（参数id，参数从属） ##加载库中参数项数据（参数id，参数从属）
if assign_params_list:
pass
else:
cursor_zi_new.execute(f"select * from vw_property where name = '{category_name}'") cursor_zi_new.execute(f"select * from vw_property where name = '{category_name}'")
db_params = pd.DataFrame(cursor_zi_new.fetchall(), columns=[tuple[0] for tuple in cursor_zi_new.description]) db_params = pd.DataFrame(cursor_zi_new.fetchall(), columns=[tuple[0] for tuple in cursor_zi_new.description])
db_params['standard_param'] = db_params['identy'].apply(lambda x: x[2]) db_params['standard_param'] = db_params['identy'].apply(lambda x: x[2])
db_params = db_params[db_params['standard_param'] != '0'] db_params = db_params[db_params['standard_param'] != '0']
db_params['subtitle'] = db_params['subtitle'].apply(lambda x: x.strip()) db_params['subtitle'] = db_params['subtitle'].apply(lambda x: x.strip())
assign_params_list = db_params['subtitle'].tolist()
#subtitle_id_list = dict(zip(db_params['subtitle'].tolist(),db_params['subtitleid'].tolist())) #subtitle_id_list = dict(zip(db_params['subtitle'].tolist(),db_params['subtitleid'].tolist()))
#subtitle_type_list = dict(zip(db_params['subtitle'].tolist(),db_params['skuorspu'].tolist())) #subtitle_type_list = dict(zip(db_params['subtitle'].tolist(),db_params['skuorspu'].tolist()))
params_list = df.columns.tolist() col_list = df.columns.tolist()
recover_params_list = [params.replace("*","") for params in df.columns.tolist()] recover_params_list = [params.replace("*","") for params in df.columns.tolist()]
rename_dict = dict(zip(params_list,recover_params_list)) rename_dict = dict(zip(col_list,recover_params_list))
df = df.rename(columns = rename_dict) df = df.rename(columns = rename_dict)
...@@ -233,7 +260,7 @@ def check_data(category_name,path): ...@@ -233,7 +260,7 @@ def check_data(category_name,path):
temp_key_list = list() temp_key_list = list()
temp_value_list = list() temp_value_list = list()
for db_param in db_params['subtitle'].tolist(): for db_param in assign_params_list:
if db_param in ['产品型号', 'CPU属性']: if db_param in ['产品型号', 'CPU属性']:
continue continue
temp_key_list.append(db_param) temp_key_list.append(db_param)
...@@ -271,7 +298,7 @@ def check_data(category_name,path): ...@@ -271,7 +298,7 @@ def check_data(category_name,path):
pass pass
eg_brand_name = row['FINISH_P_BRAND_NAME'] eg_brand_name = row['FINISH_P_BRAND_NAME']
eg_product_name = row['SOURCE_P_NAME'].replace("'","") eg_product_name = row['SOURCE_P_NAME'].replace("'","")
for db_param in db_params['subtitle'].tolist(): for db_param in assign_params_list:
if db_param in ['产品型号', 'CPU属性']:#CPU属性为衍生属性，需要特殊处理 if db_param in ['产品型号', 'CPU属性']:#CPU属性为衍生属性，需要特殊处理
continue continue
...@@ -335,7 +362,7 @@ def check_data(category_name,path): ...@@ -335,7 +362,7 @@ def check_data(category_name,path):
print(f"请补充数据字典，并导回。数据字典文件路径：{category_name}数据字典补充.xlsx") print(f"请补充数据字典，并导回。数据字典文件路径：{category_name}数据字典补充.xlsx")
else: else:
for db_param in db_params['subtitle'].tolist(): for db_param in assign_params_list:
if db_param == '产品型号': if db_param == '产品型号':
continue continue
param_var = '_' + ''.join(db_param.split()).replace('（','').replace('）','').replace('/','').replace('(','').replace(')','').replace('+','').replace('-','').replace('*','').replace('.','') param_var = '_' + ''.join(db_param.split()).replace('（','').replace('）','').replace('/','').replace('(','').replace(')','').replace('+','').replace('-','').replace('*','').replace('.','')
...@@ -348,7 +375,7 @@ def check_data(category_name,path): ...@@ -348,7 +375,7 @@ def check_data(category_name,path):
remark = 'nan' remark = 'nan'
id_ = int(row['DOC_ID']) id_ = int(row['DOC_ID'])
for db_param in db_params['subtitle'].tolist(): for db_param in assign_params_list:
param_var = '_' + ''.join(db_param.split()).replace('（','').replace('）','').replace('/','').replace('(','').replace(')','').replace('+','').replace('-','').replace('*','').replace('.','') param_var = '_' + ''.join(db_param.split()).replace('（','').replace('）','').replace('/','').replace('(','').replace(')','').replace('+','').replace('-','').replace('*','').replace('.','')
if db_param == '产品型号': if db_param == '产品型号':
continue continue
...@@ -370,7 +397,7 @@ def check_data(category_name,path): ...@@ -370,7 +397,7 @@ def check_data(category_name,path):
exec("%s_list.append('%s')"%(param_var,value.strip())) exec("%s_list.append('%s')"%(param_var,value.strip()))
for db_param in db_params['subtitle'].tolist(): for db_param in assign_params_list:
if db_param == '产品型号': if db_param == '产品型号':
continue continue
param_var = '_' + ''.join(db_param.split()).replace('（','').replace('）','').replace('/','').replace('(','').replace(')','').replace('+','').replace('-','').replace('*','').replace('.','') param_var = '_' + ''.join(db_param.split()).replace('（','').replace('）','').replace('/','').replace('(','').replace(')','').replace('+','').replace('-','').replace('*','').replace('.','')
...@@ -423,11 +450,14 @@ def get_configure_price(): ...@@ -423,11 +450,14 @@ def get_configure_price():
print('连接失败，重新连接') print('连接失败，重新连接')
return get_configure_price() return get_configure_price()
def check_configure_price(category,path,params_list):
def check_configure_price(category,path):
if category not in ['笔记本','台式机','一体电脑','工作站']: if category not in ['笔记本','台式机','一体电脑','工作站']:
return "该类别无配件，无需校验配件价格" return "该类别无配件，无需校验配件价格"
params_list = get_category_params_list(category)
conn = pymssql.connect(host='123.56.115.207', user='zgcindex', password='jiayou202006', database='ZI_NEW',autocommit=True) conn = pymssql.connect(host='123.56.115.207', user='zgcindex', password='jiayou202006', database='ZI_NEW',autocommit=True)
cursor = conn.cursor() cursor = conn.cursor()
...@@ -491,7 +521,7 @@ def check_configure_price(category,path,params_list): ...@@ -491,7 +521,7 @@ def check_configure_price(category,path,params_list):
def import_params_data(es,category_name,path,params_list): def import_params_data(es,category_name,path):
#更新补参数据 #更新补参数据
df = pd.read_excel(path,converters = {'CPU型号':str}) df = pd.read_excel(path,converters = {'CPU型号':str})
...@@ -500,6 +530,8 @@ def import_params_data(es,category_name,path,params_list): ...@@ -500,6 +530,8 @@ def import_params_data(es,category_name,path,params_list):
except: except:
df['驳回原因'] = ['nan' for i in range(len(df))] df['驳回原因'] = ['nan' for i in range(len(df))]
params_list = get_category_params_list(category_name)
process_index = 0 process_index = 0
index_ = Index() index_ = Index()
...@@ -531,7 +563,6 @@ def import_params_data(es,category_name,path,params_list): ...@@ -531,7 +563,6 @@ def import_params_data(es,category_name,path,params_list):
es.UpdateParamsDealData(category_name,search_field,search_field_value,params_info) es.UpdateParamsDealData(category_name,search_field,search_field_value,params_info)
def get_oceanbase_res_data(batch_list): def get_oceanbase_res_data(batch_list):
try: try:
...@@ -554,52 +585,33 @@ def get_oceanbase_res_data(batch_list): ...@@ -554,52 +585,33 @@ def get_oceanbase_res_data(batch_list):
#创建ES客户端 #创建ES客户端
host = "http://123.56.114.138:9200/" host = "http://123.56.114.138:9200/"
index_name = "model_params_test" index_name = "model_params_test"
#index_name = "model_params_test_copy_1118"
es = ES_Client(host,index_name) es = ES_Client(host,index_name)
#导出单个类补参数据
filter_col = "DATA_PROJECT_NAME" # DATA_PROJECT_NAME 项目名称 / DATA_BATCH 批次号
filter_value = ['徽采商城数据入库'] # 项目名称/批次号 eg. 全渠道数据入库/202111171554120001/徽采商城数据入库
category_name = '笔记本'
filter_col = "DATA_BATCH" # DATA_PROJECT_NAME 项目名称 / DATA_BATCH 批次号 #导出数据
filter_value = ['202111301656160001'] # 项目名称/批次号 eg. 全渠道数据入库/202111171554120001/徽采商城数据入库 export_params_data(es,filter_col,filter_value,category_name)
category_name = '空调'
if category_name == '笔记本':#要导出的类别名称
params_list = ['CPU型号', '产品子系列', '产品系列', '内存容量', '固态硬盘容量', '屏幕尺寸', '操作系统', '显存容量', '机械硬盘容量', '质保时间']#一定按照页面上展示的参数顺序列出
elif category_name == '投影机':#要导出的类别名称
params_list = ['投影技术', '标配外服务及配件']#一定按照页面上展示的参数顺序列出
elif category_name == '碎纸机':#要导出的类别名称
params_list = ['碎纸效果']#一定按照页面上展示的参数顺序列出
elif category_name == '台式机':#要导出的类别名称
params_list = ['CPU型号', '产品子系列', '产品系列', '内存容量', '固态硬盘容量', '屏幕尺寸', '操作系统', '显存容量', '机械硬盘容量', '质保时间']#一定按照页面上展示的参数顺序列出
elif category_name == '扫描仪':#要导出的类别名称
params_list = ['幅面','类型','质保时间']#一定按照页面上展示的参数顺序列出
elif category_name == '针式打印机':#要导出的类别名称
params_list = ['幅面']#一定按照页面上展示的参数顺序列出
elif category_name == '喷墨打印机':#要导出的类别名称
params_list = ['分辨率','幅面','彩色打印速度','标配外服务及配件','黑白打印速度']#一定按照页面上展示的参数顺序列出
elif category_name == '一体电脑':#要导出的类别名称
params_list = ['CPU型号', '产品子系列', '产品系列', '内存容量', '固态硬盘容量', '屏幕尺寸', '操作系统', '显存容量', '机械硬盘容量', '质保时间']#一定按照页面上展示的参数顺序列出
elif category_name == '复印纸':#要导出的类别名称
params_list = ['产品系列','单包张数（张/包）','规格（幅面尺寸）','计价规格（包/箱或单包装）','重量（G）']
elif category_name == '复印机':#要导出的类别名称
params_list = ['双面器','双面输稿器','标配外服务及配件','纸盒容量','网络打印']
elif category_name == '激光打印机':#要导出的类别名称
params_list = ['产品类型','双面打印','彩色打印速度','打印分辨率','最大打印幅面','标配外服务及配件','标配外耗材','网络打印','质保时间','黑白打印速度']
elif category_name == '空调':#要导出的类别名称
params_list = ['冷暖类型','匹数','变频/定频','标配外服务及配件','空调类别']
#导出多个类补参数据
#category_name_list = ['空调','摄像机','多功能一体机','U盘','移动硬盘','硒鼓/墨粉']
#导出数据 #导出数据
export_params_data(es,filter_col,filter_value,category_name,params_list) #for category_name in category_name_list:
# export_params_data(es,filter_col,filter_value,category_name)
export_all_project_params_data(es,filter_col,filter_value)
#校验补充后的数据（若有驳回原因，请添加列名为：“驳回原因”列，填写驳回原因） #校验补充后的数据（若有驳回原因，请添加列名为：“驳回原因”列，填写驳回原因）
#如果需要指定校验的参数项，请在assign_params_list中填写参数项。eg:['CPU型号','内存容量']
#如果不需要指定，则使用系统配置的参数项进行校验assign_params_list 为空即可。eg:[]
complicated_path = "/Users/rico/Downloads/徽采台式机参数补充2021-12-01-已补充.xlsx" complicated_path = "/Users/rico/Downloads/徽采台式机参数补充2021-12-01-已补充.xlsx"
check_data(category_name,complicated_path) assign_params_list = []
check_data(category_name,complicated_path,assign_params_list)
#更新数据字典 #更新数据字典
path = '/Users/rico/Downloads/台式机数据字典补充(3)(1).xlsx' path = '/Users/rico/Downloads/台式机数据字典补充(3)(1).xlsx'
...@@ -607,11 +619,11 @@ ComplicatedDataDict(path) ...@@ -607,11 +619,11 @@ ComplicatedDataDict(path)
#校验通过的数据需要进行配件价格校验 #校验通过的数据需要进行配件价格校验
checked_path = "/Users/rico/Downloads/激光打印机参数补充回导.xlsx" checked_path = "/Users/rico/Downloads/激光打印机参数补充回导.xlsx"
check_configure_price(category_name,checked_path,params_list) check_configure_price(category_name,checked_path)
#导入数据 #导入数据
import_params_data(es,category_name,checked_path,params_list) import_params_data(es,category_name,checked_path)
#更新相应字段，以跳过必填参数项补充节点 #更新相应字段，以跳过必填参数项补充节点
...@@ -628,10 +640,10 @@ update_field_value = "SYS_GET_PARAMS_SUCCESS_END" ...@@ -628,10 +640,10 @@ update_field_value = "SYS_GET_PARAMS_SUCCESS_END"
es.UpdateByMultipleThreeCondition(search_field_list,search_field_value_list,update_field,update_field_value) es.UpdateByMultipleThreeCondition(search_field_list,search_field_value_list,update_field,update_field_value)
#按批次号导出结果数据 #按批次号导出结果数据
res = es.SearchByCondition("DATA_BATCH",["202112011048580001"]) res = es.SearchByCondition("DATA_BATCH",["202111231900550001"])
#按项目导出结果数据 #按项目导出结果数据
#res = es.SearchAimAllByCondition("DATA_PROJECT_NAME","徽采商城数据入库") #res = es.SearchAimAllByCondition("DATA_PROJECT_NAME","徽采商城数据入库")
#a = res[res['FINISH_P_LASTCATEGORY_NAME'] == '激光打印机']['FLOW_NODE_STATUS'].unique().tolist() #res = res[res['FINISH_P_LASTCATEGORY_NAME'] == '台式机']['FLOW_NODE_STATUS'].unique().tolist()
''' '''
res = es.SearchAimAllByCondition("SOURCE_CHANNEL_SNAME","SHANXI") res = es.SearchAimAllByCondition("SOURCE_CHANNEL_SNAME","SHANXI")
#res = es.SearchAllByProjectName('徽采商城数据入库') #res = es.SearchAllByProjectName('徽采商城数据入库')
...@@ -661,11 +673,19 @@ res[['DOC_ID','FLOW_NODE_STATUS','FINISH_P_REMARK']].groupby(['FLOW_NODE_STATUS' ...@@ -661,11 +673,19 @@ res[['DOC_ID','FLOW_NODE_STATUS','FINISH_P_REMARK']].groupby(['FLOW_NODE_STATUS'
#校验是否有未命名的商品 #校验是否有未命名的商品
search_field_list = ['FINISH_P_LASTCATEGORY_NAME','FLOW_NODE_STATUS','FINISH_P_NAME','FLOW_FINISH_STATUS'] search_field_list = ['FINISH_P_LASTCATEGORY_NAME','FLOW_NODE_STATUS','FINISH_P_NAME','FLOW_FINISH_STATUS']
search_field_value_list = ['笔记本','SYS_FINISH_MATCH','','Y'] search_field_value_list = ['台式机','SYS_FINISH_MATCH','','Y']
update_field = "FLOW_FINISH_STATUS"
update_field_value = "N"
es.UpdateByMultipleFourCondition(search_field_list,search_field_value_list,update_field,update_field_value)
search_field_list = ['FINISH_P_LASTCATEGORY_NAME','FLOW_NODE_STATUS','FINISH_P_NAME','FLOW_FINISH_STATUS']
search_field_value_list = ['台式机','SYS_FINISH_MATCH','','N']
update_field = "FLOW_NODE_STATUS" update_field = "FLOW_NODE_STATUS"
update_field_value = "SYS_GET_PARAMS_FAILD_REDUCTIVE" update_field_value = "SYS_GET_PARAMS_SUCCESS_REDUCTIVE"
es.UpdateByMultipleFourCondition(search_field_list,search_field_value_list,update_field,update_field_value) es.UpdateByMultipleFourCondition(search_field_list,search_field_value_list,update_field,update_field_value)
search_field_list = ['FINISH_P_LASTCATEGORY_NAME','FLOW_NODE_STATUS','FINISH_P_NAME','FLOW_FINISH_STATUS'] search_field_list = ['FINISH_P_LASTCATEGORY_NAME','FLOW_NODE_STATUS','FINISH_P_NAME','FLOW_FINISH_STATUS']
search_field_value_list = ['笔记本','SYS_GET_PARAMS_SUCCESS_END','','N'] search_field_value_list = ['笔记本','SYS_GET_PARAMS_SUCCESS_END','','N']
update_field = "FLOW_NODE_STATUS" update_field = "FLOW_NODE_STATUS"
...@@ -682,7 +702,7 @@ es.UpdateByMultipleFourCondition(search_field_list,search_field_value_list,updat ...@@ -682,7 +702,7 @@ es.UpdateByMultipleFourCondition(search_field_list,search_field_value_list,updat
search_field_list = ['FINISH_P_LASTCATEGORY_NAME','FLOW_NODE_STATUS','FINISH_P_NAME','FLOW_FINISH_STATUS'] search_field_list = ['FINISH_P_LASTCATEGORY_NAME','FLOW_NODE_STATUS','FINISH_P_NAME','FLOW_FINISH_STATUS']
search_field_value_list = ['激光打印机','SYS_GET_PARAMS_FAILD_END','','N'] search_field_value_list = ['激光打印机','SYS_FINISH_MATCH','','N']
update_field = "FLOW_NODE_STATUS" update_field = "FLOW_NODE_STATUS"
update_field_value = "SYS_GET_PARAMS_SUCCESS_NONREDUCTIVE" update_field_value = "SYS_GET_PARAMS_SUCCESS_NONREDUCTIVE"
es.UpdateByMultipleFourCondition(search_field_list,search_field_value_list,update_field,update_field_value) es.UpdateByMultipleFourCondition(search_field_list,search_field_value_list,update_field,update_field_value)
......
数据治理平台线下处理/线下处理/参数补充线下处理导出导入脚本_1(导出).py 0 → 100755
View file @ afe6589d
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Dec 3 10:44:20 2021
@author: rico
"""
import sys
sys.path.append(os.path.dirname(os.getcwd()))
from ESCore.ES import ES_Client
import datetime
import pandas as pd
import pymssql
import pymysql
from public import zgc_api,Index
from SimilarCharactor.string_similarity import similarity_cn,similarity_en
#获取指定类别的参数项
def get_category_params_list(category_name):
conn_zi_new = pymssql.connect(host='123.56.115.207', user='zgcprice3311',password='zgcprice20200628',database= 'ZI_NEW',autocommit=True)
cursor_zi_new = conn_zi_new.cursor()
cursor_zi_new.execute(f"select subtitle,identy from vw_property where name = '{category_name}' and subtitle != '产品型号'")
db_params = pd.DataFrame(cursor_zi_new.fetchall(), columns=[tuple[0] for tuple in cursor_zi_new.description])
db_params['require_param'] = db_params['identy'].apply(lambda x: x[0])
db_params['standard_param'] = db_params['identy'].apply(lambda x: x[2])
db_params = db_params[(db_params['standard_param'] != '0') & (db_params['require_param'] != '0')]
db_params['subtitle'] = db_params['subtitle'].apply(lambda x: x.strip())
params_list = db_params['subtitle'].tolist()
return params_list
def export_params_data(es,filter_col,filter_value,category_name):
#导出指定项目数据
if filter_col == 'DATA_BATCH':
df = es.SearchByCondition(filter_col,filter_value)
else:
filter_value = filter_value[0]
df = es.SearchAimAllByCondition(filter_col,filter_value)
#df = df[df['DATA_BATCH'] == "202111160940430001"]
#导出补参环节数据
if category_name in ['笔记本','台式机','一体电脑','复印纸','工作站']:
FLOW_NODE_STATUS = "SYS_GET_PARAMS_FAILD_REDUCTIVE"
COL_LIST = ['DOC_ID','SOURCE_P_NAME','FINISH_P_LASTCATEGORY_NAME','FINISH_P_BRAND_NAME','SOURCE_P_PARAMS','SOURCE_P_URL']
else:
FLOW_NODE_STATUS = "SYS_GET_PARAMS_FAILD_NONREDUCTIVE"
COL_LIST = ['DOC_ID','SOURCE_P_NAME','FINISH_P_LASTCATEGORY_NAME','FINISH_P_BRAND_NAME','FINISH_P_MODEL','SOURCE_P_PARAMS','SOURCE_P_URL']
cat_df = df[(df['FINISH_P_LASTCATEGORY_NAME'] == category_name) & (df['FLOW_NODE_STATUS'] == FLOW_NODE_STATUS) & (df['FLOW_FINISH_STATUS'] == 'N')]
params_list = get_category_params_list(category_name)
for param in params_list:
param_var = '_' + ''.join(param.split()).replace('（','').replace('）','').replace('/','').replace('(','').replace(')','').replace('+','').replace('-','').replace('*','').replace('.','')
exec('%s_list=[]'%param_var)
for index,row in cat_df.iterrows():
#params_dict = eval(row['DEAL_UNSTD_PARAMS_DICT'])#取第一行的数据
params_dict = eval(row['DEAL_STD_PARAMS_DICT'])
for param in params_list:
param_var = '_' + ''.join(param.split()).replace('（','').replace('）','').replace('/','').replace('(','').replace(')','').replace('+','').replace('-','').replace('*','').replace('.','')
exec("%s_list.append('%s')"%(param_var,params_dict[param].replace("\\","需人工补充")))
cat_df = cat_df[COL_LIST]#取指定的字段信息
for param in params_list:
param_var = '_' + ''.join(param.split()).replace('（','').replace('）','').replace('/','').replace('(','').replace(')','').replace('+','').replace('-','').replace('*','').replace('.','')
exec("cat_df['%s']=%s_list"%(param,param_var))
#结果导出excel
date = str(datetime.datetime.now()).split(" ")[0]
if "/" in category_name:
category_name = category_name.replace("/","_")
cat_df.to_excel(f"{category_name}参数补充{date}.xlsx")
#创建ES客户端
host = "http://123.56.114.138:9200/"
index_name = "model_params_test"
es = ES_Client(host,index_name)
#导出单个类补参数据
filter_col = "DATA_PROJECT_NAME" # DATA_PROJECT_NAME 项目名称 / DATA_BATCH 批次号
filter_value = ['徽采商城数据入库'] # 项目名称/批次号 eg. 全渠道数据入库/202111171554120001/徽采商城数据入库
category_name = '笔记本'
#导出数据
export_params_data(es,filter_col,filter_value,category_name)
#导出多个类补参数据
#category_name_list = ['空调','摄像机','多功能一体机','U盘','移动硬盘','硒鼓/墨粉']
#导出数据
#for category_name in category_name_list:
# export_params_data(es,filter_col,filter_value,category_name)
数据治理平台线下处理/线下处理/参数补充线下处理导出导入脚本_2(校验).py 0 → 100755
View file @ afe6589d
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Dec 3 10:51:23 2021
@author: rico
"""
import sys
sys.path.append(os.path.dirname(os.getcwd()))
from ESCore.ES import ES_Client
import datetime
import pandas as pd
import pymssql
import pymysql
from public import zgc_api,Index
from SimilarCharactor.string_similarity import similarity_cn,similarity_en
def get_zgc_param_values(zgc_categoryname,id_list,param_value_dict_list):
#获取zgc参数值
try:
conn_zi_new = pymssql.connect(host='123.56.115.207', user='zgcprice3311',password='zgcprice20200628',database= 'ZI_NEW',autocommit=True)
cursor_zi_new = conn_zi_new.cursor()
cursor_zi_new.execute(f"select * from ShuJuZiDian_Cfg where categoryname = '{zgc_categoryname}' and stdvalue is not Null")
data = cursor_zi_new.fetchall()
cat_data_dict = pd.DataFrame(data, columns=[tuple[0] for tuple in cursor_zi_new.description])
cat_data_dict['stdvalue'] = cat_data_dict['stdvalue'].apply(lambda x:str(x).strip()).tolist()
cat_data_dict['primitive'] = cat_data_dict['primitive'].apply(lambda x:str(x).strip().upper()).tolist()
cat_data_dict['simplevalue'] = cat_data_dict['simplevalue'].apply(lambda x:str(x).strip().upper()).tolist()
final_res_list = list()
for id_,param_value_dict in zip(id_list,param_value_dict_list):
res = list()
for subtitle,value_ori in zip(param_value_dict.keys(), param_value_dict.values()):
res_value_list = list()
res_flag_list = list()
value = str(value_ori).strip().upper()
value_df = cat_data_dict[cat_data_dict['subtitle'] == subtitle][['stdvalue','primitive','simplevalue']]
flag = False
for value_std in value_df['stdvalue'].tolist():
if value_std and value_std != 'None':
value_std = str(value_std).upper()
if value == value_std:
flag = True
res_value = value_std
break
if flag == False:
for value_std,value_pri in zip(value_df['stdvalue'].tolist(),value_df['primitive'].tolist()):
if value_std and value_std != 'None':
value_std = str(value_std).upper()
value_pri = str(value_pri).upper()
if value == value_pri:
flag = True
res_value = value_std
break
if flag == False:
for value_std,value_sim in zip(value_df['stdvalue'].tolist(),value_df['simplevalue'].tolist()):
if value_std and value_std != 'None':
value_std = str(value_std).upper()
value_sim = str(value_sim).upper()
if value == value_sim:
flag = True
res_value = value_std
break
if flag == False:
#res = '请添加数据字典'
res_value = value_ori
if type(res_value) != str:
res_value = ""
res_value_list.append(res_value)
res_flag_list.append(flag)
res.append(dict(zip(res_value_list, res_flag_list)))
res_dict = dict(zip(param_value_dict.keys(), res))
final_res_list.append(res_dict)
return_data = dict(zip(id_list,final_res_list))
conn_zi_new.close()
code = 1
msg = "success"
res = {'code':code,'msg':msg,'paramsValue_dict':return_data}
return res
except Exception as e:
conn_zi_new.close()
code = 0
msg = str(e)
res = {'code':code,'msg':msg,'paramsValue_dict':{}}
return res
def check_data(category_name,path,assign_params_list):
df = pd.read_excel(path,converters = {'产品编码':str})
try:
df['驳回原因'] = df['驳回原因'].apply(lambda x:str(x))
except:
pass
if '无参数，需补充' in df.values:
print("仍有参数未补充，请补充完整后再继续处理。")
return False
#创建新产品库链接
conn_zi_new = pymssql.connect(host='123.56.115.207', user='zgcprice3311',password='zgcprice20200628',database= 'ZI_NEW',autocommit=True)
cursor_zi_new = conn_zi_new.cursor()
#加载数据字典
cursor_zi_new.execute(f"select subtitle,stdvalue,primitive,simplevalue from ShuJuZiDian_Cfg where categoryname = '{category_name}' and stdvalue is not Null")
data_dict = pd.DataFrame(cursor_zi_new.fetchall(), columns=[tuple[0] for tuple in cursor_zi_new.description])
#进行非标转标
##加载库中参数项数据（参数id，参数从属）
if assign_params_list:
pass
else:
cursor_zi_new.execute(f"select * from vw_property where name = '{category_name}'")
db_params = pd.DataFrame(cursor_zi_new.fetchall(), columns=[tuple[0] for tuple in cursor_zi_new.description])
db_params['standard_param'] = db_params['identy'].apply(lambda x: x[2])
db_params = db_params[db_params['standard_param'] != '0']
db_params['subtitle'] = db_params['subtitle'].apply(lambda x: x.strip())
assign_params_list = db_params['subtitle'].tolist()
#subtitle_id_list = dict(zip(db_params['subtitle'].tolist(),db_params['subtitleid'].tolist()))
#subtitle_type_list = dict(zip(db_params['subtitle'].tolist(),db_params['skuorspu'].tolist()))
col_list = df.columns.tolist()
recover_params_list = [params.replace("*","") for params in df.columns.tolist()]
rename_dict = dict(zip(col_list,recover_params_list))
df = df.rename(columns = rename_dict)
##数据准备
id_list = list()
params_list = list()
for index,row in df.iterrows():
try:
remark = row['驳回原因']
if remark != 'nan':
continue
except:
pass
id_ = row['DOC_ID']
id_list.append(id_)
temp_key_list = list()
temp_value_list = list()
for db_param in assign_params_list:
if db_param in ['产品型号', 'CPU属性']:
continue
temp_key_list.append(db_param)
temp_value_list.append(row[db_param])
params_list.append(dict(zip(temp_key_list, temp_value_list)))
print(f"{category_name}数据准备完成")
##获取标准参数值
res = get_zgc_param_values(category_name,id_list,params_list)
res_value_dict = res['paramsValue_dict']
#非标准值添加数据字典
flag = False
dict_id_list = list()
process_index = 0
index_ = Index()
for index,row in df.iterrows():
try:
print(index_(process_index,len(df)-1), end='%')
except:
print(index_(process_index,1), end='%')
process_index+=1
id_ = row['DOC_ID']
try:
remark = row['驳回原因']
if remark != 'nan':
continue
except:
pass
eg_brand_name = row['FINISH_P_BRAND_NAME']
eg_product_name = row['SOURCE_P_NAME'].replace("'","")
for db_param in assign_params_list:
if db_param in ['产品型号', 'CPU属性']:#CPU属性为衍生属性，需要特殊处理
continue
pri_value = str(row[db_param])
value, value_flag = tuple(res_value_dict[id_][db_param].items())[0]
if value_flag == False:
#print(f"select * from ShuJuZiDian_Cfg where categoryname = '{category_name}' and subtitle = '{db_param}' and primitive = '{pri_value}' and stdvalue is not Null")
cursor_zi_new.execute(f"select * from ShuJuZiDian_Cfg where categoryname = '{category_name}' and subtitle = '{db_param}' and primitive = '{pri_value}' and stdvalue is Null")
check_df = pd.DataFrame(cursor_zi_new.fetchall(),
columns=[tuple[0] for tuple in cursor_zi_new.description])
if check_df.empty:
#推荐数据字典值
init_fraction = 0
recommend_pri = ''
for dict_pri in data_dict[data_dict['subtitle'] == db_param]['primitive'].unique().tolist():
dict_pri = str(dict_pri)
try:
fraction_part1 = similarity_cn(pri_value,dict_pri)
except:
#print(f"无法比较 '{pri_value}'与 '{dict_pri}'")
#fraction = 0
fraction_part1 = 0
fraction_part2 = similarity_en(pri_value,dict_pri)
fraction = fraction_part1 * 0.5 + fraction_part2 * 0.5
if fraction > init_fraction:
init_fraction = fraction
recommend_pri = dict_pri
try:
recommend_stdvalue = data_dict[data_dict['primitive'] == recommend_pri]['stdvalue'].tolist()[0]
recommend_simplevalue = data_dict[data_dict['primitive'] == recommend_pri]['simplevalue'].tolist()[0]
except:
recommend_stdvalue = '无'
recommend_simplevalue = '无'
flag = True
cursor_zi_new.execute(f"insert into ShuJuZiDian_Cfg (categoryname,subtitle,primitive,eg_brand_name,eg_product_name,recommend_primitive,recommend_stdvalue,recommend_simplevalue) \
values ('{category_name}','{db_param}','{pri_value}','{eg_brand_name}','{eg_product_name}','{recommend_pri}','{recommend_stdvalue}','{recommend_simplevalue}')")
cursor_zi_new.execute(f"select id from ShuJuZiDian_Cfg where categoryname = '{category_name}' and subtitle = '{db_param}' and primitive = '{pri_value}'")
dict_id = cursor_zi_new.fetchone()[0]
dict_id_list.append(dict_id)
else:
stdvalue = check_df['stdvalue'].tolist()[0]
res_value_dict[id_].update({f"{db_param}":{f"{stdvalue}":True}})
if flag:
if len(dict_id_list) == 1:
_id = dict_id_list[0]
cursor_zi_new.execute(f"select * from ShuJuZiDian_Cfg where stdvalue is Null and id = {_id}")
else:
cursor_zi_new.execute("select * from ShuJuZiDian_Cfg where stdvalue is Null and id in (%s)" % ','.join(['%s'] * len(dict_id_list)),tuple(dict_id_list))
fill_shujuzidian_df = pd.DataFrame(cursor_zi_new.fetchall(), columns=[tuple[0] for tuple in cursor_zi_new.description])
fill_shujuzidian_df.to_excel(f"{category_name}数据字典补充.xlsx")
print(f"请补充数据字典，并导回。数据字典文件路径：{category_name}数据字典补充.xlsx")
else:
for db_param in assign_params_list:
if db_param == '产品型号':
continue
param_var = '_' + ''.join(db_param.split()).replace('（','').replace('）','').replace('/','').replace('(','').replace(')','').replace('+','').replace('-','').replace('*','').replace('.','')
exec('%s_list=[]'%param_var)
for index,row in df.iterrows():
try:
remark = row['驳回原因']
except:
remark = 'nan'
id_ = int(row['DOC_ID'])
for db_param in assign_params_list:
param_var = '_' + ''.join(db_param.split()).replace('（','').replace('）','').replace('/','').replace('(','').replace(')','').replace('+','').replace('-','').replace('*','').replace('.','')
if db_param == '产品型号':
continue
elif db_param == 'CPU属性':
if remark == 'nan':
value,value_flag = tuple(res_value_dict[id_]['CPU型号'].items())[0]
if '飞腾' in value or '龙芯' in value or '兆芯' in value:
pass
else:
value = value.split('-')[0]
else:
value = "驳回数据"
else:
if remark == 'nan':
value,value_flag = tuple(res_value_dict[id_][db_param].items())[0]
else:
value = "驳回数据"
exec("%s_list.append('%s')"%(param_var,value.strip()))
for db_param in assign_params_list:
if db_param == '产品型号':
continue
param_var = '_' + ''.join(db_param.split()).replace('（','').replace('）','').replace('/','').replace('(','').replace(')','').replace('+','').replace('-','').replace('*','').replace('.','')
exec("df['%s']=%s_list"%(db_param,param_var))
pass_path = path.split('.xlsx')[0] + '(校验通过).xlsx'
df.to_excel(pass_path,index=False)
print(f"{category_name}数据通过校验，请使用'{pass_path}'更新库内参数")
#校验补充后的数据（若有驳回原因，请添加列名为：“驳回原因”列，填写驳回原因）
#如果需要指定校验的参数项，请在assign_params_list中填写参数项。eg:['CPU型号','内存容量']
#如果不需要指定，则使用系统配置的参数项进行校验assign_params_list 为空即可。eg:[]
category_name = '笔记本'
complicated_path = "/Users/rico/Downloads/徽采台式机参数补充2021-12-01-已补充.xlsx"
assign_params_list = []
check_data(category_name,complicated_path,assign_params_list)
数据治理平台线下处理/线下处理/参数补充线下处理导出导入脚本_3(更新数据字典).py 0 → 100755
View file @ afe6589d
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Dec 3 11:12:31 2021
@author: rico
"""
import sys
sys.path.append(os.path.dirname(os.getcwd()))
from ESCore.ES import ES_Client
import datetime
import pandas as pd
import pymssql
import pymysql
from public import zgc_api,Index
from SimilarCharactor.string_similarity import similarity_cn,similarity_en
#上传数据字典补充数据
def ComplicatedDataDict(path):
#创建新产品库链接
conn_zi_new = pymssql.connect(host='123.56.115.207', user='zgcprice3311',password='zgcprice20200628',database= 'ZI_NEW',autocommit=True)
cursor_zi_new = conn_zi_new.cursor()
df = pd.read_excel(path)
process_index = 0
index_ = Index()
for index,row in df.iterrows():
try:
print(index_(process_index,len(df)-1), end='%')
except:
print(index_(process_index,1), end='%')
process_index+=1
cursor_zi_new.execute(f"update ShuJuZiDian_Cfg set stdvalue = '{row['stdvalue']}',simplevalue = '{row['simplevalue']}' where id = {row['id']}")
conn_zi_new.close()
#更新数据字典
path = '/Users/rico/Downloads/台式机数据字典补充(3)(1).xlsx'
ComplicatedDataDict(path)
数据治理平台线下处理/线下处理/参数补充线下处理导出导入脚本_4(校验配件价格).py 0 → 100755
View file @ afe6589d
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Dec 3 11:14:31 2021
@author: rico
"""
import sys
sys.path.append(os.path.dirname(os.getcwd()))
from ESCore.ES import ES_Client
import datetime
import pandas as pd
import pymssql
import pymysql
from public import zgc_api,Index
from SimilarCharactor.string_similarity import similarity_cn,similarity_en
#获取指定类别的参数项
def get_category_params_list(category_name):
conn_zi_new = pymssql.connect(host='123.56.115.207', user='zgcprice3311',password='zgcprice20200628',database= 'ZI_NEW',autocommit=True)
cursor_zi_new = conn_zi_new.cursor()
cursor_zi_new.execute(f"select subtitle,identy from vw_property where name = '{category_name}' and subtitle != '产品型号'")
db_params = pd.DataFrame(cursor_zi_new.fetchall(), columns=[tuple[0] for tuple in cursor_zi_new.description])
db_params['require_param'] = db_params['identy'].apply(lambda x: x[0])
db_params['standard_param'] = db_params['identy'].apply(lambda x: x[2])
db_params = db_params[(db_params['standard_param'] != '0') & (db_params['require_param'] != '0')]
db_params['subtitle'] = db_params['subtitle'].apply(lambda x: x.strip())
params_list = db_params['subtitle'].tolist()
return params_list
def check_configure_price(category,path):
if category not in ['笔记本','台式机','一体电脑','工作站']:
return "该类别无配件，无需校验配件价格"
params_list = get_category_params_list(category)
conn = pymssql.connect(host='123.56.115.207', user='zgcindex', password='jiayou202006', database='ZI_NEW',autocommit=True)
cursor = conn.cursor()
cursor.execute(f"select name,id from p_category where name in ('笔记本','台式机','一体电脑','工作站')")
category_df = pd.DataFrame([v for v in cursor.fetchall()],columns=[tuple[0] for tuple in cursor.description])
category_dict = dict(zip(category_df['name'].tolist(),category_df['id'].tolist()))
date = str(datetime.datetime.now()).split(" ")[0]
category_list = [category]
configure_price_df = get_configure_price()
cat_params_df = pd.read_excel(path)
try:
cat_params_df['驳回原因'] = cat_params_df['驳回原因'].apply(lambda x:str(x))
except:
cat_params_df['驳回原因'] = ['nan' for i in range(len(cat_params_df))]
cat_configure_price_df = configure_price_df[configure_price_df['category_name'] == category]
process_index = 0
index_ = Index()
res_list = []
for index,row in cat_params_df.iterrows():
try:
print(index_(process_index,len(cat_params_df)-1), end='%')
except:
print(index_(process_index,1), end='%')
process_index+=1
remark = row['驳回原因']
if remark != 'nan':
continue
for params in params_list:
value = row[params]
if params not in list(cat_configure_price_df['part']):
continue
else:
try:
current_price = cat_configure_price_df[(cat_configure_price_df['part'] == params) & (cat_configure_price_df['detail'] == value)]['price'].tolist()[0]
except:
mark = "无该配件价格：" + category + " " + params + " " + value
if mark in res_list:
continue
else:
res_list.append(mark)
category_code = category_dict[category]
cursor.execute(f"insert into configure_price (category_code,part,detail,create_date,category_name) \
values ('{category_code}','{params}','{value}','{date}','{category}')")
if res_list:
print("请补充配件价格")
print(res_list)
else:
print("配件价格齐全，无需补充")
return res_list
#校验通过的数据需要进行配件价格校验
category_name = '笔记本'
checked_path = "/Users/rico/Downloads/激光打印机参数补充回导.xlsx"
check_configure_price(category_name,checked_path)
数据治理平台线下处理/线下处理/参数补充线下处理导出导入脚本_5(导入).py 0 → 100755
View file @ afe6589d
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Dec 3 11:17:16 2021
@author: rico
"""
import sys
sys.path.append(os.path.dirname(os.getcwd()))
from ESCore.ES import ES_Client
import datetime
import pandas as pd
import pymssql
import pymysql
from public import zgc_api,Index
from SimilarCharactor.string_similarity import similarity_cn,similarity_en
#获取指定类别的参数项
def get_category_params_list(category_name):
conn_zi_new = pymssql.connect(host='123.56.115.207', user='zgcprice3311',password='zgcprice20200628',database= 'ZI_NEW',autocommit=True)
cursor_zi_new = conn_zi_new.cursor()
cursor_zi_new.execute(f"select subtitle,identy from vw_property where name = '{category_name}' and subtitle != '产品型号'")
db_params = pd.DataFrame(cursor_zi_new.fetchall(), columns=[tuple[0] for tuple in cursor_zi_new.description])
db_params['require_param'] = db_params['identy'].apply(lambda x: x[0])
db_params['standard_param'] = db_params['identy'].apply(lambda x: x[2])
db_params = db_params[(db_params['standard_param'] != '0') & (db_params['require_param'] != '0')]
db_params['subtitle'] = db_params['subtitle'].apply(lambda x: x.strip())
params_list = db_params['subtitle'].tolist()
return params_list
def import_params_data(es,category_name,path):
#更新补参数据
df = pd.read_excel(path,converters = {'CPU型号':str})
try:
df['驳回原因'] = df['驳回原因'].apply(lambda x:str(x))
except:
df['驳回原因'] = ['nan' for i in range(len(df))]
params_list = get_category_params_list(category_name)
process_index = 0
index_ = Index()
for index,row in df.iterrows():
try:
print(index_(process_index,len(df)-1), end='%')
except:
print(index_(process_index,1), end='%')
process_index+=1
search_field = "DOC_ID"
search_field_value = str(row['DOC_ID'])
remark = row['驳回原因']
if remark != 'nan':
now_time = str(datetime.datetime.now()).replace(' ','T').split('.')[0]+"+08:00"
es.UpdateParamsReturnData(search_field,search_field_value,remark,now_time)
continue
#update_field = "DEAL_INPUT_PARAMS_DICT" #DEAL_STD_PARAMS_DICT DEAL_INPUT_PARAMS_DICT FLOW_NODE_STATUS
#update_field_value = "SYS_GET_CATEGORY_SUCCESS_REDUCTIVE"
update_dict = {}
for params in params_list:
update_dict.update({params:str(row[params]).replace('\xa0',' ')})
params_info = str(update_dict).replace("'",'\"')
es.UpdateParamsDealData(category_name,search_field,search_field_value,params_info)
#导入数据
host = "http://123.56.114.138:9200/"
index_name = "model_params_test"
es = ES_Client(host,index_name)
category_name = '笔记本'
checked_path = "/Users/rico/Downloads/激光打印机参数补充回导.xlsx"
import_params_data(es,category_name,checked_path)
数据治理平台线下处理/线下处理/参数补充线下处理导出导入脚本_6(查看处理结果并导出确认).py 0 → 100755
View file @ afe6589d
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Dec 3 11:29:33 2021
@author: rico
"""
import sys
sys.path.append(os.path.dirname(os.getcwd()))
from ESCore.ES import ES_Client
import datetime
import pandas as pd
import pymssql
import pymysql
from public import zgc_api,Index
from SimilarCharactor.string_similarity import similarity_cn,similarity_en
host = "http://123.56.114.138:9200/"
index_name = "model_params_test"
es = ES_Client(host,index_name)
#按批次号检查结果数据
res = es.SearchByCondition("DATA_BATCH",["202111231900550001"])
#按项目检查结果数据
#res = es.SearchAimAllByCondition("DATA_PROJECT_NAME","徽采商城数据入库")
##按批次下或者项目下的指定类检查结果数据
#res = res[res['FINISH_P_LASTCATEGORY_NAME'] == '台式机']
'''
#状态总揽
FLOW_NODE_STATUS
INIT：初始化状态
STOP_WORD_FAILD：停用词失败
SKU_EXIST：SKU匹配成功（终止态）
URL_EXIST：URL匹配成功（终止态）
FIRST_CHECK_SUCCESS：初步审核成功
SYS_GET_BRAND_SUCCESS：品牌提取成功
SYS_GET_BRAND_FAILD：品牌提取失败
SYS_GET_CATEGORY_SUCCESS_NONPOINT：类别提取成功-非重点类
SYS_GET_CATEGORY_SUCCESS_NONREDUCTIVE：类别提取成功-非还原类
SYS_GET_CATEGORY_SUCCESS_REDUCTIVE：类别提取成功-还原类
SYS_GET_CATEGORY_FAILD：类别提取失败
SYS_GET_MODEL_SUCCESS_NONPOINT：型号提取成功-非重点类
SYS_GET_MODEL_SUCCESSS_NONREDUCTIVE：型号提取成功-非还原类
SYS_GET_MODEL_FAILD_NONPOINT：型号提取失败-非重点类
SYS_GET_MODEL_FAILD_NONREDUCTIVE：型号提取失败-非还原类
SYS_GET_PARAMS_SUCCESS_NONREDUCTIVE：匹配标准参数提取成功-非还原类
SYS_GET_PARAMS_SUCCESS_REDUCTIVE：匹配标准参数提取成功-还原类
SYS_GET_PARAMS_FAILD_NONREDUCTIVE：匹配标准参数提取失败-非还原类
SYS_GET_PARAMS_FAILD_REDUCTIVE：匹配标准参数提取失败-还原类
SYS_FINISH_MATCH：产品名称或匹配项匹配成功（终止态）
SYS_GET_PARAMS_SUCCESS_END：必填项参数提取成功（终止态）
SYS_GET_PARAMS_FAILD_END：必填项参数提取失败
'''
#分析原则：确保下方统计的数据除了驳回的以外其它数据都处于终止态。否则去页面查看处理，或者反馈北京
#数据总揽
print(res[['DOC_ID','FLOW_NODE_STATUS','FINISH_P_LASTCATEGORY_NAME']].groupby(['FLOW_NODE_STATUS','FINISH_P_LASTCATEGORY_NAME']).count())
#各节点驳回的
print(res[['DOC_ID','FLOW_NODE_STATUS','FINISH_P_REMARK']].groupby(['FLOW_NODE_STATUS','FINISH_P_REMARK']).count())
#校验是否有未命名的商品
category_name = '台式机'
if category_name:
check_flag = True
if category_name in ['笔记本','台式机','一体电脑','复印纸','工作站']:
FLOW_NODE_STATUS = "SYS_GET_PARAMS_SUCCESS_REDUCTIVE"
else:
FLOW_NODE_STATUS = "SYS_GET_PARAMS_SUCCESS_NONREDUCTIVE"
search_field_list = ['FINISH_P_LASTCATEGORY_NAME','FLOW_NODE_STATUS','FINISH_P_NAME','FLOW_FINISH_STATUS']
search_field_value_list = [category_name,'SYS_FINISH_MATCH','','Y']
update_field = "FLOW_FINISH_STATUS"
update_field_value = "N"
update_res_1 = es.UpdateByMultipleFourCondition(search_field_list,search_field_value_list,update_field,update_field_value)
if update_res_1['updated'] != 0:
check_flag = False
search_field_list = ['FINISH_P_LASTCATEGORY_NAME','FLOW_NODE_STATUS','FINISH_P_NAME','FLOW_FINISH_STATUS']
search_field_value_list = [category_name,'SYS_FINISH_MATCH','','N']
update_field = "FLOW_NODE_STATUS"
update_field_value = FLOW_NODE_STATUS
update_res_2 = es.UpdateByMultipleFourCondition(search_field_list,search_field_value_list,update_field,update_field_value)
if update_res_2['updated'] != 0:
check_flag = False
search_field_list = ['FINISH_P_LASTCATEGORY_NAME','FLOW_NODE_STATUS','FINISH_P_NAME','FLOW_FINISH_STATUS']
search_field_value_list = [category_name,'SYS_GET_PARAMS_SUCCESS_END','','Y']
update_field = "FLOW_FINISH_STATUS"
update_field_value = "N"
update_res_3 = es.UpdateByMultipleFourCondition(search_field_list,search_field_value_list,update_field,update_field_value)
if update_res_3['updated'] != 0:
check_flag = False
search_field_list = ['FINISH_P_LASTCATEGORY_NAME','FLOW_NODE_STATUS','FINISH_P_NAME','FLOW_FINISH_STATUS']
search_field_value_list = [category_name,'SYS_GET_PARAMS_SUCCESS_END','','N']
update_field = "FLOW_NODE_STATUS"
update_field_value = FLOW_NODE_STATUS
update_res_4 = es.UpdateByMultipleFourCondition(search_field_list,search_field_value_list,update_field,update_field_value)
if update_res_4['updated'] != 0:
check_flag = False
if check_flag:
date = str(datetime.datetime.now()).split(".")[0]
res.to_excel(f"（未确认）{date}数据.xlsx")
数据治理平台线下处理/线下处理/基础信息线下处理脚本.py → 数据治理平台线下处理/线下处理/基础信息线下处理导入脚本.py
View file @ afe6589d
...@@ -239,7 +239,10 @@ def check_data(es,type,path): ...@@ -239,7 +239,10 @@ def check_data(es,type,path):
return False,path return False,path
'''
品牌、类别、型号补充完成后导入脚本
根据数据情况，修改type和path参数即可
'''
#创建ES客户端 #创建ES客户端
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment