Skip to content

中
中电中采
Commit 2e6bc4f1 authored by rico.liu's avatar rico.liu
Browse files
Options

update new

parent f2fcfa4a
Hide whitespace changes
Inline Side-by-side
Showing with 77 additions and 46 deletions

Too many changes to show.
Plain diff Email patch

To preserve performance only 1000 of 1000+ files are displayed.

API数据处理.py
View file @ 2e6bc4f1
......@@ -7,9 +7,10 @@ Created on Tue Mar 30 15:27:36 2021
"""
from API数据处理func import *
channel_list = ["CG","LX","LXWL","STB","ZKH","HCW","SGGJ","XY","OFS","YXDT"]
batch='2021-05-07'
channel_list = ["CG","LX","LXWL","STB","ZKH","HCW","SGGJ","XY","XFS","OFS","YXDT"]
batch='2021-03-25'
#上传数据
upload_data(channel_list)
......@@ -35,7 +36,7 @@ analyse_basic_info(df)
export_2_excel_basic_confirm(channel_list,batch)
#返回数据并更新
path = r"E:\work\刘晓琦代码\客户数据处理（调接口）\原始客户数据\山西路桥\20210326\山西路桥基础信息确认3.26（1）.xlsx"
path = r"/Users/rico/Downloads/1 API-过程数据-2021-05-07.xlsx"
update_brand_category_data(path)
......@@ -49,7 +50,7 @@ df = get_nonpoint_category_data_all(channel_list)
match_result = get_match_brand_product(df)
writer = pd.ExcelWriter(fr"E:\work\刘晓琦代码\客户数据处理（调接口）\原始客户数据\山西路桥\20210326\SXLQ非重点类匹配结果确认及命名-批量-{batch}.xlsx")
writer = pd.ExcelWriter(f"SXLQ非重点类匹配结果确认及命名-批量-'{batch}'.xlsx")
export_excel_1 = match_result[match_result['fuzzy_match_result'] == 1]
export_excel_1.to_excel(writer,'匹配到的数据')
......@@ -59,7 +60,7 @@ writer.save()
#处理重点类数据
path = fr"E:\work\刘晓琦代码\客户数据处理（调接口）\原始客户数据\山西路桥\20210323\SXLQ重点类参数确认-2021-03-23（1）.xlsx"
path = fr"/Users/rico/Work Space/1_Project/Company/中电中采/DDMP数据处理流/offline_code/4 反馈-SXLQ重点类参数确认-2021-05-07.xlsx"
##校验参数数据
std_value_dict = check_params_data(path)
##导入补充的数据字典
......@@ -99,7 +100,7 @@ export_excel_2.to_excel(writer,'未匹配数据处理')
writer.save()
'''
path = r"E:\work\刘晓琦代码\客户数据处理（调接口）\原始客户数据\山西路桥\20210326\SXLQ非重点类匹配结果确认及命名-批量-2021-03-25（1）.xlsx"
path = r"/Users/rico/Work Space/1_Project/Company/中电中采/DDMP数据处理流/offline_code/5 反馈-SXLQ非重点类匹配结果确认及命名-批量-'2021-05-07'.xlsx"
mode = 'all_data'
update_non_point_to_db(path,mode)
......@@ -116,6 +117,6 @@ get_name(df)
'''
#导出处理结果
path = r'E:\work\刘晓琦代码\客户数据处理（调接口）\原始客户数据\山西路桥\20210326\山西路桥基础信息确认3.26（1）.xlsx'
path = r'/Users/rico/Downloads/1 API-过程数据-2021-05-07.xlsx'
export_result_data(path)
API数据处理func.py
View file @ 2e6bc4f1
......@@ -340,7 +340,7 @@ def export_2_excel_basic_confirm(channel_list,batch):
def update_brand_category_data(path):
df = pd.read_excel(path,sheetname = "Sheet1",converters={'zi_brandcode':str,'zi_subcategorycode':str})
df = pd.read_excel(path,sheet_name = "Sheet1",converters={'zi_brandcode':str,'zi_subcategorycode':str})
df['zi_brandcode'] = df['zi_brandcode'].apply(lambda x:str(x))
df['zi_subcategorycode'] = df['zi_subcategorycode'].apply(lambda x:str(x))
df['remark'] = df['remark'].apply(lambda x:str(x))
......@@ -931,7 +931,7 @@ def check_params_data(path):
try:
point_category = category.replace('/','_')
cat_df = pd.read_excel(path,sheetname = f"{point_category}参数数据",converters = {'productcode':str})
cat_df = pd.read_excel(path,sheet_name = f"{point_category}参数数据",converters = {'productcode':str})
except:
continue
......@@ -1023,8 +1023,6 @@ def check_params_data(path):
init_fraction = 0
recommend_pri = ''
for dict_pri in data_dict[data_dict['subtitle'] == db_param]['primitive'].unique().tolist():
pri_value = str(pri_value)
dict_pri = str(dict_pri)
try:
fraction_part1 = similarity_cn(pri_value,dict_pri)
except:
......@@ -1202,7 +1200,7 @@ def create_to_db(path,std_value_dict):
for category in point_category_list:
try:
point_category = category.replace('/','_')
cat_df = pd.read_excel(path,sheetname = f"{point_category}参数数据",converters = {'productcode':str})
cat_df = pd.read_excel(path,sheet_name = f"{point_category}参数数据",converters = {'productcode':str})
#print(category)
channel_list = cat_df['channel_alias'].unique().tolist()
......@@ -1251,7 +1249,8 @@ def create_to_db(path,std_value_dict):
id_ = int(row['id'])
skuname = ''
brandname = row['zi_brandname']
cursor_zi_new.execute(f"select pid from p_brand where name = '{brandname}'")
father_brand_code = cursor_zi_new.fetchone()[0]
for element in named_rule.split(" "):
if element == '品牌名称':
......@@ -1705,8 +1704,11 @@ def create_to_db(path,std_value_dict):
new_df_combine_list.append(export_df)
conn_zi_lq.close()
new_df_combine = pd.concat(new_df_combine_list)
try:
new_df_combine = pd.concat(new_df_combine_list)
except:
new_df_combine = pd.DataFrame()
conn_zi_new.close()
conn_zi_price.close()
......@@ -1879,7 +1881,7 @@ def calculate_spu_price(new_df_all):
def update_non_point_to_db(path,mode):
df_match = pd.read_excel(path,sheetname = '匹配到的数据',converters = {'fuzzy_match_code':str,'zi_subcategorycode':str})
df_match = pd.read_excel(path,sheet_name = '匹配到的数据',converters = {'fuzzy_match_code':str,'zi_subcategorycode':str})
df_match['remark'] = df_match['remark'].apply(lambda x : str(x))
df_match['fuzzy_match_code'] = df_match['fuzzy_match_code'].apply(lambda x : str(x))
#del df_match['Unnamed: 0']
......@@ -1925,7 +1927,7 @@ def update_non_point_to_db(path,mode):
if mode == 'match_data':
return "完成数据更新"
df_unmatch = pd.read_excel(path,sheetname = '未匹配数据处理',converters = {'fuzzy_match_code':str,'zi_subcategorycode':str})
df_unmatch = pd.read_excel(path,sheet_name = '未匹配数据处理',converters = {'fuzzy_match_code':str,'zi_subcategorycode':str})
df_unmatch['remark'] = df_unmatch['remark'].apply(lambda x : str(x))
df_unmatch['fuzzy_match_code'] = df_unmatch['fuzzy_match_code'].apply(lambda x : str(x))
......@@ -1967,7 +1969,7 @@ def update_non_point_to_db(path,mode):
#导出处理结果
def export_result_data(path):
df = pd.read_excel(path,sheetname='Sheet1')
df = pd.read_excel(path,sheet_name='Sheet1')
channel_list = df['channel'].unique().tolist()
conn = create_engine('mssql+pymssql://zgcprice:zgcprice20200708@123.57.45.119/SXLQ_DS')
......@@ -1984,7 +1986,7 @@ def export_result_data(path):
_id = row['id']
_remark = row['remark']
print(index+1,len(channel_df))
#print(index+1,len(channel_df))
if pd.isnull(_remark):
cursor_zi_lq.execute(f"select sku,productcode,channel_alias,name,new_name,price from product_all where id = {_id} and state = '9'")
......@@ -2005,7 +2007,7 @@ def export_result_data(path):
df_sys = pd.concat(df_sys_list)
df_sys.to_sql(name="product_all", con=conn, if_exists='append',index = False)
'''
res = pd.concat(df_list)
res['sku（产品名称）'] = ''
......@@ -2022,9 +2024,10 @@ def export_result_data(path):
data_sku = data['sku']
data_code = data['productcode']
data_dbname = data['channel_alias']
print(data_dbname)
#print(i+1,len(res),data_dbname)
conn_zi_xt = pymssql.connect(host='123.57.45.119', user='zgcprice', password='zgcprice20200708',database= f"{data_dbname}",autocommit=True)
cursor_zi_xt = conn_zi_lq.cursor()
cursor_zi_xt = conn_zi_xt.cursor()
#print(f"select name from {data_dbname}.dbo.product_all where sku='{data_sku}'")
cursor_zi_xt.execute(f"select name from product_all where sku='{data_sku}'")
try:
......@@ -2044,5 +2047,6 @@ def export_result_data(path):
conn_xt.close()
cursor_xt.close()
del df['Unnamed: 0']
res.to_excel(r'SXLQ全量结果.xlsx')
\ No newline at end of file
'''
df_sys.to_excel(f"SXLQ全量结果.xlsx")
#res.to_excel(r'SXLQ全量结果.xlsx')
\ No newline at end of file
Excel数据处理.py
View file @ 2e6bc4f1
......@@ -9,9 +9,9 @@ Created on Tue Mar 30 13:52:36 2021
from Excel数据处理func import *
path = r'E:\work\刘晓琦代码\客户数据处理（调接口）\原始客户数据\泰康人寿TKRS\20210315\泰康入库模板.xlsx'
channel_alias = 'AH'
batch = '2021-03-31'
path = r'/Users/rico/Downloads/0 四川4月入库模板.xlsx'
channel_alias = 'SC_ZDJ'
batch = '2021-05-13'
#初始化数据
InitializeData(path,channel_alias)
......@@ -32,7 +32,7 @@ AnalyseBasicInfo(df)
ExportToExcelBasicConfirm(batch,channel_alias)
#返回数据并更新
path = r"E:\work\刘晓琦代码\客户数据处理（调接口）\原始客户数据\泰康人寿TKRS\20210315\TKRS基础信息确认2021-03-15（1）.xlsx"
path = r"/Users/rico/Downloads/SC基础信息确认2021-05-13.xlsx"
UpdateBasicData(path)
......@@ -58,12 +58,12 @@ writer.save()
#处理重点类数据
path = r"E:\work\刘晓琦代码\客户数据处理（调接口）\原始客户数据\泰康人寿TKRS\20210315\TKRS重点类参数确认-2021-03-15（1）.xlsx"
path = r"/Users/rico/Downloads/反馈-SC重点类参数确认-2021-05-13(3).xlsx"
##校验参数数据
std_value_dict = CheckPointParamsData(path)
##导入补充的数据字典
data_dict_path = r"E:\work\刘晓琦代码\客户数据处理（调接口）\原始客户数据\泰康人寿TKRS\20210315\TKRS重点类参数确认-2021-03-15（1）(数据字典补充)(1).xlsx"
data_dict_path = r"/Users/rico/Downloads/确认-SC重点类参数确认-2021-05-13(3)(数据字典补充).xlsx"
ComplicatedDataDict(data_dict_path)
##入库(std_value_dict为空时不可往下执行)
......@@ -84,7 +84,7 @@ add_configure_price_difference(new_df_all)
#处理非重点类数据
##更新非重点类数据
path = r"E:\work\刘晓琦代码\客户数据处理（调接口）\原始客户数据\山西路桥\20210308入库\2021-03-08-sxlq_pp非重点类匹配结果确认及命名(1).xlsx"
path = r"/Users/rico/Downloads/反馈-2021-05-13-SC非重点类匹配结果确认及命名(11).xlsx"
mode = 'all_data'
update_non_point_to_db(path,mode)
......
Excel数据处理func.py
View file @ 2e6bc4f1
......@@ -28,24 +28,24 @@ from public.SKUParamsInfo.getParamsInfo import *
def InitializeData(path,channel_alias):
#加载上传数据
df_product = pd.read_excel(path,sheetname = '产品表',converters = {'sku':str})
df_product = pd.read_excel(path,sheet_name = '产品表',converters = {'sku':str})
df_product = df_product.fillna('无')
try:
df_params = pd.read_excel(path,sheetname = '参数表',converters = {'sku':str})
df_params = pd.read_excel(path,sheet_name = '参数表',converters = {'sku':str})
df_params = df_params.fillna('无')
except:
print('无参数表')
df_params = pd.DataFrame()
try:
df_category_map = pd.read_excel(path,sheetname = '类别对应表')
df_category_map = pd.read_excel(path,sheet_name = '类别对应表')
except:
print('无类别对应表')
df_category_map = pd.DataFrame()
try:
df_attr_map = pd.read_excel(path,sheetname = '参数项对应表')
df_attr_map = pd.read_excel(path,sheet_name = '参数项对应表')
except:
print('无参数对应表')
df_attr_map = pd.DataFrame()
......@@ -241,15 +241,16 @@ def SkuMatch(df,channel_alias,engine):
cursor_zi_new.execute(f"select skuname from p_sku where sku = '{code}'")
try:
input_name = cursor_zi_new.fetchone()[0]
input_name = input_name.replace("'","''")
except:
input_name = '未查询到数据'
if name == name_his:
print(f"update product_all set state = '9',productcode = '{code}',new_name = '{input_name}' ,pic_state = '0',is_different = '0' where id = {id_}")
cursor_zi_service.execute(f"update product_all set state = '9',productcode = '{code}',new_name = '{input_name}' ,pic_state = '0',is_different = '0' where id = {id_}")
else:
print(f"update product_all set state = '9',productcode = '{code}',new_name = '{input_name}' ,pic_state = '0',is_different = '1' where id = {id_}")
cursor_zi_service.execute(f"update product_all set state = '9',productcode = '{code}',new_name = '{input_name}' ,pic_state = '0',is_different = '1' where id = {id_}")
mssql.Close()
......@@ -403,7 +404,7 @@ def ExportToExcelBasicConfirm(batch,channel_alias):
def UpdateBasicData(path):
#读取Excel数据
df = pd.read_excel(path,sheetname = "Sheet1",converters={'zi_brandcode':str,'zi_subcategorycode':str,'zi_subcategoryname':str})
df = pd.read_excel(path,sheet_name = "Sheet1",converters={'zi_brandcode':str,'zi_subcategorycode':str,'zi_subcategoryname':str})
df['zi_brandcode'] = df['zi_brandcode'].apply(lambda x:str(x))
df['zi_subcategorycode'] = df['zi_subcategorycode'].apply(lambda x:str(x))
df['remark'] = df['remark'].apply(lambda x:str(x))
......@@ -1058,7 +1059,7 @@ def CheckPointParamsData(path):
try:
point_category = category.replace('/','_')
cat_df = pd.read_excel(path,sheetname = f"{point_category}参数数据",converters = {'productcode':str})
cat_df = pd.read_excel(path,sheet_name = f"{point_category}参数数据",converters = {'productcode':str})
except:
continue
......@@ -1165,10 +1166,12 @@ def CheckPointParamsData(path):
if fraction > init_fraction:
init_fraction = fraction
recommend_pri = dict_pri
recommend_stdvalue = data_dict[data_dict['primitive'] == recommend_pri]['stdvalue'].tolist()[0]
recommend_simplevalue = data_dict[data_dict['primitive'] == recommend_pri]['simplevalue'].tolist()[0]
try:
recommend_stdvalue = data_dict[data_dict['primitive'] == recommend_pri]['stdvalue'].tolist()[0]
recommend_simplevalue = data_dict[data_dict['primitive'] == recommend_pri]['simplevalue'].tolist()[0]
except:
recommend_stdvalue = '无'
recommend_simplevalue = '无'
flag = False
cursor_zi_new.execute(f"insert into ShuJuZiDian_Cfg (categoryname,subtitle,primitive,eg_brand_name,eg_product_name,eg_product_url,recommend_primitive,recommend_stdvalue,recommend_simplevalue) \
......@@ -1335,7 +1338,7 @@ def create_to_db(path,std_value_dict):
try:
point_category = category.replace('/','_')
cat_df = pd.read_excel(path,sheetname = f"{point_category}参数数据",converters = {'productcode':str})
cat_df = pd.read_excel(path,sheet_name = f"{point_category}参数数据",converters = {'productcode':str})
channel_list = cat_df['channel_alias'].unique().tolist()
......@@ -2021,7 +2024,7 @@ def update_non_point_to_db(path,mode):
cursor_zi_service = mssql._cur
df_match = pd.read_excel(path,sheetname = '匹配到的数据',converters = {'fuzzy_match_code':str,'zi_subcategorycode':str})
df_match = pd.read_excel(path,sheet_name = '匹配到的数据',converters = {'fuzzy_match_code':str,'zi_subcategorycode':str})
df_match['remark'] = df_match['remark'].apply(lambda x : str(x))
df_match['fuzzy_match_code'] = df_match['fuzzy_match_code'].apply(lambda x : str(x))
#del df_match['Unnamed: 0']
......@@ -2064,7 +2067,7 @@ def update_non_point_to_db(path,mode):
mssql.Close()
return "完成数据更新"
df_unmatch = pd.read_excel(path,sheetname = '未匹配数据处理',converters = {'fuzzy_match_code':str,'zi_subcategorycode':str})
df_unmatch = pd.read_excel(path,sheet_name = '未匹配数据处理',converters = {'fuzzy_match_code':str,'zi_subcategorycode':str})
df_unmatch['remark'] = df_unmatch['remark'].apply(lambda x : str(x))
df_unmatch['fuzzy_match_code'] = df_unmatch['fuzzy_match_code'].apply(lambda x : str(x))
......
pic/.DS_Store 0 → 100644
View file @ 2e6bc4f1
File added
pic/2021-05-08-2/20210508162159882-1.jpg 0 → 100644
View file @ 2e6bc4f1
{"Code":"NoSuchURL","Msg":"The specified URL does not exist."}
\ No newline at end of file
pic/2021-05-08-2/20210508162159882-2.jpg 0 → 100644
View file @ 2e6bc4f1
{"Code":"NoSuchURL","Msg":"The specified URL does not exist."}
\ No newline at end of file
pic/2021-05-08-2/20210508162159882-3.jpg 0 → 100644
View file @ 2e6bc4f1
{"Code":"NoSuchURL","Msg":"The specified URL does not exist."}
\ No newline at end of file
pic/2021-05-08-2/20210508162159882-4.jpg 0 → 100644
View file @ 2e6bc4f1
{"Code":"NoSuchURL","Msg":"The specified URL does not exist."}
\ No newline at end of file
pic/2021-05-08-2/20210508162159983-1.jpg 0 → 100644
View file @ 2e6bc4f1
{"Code":"NoSuchURL","Msg":"The specified URL does not exist."}
\ No newline at end of file
pic/2021-05-08-2/20210508162159983-2.jpg 0 → 100644
View file @ 2e6bc4f1
{"Code":"NoSuchURL","Msg":"The specified URL does not exist."}
\ No newline at end of file
pic/2021-05-08-2/20210508162159983-3.jpg 0 → 100644
View file @ 2e6bc4f1
{"Code":"NoSuchURL","Msg":"The specified URL does not exist."}
\ No newline at end of file
pic/2021-05-08-2/20210508162159983-4.jpg 0 → 100644
View file @ 2e6bc4f1
{"Code":"NoSuchURL","Msg":"The specified URL does not exist."}
\ No newline at end of file
pic/2021-05-08-2/20210508162200066-1.jpg 0 → 100644
View file @ 2e6bc4f1
{"Code":"NoSuchURL","Msg":"The specified URL does not exist."}
\ No newline at end of file
pic/2021-05-08-2/20210508162200066-2.jpg 0 → 100644
View file @ 2e6bc4f1
{"Code":"NoSuchURL","Msg":"The specified URL does not exist."}
\ No newline at end of file
pic/2021-05-08-2/20210508162200066-3.jpg 0 → 100644
View file @ 2e6bc4f1
{"Code":"NoSuchURL","Msg":"The specified URL does not exist."}
\ No newline at end of file
pic/2021-05-08-2/20210508162200066-4.jpg 0 → 100644
View file @ 2e6bc4f1
{"Code":"NoSuchURL","Msg":"The specified URL does not exist."}
\ No newline at end of file
pic/2021-05-08-2/20210508162200146-1.jpg 0 → 100644
View file @ 2e6bc4f1
{"Code":"NoSuchURL","Msg":"The specified URL does not exist."}
\ No newline at end of file
pic/2021-05-08-2/20210508162200146-2.jpg 0 → 100644
View file @ 2e6bc4f1
{"Code":"NoSuchURL","Msg":"The specified URL does not exist."}
\ No newline at end of file
pic/2021-05-08-2/20210508162200146-3.jpg 0 → 100644
View file @ 2e6bc4f1
{"Code":"NoSuchURL","Msg":"The specified URL does not exist."}
\ No newline at end of file
pic/2021-05-08-2/20210508162200146-4.jpg 0 → 100644
View file @ 2e6bc4f1
{"Code":"NoSuchURL","Msg":"The specified URL does not exist."}
\ No newline at end of file
pic/2021-05-25-1/20210525223553144-2.jpg 0 → 100644
View file @ 2e6bc4f1
{"Code":"NoSuchURL","Msg":"The specified URL does not exist."}
\ No newline at end of file
pic/2021-05-25-1/20210525223553144-3.jpg 0 → 100644
View file @ 2e6bc4f1
{"Code":"NoSuchURL","Msg":"The specified URL does not exist."}
\ No newline at end of file
pic/2021-05-25-1/20210525223553144-4.jpg 0 → 100644
View file @ 2e6bc4f1
{"Code":"NoSuchURL","Msg":"The specified URL does not exist."}
\ No newline at end of file
pic/2021-05-25-1/20210525223553144-5.jpg 0 → 100644
View file @ 2e6bc4f1
{"Code":"NoSuchURL","Msg":"The specified URL does not exist."}
\ No newline at end of file
pic/2021-05-25-1/20210525223553144-6.jpg 0 → 100644
View file @ 2e6bc4f1
{"Code":"NoSuchURL","Msg":"The specified URL does not exist."}
\ No newline at end of file
pic/2021-05-25-1/20210525223553144-7.jpg 0 → 100644
View file @ 2e6bc4f1
{"Code":"NoSuchURL","Msg":"The specified URL does not exist."}
\ No newline at end of file
pic/2021-05-25-1/20210525223553144-8.jpg 0 → 100644
View file @ 2e6bc4f1
{"Code":"NoSuchURL","Msg":"The specified URL does not exist."}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment