Commit ea2e97cd authored by rico.liu's avatar rico.liu

update

parent 2950afa8
...@@ -404,7 +404,7 @@ def GetParamsinfoAndPic(df): ...@@ -404,7 +404,7 @@ def GetParamsinfoAndPic(df):
url_params_list = [] url_params_list = []
for element,url_params in zip(res,crawl_params_list): for element,url_params in zip(res,crawl_params_list):
params_dict = eval(url_params) params_dict = eval(url_params)
params_dict.update({'爬取链接':element['url']}) #params_dict.update({'爬取链接':element['url']})
url_params_list.append(str(params_dict)) url_params_list.append(str(params_dict))
df['url_params'] = url_params_list df['url_params'] = url_params_list
...@@ -520,8 +520,10 @@ def GetCollectDataDetail(df,channel_alias,batch): ...@@ -520,8 +520,10 @@ def GetCollectDataDetail(df,channel_alias,batch):
#将爬取信息放到最后 #将爬取信息放到最后
param_list_all = requier_param_list + non_requier_param_list param_list_all = requier_param_list + non_requier_param_list
try: try:
param_list_all.remove("原始参数及链接信息") #param_list_all.remove("原始参数及链接信息")
param_list_all.append("原始参数及链接信息") #param_list_all.append("原始参数及链接信息")
param_list_all.remove("详细参数信息")
param_list_all.append("详细参数信息")
except: except:
pass pass
...@@ -577,7 +579,8 @@ def GetCollectDataDetail(df,channel_alias,batch): ...@@ -577,7 +579,8 @@ def GetCollectDataDetail(df,channel_alias,batch):
param_list.append(param) param_list.append(param)
new_name_list.append('') new_name_list.append('')
if param == '原始参数及链接信息': #if param == '原始参数及链接信息':
if param == '详细参数信息':
value = row['url_params'] value = row['url_params']
else: else:
try: try:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment