Commit 70a80df7 authored by huangziyu's avatar huangziyu

1

parent 962a3e54
...@@ -80,7 +80,7 @@ def get_cookie(): ...@@ -80,7 +80,7 @@ def get_cookie():
count = count + 1 count = count + 1
print(f"需要登录") print(f"需要登录")
time.sleep(1) time.sleep(1)
query_spider_sql = f"SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and project_id = '41' and data_batch ='202404180000000004' limit 30000" query_spider_sql = f"SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and project_id = '41' limit 30000"
query_spider = mysql_zdzs_proxy(query_spider_sql) query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0 index = 0
len_ = len(query_spider) len_ = len(query_spider)
......
...@@ -91,7 +91,7 @@ def get_cookie(username=None, password=None, driver=None): ...@@ -91,7 +91,7 @@ def get_cookie(username=None, password=None, driver=None):
count = count + 1 count = count + 1
print(f"需要登录") print(f"需要登录")
time.sleep(1) time.sleep(1)
query_spider_sql = f"SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and project_id = '41' and data_batch ='202404180000000004' and id like '%0' limit 30000" query_spider_sql = f"SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and project_id = '41' and id like '%0' limit 30000"
query_spider = mysql_zdzs_proxy(query_spider_sql) query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0 index = 0
len_ = len(query_spider) len_ = len(query_spider)
......
...@@ -91,7 +91,7 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -91,7 +91,7 @@ def get_cookie(username=None,password=None,driver=None):
count = count + 1 count = count + 1
print(f"需要登录") print(f"需要登录")
time.sleep(1) time.sleep(1)
query_spider_sql = f"SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and project_id = '41' and data_batch ='202404180000000004' and id like '%1' limit 30000" query_spider_sql = f"SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and project_id = '41' and id like '%1' limit 30000"
query_spider = mysql_zdzs_proxy(query_spider_sql) query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0 index = 0
len_ = len(query_spider) len_ = len(query_spider)
......
...@@ -91,7 +91,7 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -91,7 +91,7 @@ def get_cookie(username=None,password=None,driver=None):
count = count + 1 count = count + 1
print(f"需要登录") print(f"需要登录")
time.sleep(1) time.sleep(1)
query_spider_sql = f"SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and project_id = '41' and data_batch ='202404180000000004' and id like '%2' limit 30000" query_spider_sql = f"SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and project_id = '41' and id like '%2' limit 30000"
query_spider = mysql_zdzs_proxy(query_spider_sql) query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0 index = 0
len_ = len(query_spider) len_ = len(query_spider)
......
...@@ -91,7 +91,7 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -91,7 +91,7 @@ def get_cookie(username=None,password=None,driver=None):
count = count + 1 count = count + 1
print(f"需要登录") print(f"需要登录")
time.sleep(1) time.sleep(1)
query_spider_sql = f"SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and project_id = '41' and data_batch ='202404180000000004' and id like '%3' limit 30000" query_spider_sql = f"SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and project_id = '41' and id like '%3' limit 30000"
query_spider = mysql_zdzs_proxy(query_spider_sql) query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0 index = 0
len_ = len(query_spider) len_ = len(query_spider)
......
...@@ -91,7 +91,7 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -91,7 +91,7 @@ def get_cookie(username=None,password=None,driver=None):
count = count + 1 count = count + 1
print(f"需要登录") print(f"需要登录")
time.sleep(1) time.sleep(1)
query_spider_sql = f"SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and project_id = '41' and data_batch ='202404180000000004' and id like '%4' limit 30000" query_spider_sql = f"SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and project_id = '41' and id like '%4' limit 30000"
query_spider = mysql_zdzs_proxy(query_spider_sql) query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0 index = 0
len_ = len(query_spider) len_ = len(query_spider)
......
...@@ -91,7 +91,7 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -91,7 +91,7 @@ def get_cookie(username=None,password=None,driver=None):
count = count + 1 count = count + 1
print(f"需要登录") print(f"需要登录")
time.sleep(1) time.sleep(1)
query_spider_sql = f"SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and project_id = '41' and data_batch ='202404180000000004' and id like '%5' limit 30000" query_spider_sql = f"SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and project_id = '41' and id like '%5' limit 30000"
query_spider = mysql_zdzs_proxy(query_spider_sql) query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0 index = 0
len_ = len(query_spider) len_ = len(query_spider)
......
...@@ -91,7 +91,7 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -91,7 +91,7 @@ def get_cookie(username=None,password=None,driver=None):
count = count + 1 count = count + 1
print(f"需要登录") print(f"需要登录")
time.sleep(1) time.sleep(1)
query_spider_sql = f"SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and project_id = '41' and data_batch ='202404180000000004' and id like '%6' limit 30000" query_spider_sql = f"SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and project_id = '41' and id like '%6' limit 30000"
query_spider = mysql_zdzs_proxy(query_spider_sql) query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0 index = 0
len_ = len(query_spider) len_ = len(query_spider)
......
...@@ -91,7 +91,7 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -91,7 +91,7 @@ def get_cookie(username=None,password=None,driver=None):
count = count + 1 count = count + 1
print(f"需要登录") print(f"需要登录")
time.sleep(1) time.sleep(1)
query_spider_sql = f"SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and project_id = '41' and data_batch ='202404180000000004' and id like '%7' limit 30000" query_spider_sql = f"SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and project_id = '41' and id like '%7' limit 30000"
query_spider = mysql_zdzs_proxy(query_spider_sql) query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0 index = 0
len_ = len(query_spider) len_ = len(query_spider)
......
...@@ -91,7 +91,7 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -91,7 +91,7 @@ def get_cookie(username=None,password=None,driver=None):
count = count + 1 count = count + 1
print(f"需要登录") print(f"需要登录")
time.sleep(1) time.sleep(1)
query_spider_sql = f"SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and project_id = '41' and data_batch ='202404180000000004' and id like '%8' limit 30000" query_spider_sql = f"SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and project_id = '41' and id like '%8' limit 30000"
query_spider = mysql_zdzs_proxy(query_spider_sql) query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0 index = 0
len_ = len(query_spider) len_ = len(query_spider)
......
...@@ -91,7 +91,7 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -91,7 +91,7 @@ def get_cookie(username=None,password=None,driver=None):
count = count + 1 count = count + 1
print(f"需要登录") print(f"需要登录")
time.sleep(1) time.sleep(1)
query_spider_sql = f"SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and project_id = '41' and data_batch ='202404180000000004' and id like '%9' limit 30000" query_spider_sql = f"SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and project_id = '41' and id like '%9' limit 30000"
query_spider = mysql_zdzs_proxy(query_spider_sql) query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0 index = 0
len_ = len(query_spider) len_ = len(query_spider)
......
...@@ -361,7 +361,7 @@ def get_cookie(): ...@@ -361,7 +361,7 @@ def get_cookie():
# query_spider_sql = f"SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and data_batch ='202403051815460001' order by id" \ # query_spider_sql = f"SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and data_batch ='202403051815460001' order by id" \
# f" limit 100000" # f" limit 100000"
# query_spider_sql = f"-- SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and project_id = '41' and (id like '%5' or id like '%6' or id like '%7' or id like '%8' or id like '%9' ) limit 10000" # query_spider_sql = f"-- SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and project_id = '41' and (id like '%5' or id like '%6' or id like '%7' or id like '%8' or id like '%9' ) limit 10000"
query_spider_sql = f"SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and project_id = '41' and data_batch ='202404180000000004' " query_spider_sql = f"SELECT id,url,mallId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' and project_id = '41' "
query_spider = mysql_zdzs_proxy(query_spider_sql) query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0 index = 0
len_ = len(query_spider) len_ = len(query_spider)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment