Skip to content

京
京东搜索采集自动验证
Commit 2aa7118d authored by huangziyu's avatar huangziyu
Browse files
Options

1

parent 60847175
Expand all Show whitespace changes
Inline Side-by-side
Showing with 365 additions and 166 deletions
config.py
View file @ 2aa7118d
This diff is collapsed.
京东搜索_手动验证版_0.py
View file @ 2aa7118d
...@@ -96,6 +96,7 @@ def get_cookie(username=None, password=None, driver=None): ...@@ -96,6 +96,7 @@ def get_cookie(username=None, password=None, driver=None):
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%0' " query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%0' "
query_spider = mysql_zdzs_proxy(query_spider_sql) query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0 index = 0
error_count = 0
len_ = len(query_spider) len_ = len(query_spider)
print(f"len={len_},index={index}") print(f"len={len_},index={index}")
if len_ == 0: if len_ == 0:
...@@ -111,8 +112,12 @@ def get_cookie(username=None, password=None, driver=None): ...@@ -111,8 +112,12 @@ def get_cookie(username=None, password=None, driver=None):
driver.get(url) driver.get(url)
while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！': while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print(url) print(url)
print("跳转首页了，结束，切换账号") error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return return
time.sleep(1)
continue
text = driver.page_source text = driver.page_source
while ( while (
...@@ -122,12 +127,21 @@ def get_cookie(username=None, password=None, driver=None): ...@@ -122,12 +127,21 @@ def get_cookie(username=None, password=None, driver=None):
text = driver.page_source text = driver.page_source
# 判断是否跳转了登录 # 判断是否跳转了登录
if '<title>京东-欢迎登录</title>' in text: if '<title>京东-欢迎登录</title>' in text:
print("需要登录，结束，切换账号") error_count += 1
if error_count >= 5:
print("需要登录，重试五次结束，切换账号")
return return
time.sleep(1)
continue
if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！': if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print("验证后还是首页，结束，切换账号") error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return return
time.sleep(1)
continue
error_count = 0
Response = etree.HTML(text) Response = etree.HTML(text)
dataList = [] dataList = []
...@@ -179,12 +193,18 @@ if __name__ == "__main__": ...@@ -179,12 +193,18 @@ if __name__ == "__main__":
driver.set_window_size(1200, 600) driver.set_window_size(1200, 600)
# get_cookie(username=config.account_list[0]['username'],password=config.account_list[0]['password'],driver=driver) # get_cookie(username=config.account_list[0]['username'],password=config.account_list[0]['password'],driver=driver)
account_start_index = 0 account_start_index = 0
get_cookie(username=config.account_list[account_start_index]['username'],password=config.account_list[account_start_index]['password'],driver=driver)
while True:
for account in config.account_list: for account in config.account_list:
username = account['username'] username = account['username']
# 登陆密码 # 登陆密码
password = account['password'] password = account['password']
print(f"切换账号：{username}") print(f"切换账号：{username}")
get_cookie(username=username, password=password, driver=driver) get_cookie(username=username, password=password, driver=driver)
# get_cookie(username=config.account_list[account_start_index]['username'],password=config.account_list[account_start_index]['password'],driver=driver)
# while True:
# for account in config.account_list:
# username = account['username']
# # 登陆密码
# password = account['password']
# print(f"切换账号：{username}")
# get_cookie(username=username, password=password, driver=driver)
driver.quit() driver.quit()
京东搜索_手动验证版_1.py
View file @ 2aa7118d
...@@ -12,6 +12,7 @@ from selenium.common.exceptions import NoSuchElementException ...@@ -12,6 +12,7 @@ from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver import ChromeOptions from selenium.webdriver import ChromeOptions
from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support import expected_conditions as EC
import traceback
import pymysql import pymysql
from lxml import etree from lxml import etree
# 登陆账号 # 登陆账号
...@@ -67,7 +68,7 @@ def mysql_zdzs_proxy(sql=None): ...@@ -67,7 +68,7 @@ def mysql_zdzs_proxy(sql=None):
return [] return []
def get_cookie(username=None,password=None,driver=None): def get_cookie(username=None, password=None, driver=None):
# driver.maximize_window() # driver.maximize_window()
driver.get('https://passport.jd.com/uc/login') driver.get('https://passport.jd.com/uc/login')
print(f"清空文本框开始") print(f"清空文本框开始")
...@@ -92,9 +93,10 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -92,9 +93,10 @@ def get_cookie(username=None,password=None,driver=None):
count = count + 1 count = count + 1
print(f"需要登录") print(f"需要登录")
time.sleep(1) time.sleep(1)
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%1'" query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%1' "
query_spider = mysql_zdzs_proxy(query_spider_sql) query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0 index = 0
error_count = 0
len_ = len(query_spider) len_ = len(query_spider)
print(f"len={len_},index={index}") print(f"len={len_},index={index}")
if len_ == 0: if len_ == 0:
...@@ -105,12 +107,17 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -105,12 +107,17 @@ def get_cookie(username=None,password=None,driver=None):
index += 1 index += 1
print(f"len={len_},index={index}") print(f"len={len_},index={index}")
url = row['url'] url = row['url']
print(row['docId'])
print(url) print(url)
driver.get(url) driver.get(url)
while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！': while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print(url) print(url)
print("跳转首页了，结束，切换账号") error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return return
time.sleep(1)
continue
text = driver.page_source text = driver.page_source
while ( while (
...@@ -120,12 +127,21 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -120,12 +127,21 @@ def get_cookie(username=None,password=None,driver=None):
text = driver.page_source text = driver.page_source
# 判断是否跳转了登录 # 判断是否跳转了登录
if '<title>京东-欢迎登录</title>' in text: if '<title>京东-欢迎登录</title>' in text:
print("需要登录，结束，切换账号") error_count += 1
if error_count >= 5:
print("需要登录，重试五次结束，切换账号")
return return
time.sleep(1)
continue
if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！': if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print("验证后还是首页，结束，切换账号") error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return return
time.sleep(1)
continue
error_count = 0
Response = etree.HTML(text) Response = etree.HTML(text)
dataList = [] dataList = []
...@@ -157,8 +173,9 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -157,8 +173,9 @@ def get_cookie(username=None,password=None,driver=None):
""" """
mysql_zdzs_proxy(insert_sql) mysql_zdzs_proxy(insert_sql)
except Exception as e: except Exception as e:
print(e)
print("保存代码异常") print("保存代码异常")
print(e)
print(traceback.format_exc())
continue continue
print("保存成功") print("保存成功")
pass pass
...@@ -174,15 +191,20 @@ if __name__ == "__main__": ...@@ -174,15 +191,20 @@ if __name__ == "__main__":
driver = webdriver.Chrome(options=option) driver = webdriver.Chrome(options=option)
option.add_argument("--disable-blink-features=AutomationControlled") option.add_argument("--disable-blink-features=AutomationControlled")
driver.set_window_size(1200, 600) driver.set_window_size(1200, 600)
# account_start_index为0到5的账号 # get_cookie(username=config.account_list[0]['username'],password=config.account_list[0]['password'],driver=driver)
account_start_index = 1 account_start_index = 0
get_cookie(username=config.account_list[account_start_index]['username'],
password=config.account_list[account_start_index]['password'], driver=driver)
while True:
for account in config.account_list: for account in config.account_list:
username = account['username'] username = account['username']
# 登陆密码 # 登陆密码
password = account['password'] password = account['password']
print(f"切换账号：{username}") print(f"切换账号：{username}")
get_cookie(username=username,password=password,driver=driver) get_cookie(username=username, password=password, driver=driver)
# get_cookie(username=config.account_list[account_start_index]['username'],password=config.account_list[account_start_index]['password'],driver=driver)
# while True:
# for account in config.account_list:
# username = account['username']
# # 登陆密码
# password = account['password']
# print(f"切换账号：{username}")
# get_cookie(username=username, password=password, driver=driver)
driver.quit() driver.quit()
京东搜索_手动验证版_2.py
View file @ 2aa7118d
...@@ -12,6 +12,7 @@ from selenium.common.exceptions import NoSuchElementException ...@@ -12,6 +12,7 @@ from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver import ChromeOptions from selenium.webdriver import ChromeOptions
from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support import expected_conditions as EC
import traceback
import pymysql import pymysql
from lxml import etree from lxml import etree
# 登陆账号 # 登陆账号
...@@ -67,7 +68,7 @@ def mysql_zdzs_proxy(sql=None): ...@@ -67,7 +68,7 @@ def mysql_zdzs_proxy(sql=None):
return [] return []
def get_cookie(username=None,password=None,driver=None): def get_cookie(username=None, password=None, driver=None):
# driver.maximize_window() # driver.maximize_window()
driver.get('https://passport.jd.com/uc/login') driver.get('https://passport.jd.com/uc/login')
print(f"清空文本框开始") print(f"清空文本框开始")
...@@ -95,6 +96,7 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -95,6 +96,7 @@ def get_cookie(username=None,password=None,driver=None):
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%2' " query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%2' "
query_spider = mysql_zdzs_proxy(query_spider_sql) query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0 index = 0
error_count = 0
len_ = len(query_spider) len_ = len(query_spider)
print(f"len={len_},index={index}") print(f"len={len_},index={index}")
if len_ == 0: if len_ == 0:
...@@ -105,12 +107,17 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -105,12 +107,17 @@ def get_cookie(username=None,password=None,driver=None):
index += 1 index += 1
print(f"len={len_},index={index}") print(f"len={len_},index={index}")
url = row['url'] url = row['url']
print(row['docId'])
print(url) print(url)
driver.get(url) driver.get(url)
while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！': while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print(url) print(url)
print("跳转首页了，结束，切换账号") error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return return
time.sleep(1)
continue
text = driver.page_source text = driver.page_source
while ( while (
...@@ -120,12 +127,21 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -120,12 +127,21 @@ def get_cookie(username=None,password=None,driver=None):
text = driver.page_source text = driver.page_source
# 判断是否跳转了登录 # 判断是否跳转了登录
if '<title>京东-欢迎登录</title>' in text: if '<title>京东-欢迎登录</title>' in text:
print("需要登录，结束，切换账号") error_count += 1
if error_count >= 5:
print("需要登录，重试五次结束，切换账号")
return return
time.sleep(1)
continue
if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！': if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print("验证后还是首页，结束，切换账号") error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return return
time.sleep(1)
continue
error_count = 0
Response = etree.HTML(text) Response = etree.HTML(text)
dataList = [] dataList = []
...@@ -157,8 +173,9 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -157,8 +173,9 @@ def get_cookie(username=None,password=None,driver=None):
""" """
mysql_zdzs_proxy(insert_sql) mysql_zdzs_proxy(insert_sql)
except Exception as e: except Exception as e:
print(e)
print("保存代码异常") print("保存代码异常")
print(e)
print(traceback.format_exc())
continue continue
print("保存成功") print("保存成功")
pass pass
...@@ -174,14 +191,20 @@ if __name__ == "__main__": ...@@ -174,14 +191,20 @@ if __name__ == "__main__":
driver = webdriver.Chrome(options=option) driver = webdriver.Chrome(options=option)
option.add_argument("--disable-blink-features=AutomationControlled") option.add_argument("--disable-blink-features=AutomationControlled")
driver.set_window_size(1200, 600) driver.set_window_size(1200, 600)
# get_cookie(username=config.account_list[0]['username'],password=config.account_list[0]['password'],driver=driver)
account_start_index = 0 account_start_index = 0
get_cookie(username=config.account_list[account_start_index]['username'],
password=config.account_list[account_start_index]['password'], driver=driver)
while True:
for account in config.account_list: for account in config.account_list:
username = account['username'] username = account['username']
# 登陆密码 # 登陆密码
password = account['password'] password = account['password']
print(f"切换账号：{username}") print(f"切换账号：{username}")
get_cookie(username=username,password=password,driver=driver) get_cookie(username=username, password=password, driver=driver)
# get_cookie(username=config.account_list[account_start_index]['username'],password=config.account_list[account_start_index]['password'],driver=driver)
# while True:
# for account in config.account_list:
# username = account['username']
# # 登陆密码
# password = account['password']
# print(f"切换账号：{username}")
# get_cookie(username=username, password=password, driver=driver)
driver.quit() driver.quit()
京东搜索_手动验证版_3.py
View file @ 2aa7118d
...@@ -12,6 +12,7 @@ from selenium.common.exceptions import NoSuchElementException ...@@ -12,6 +12,7 @@ from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver import ChromeOptions from selenium.webdriver import ChromeOptions
from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support import expected_conditions as EC
import traceback
import pymysql import pymysql
from lxml import etree from lxml import etree
# 登陆账号 # 登陆账号
...@@ -67,7 +68,7 @@ def mysql_zdzs_proxy(sql=None): ...@@ -67,7 +68,7 @@ def mysql_zdzs_proxy(sql=None):
return [] return []
def get_cookie(username=None,password=None,driver=None): def get_cookie(username=None, password=None, driver=None):
# driver.maximize_window() # driver.maximize_window()
driver.get('https://passport.jd.com/uc/login') driver.get('https://passport.jd.com/uc/login')
print(f"清空文本框开始") print(f"清空文本框开始")
...@@ -95,6 +96,7 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -95,6 +96,7 @@ def get_cookie(username=None,password=None,driver=None):
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%3' " query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%3' "
query_spider = mysql_zdzs_proxy(query_spider_sql) query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0 index = 0
error_count = 0
len_ = len(query_spider) len_ = len(query_spider)
print(f"len={len_},index={index}") print(f"len={len_},index={index}")
if len_ == 0: if len_ == 0:
...@@ -105,12 +107,17 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -105,12 +107,17 @@ def get_cookie(username=None,password=None,driver=None):
index += 1 index += 1
print(f"len={len_},index={index}") print(f"len={len_},index={index}")
url = row['url'] url = row['url']
print(row['docId'])
print(url) print(url)
driver.get(url) driver.get(url)
while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！': while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print(url) print(url)
print("跳转首页了，结束，切换账号") error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return return
time.sleep(1)
continue
text = driver.page_source text = driver.page_source
while ( while (
...@@ -120,12 +127,21 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -120,12 +127,21 @@ def get_cookie(username=None,password=None,driver=None):
text = driver.page_source text = driver.page_source
# 判断是否跳转了登录 # 判断是否跳转了登录
if '<title>京东-欢迎登录</title>' in text: if '<title>京东-欢迎登录</title>' in text:
print("需要登录，结束，切换账号") error_count += 1
if error_count >= 5:
print("需要登录，重试五次结束，切换账号")
return return
time.sleep(1)
continue
if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！': if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print("验证后还是首页，结束，切换账号") error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return return
time.sleep(1)
continue
error_count = 0
Response = etree.HTML(text) Response = etree.HTML(text)
dataList = [] dataList = []
...@@ -157,8 +173,9 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -157,8 +173,9 @@ def get_cookie(username=None,password=None,driver=None):
""" """
mysql_zdzs_proxy(insert_sql) mysql_zdzs_proxy(insert_sql)
except Exception as e: except Exception as e:
print(e)
print("保存代码异常") print("保存代码异常")
print(e)
print(traceback.format_exc())
continue continue
print("保存成功") print("保存成功")
pass pass
...@@ -174,14 +191,20 @@ if __name__ == "__main__": ...@@ -174,14 +191,20 @@ if __name__ == "__main__":
driver = webdriver.Chrome(options=option) driver = webdriver.Chrome(options=option)
option.add_argument("--disable-blink-features=AutomationControlled") option.add_argument("--disable-blink-features=AutomationControlled")
driver.set_window_size(1200, 600) driver.set_window_size(1200, 600)
account_start_index = 1 # get_cookie(username=config.account_list[0]['username'],password=config.account_list[0]['password'],driver=driver)
get_cookie(username=config.account_list[account_start_index]['username'], account_start_index = 0
password=config.account_list[account_start_index]['password'], driver=driver)
while True:
for account in config.account_list: for account in config.account_list:
username = account['username'] username = account['username']
# 登陆密码 # 登陆密码
password = account['password'] password = account['password']
print(f"切换账号：{username}") print(f"切换账号：{username}")
get_cookie(username=username,password=password,driver=driver) get_cookie(username=username, password=password, driver=driver)
# get_cookie(username=config.account_list[account_start_index]['username'],password=config.account_list[account_start_index]['password'],driver=driver)
# while True:
# for account in config.account_list:
# username = account['username']
# # 登陆密码
# password = account['password']
# print(f"切换账号：{username}")
# get_cookie(username=username, password=password, driver=driver)
driver.quit() driver.quit()
京东搜索_手动验证版_4.py
View file @ 2aa7118d
...@@ -12,6 +12,7 @@ from selenium.common.exceptions import NoSuchElementException ...@@ -12,6 +12,7 @@ from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver import ChromeOptions from selenium.webdriver import ChromeOptions
from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support import expected_conditions as EC
import traceback
import pymysql import pymysql
from lxml import etree from lxml import etree
# 登陆账号 # 登陆账号
...@@ -67,7 +68,7 @@ def mysql_zdzs_proxy(sql=None): ...@@ -67,7 +68,7 @@ def mysql_zdzs_proxy(sql=None):
return [] return []
def get_cookie(username=None,password=None,driver=None): def get_cookie(username=None, password=None, driver=None):
# driver.maximize_window() # driver.maximize_window()
driver.get('https://passport.jd.com/uc/login') driver.get('https://passport.jd.com/uc/login')
print(f"清空文本框开始") print(f"清空文本框开始")
...@@ -95,6 +96,7 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -95,6 +96,7 @@ def get_cookie(username=None,password=None,driver=None):
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%4' " query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%4' "
query_spider = mysql_zdzs_proxy(query_spider_sql) query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0 index = 0
error_count = 0
len_ = len(query_spider) len_ = len(query_spider)
print(f"len={len_},index={index}") print(f"len={len_},index={index}")
if len_ == 0: if len_ == 0:
...@@ -105,12 +107,17 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -105,12 +107,17 @@ def get_cookie(username=None,password=None,driver=None):
index += 1 index += 1
print(f"len={len_},index={index}") print(f"len={len_},index={index}")
url = row['url'] url = row['url']
print(row['docId'])
print(url) print(url)
driver.get(url) driver.get(url)
while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！': while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print(url) print(url)
print("跳转首页了，结束，切换账号") error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return return
time.sleep(1)
continue
text = driver.page_source text = driver.page_source
while ( while (
...@@ -120,12 +127,21 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -120,12 +127,21 @@ def get_cookie(username=None,password=None,driver=None):
text = driver.page_source text = driver.page_source
# 判断是否跳转了登录 # 判断是否跳转了登录
if '<title>京东-欢迎登录</title>' in text: if '<title>京东-欢迎登录</title>' in text:
print("需要登录，结束，切换账号") error_count += 1
if error_count >= 5:
print("需要登录，重试五次结束，切换账号")
return return
time.sleep(1)
continue
if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！': if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print("验证后还是首页，结束，切换账号") error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return return
time.sleep(1)
continue
error_count = 0
Response = etree.HTML(text) Response = etree.HTML(text)
dataList = [] dataList = []
...@@ -157,8 +173,9 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -157,8 +173,9 @@ def get_cookie(username=None,password=None,driver=None):
""" """
mysql_zdzs_proxy(insert_sql) mysql_zdzs_proxy(insert_sql)
except Exception as e: except Exception as e:
print(e)
print("保存代码异常") print("保存代码异常")
print(e)
print(traceback.format_exc())
continue continue
print("保存成功") print("保存成功")
pass pass
...@@ -174,14 +191,20 @@ if __name__ == "__main__": ...@@ -174,14 +191,20 @@ if __name__ == "__main__":
driver = webdriver.Chrome(options=option) driver = webdriver.Chrome(options=option)
option.add_argument("--disable-blink-features=AutomationControlled") option.add_argument("--disable-blink-features=AutomationControlled")
driver.set_window_size(1200, 600) driver.set_window_size(1200, 600)
account_start_index = 4 # get_cookie(username=config.account_list[0]['username'],password=config.account_list[0]['password'],driver=driver)
get_cookie(username=config.account_list[account_start_index]['username'], account_start_index = 0
password=config.account_list[account_start_index]['password'], driver=driver)
while True:
for account in config.account_list: for account in config.account_list:
username = account['username'] username = account['username']
# 登陆密码 # 登陆密码
password = account['password'] password = account['password']
print(f"切换账号：{username}") print(f"切换账号：{username}")
get_cookie(username=username,password=password,driver=driver) get_cookie(username=username, password=password, driver=driver)
# get_cookie(username=config.account_list[account_start_index]['username'],password=config.account_list[account_start_index]['password'],driver=driver)
# while True:
# for account in config.account_list:
# username = account['username']
# # 登陆密码
# password = account['password']
# print(f"切换账号：{username}")
# get_cookie(username=username, password=password, driver=driver)
driver.quit() driver.quit()
京东搜索_手动验证版_5.py
View file @ 2aa7118d
...@@ -12,6 +12,7 @@ from selenium.common.exceptions import NoSuchElementException ...@@ -12,6 +12,7 @@ from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver import ChromeOptions from selenium.webdriver import ChromeOptions
from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support import expected_conditions as EC
import traceback
import pymysql import pymysql
from lxml import etree from lxml import etree
# 登陆账号 # 登陆账号
...@@ -67,7 +68,7 @@ def mysql_zdzs_proxy(sql=None): ...@@ -67,7 +68,7 @@ def mysql_zdzs_proxy(sql=None):
return [] return []
def get_cookie(username=None,password=None,driver=None): def get_cookie(username=None, password=None, driver=None):
# driver.maximize_window() # driver.maximize_window()
driver.get('https://passport.jd.com/uc/login') driver.get('https://passport.jd.com/uc/login')
print(f"清空文本框开始") print(f"清空文本框开始")
...@@ -95,6 +96,7 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -95,6 +96,7 @@ def get_cookie(username=None,password=None,driver=None):
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%5' " query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%5' "
query_spider = mysql_zdzs_proxy(query_spider_sql) query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0 index = 0
error_count = 0
len_ = len(query_spider) len_ = len(query_spider)
print(f"len={len_},index={index}") print(f"len={len_},index={index}")
if len_ == 0: if len_ == 0:
...@@ -105,12 +107,17 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -105,12 +107,17 @@ def get_cookie(username=None,password=None,driver=None):
index += 1 index += 1
print(f"len={len_},index={index}") print(f"len={len_},index={index}")
url = row['url'] url = row['url']
print(row['docId'])
print(url) print(url)
driver.get(url) driver.get(url)
while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！': while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print(url) print(url)
print("跳转首页了，结束，切换账号") error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return return
time.sleep(1)
continue
text = driver.page_source text = driver.page_source
while ( while (
...@@ -120,12 +127,21 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -120,12 +127,21 @@ def get_cookie(username=None,password=None,driver=None):
text = driver.page_source text = driver.page_source
# 判断是否跳转了登录 # 判断是否跳转了登录
if '<title>京东-欢迎登录</title>' in text: if '<title>京东-欢迎登录</title>' in text:
print("需要登录，结束，切换账号") error_count += 1
if error_count >= 5:
print("需要登录，重试五次结束，切换账号")
return return
time.sleep(1)
continue
if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！': if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print("验证后还是首页，结束，切换账号") error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return return
time.sleep(1)
continue
error_count = 0
Response = etree.HTML(text) Response = etree.HTML(text)
dataList = [] dataList = []
...@@ -157,8 +173,9 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -157,8 +173,9 @@ def get_cookie(username=None,password=None,driver=None):
""" """
mysql_zdzs_proxy(insert_sql) mysql_zdzs_proxy(insert_sql)
except Exception as e: except Exception as e:
print(e)
print("保存代码异常") print("保存代码异常")
print(e)
print(traceback.format_exc())
continue continue
print("保存成功") print("保存成功")
pass pass
...@@ -174,14 +191,20 @@ if __name__ == "__main__": ...@@ -174,14 +191,20 @@ if __name__ == "__main__":
driver = webdriver.Chrome(options=option) driver = webdriver.Chrome(options=option)
option.add_argument("--disable-blink-features=AutomationControlled") option.add_argument("--disable-blink-features=AutomationControlled")
driver.set_window_size(1200, 600) driver.set_window_size(1200, 600)
account_start_index = 4 # get_cookie(username=config.account_list[0]['username'],password=config.account_list[0]['password'],driver=driver)
get_cookie(username=config.account_list[account_start_index]['username'], account_start_index = 0
password=config.account_list[account_start_index]['password'], driver=driver)
while True:
for account in config.account_list: for account in config.account_list:
username = account['username'] username = account['username']
# 登陆密码 # 登陆密码
password = account['password'] password = account['password']
print(f"切换账号：{username}") print(f"切换账号：{username}")
get_cookie(username=username,password=password,driver=driver) get_cookie(username=username, password=password, driver=driver)
# get_cookie(username=config.account_list[account_start_index]['username'],password=config.account_list[account_start_index]['password'],driver=driver)
# while True:
# for account in config.account_list:
# username = account['username']
# # 登陆密码
# password = account['password']
# print(f"切换账号：{username}")
# get_cookie(username=username, password=password, driver=driver)
driver.quit() driver.quit()
京东搜索_手动验证版_6.py
View file @ 2aa7118d
...@@ -12,6 +12,7 @@ from selenium.common.exceptions import NoSuchElementException ...@@ -12,6 +12,7 @@ from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver import ChromeOptions from selenium.webdriver import ChromeOptions
from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support import expected_conditions as EC
import traceback
import pymysql import pymysql
from lxml import etree from lxml import etree
# 登陆账号 # 登陆账号
...@@ -67,7 +68,7 @@ def mysql_zdzs_proxy(sql=None): ...@@ -67,7 +68,7 @@ def mysql_zdzs_proxy(sql=None):
return [] return []
def get_cookie(username=None,password=None,driver=None): def get_cookie(username=None, password=None, driver=None):
# driver.maximize_window() # driver.maximize_window()
driver.get('https://passport.jd.com/uc/login') driver.get('https://passport.jd.com/uc/login')
print(f"清空文本框开始") print(f"清空文本框开始")
...@@ -95,6 +96,7 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -95,6 +96,7 @@ def get_cookie(username=None,password=None,driver=None):
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%6' " query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%6' "
query_spider = mysql_zdzs_proxy(query_spider_sql) query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0 index = 0
error_count = 0
len_ = len(query_spider) len_ = len(query_spider)
print(f"len={len_},index={index}") print(f"len={len_},index={index}")
if len_ == 0: if len_ == 0:
...@@ -105,12 +107,17 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -105,12 +107,17 @@ def get_cookie(username=None,password=None,driver=None):
index += 1 index += 1
print(f"len={len_},index={index}") print(f"len={len_},index={index}")
url = row['url'] url = row['url']
print(row['docId'])
print(url) print(url)
driver.get(url) driver.get(url)
while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！': while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print(url) print(url)
print("跳转首页了，结束，切换账号") error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return return
time.sleep(1)
continue
text = driver.page_source text = driver.page_source
while ( while (
...@@ -120,12 +127,21 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -120,12 +127,21 @@ def get_cookie(username=None,password=None,driver=None):
text = driver.page_source text = driver.page_source
# 判断是否跳转了登录 # 判断是否跳转了登录
if '<title>京东-欢迎登录</title>' in text: if '<title>京东-欢迎登录</title>' in text:
print("需要登录，结束，切换账号") error_count += 1
if error_count >= 5:
print("需要登录，重试五次结束，切换账号")
return return
time.sleep(1)
continue
if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！': if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print("验证后还是首页，结束，切换账号") error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return return
time.sleep(1)
continue
error_count = 0
Response = etree.HTML(text) Response = etree.HTML(text)
dataList = [] dataList = []
...@@ -157,8 +173,9 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -157,8 +173,9 @@ def get_cookie(username=None,password=None,driver=None):
""" """
mysql_zdzs_proxy(insert_sql) mysql_zdzs_proxy(insert_sql)
except Exception as e: except Exception as e:
print(e)
print("保存代码异常") print("保存代码异常")
print(e)
print(traceback.format_exc())
continue continue
print("保存成功") print("保存成功")
pass pass
...@@ -174,14 +191,20 @@ if __name__ == "__main__": ...@@ -174,14 +191,20 @@ if __name__ == "__main__":
driver = webdriver.Chrome(options=option) driver = webdriver.Chrome(options=option)
option.add_argument("--disable-blink-features=AutomationControlled") option.add_argument("--disable-blink-features=AutomationControlled")
driver.set_window_size(1200, 600) driver.set_window_size(1200, 600)
account_start_index = 6 # get_cookie(username=config.account_list[0]['username'],password=config.account_list[0]['password'],driver=driver)
get_cookie(username=config.account_list[account_start_index]['username'], account_start_index = 0
password=config.account_list[account_start_index]['password'], driver=driver)
while True:
for account in config.account_list: for account in config.account_list:
username = account['username'] username = account['username']
# 登陆密码 # 登陆密码
password = account['password'] password = account['password']
print(f"切换账号：{username}") print(f"切换账号：{username}")
get_cookie(username=username,password=password,driver=driver) get_cookie(username=username, password=password, driver=driver)
# get_cookie(username=config.account_list[account_start_index]['username'],password=config.account_list[account_start_index]['password'],driver=driver)
# while True:
# for account in config.account_list:
# username = account['username']
# # 登陆密码
# password = account['password']
# print(f"切换账号：{username}")
# get_cookie(username=username, password=password, driver=driver)
driver.quit() driver.quit()
京东搜索_手动验证版_7.py
View file @ 2aa7118d
...@@ -12,6 +12,7 @@ from selenium.common.exceptions import NoSuchElementException ...@@ -12,6 +12,7 @@ from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver import ChromeOptions from selenium.webdriver import ChromeOptions
from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support import expected_conditions as EC
import traceback
import pymysql import pymysql
from lxml import etree from lxml import etree
# 登陆账号 # 登陆账号
...@@ -67,7 +68,7 @@ def mysql_zdzs_proxy(sql=None): ...@@ -67,7 +68,7 @@ def mysql_zdzs_proxy(sql=None):
return [] return []
def get_cookie(username=None,password=None,driver=None): def get_cookie(username=None, password=None, driver=None):
# driver.maximize_window() # driver.maximize_window()
driver.get('https://passport.jd.com/uc/login') driver.get('https://passport.jd.com/uc/login')
print(f"清空文本框开始") print(f"清空文本框开始")
...@@ -95,6 +96,7 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -95,6 +96,7 @@ def get_cookie(username=None,password=None,driver=None):
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%7' " query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%7' "
query_spider = mysql_zdzs_proxy(query_spider_sql) query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0 index = 0
error_count = 0
len_ = len(query_spider) len_ = len(query_spider)
print(f"len={len_},index={index}") print(f"len={len_},index={index}")
if len_ == 0: if len_ == 0:
...@@ -105,12 +107,17 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -105,12 +107,17 @@ def get_cookie(username=None,password=None,driver=None):
index += 1 index += 1
print(f"len={len_},index={index}") print(f"len={len_},index={index}")
url = row['url'] url = row['url']
print(row['docId'])
print(url) print(url)
driver.get(url) driver.get(url)
while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！': while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print(url) print(url)
print("跳转首页了，结束，切换账号") error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return return
time.sleep(1)
continue
text = driver.page_source text = driver.page_source
while ( while (
...@@ -120,12 +127,21 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -120,12 +127,21 @@ def get_cookie(username=None,password=None,driver=None):
text = driver.page_source text = driver.page_source
# 判断是否跳转了登录 # 判断是否跳转了登录
if '<title>京东-欢迎登录</title>' in text: if '<title>京东-欢迎登录</title>' in text:
print("需要登录，结束，切换账号") error_count += 1
if error_count >= 5:
print("需要登录，重试五次结束，切换账号")
return return
time.sleep(1)
continue
if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！': if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print("验证后还是首页，结束，切换账号") error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return return
time.sleep(1)
continue
error_count = 0
Response = etree.HTML(text) Response = etree.HTML(text)
dataList = [] dataList = []
...@@ -157,8 +173,9 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -157,8 +173,9 @@ def get_cookie(username=None,password=None,driver=None):
""" """
mysql_zdzs_proxy(insert_sql) mysql_zdzs_proxy(insert_sql)
except Exception as e: except Exception as e:
print(e)
print("保存代码异常") print("保存代码异常")
print(e)
print(traceback.format_exc())
continue continue
print("保存成功") print("保存成功")
pass pass
...@@ -174,14 +191,20 @@ if __name__ == "__main__": ...@@ -174,14 +191,20 @@ if __name__ == "__main__":
driver = webdriver.Chrome(options=option) driver = webdriver.Chrome(options=option)
option.add_argument("--disable-blink-features=AutomationControlled") option.add_argument("--disable-blink-features=AutomationControlled")
driver.set_window_size(1200, 600) driver.set_window_size(1200, 600)
account_start_index = 7 # get_cookie(username=config.account_list[0]['username'],password=config.account_list[0]['password'],driver=driver)
get_cookie(username=config.account_list[account_start_index]['username'], account_start_index = 0
password=config.account_list[account_start_index]['password'], driver=driver)
while True:
for account in config.account_list: for account in config.account_list:
username = account['username'] username = account['username']
# 登陆密码 # 登陆密码
password = account['password'] password = account['password']
print(f"切换账号：{username}") print(f"切换账号：{username}")
get_cookie(username=username,password=password,driver=driver) get_cookie(username=username, password=password, driver=driver)
# get_cookie(username=config.account_list[account_start_index]['username'],password=config.account_list[account_start_index]['password'],driver=driver)
# while True:
# for account in config.account_list:
# username = account['username']
# # 登陆密码
# password = account['password']
# print(f"切换账号：{username}")
# get_cookie(username=username, password=password, driver=driver)
driver.quit() driver.quit()
京东搜索_手动验证版_8.py
View file @ 2aa7118d
...@@ -12,6 +12,7 @@ from selenium.common.exceptions import NoSuchElementException ...@@ -12,6 +12,7 @@ from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver import ChromeOptions from selenium.webdriver import ChromeOptions
from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support import expected_conditions as EC
import traceback
import pymysql import pymysql
from lxml import etree from lxml import etree
# 登陆账号 # 登陆账号
...@@ -67,7 +68,7 @@ def mysql_zdzs_proxy(sql=None): ...@@ -67,7 +68,7 @@ def mysql_zdzs_proxy(sql=None):
return [] return []
def get_cookie(username=None,password=None,driver=None): def get_cookie(username=None, password=None, driver=None):
# driver.maximize_window() # driver.maximize_window()
driver.get('https://passport.jd.com/uc/login') driver.get('https://passport.jd.com/uc/login')
print(f"清空文本框开始") print(f"清空文本框开始")
...@@ -95,6 +96,7 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -95,6 +96,7 @@ def get_cookie(username=None,password=None,driver=None):
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%8' " query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%8' "
query_spider = mysql_zdzs_proxy(query_spider_sql) query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0 index = 0
error_count = 0
len_ = len(query_spider) len_ = len(query_spider)
print(f"len={len_},index={index}") print(f"len={len_},index={index}")
if len_ == 0: if len_ == 0:
...@@ -105,12 +107,17 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -105,12 +107,17 @@ def get_cookie(username=None,password=None,driver=None):
index += 1 index += 1
print(f"len={len_},index={index}") print(f"len={len_},index={index}")
url = row['url'] url = row['url']
print(row['docId'])
print(url) print(url)
driver.get(url) driver.get(url)
while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！': while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print(url) print(url)
print("跳转首页了，结束，切换账号") error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return return
time.sleep(1)
continue
text = driver.page_source text = driver.page_source
while ( while (
...@@ -120,12 +127,21 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -120,12 +127,21 @@ def get_cookie(username=None,password=None,driver=None):
text = driver.page_source text = driver.page_source
# 判断是否跳转了登录 # 判断是否跳转了登录
if '<title>京东-欢迎登录</title>' in text: if '<title>京东-欢迎登录</title>' in text:
print("需要登录，结束，切换账号") error_count += 1
if error_count >= 5:
print("需要登录，重试五次结束，切换账号")
return return
time.sleep(1)
continue
if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！': if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print("验证后还是首页，结束，切换账号") error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return return
time.sleep(1)
continue
error_count = 0
Response = etree.HTML(text) Response = etree.HTML(text)
dataList = [] dataList = []
...@@ -157,8 +173,9 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -157,8 +173,9 @@ def get_cookie(username=None,password=None,driver=None):
""" """
mysql_zdzs_proxy(insert_sql) mysql_zdzs_proxy(insert_sql)
except Exception as e: except Exception as e:
print(e)
print("保存代码异常") print("保存代码异常")
print(e)
print(traceback.format_exc())
continue continue
print("保存成功") print("保存成功")
pass pass
...@@ -174,14 +191,20 @@ if __name__ == "__main__": ...@@ -174,14 +191,20 @@ if __name__ == "__main__":
driver = webdriver.Chrome(options=option) driver = webdriver.Chrome(options=option)
option.add_argument("--disable-blink-features=AutomationControlled") option.add_argument("--disable-blink-features=AutomationControlled")
driver.set_window_size(1200, 600) driver.set_window_size(1200, 600)
account_start_index = 1 # get_cookie(username=config.account_list[0]['username'],password=config.account_list[0]['password'],driver=driver)
get_cookie(username=config.account_list[account_start_index]['username'], account_start_index = 0
password=config.account_list[account_start_index]['password'], driver=driver)
while True:
for account in config.account_list: for account in config.account_list:
username = account['username'] username = account['username']
# 登陆密码 # 登陆密码
password = account['password'] password = account['password']
print(f"切换账号：{username}") print(f"切换账号：{username}")
get_cookie(username=username,password=password,driver=driver) get_cookie(username=username, password=password, driver=driver)
# get_cookie(username=config.account_list[account_start_index]['username'],password=config.account_list[account_start_index]['password'],driver=driver)
# while True:
# for account in config.account_list:
# username = account['username']
# # 登陆密码
# password = account['password']
# print(f"切换账号：{username}")
# get_cookie(username=username, password=password, driver=driver)
driver.quit() driver.quit()
京东搜索_手动验证版_9.py
View file @ 2aa7118d
...@@ -12,6 +12,7 @@ from selenium.common.exceptions import NoSuchElementException ...@@ -12,6 +12,7 @@ from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver import ChromeOptions from selenium.webdriver import ChromeOptions
from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support import expected_conditions as EC
import traceback
import pymysql import pymysql
from lxml import etree from lxml import etree
# 登陆账号 # 登陆账号
...@@ -67,7 +68,7 @@ def mysql_zdzs_proxy(sql=None): ...@@ -67,7 +68,7 @@ def mysql_zdzs_proxy(sql=None):
return [] return []
def get_cookie(username=None,password=None,driver=None): def get_cookie(username=None, password=None, driver=None):
# driver.maximize_window() # driver.maximize_window()
driver.get('https://passport.jd.com/uc/login') driver.get('https://passport.jd.com/uc/login')
print(f"清空文本框开始") print(f"清空文本框开始")
...@@ -89,13 +90,13 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -89,13 +90,13 @@ def get_cookie(username=None,password=None,driver=None):
# 一直循环直到登录成功位置，超过一百次就算失败。 # 一直循环直到登录成功位置，超过一百次就算失败。
count = 0 count = 0
while driver.title != '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！': while driver.title != '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
count = count + 1
count = count + 1 count = count + 1
print(f"需要登录") print(f"需要登录")
time.sleep(1) time.sleep(1)
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%9' " query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%9' "
query_spider = mysql_zdzs_proxy(query_spider_sql) query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0 index = 0
error_count = 0
len_ = len(query_spider) len_ = len(query_spider)
print(f"len={len_},index={index}") print(f"len={len_},index={index}")
if len_ == 0: if len_ == 0:
...@@ -106,12 +107,17 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -106,12 +107,17 @@ def get_cookie(username=None,password=None,driver=None):
index += 1 index += 1
print(f"len={len_},index={index}") print(f"len={len_},index={index}")
url = row['url'] url = row['url']
print(row['docId'])
print(url) print(url)
driver.get(url) driver.get(url)
while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！': while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print(url) print(url)
print("跳转首页了，结束，切换账号") error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return return
time.sleep(1)
continue
text = driver.page_source text = driver.page_source
while ( while (
...@@ -121,12 +127,21 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -121,12 +127,21 @@ def get_cookie(username=None,password=None,driver=None):
text = driver.page_source text = driver.page_source
# 判断是否跳转了登录 # 判断是否跳转了登录
if '<title>京东-欢迎登录</title>' in text: if '<title>京东-欢迎登录</title>' in text:
print("需要登录，结束，切换账号") error_count += 1
if error_count >= 5:
print("需要登录，重试五次结束，切换账号")
return return
time.sleep(1)
continue
if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！': if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print("验证后还是首页，结束，切换账号") error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return return
time.sleep(1)
continue
error_count = 0
Response = etree.HTML(text) Response = etree.HTML(text)
dataList = [] dataList = []
...@@ -158,8 +173,9 @@ def get_cookie(username=None,password=None,driver=None): ...@@ -158,8 +173,9 @@ def get_cookie(username=None,password=None,driver=None):
""" """
mysql_zdzs_proxy(insert_sql) mysql_zdzs_proxy(insert_sql)
except Exception as e: except Exception as e:
print(e)
print("保存代码异常") print("保存代码异常")
print(e)
print(traceback.format_exc())
continue continue
print("保存成功") print("保存成功")
pass pass
...@@ -175,14 +191,20 @@ if __name__ == "__main__": ...@@ -175,14 +191,20 @@ if __name__ == "__main__":
driver = webdriver.Chrome(options=option) driver = webdriver.Chrome(options=option)
option.add_argument("--disable-blink-features=AutomationControlled") option.add_argument("--disable-blink-features=AutomationControlled")
driver.set_window_size(1200, 600) driver.set_window_size(1200, 600)
account_start_index = 1 # get_cookie(username=config.account_list[0]['username'],password=config.account_list[0]['password'],driver=driver)
get_cookie(username=config.account_list[account_start_index]['username'], account_start_index = 0
password=config.account_list[account_start_index]['password'], driver=driver)
while True:
for account in config.account_list: for account in config.account_list:
username = account['username'] username = account['username']
# 登陆密码 # 登陆密码
password = account['password'] password = account['password']
print(f"切换账号：{username}") print(f"切换账号：{username}")
get_cookie(username=username,password=password,driver=driver) get_cookie(username=username, password=password, driver=driver)
# get_cookie(username=config.account_list[account_start_index]['username'],password=config.account_list[account_start_index]['password'],driver=driver)
# while True:
# for account in config.account_list:
# username = account['username']
# # 登陆密码
# password = account['password']
# print(f"切换账号：{username}")
# get_cookie(username=username, password=password, driver=driver)
driver.quit() driver.quit()
查询剩余京东搜索链接量.py
View file @ 2aa7118d
...@@ -97,5 +97,6 @@ if __name__ == '__main__': ...@@ -97,5 +97,6 @@ if __name__ == '__main__':
# SELECT * FROM `DMP_SPIDER` where docId = '56033241'; # SELECT * FROM `DMP_SPIDER` where docId = '56033241';
# status = 0 就是若煜没跑 # status = 0 就是若煜没跑
# and id like '%9' # and id like '%9'
# count(*) # count(*)0
# limit 10 # limit 10
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment