Skip to content

京
京东搜索采集自动验证
Commit 2aa7118d authored by huangziyu's avatar huangziyu
Browse files
Options

1

parent 60847175
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 491 additions and 301 deletions
config.py
View file @ 2aa7118d
This diff is collapsed.
京东搜索_手动验证版_0.py
View file @ 2aa7118d
......@@ -96,6 +96,7 @@ def get_cookie(username=None, password=None, driver=None):
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%0' "
query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0
error_count = 0
len_ = len(query_spider)
print(f"len={len_},index={index}")
if len_ == 0:
......@@ -111,8 +112,12 @@ def get_cookie(username=None, password=None, driver=None):
driver.get(url)
while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print(url)
print("跳转首页了，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return
time.sleep(1)
continue
text = driver.page_source
while (
......@@ -122,12 +127,21 @@ def get_cookie(username=None, password=None, driver=None):
text = driver.page_source
# 判断是否跳转了登录
if '<title>京东-欢迎登录</title>' in text:
print("需要登录，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("需要登录，重试五次结束，切换账号")
return
time.sleep(1)
continue
if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print("验证后还是首页，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return
time.sleep(1)
continue
error_count = 0
Response = etree.HTML(text)
dataList = []
......@@ -179,12 +193,18 @@ if __name__ == "__main__":
driver.set_window_size(1200, 600)
# get_cookie(username=config.account_list[0]['username'],password=config.account_list[0]['password'],driver=driver)
account_start_index = 0
get_cookie(username=config.account_list[account_start_index]['username'],password=config.account_list[account_start_index]['password'],driver=driver)
while True:
for account in config.account_list:
username = account['username']
# 登陆密码
password = account['password']
print(f"切换账号：{username}")
get_cookie(username=username, password=password, driver=driver)
for account in config.account_list:
username = account['username']
# 登陆密码
password = account['password']
print(f"切换账号：{username}")
get_cookie(username=username, password=password, driver=driver)
# get_cookie(username=config.account_list[account_start_index]['username'],password=config.account_list[account_start_index]['password'],driver=driver)
# while True:
# for account in config.account_list:
# username = account['username']
# # 登陆密码
# password = account['password']
# print(f"切换账号：{username}")
# get_cookie(username=username, password=password, driver=driver)
driver.quit()
京东搜索_手动验证版_1.py
View file @ 2aa7118d
......@@ -12,6 +12,7 @@ from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver import ChromeOptions
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import traceback
import pymysql
from lxml import etree
# 登陆账号
......@@ -67,7 +68,7 @@ def mysql_zdzs_proxy(sql=None):
return []
def get_cookie(username=None,password=None,driver=None):
def get_cookie(username=None, password=None, driver=None):
# driver.maximize_window()
driver.get('https://passport.jd.com/uc/login')
print(f"清空文本框开始")
......@@ -92,9 +93,10 @@ def get_cookie(username=None,password=None,driver=None):
count = count + 1
print(f"需要登录")
time.sleep(1)
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%1'"
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%1' "
query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0
error_count = 0
len_ = len(query_spider)
print(f"len={len_},index={index}")
if len_ == 0:
......@@ -105,12 +107,17 @@ def get_cookie(username=None,password=None,driver=None):
index += 1
print(f"len={len_},index={index}")
url = row['url']
print(row['docId'])
print(url)
driver.get(url)
while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print(url)
print("跳转首页了，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return
time.sleep(1)
continue
text = driver.page_source
while (
......@@ -120,24 +127,33 @@ def get_cookie(username=None,password=None,driver=None):
text = driver.page_source
# 判断是否跳转了登录
if '<title>京东-欢迎登录</title>' in text:
print("需要登录，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("需要登录，重试五次结束，切换账号")
return
time.sleep(1)
continue
if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print("验证后还是首页，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return
time.sleep(1)
continue
error_count = 0
Response = etree.HTML(text)
dataList = []
try:
for x in Response.xpath("//div[@id='J_goodsList']/ul/li"):
is_self_operate_div = ''
if x.xpath(".//div[@class='p-name p-name-type-2']") and x.xpath(".//div[@class='p-price']//i//text()"):
shop_name = x.xpath(".//span[@class='J_im_icon']/a/@title")[0] if x.xpath(
".//span[@class='J_im_icon']/a/@title") else ''
dataList.append(
{
'is_self_operate_div': is_self_operate_div,
......@@ -149,7 +165,7 @@ def get_cookie(username=None,password=None,driver=None):
"'")).strip(),
'sku_price': x.xpath(".//div[@class='p-price']//i//text()")[0]
}
)
insert_sql = f"""
INSERT INTO DMP_SEARCH_DATA(id,result,mallId)
......@@ -157,8 +173,9 @@ def get_cookie(username=None,password=None,driver=None):
"""
mysql_zdzs_proxy(insert_sql)
except Exception as e:
print(e)
print("保存代码异常")
print(e)
print(traceback.format_exc())
continue
print("保存成功")
pass
......@@ -174,15 +191,20 @@ if __name__ == "__main__":
driver = webdriver.Chrome(options=option)
option.add_argument("--disable-blink-features=AutomationControlled")
driver.set_window_size(1200, 600)
# account_start_index为0到5的账号
account_start_index = 1
get_cookie(username=config.account_list[account_start_index]['username'],
password=config.account_list[account_start_index]['password'], driver=driver)
while True:
for account in config.account_list:
username = account['username']
# 登陆密码
password = account['password']
print(f"切换账号：{username}")
get_cookie(username=username,password=password,driver=driver)
driver.quit()
\ No newline at end of file
# get_cookie(username=config.account_list[0]['username'],password=config.account_list[0]['password'],driver=driver)
account_start_index = 0
for account in config.account_list:
username = account['username']
# 登陆密码
password = account['password']
print(f"切换账号：{username}")
get_cookie(username=username, password=password, driver=driver)
# get_cookie(username=config.account_list[account_start_index]['username'],password=config.account_list[account_start_index]['password'],driver=driver)
# while True:
# for account in config.account_list:
# username = account['username']
# # 登陆密码
# password = account['password']
# print(f"切换账号：{username}")
# get_cookie(username=username, password=password, driver=driver)
driver.quit()
京东搜索_手动验证版_2.py
View file @ 2aa7118d
......@@ -12,6 +12,7 @@ from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver import ChromeOptions
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import traceback
import pymysql
from lxml import etree
# 登陆账号
......@@ -67,7 +68,7 @@ def mysql_zdzs_proxy(sql=None):
return []
def get_cookie(username=None,password=None,driver=None):
def get_cookie(username=None, password=None, driver=None):
# driver.maximize_window()
driver.get('https://passport.jd.com/uc/login')
print(f"清空文本框开始")
......@@ -92,9 +93,10 @@ def get_cookie(username=None,password=None,driver=None):
count = count + 1
print(f"需要登录")
time.sleep(1)
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%2' "
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%2' "
query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0
error_count = 0
len_ = len(query_spider)
print(f"len={len_},index={index}")
if len_ == 0:
......@@ -105,12 +107,17 @@ def get_cookie(username=None,password=None,driver=None):
index += 1
print(f"len={len_},index={index}")
url = row['url']
print(row['docId'])
print(url)
driver.get(url)
while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print(url)
print("跳转首页了，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return
time.sleep(1)
continue
text = driver.page_source
while (
......@@ -120,12 +127,21 @@ def get_cookie(username=None,password=None,driver=None):
text = driver.page_source
# 判断是否跳转了登录
if '<title>京东-欢迎登录</title>' in text:
print("需要登录，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("需要登录，重试五次结束，切换账号")
return
time.sleep(1)
continue
if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print("验证后还是首页，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return
time.sleep(1)
continue
error_count = 0
Response = etree.HTML(text)
dataList = []
......@@ -157,8 +173,9 @@ def get_cookie(username=None,password=None,driver=None):
"""
mysql_zdzs_proxy(insert_sql)
except Exception as e:
print(e)
print("保存代码异常")
print(e)
print(traceback.format_exc())
continue
print("保存成功")
pass
......@@ -174,14 +191,20 @@ if __name__ == "__main__":
driver = webdriver.Chrome(options=option)
option.add_argument("--disable-blink-features=AutomationControlled")
driver.set_window_size(1200, 600)
# get_cookie(username=config.account_list[0]['username'],password=config.account_list[0]['password'],driver=driver)
account_start_index = 0
get_cookie(username=config.account_list[account_start_index]['username'],
password=config.account_list[account_start_index]['password'], driver=driver)
while True:
for account in config.account_list:
username = account['username']
# 登陆密码
password = account['password']
print(f"切换账号：{username}")
get_cookie(username=username,password=password,driver=driver)
driver.quit()
\ No newline at end of file
for account in config.account_list:
username = account['username']
# 登陆密码
password = account['password']
print(f"切换账号：{username}")
get_cookie(username=username, password=password, driver=driver)
# get_cookie(username=config.account_list[account_start_index]['username'],password=config.account_list[account_start_index]['password'],driver=driver)
# while True:
# for account in config.account_list:
# username = account['username']
# # 登陆密码
# password = account['password']
# print(f"切换账号：{username}")
# get_cookie(username=username, password=password, driver=driver)
driver.quit()
京东搜索_手动验证版_3.py
View file @ 2aa7118d
......@@ -12,6 +12,7 @@ from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver import ChromeOptions
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import traceback
import pymysql
from lxml import etree
# 登陆账号
......@@ -67,7 +68,7 @@ def mysql_zdzs_proxy(sql=None):
return []
def get_cookie(username=None,password=None,driver=None):
def get_cookie(username=None, password=None, driver=None):
# driver.maximize_window()
driver.get('https://passport.jd.com/uc/login')
print(f"清空文本框开始")
......@@ -92,9 +93,10 @@ def get_cookie(username=None,password=None,driver=None):
count = count + 1
print(f"需要登录")
time.sleep(1)
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%3' "
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%3' "
query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0
error_count = 0
len_ = len(query_spider)
print(f"len={len_},index={index}")
if len_ == 0:
......@@ -105,12 +107,17 @@ def get_cookie(username=None,password=None,driver=None):
index += 1
print(f"len={len_},index={index}")
url = row['url']
print(row['docId'])
print(url)
driver.get(url)
while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print(url)
print("跳转首页了，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return
time.sleep(1)
continue
text = driver.page_source
while (
......@@ -120,24 +127,33 @@ def get_cookie(username=None,password=None,driver=None):
text = driver.page_source
# 判断是否跳转了登录
if '<title>京东-欢迎登录</title>' in text:
print("需要登录，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("需要登录，重试五次结束，切换账号")
return
time.sleep(1)
continue
if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print("验证后还是首页，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return
time.sleep(1)
continue
error_count = 0
Response = etree.HTML(text)
dataList = []
try:
for x in Response.xpath("//div[@id='J_goodsList']/ul/li"):
is_self_operate_div = ''
if x.xpath(".//div[@class='p-name p-name-type-2']") and x.xpath(".//div[@class='p-price']//i//text()"):
shop_name = x.xpath(".//span[@class='J_im_icon']/a/@title")[0] if x.xpath(
".//span[@class='J_im_icon']/a/@title") else ''
dataList.append(
{
'is_self_operate_div': is_self_operate_div,
......@@ -149,7 +165,7 @@ def get_cookie(username=None,password=None,driver=None):
"'")).strip(),
'sku_price': x.xpath(".//div[@class='p-price']//i//text()")[0]
}
)
insert_sql = f"""
INSERT INTO DMP_SEARCH_DATA(id,result,mallId)
......@@ -157,8 +173,9 @@ def get_cookie(username=None,password=None,driver=None):
"""
mysql_zdzs_proxy(insert_sql)
except Exception as e:
print(e)
print("保存代码异常")
print(e)
print(traceback.format_exc())
continue
print("保存成功")
pass
......@@ -174,14 +191,20 @@ if __name__ == "__main__":
driver = webdriver.Chrome(options=option)
option.add_argument("--disable-blink-features=AutomationControlled")
driver.set_window_size(1200, 600)
account_start_index = 1
get_cookie(username=config.account_list[account_start_index]['username'],
password=config.account_list[account_start_index]['password'], driver=driver)
while True:
for account in config.account_list:
username = account['username']
# 登陆密码
password = account['password']
print(f"切换账号：{username}")
get_cookie(username=username,password=password,driver=driver)
driver.quit()
\ No newline at end of file
# get_cookie(username=config.account_list[0]['username'],password=config.account_list[0]['password'],driver=driver)
account_start_index = 0
for account in config.account_list:
username = account['username']
# 登陆密码
password = account['password']
print(f"切换账号：{username}")
get_cookie(username=username, password=password, driver=driver)
# get_cookie(username=config.account_list[account_start_index]['username'],password=config.account_list[account_start_index]['password'],driver=driver)
# while True:
# for account in config.account_list:
# username = account['username']
# # 登陆密码
# password = account['password']
# print(f"切换账号：{username}")
# get_cookie(username=username, password=password, driver=driver)
driver.quit()
京东搜索_手动验证版_4.py
View file @ 2aa7118d
......@@ -12,6 +12,7 @@ from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver import ChromeOptions
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import traceback
import pymysql
from lxml import etree
# 登陆账号
......@@ -67,7 +68,7 @@ def mysql_zdzs_proxy(sql=None):
return []
def get_cookie(username=None,password=None,driver=None):
def get_cookie(username=None, password=None, driver=None):
# driver.maximize_window()
driver.get('https://passport.jd.com/uc/login')
print(f"清空文本框开始")
......@@ -92,9 +93,10 @@ def get_cookie(username=None,password=None,driver=None):
count = count + 1
print(f"需要登录")
time.sleep(1)
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%4' "
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%4' "
query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0
error_count = 0
len_ = len(query_spider)
print(f"len={len_},index={index}")
if len_ == 0:
......@@ -105,12 +107,17 @@ def get_cookie(username=None,password=None,driver=None):
index += 1
print(f"len={len_},index={index}")
url = row['url']
print(row['docId'])
print(url)
driver.get(url)
while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print(url)
print("跳转首页了，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return
time.sleep(1)
continue
text = driver.page_source
while (
......@@ -120,24 +127,33 @@ def get_cookie(username=None,password=None,driver=None):
text = driver.page_source
# 判断是否跳转了登录
if '<title>京东-欢迎登录</title>' in text:
print("需要登录，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("需要登录，重试五次结束，切换账号")
return
time.sleep(1)
continue
if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print("验证后还是首页，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return
time.sleep(1)
continue
error_count = 0
Response = etree.HTML(text)
dataList = []
try:
for x in Response.xpath("//div[@id='J_goodsList']/ul/li"):
is_self_operate_div = ''
if x.xpath(".//div[@class='p-name p-name-type-2']") and x.xpath(".//div[@class='p-price']//i//text()"):
shop_name = x.xpath(".//span[@class='J_im_icon']/a/@title")[0] if x.xpath(
".//span[@class='J_im_icon']/a/@title") else ''
dataList.append(
{
'is_self_operate_div': is_self_operate_div,
......@@ -149,7 +165,7 @@ def get_cookie(username=None,password=None,driver=None):
"'")).strip(),
'sku_price': x.xpath(".//div[@class='p-price']//i//text()")[0]
}
)
insert_sql = f"""
INSERT INTO DMP_SEARCH_DATA(id,result,mallId)
......@@ -157,8 +173,9 @@ def get_cookie(username=None,password=None,driver=None):
"""
mysql_zdzs_proxy(insert_sql)
except Exception as e:
print(e)
print("保存代码异常")
print(e)
print(traceback.format_exc())
continue
print("保存成功")
pass
......@@ -174,14 +191,20 @@ if __name__ == "__main__":
driver = webdriver.Chrome(options=option)
option.add_argument("--disable-blink-features=AutomationControlled")
driver.set_window_size(1200, 600)
account_start_index = 4
get_cookie(username=config.account_list[account_start_index]['username'],
password=config.account_list[account_start_index]['password'], driver=driver)
while True:
for account in config.account_list:
username = account['username']
# 登陆密码
password = account['password']
print(f"切换账号：{username}")
get_cookie(username=username,password=password,driver=driver)
driver.quit()
\ No newline at end of file
# get_cookie(username=config.account_list[0]['username'],password=config.account_list[0]['password'],driver=driver)
account_start_index = 0
for account in config.account_list:
username = account['username']
# 登陆密码
password = account['password']
print(f"切换账号：{username}")
get_cookie(username=username, password=password, driver=driver)
# get_cookie(username=config.account_list[account_start_index]['username'],password=config.account_list[account_start_index]['password'],driver=driver)
# while True:
# for account in config.account_list:
# username = account['username']
# # 登陆密码
# password = account['password']
# print(f"切换账号：{username}")
# get_cookie(username=username, password=password, driver=driver)
driver.quit()
京东搜索_手动验证版_5.py
View file @ 2aa7118d
......@@ -12,6 +12,7 @@ from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver import ChromeOptions
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import traceback
import pymysql
from lxml import etree
# 登陆账号
......@@ -67,7 +68,7 @@ def mysql_zdzs_proxy(sql=None):
return []
def get_cookie(username=None,password=None,driver=None):
def get_cookie(username=None, password=None, driver=None):
# driver.maximize_window()
driver.get('https://passport.jd.com/uc/login')
print(f"清空文本框开始")
......@@ -92,9 +93,10 @@ def get_cookie(username=None,password=None,driver=None):
count = count + 1
print(f"需要登录")
time.sleep(1)
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%5' "
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%5' "
query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0
error_count = 0
len_ = len(query_spider)
print(f"len={len_},index={index}")
if len_ == 0:
......@@ -105,12 +107,17 @@ def get_cookie(username=None,password=None,driver=None):
index += 1
print(f"len={len_},index={index}")
url = row['url']
print(row['docId'])
print(url)
driver.get(url)
while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print(url)
print("跳转首页了，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return
time.sleep(1)
continue
text = driver.page_source
while (
......@@ -120,12 +127,21 @@ def get_cookie(username=None,password=None,driver=None):
text = driver.page_source
# 判断是否跳转了登录
if '<title>京东-欢迎登录</title>' in text:
print("需要登录，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("需要登录，重试五次结束，切换账号")
return
time.sleep(1)
continue
if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print("验证后还是首页，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return
time.sleep(1)
continue
error_count = 0
Response = etree.HTML(text)
dataList = []
......@@ -157,8 +173,9 @@ def get_cookie(username=None,password=None,driver=None):
"""
mysql_zdzs_proxy(insert_sql)
except Exception as e:
print(e)
print("保存代码异常")
print(e)
print(traceback.format_exc())
continue
print("保存成功")
pass
......@@ -174,14 +191,20 @@ if __name__ == "__main__":
driver = webdriver.Chrome(options=option)
option.add_argument("--disable-blink-features=AutomationControlled")
driver.set_window_size(1200, 600)
account_start_index = 4
get_cookie(username=config.account_list[account_start_index]['username'],
password=config.account_list[account_start_index]['password'], driver=driver)
while True:
for account in config.account_list:
username = account['username']
# 登陆密码
password = account['password']
print(f"切换账号：{username}")
get_cookie(username=username,password=password,driver=driver)
driver.quit()
\ No newline at end of file
# get_cookie(username=config.account_list[0]['username'],password=config.account_list[0]['password'],driver=driver)
account_start_index = 0
for account in config.account_list:
username = account['username']
# 登陆密码
password = account['password']
print(f"切换账号：{username}")
get_cookie(username=username, password=password, driver=driver)
# get_cookie(username=config.account_list[account_start_index]['username'],password=config.account_list[account_start_index]['password'],driver=driver)
# while True:
# for account in config.account_list:
# username = account['username']
# # 登陆密码
# password = account['password']
# print(f"切换账号：{username}")
# get_cookie(username=username, password=password, driver=driver)
driver.quit()
京东搜索_手动验证版_6.py
View file @ 2aa7118d
......@@ -12,6 +12,7 @@ from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver import ChromeOptions
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import traceback
import pymysql
from lxml import etree
# 登陆账号
......@@ -67,7 +68,7 @@ def mysql_zdzs_proxy(sql=None):
return []
def get_cookie(username=None,password=None,driver=None):
def get_cookie(username=None, password=None, driver=None):
# driver.maximize_window()
driver.get('https://passport.jd.com/uc/login')
print(f"清空文本框开始")
......@@ -92,9 +93,10 @@ def get_cookie(username=None,password=None,driver=None):
count = count + 1
print(f"需要登录")
time.sleep(1)
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%6' "
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%6' "
query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0
error_count = 0
len_ = len(query_spider)
print(f"len={len_},index={index}")
if len_ == 0:
......@@ -105,12 +107,17 @@ def get_cookie(username=None,password=None,driver=None):
index += 1
print(f"len={len_},index={index}")
url = row['url']
print(row['docId'])
print(url)
driver.get(url)
while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print(url)
print("跳转首页了，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return
time.sleep(1)
continue
text = driver.page_source
while (
......@@ -120,24 +127,33 @@ def get_cookie(username=None,password=None,driver=None):
text = driver.page_source
# 判断是否跳转了登录
if '<title>京东-欢迎登录</title>' in text:
print("需要登录，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("需要登录，重试五次结束，切换账号")
return
time.sleep(1)
continue
if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print("验证后还是首页，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return
time.sleep(1)
continue
error_count = 0
Response = etree.HTML(text)
dataList = []
try:
for x in Response.xpath("//div[@id='J_goodsList']/ul/li"):
is_self_operate_div = ''
if x.xpath(".//div[@class='p-name p-name-type-2']") and x.xpath(".//div[@class='p-price']//i//text()"):
shop_name = x.xpath(".//span[@class='J_im_icon']/a/@title")[0] if x.xpath(
".//span[@class='J_im_icon']/a/@title") else ''
dataList.append(
{
'is_self_operate_div': is_self_operate_div,
......@@ -149,7 +165,7 @@ def get_cookie(username=None,password=None,driver=None):
"'")).strip(),
'sku_price': x.xpath(".//div[@class='p-price']//i//text()")[0]
}
)
insert_sql = f"""
INSERT INTO DMP_SEARCH_DATA(id,result,mallId)
......@@ -157,8 +173,9 @@ def get_cookie(username=None,password=None,driver=None):
"""
mysql_zdzs_proxy(insert_sql)
except Exception as e:
print(e)
print("保存代码异常")
print(e)
print(traceback.format_exc())
continue
print("保存成功")
pass
......@@ -174,14 +191,20 @@ if __name__ == "__main__":
driver = webdriver.Chrome(options=option)
option.add_argument("--disable-blink-features=AutomationControlled")
driver.set_window_size(1200, 600)
account_start_index = 6
get_cookie(username=config.account_list[account_start_index]['username'],
password=config.account_list[account_start_index]['password'], driver=driver)
while True:
for account in config.account_list:
username = account['username']
# 登陆密码
password = account['password']
print(f"切换账号：{username}")
get_cookie(username=username,password=password,driver=driver)
driver.quit()
\ No newline at end of file
# get_cookie(username=config.account_list[0]['username'],password=config.account_list[0]['password'],driver=driver)
account_start_index = 0
for account in config.account_list:
username = account['username']
# 登陆密码
password = account['password']
print(f"切换账号：{username}")
get_cookie(username=username, password=password, driver=driver)
# get_cookie(username=config.account_list[account_start_index]['username'],password=config.account_list[account_start_index]['password'],driver=driver)
# while True:
# for account in config.account_list:
# username = account['username']
# # 登陆密码
# password = account['password']
# print(f"切换账号：{username}")
# get_cookie(username=username, password=password, driver=driver)
driver.quit()
京东搜索_手动验证版_7.py
View file @ 2aa7118d
......@@ -12,6 +12,7 @@ from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver import ChromeOptions
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import traceback
import pymysql
from lxml import etree
# 登陆账号
......@@ -67,7 +68,7 @@ def mysql_zdzs_proxy(sql=None):
return []
def get_cookie(username=None,password=None,driver=None):
def get_cookie(username=None, password=None, driver=None):
# driver.maximize_window()
driver.get('https://passport.jd.com/uc/login')
print(f"清空文本框开始")
......@@ -92,9 +93,10 @@ def get_cookie(username=None,password=None,driver=None):
count = count + 1
print(f"需要登录")
time.sleep(1)
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%7' "
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%7' "
query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0
error_count = 0
len_ = len(query_spider)
print(f"len={len_},index={index}")
if len_ == 0:
......@@ -105,12 +107,17 @@ def get_cookie(username=None,password=None,driver=None):
index += 1
print(f"len={len_},index={index}")
url = row['url']
print(row['docId'])
print(url)
driver.get(url)
while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print(url)
print("跳转首页了，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return
time.sleep(1)
continue
text = driver.page_source
while (
......@@ -120,24 +127,33 @@ def get_cookie(username=None,password=None,driver=None):
text = driver.page_source
# 判断是否跳转了登录
if '<title>京东-欢迎登录</title>' in text:
print("需要登录，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("需要登录，重试五次结束，切换账号")
return
time.sleep(1)
continue
if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print("验证后还是首页，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return
time.sleep(1)
continue
error_count = 0
Response = etree.HTML(text)
dataList = []
try:
for x in Response.xpath("//div[@id='J_goodsList']/ul/li"):
is_self_operate_div = ''
if x.xpath(".//div[@class='p-name p-name-type-2']") and x.xpath(".//div[@class='p-price']//i//text()"):
shop_name = x.xpath(".//span[@class='J_im_icon']/a/@title")[0] if x.xpath(
".//span[@class='J_im_icon']/a/@title") else ''
dataList.append(
{
'is_self_operate_div': is_self_operate_div,
......@@ -149,7 +165,7 @@ def get_cookie(username=None,password=None,driver=None):
"'")).strip(),
'sku_price': x.xpath(".//div[@class='p-price']//i//text()")[0]
}
)
insert_sql = f"""
INSERT INTO DMP_SEARCH_DATA(id,result,mallId)
......@@ -157,8 +173,9 @@ def get_cookie(username=None,password=None,driver=None):
"""
mysql_zdzs_proxy(insert_sql)
except Exception as e:
print(e)
print("保存代码异常")
print(e)
print(traceback.format_exc())
continue
print("保存成功")
pass
......@@ -174,14 +191,20 @@ if __name__ == "__main__":
driver = webdriver.Chrome(options=option)
option.add_argument("--disable-blink-features=AutomationControlled")
driver.set_window_size(1200, 600)
account_start_index = 7
get_cookie(username=config.account_list[account_start_index]['username'],
password=config.account_list[account_start_index]['password'], driver=driver)
while True:
for account in config.account_list:
username = account['username']
# 登陆密码
password = account['password']
print(f"切换账号：{username}")
get_cookie(username=username,password=password,driver=driver)
driver.quit()
\ No newline at end of file
# get_cookie(username=config.account_list[0]['username'],password=config.account_list[0]['password'],driver=driver)
account_start_index = 0
for account in config.account_list:
username = account['username']
# 登陆密码
password = account['password']
print(f"切换账号：{username}")
get_cookie(username=username, password=password, driver=driver)
# get_cookie(username=config.account_list[account_start_index]['username'],password=config.account_list[account_start_index]['password'],driver=driver)
# while True:
# for account in config.account_list:
# username = account['username']
# # 登陆密码
# password = account['password']
# print(f"切换账号：{username}")
# get_cookie(username=username, password=password, driver=driver)
driver.quit()
京东搜索_手动验证版_8.py
View file @ 2aa7118d
......@@ -12,6 +12,7 @@ from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver import ChromeOptions
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import traceback
import pymysql
from lxml import etree
# 登陆账号
......@@ -67,7 +68,7 @@ def mysql_zdzs_proxy(sql=None):
return []
def get_cookie(username=None,password=None,driver=None):
def get_cookie(username=None, password=None, driver=None):
# driver.maximize_window()
driver.get('https://passport.jd.com/uc/login')
print(f"清空文本框开始")
......@@ -92,9 +93,10 @@ def get_cookie(username=None,password=None,driver=None):
count = count + 1
print(f"需要登录")
time.sleep(1)
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%8' "
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%8' "
query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0
error_count = 0
len_ = len(query_spider)
print(f"len={len_},index={index}")
if len_ == 0:
......@@ -105,12 +107,17 @@ def get_cookie(username=None,password=None,driver=None):
index += 1
print(f"len={len_},index={index}")
url = row['url']
print(row['docId'])
print(url)
driver.get(url)
while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print(url)
print("跳转首页了，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return
time.sleep(1)
continue
text = driver.page_source
while (
......@@ -120,24 +127,33 @@ def get_cookie(username=None,password=None,driver=None):
text = driver.page_source
# 判断是否跳转了登录
if '<title>京东-欢迎登录</title>' in text:
print("需要登录，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("需要登录，重试五次结束，切换账号")
return
time.sleep(1)
continue
if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print("验证后还是首页，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return
time.sleep(1)
continue
error_count = 0
Response = etree.HTML(text)
dataList = []
try:
for x in Response.xpath("//div[@id='J_goodsList']/ul/li"):
is_self_operate_div = ''
if x.xpath(".//div[@class='p-name p-name-type-2']") and x.xpath(".//div[@class='p-price']//i//text()"):
shop_name = x.xpath(".//span[@class='J_im_icon']/a/@title")[0] if x.xpath(
".//span[@class='J_im_icon']/a/@title") else ''
dataList.append(
{
'is_self_operate_div': is_self_operate_div,
......@@ -149,7 +165,7 @@ def get_cookie(username=None,password=None,driver=None):
"'")).strip(),
'sku_price': x.xpath(".//div[@class='p-price']//i//text()")[0]
}
)
insert_sql = f"""
INSERT INTO DMP_SEARCH_DATA(id,result,mallId)
......@@ -157,8 +173,9 @@ def get_cookie(username=None,password=None,driver=None):
"""
mysql_zdzs_proxy(insert_sql)
except Exception as e:
print(e)
print("保存代码异常")
print(e)
print(traceback.format_exc())
continue
print("保存成功")
pass
......@@ -174,14 +191,20 @@ if __name__ == "__main__":
driver = webdriver.Chrome(options=option)
option.add_argument("--disable-blink-features=AutomationControlled")
driver.set_window_size(1200, 600)
account_start_index = 1
get_cookie(username=config.account_list[account_start_index]['username'],
password=config.account_list[account_start_index]['password'], driver=driver)
while True:
for account in config.account_list:
username = account['username']
# 登陆密码
password = account['password']
print(f"切换账号：{username}")
get_cookie(username=username,password=password,driver=driver)
driver.quit()
\ No newline at end of file
# get_cookie(username=config.account_list[0]['username'],password=config.account_list[0]['password'],driver=driver)
account_start_index = 0
for account in config.account_list:
username = account['username']
# 登陆密码
password = account['password']
print(f"切换账号：{username}")
get_cookie(username=username, password=password, driver=driver)
# get_cookie(username=config.account_list[account_start_index]['username'],password=config.account_list[account_start_index]['password'],driver=driver)
# while True:
# for account in config.account_list:
# username = account['username']
# # 登陆密码
# password = account['password']
# print(f"切换账号：{username}")
# get_cookie(username=username, password=password, driver=driver)
driver.quit()
京东搜索_手动验证版_9.py
View file @ 2aa7118d
......@@ -12,6 +12,7 @@ from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver import ChromeOptions
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import traceback
import pymysql
from lxml import etree
# 登陆账号
......@@ -67,7 +68,7 @@ def mysql_zdzs_proxy(sql=None):
return []
def get_cookie(username=None,password=None,driver=None):
def get_cookie(username=None, password=None, driver=None):
# driver.maximize_window()
driver.get('https://passport.jd.com/uc/login')
print(f"清空文本框开始")
......@@ -89,13 +90,13 @@ def get_cookie(username=None,password=None,driver=None):
# 一直循环直到登录成功位置，超过一百次就算失败。
count = 0
while driver.title != '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
count = count + 1
count = count + 1
print(f"需要登录")
time.sleep(1)
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%9' "
query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id like '%9' "
query_spider = mysql_zdzs_proxy(query_spider_sql)
index = 0
error_count = 0
len_ = len(query_spider)
print(f"len={len_},index={index}")
if len_ == 0:
......@@ -106,12 +107,17 @@ def get_cookie(username=None,password=None,driver=None):
index += 1
print(f"len={len_},index={index}")
url = row['url']
print(row['docId'])
print(url)
driver.get(url)
while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print(url)
print("跳转首页了，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return
time.sleep(1)
continue
text = driver.page_source
while (
......@@ -121,24 +127,33 @@ def get_cookie(username=None,password=None,driver=None):
text = driver.page_source
# 判断是否跳转了登录
if '<title>京东-欢迎登录</title>' in text:
print("需要登录，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("需要登录，重试五次结束，切换账号")
return
time.sleep(1)
continue
if driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
print("验证后还是首页，结束，切换账号")
return
error_count += 1
if error_count >= 5:
print("跳转首页了，重试五次后结束，切换账号")
return
time.sleep(1)
continue
error_count = 0
Response = etree.HTML(text)
dataList = []
try:
for x in Response.xpath("//div[@id='J_goodsList']/ul/li"):
is_self_operate_div = ''
if x.xpath(".//div[@class='p-name p-name-type-2']") and x.xpath(".//div[@class='p-price']//i//text()"):
shop_name = x.xpath(".//span[@class='J_im_icon']/a/@title")[0] if x.xpath(
".//span[@class='J_im_icon']/a/@title") else ''
dataList.append(
{
'is_self_operate_div': is_self_operate_div,
......@@ -150,7 +165,7 @@ def get_cookie(username=None,password=None,driver=None):
"'")).strip(),
'sku_price': x.xpath(".//div[@class='p-price']//i//text()")[0]
}
)
insert_sql = f"""
INSERT INTO DMP_SEARCH_DATA(id,result,mallId)
......@@ -158,8 +173,9 @@ def get_cookie(username=None,password=None,driver=None):
"""
mysql_zdzs_proxy(insert_sql)
except Exception as e:
print(e)
print("保存代码异常")
print(e)
print(traceback.format_exc())
continue
print("保存成功")
pass
......@@ -175,14 +191,20 @@ if __name__ == "__main__":
driver = webdriver.Chrome(options=option)
option.add_argument("--disable-blink-features=AutomationControlled")
driver.set_window_size(1200, 600)
account_start_index = 1
get_cookie(username=config.account_list[account_start_index]['username'],
password=config.account_list[account_start_index]['password'], driver=driver)
while True:
for account in config.account_list:
username = account['username']
# 登陆密码
password = account['password']
print(f"切换账号：{username}")
get_cookie(username=username,password=password,driver=driver)
driver.quit()
\ No newline at end of file
# get_cookie(username=config.account_list[0]['username'],password=config.account_list[0]['password'],driver=driver)
account_start_index = 0
for account in config.account_list:
username = account['username']
# 登陆密码
password = account['password']
print(f"切换账号：{username}")
get_cookie(username=username, password=password, driver=driver)
# get_cookie(username=config.account_list[account_start_index]['username'],password=config.account_list[account_start_index]['password'],driver=driver)
# while True:
# for account in config.account_list:
# username = account['username']
# # 登陆密码
# password = account['password']
# print(f"切换账号：{username}")
# get_cookie(username=username, password=password, driver=driver)
driver.quit()
查询剩余京东搜索链接量.py
View file @ 2aa7118d
......@@ -97,5 +97,6 @@ if __name__ == '__main__':
# SELECT * FROM `DMP_SPIDER` where docId = '56033241';
# status = 0 就是若煜没跑
# and id like '%9'
# count(*)
# count(*)0
# limit 10
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment