1

20240525_胡鹏朝表格截图/20240525_胡鹏朝表格截图.py

+import os
+# 对服务器运行的判断
+import sys
+
+sys.path.append(os.pardir)
+import json
+import time
+import openpyxl
+import requests
+from loguru import logger
+from openpyxl import Workbook
+
+import time
+from urllib import request
+import cv2
+import numpy as np
+import pyautogui
+from openpyxl.drawing.image import Image as Image2
+from selenium import webdriver
+from selenium.common.exceptions import NoSuchElementException
+from selenium.webdriver import ChromeOptions
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+import pymysql
+from docx import Document
+from docx.shared import Inches
+# 登陆账号
+from selenium.webdriver.common.by import By
+import openpyxl
+import time
+import random
+import json
+from openpyxl import Workbook
+from PIL import Image
+import re
+
+import os
+import re
+import time
+import urllib3
+from PIL import Image
+from loguru import logger
+from selenium import webdriver
+import sys
+
+sys.path.append(os.pardir)
+
+curPath = os.path.abspath(os.path.dirname(__file__))
+
+urllib3.disable_warnings()
+
+
+def generate_time_img():
+    from PIL import Image, ImageFont, ImageDraw
+    text = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
+    im = Image.new("RGB", (350, 50), (255, 255, 255))
+    dr = ImageDraw.Draw(im)
+    # print(curPath)
+    font = ImageFont.truetype(f"msyh.ttf", 32)
+    # font = ImageFont.truetype("E:\\code\\dmp_data_handling_api\\utils\\msyh.ttf", 32)
+    # if ':\\' not in curPath:
+    #     font = ImageFont.truetype(f"/data/dmp_data_handling_api/utils/spider/msyh.ttf", 32)
+    # else:
+    #     font = ImageFont.truetype("E:\\code\\dmp_data_handling_python\\utils\\spider\\msyh.ttf", 32)
+    dr.text((10, 5), text, font=font, fill="#f73131")
+    im.save("time.png")
+
+
+def watermark(file_path=None):
+    generate_time_img()
+    lp = Image.open(file_path)
+    tp = Image.open('time.png')  # 水印
+    img = tp.convert('RGBA')
+    r, g, b, alpha = img.split()
+    alpha = alpha.point(lambda i: i > 0 and 240)
+    img.putalpha(alpha)
+
+    image_x, image_y = lp.size
+    watermark_x, watermark_y = tp.size
+
+    # 水印位置  mask是通道值,240+ 是透明效果
+    lp.paste(img, (image_x - watermark_x - 100, image_y - watermark_y - 100), mask=alpha)
+    lp.save(file_path, "png")
+
+
+
+username = 'jd_72550b7449faa'
+# 登陆密码
+password = 'xiaokui666'
+
+# mysql_zdzs_proxy_host = '59.110.219.171'
+# mysql_zdzs_proxy_user = 'zgcindex'
+# mysql_zdzs_proxy_password = 'zgcprice2019'
+# mysql_zdzs_proxy_database = 'zdzs_proxy'
+# 滑块距离屏幕左上角的x, y像素, 需根据自己屏幕大小调整，利用微信截图得到xy的具体位置
+# 登录的
+# slide_x_position, slid_y_position = 850, 459
+# # 验证的
+# slide_x_position_quick_verification, slid_y_position_quick_verification = 490, 512
+# 偏移度加值(根据电脑分辨率情况)
+offset_increase = 0
+
+
+def remove_the_symbol(content=None):
+    if content is None:
+        return content
+    content = re.findall('([\u4e00-\u9fa5A-Za-z0-9]*)', content.upper())
+    return ''.join(content)
+
+
+
+def return_js():
+    return open('stealth.min.js', 'r').read()
+option = ChromeOptions()
+option.binary_location = 'D:\ChromeMaYi\guge\chrome.exe'
+# 这个不能开，开了识别不了图片
+# option.add_argument('--headless') # 无头模式，可不启用界面显示运行
+option.add_experimental_option('excludeSwitches', ['enable-automation'])
+driver = webdriver.Chrome(options=option)
+driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
+    "source": return_js()
+})
+option.add_argument('headless')
+option.add_argument('no-sandbox')
+option.add_argument('disable-dev-shm-usage')
+option.add_argument("start-maximized")
+option.add_argument("enable-automation")
+option.add_experimental_option('excludeSwitches', ['enable-automation'])
+option.add_argument("--no-sandbox")
+option.add_argument("--disable-infobars")
+option.add_argument("--disable-dev-shm-usage")
+option.add_argument("--disable-browser-side-navigation")
+option.add_argument("--disable-gpu")
+option.add_argument("--disable-blink-features=AutomationControlled")
+driver.set_window_size(1600, 900)
+# driver.maximize_window()
+driver.get('https://passport.jd.com/uc/login')
+# 切換账号密码登录
+# driver.find_element(by=By.CLASS_NAME, value='login-tab-r').click()
+# 设置账号密码
+driver.find_element(by=By.ID, value='loginname').send_keys(username)
+driver.find_element(by=By.ID, value='nloginpwd').send_keys(password)
+time.sleep(0.5)
+# 登录
+driver.find_element(by=By.ID, value='loginsubmit').click()
+print("睡眠0.5秒后开始滑动登录")
+time.sleep(0.5)
+while driver.title != '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
+    print(f"需要登录")
+    time.sleep(1)
+count = 0
+# 1.打开文件
+file_name = '京东价格查询合并'
+# wb = openpyxl.load_workbook(f"/data/chajiabao_data_processing/数据处理需求/20240514_e购数据驳回/{file_name}.xlsx", read_only=False)
+wb = openpyxl.load_workbook(f"{file_name}.xlsx", read_only=False)
+# 2.选择工作表
+ws = wb[wb.sheetnames[0]]
+# 获取表头，假设表头在第一行
+i = 0
+max_param_index = 0
+headers = {}
+for cell in ws[1]:
+    headers[cell.value] = i
+    i += 1
+time_08 = time.strftime('%Y-%m-%dT%H:%M:%S+08:00', time.localtime(time.time()))
+logger.info("表头:", headers)
+submit_doc_id = []
+# 3.遍历单元格
+for row in ws.rows:
+    count += 1
+    logger.info(f"{count}")
+    if count > 1:
+        row_dict = {}
+        for key in headers.keys():
+            row_dict[key] = row[headers[key]].value
+        url = row_dict['参考链接']
+        logger.info(f"{url}")
+        pass
+        print(url)
+        driver.get(url)
+        driver.set_window_size(1600, 900)
+        text = driver.page_source
+        while 'jd.com' in url and (
+                '快速验证' in text and '验证一下，购物无忧' in text) or '前方拥挤，请刷新重试' in text or 'class="title">加载中...</span></div></div><!----></div>' in text:
+            pass
+            print("需要验证，刷新后睡眠7秒")
+            time.sleep(2)
+            text = driver.page_source
+            # driver.get(url)
+        while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
+            print("跳转到首页了，说明链接失效。")
+            row[5].value = "-1.00"
+            break
+        if row[5].value == "-1.00":
+            continue
+        price = driver.find_element_by_class_name("p-price")
+        while price.text == '￥':
+            print("价格没刷新出来，重新刷新")
+            driver.get(url)
+            time.sleep(0.5)
+            text = driver.page_source
+
+            while 'jd.com' in url and (
+                    '快速验证' in text and '验证一下，购物无忧' in text) or '前方拥挤，请刷新重试' in text or 'class="title">加载中...</span></div></div><!----></div>' in text:
+                pass
+                print("需要验证，刷新后睡眠7秒")
+                time.sleep(2)
+                text = driver.page_source
+                # driver.get(url)
+            while driver.title == '京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！':
+                print("跳转到首页了，说明链接失效。")
+                row[5].value = "-1.00"
+                driver.execute_script("window.scrollBy(0, 30);")
+                filename = remove_the_symbol(url)
+                driver.save_screenshot('.\\{}.png'.format(filename))
+                img_path = '{}.png'.format(filename)
+                # 将图片插入到指定的单元格
+                cell = f'H{count}'
+                img = Image2(img_path)
+                img.anchor = cell
+                ws.add_image(img)
+                print("保存成功")
+                break
+            if row[5].value == "-1.00":
+                break
+            price = driver.find_element_by_class_name("p-price")
+        if row[5].value == "-1.00":
+            continue
+
+        driver.execute_script("window.scrollBy(0, 30);")
+        row[5].value = price.text
+        filename = remove_the_symbol(url)
+        driver.save_screenshot('.\\{}.png'.format(filename))
+        img_path = '{}.png'.format(filename)
+
+        # 将图片插入到指定的单元格
+        cell = f'H{count}'
+        img = Image2(img_path)
+        img.anchor = cell
+        ws.add_image(img)
+        print("保存成功")
+wb.save(f'{file_name}_结果.xlsx')
+driver.close()
+logger.info("success")
+exit(0)

京东搜索_手动验证版_0.py

@@ -93,7 +93,7 @@ def get_cookie(username=None, password=None, driver=None):
         count = count + 1
         print(f"需要登录")
         time.sleep(1)
-    query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql} and id  like '%0' "
+    query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD' {config.query_where_sql}  and id like '%0' "
     query_spider = mysql_zdzs_proxy(query_spider_sql)
     index = 0
     len_ = len(query_spider)

京东搜索_手动验证版_1.py

@@ -92,7 +92,7 @@ def get_cookie(username=None,password=None,driver=None):
         count = count + 1
         print(f"需要登录")
         time.sleep(1)
-    query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD'  {config.query_where_sql} and id  like '%1' "
+    query_spider_sql = f"SELECT id,url,mallId,docId FROM DMP_SEARCH_SPIDER WHERE ID NOT IN (SELECT ID FROM DMP_SEARCH_DATA) and mallId = 'DS-JD'  {config.query_where_sql} and id  like '%1'"
     query_spider = mysql_zdzs_proxy(query_spider_sql)
     index = 0
     len_ = len(query_spider)

京东搜索_手动验证版_8.py

@@ -174,7 +174,7 @@ if __name__ == "__main__":
     driver = webdriver.Chrome(options=option)
     option.add_argument("--disable-blink-features=AutomationControlled")
     driver.set_window_size(1200, 600)
-    account_start_index = 8
+    account_start_index = 1
     get_cookie(username=config.account_list[account_start_index]['username'],
                password=config.account_list[account_start_index]['password'], driver=driver)
     while True:

京东搜索_手动验证版_9.py

@@ -175,7 +175,7 @@ if __name__ == "__main__":
     driver = webdriver.Chrome(options=option)
     option.add_argument("--disable-blink-features=AutomationControlled")
     driver.set_window_size(1200, 600)
-    account_start_index = 8
+    account_start_index = 1
     get_cookie(username=config.account_list[account_start_index]['username'],
                password=config.account_list[account_start_index]['password'], driver=driver)
     while True: