瀏覽代碼

fix(spider): 修复请求超时配置并优化爬虫执行

- 将请求超时参数从单值改为连接超时和读取超时的元组格式 (5, 30)
- 移除 zc_demo.py 中未使用的 time 模块导入
- 注释掉 zc_demo.py 中默认执行的 zc_main 函数调用
- 修正 zc_new_daily_spider.py 中的代理请求配置格式
- 清理代码中的多余空行以提升可读性
charley 5 天之前
父節點
當前提交
5034d63769
共有 3 個文件被更改,包括 6 次插入7 次删除
  1. 3 3
      zc_spider/zc_demo.py
  2. 1 2
      zc_spider/zc_history_spider.py
  3. 2 2
      zc_spider/zc_new_daily_spider.py

+ 3 - 3
zc_spider/zc_demo.py

@@ -91,7 +91,7 @@ def make_encrypted_post_request(log, url: str, request_data: dict, extra_headers
 
     encrypted_body = CryptoHelper.encrypt_request_data(request_data)
     # print(request_headers)
-    response = requests.post(url, headers=request_headers, json=encrypted_body, timeout=30)
+    response = requests.post(url, headers=request_headers, json=encrypted_body, timeout=(5, 30))
     # response.raise_for_status()
 
     if response.status_code == 200:
@@ -519,12 +519,12 @@ def zc_main(log):
 
 if __name__ == '__main__':
     # 获取单页数据
-    # result = get_sold_single_page(logger, 'ZC10264451', page_num=1, page_size=20)
+    result = get_sold_single_page(logger, 'ZC10264451', page_num=1)
     # result = get_shop_single_page(logger, page_num=1, page_size=10)
     # result = get_player_single_page(logger, 1520, 'your_token', page_num=1, page_size=10)
     # print("单页数据:", result)
 
-    zc_main(logger)
+    # zc_main(logger)
     # get_vodeo(logger, 'your_token', 1726)
     # sql_pool = MySQLConnectionPool(log=logger)
     # get_shop_list(logger, sql_pool)

+ 1 - 2
zc_spider/zc_history_spider.py

@@ -2,7 +2,6 @@
 # Author : Charley
 # Python : 3.10.8
 # Date   : 2026/2/27 11:22
-import time
 import inspect
 import requests
 import user_agent
@@ -91,7 +90,7 @@ def make_encrypted_post_request(log, url: str, request_data: dict, extra_headers
 
     encrypted_body = CryptoHelper.encrypt_request_data(request_data)
     # print(request_headers)
-    response = requests.post(url, headers=request_headers, json=encrypted_body, timeout=30)
+    response = requests.post(url, headers=request_headers, json=encrypted_body, timeout=(5, 30))
     # response.raise_for_status()
 
     if response.status_code == 200:

+ 2 - 2
zc_spider/zc_new_daily_spider.py

@@ -92,7 +92,8 @@ def make_encrypted_post_request(log, url: str, request_data: dict, extra_headers
 
     encrypted_body = CryptoHelper.encrypt_request_data(request_data)
     # print(request_headers)
-    response = requests.post(url, headers=request_headers, json=encrypted_body, timeout=22, proxies=get_proxys(log))
+    # response = requests.post(url, headers=request_headers, json=encrypted_body, timeout=22, proxies=get_proxys(log))
+    response = requests.post(url, headers=request_headers, json=encrypted_body, timeout=(5, 30), proxies=get_proxys(log))
     # response.raise_for_status()
 
     if response.status_code == 200:
@@ -263,7 +264,6 @@ def parse_sold_data(log, token, items, sql_pool, shop_name):
         except Exception as e:
             log.error(f"Error getting video info for pid {pid}: {e}")
             live_id, live_open_time, live_close_time, video_url = None, None, None, None
-
         data_dict = {
             'shop_id': shop_id,
             'shop_name': shop_name,