Parcourir la source

fix(spider): 修复请求头信息及启动流程问题

- 更新请求头中的版本号、cid、sk 值以匹配新版本要求
- 修正判断 URL 包含关键字的条件表达式逻辑错误
- 启用启动文件中登录函数和任务线程的执行
- 配置定时任务每日更新 token 并按设定时间运行爬取任务
charley il y a 2 semaines
Parent
commit
bd347824fb
2 fichiers modifiés avec 13 ajouts et 36 suppressions
  1. 6 29
      yueka_spider/settings.py
  2. 7 7
      yueka_spider/start_yueka_spider.py

+ 6 - 29
yueka_spider/settings.py

@@ -26,36 +26,14 @@ HEADERS = {
     "systemversion": "30",
     "theme": "light",
     "lang": "zh",
-    "verse-ua": "a42b9928820764bfa432049923f874be",
-    "version": "1.1.3",
+    "verse-ua": "37ba2436e1eec03479ded38f6acb8fd9",
+    "version": "1.1.5",
     "isphysicaldevice": "true",
-    "sktime": "1774854185592",
-    "cid": "42661545",
-    "sk": "d4f95a2f040337a73979f641138deb8f" # 每次版本变化 也需要修改
+    "sktime": "1776653418408",
+    "cid": "08295441",
+    "sk": "d6fb6fcce1243c11dc940183af6cdad6"  # 每次版本变化 也需要修改
 }
 
-# headers = {
-#     "User-Agent": "Dart/3.5 (dart:io)",
-#     "Accept-Encoding": "gzip",
-#     "deviceid": "110bec91-951f-4346-8195-95700fdb8c1d",
-#     "brand": "google",
-#     "channel": "JC",
-#     "os": "android",
-#     "content-type": "application/json; charset=utf-8",
-#     # "authori-zation": "a-d9200f060d5c4e4f81146d4118b1d148",
-#     "systemversion": "30",
-#     "theme": "light",
-#     "lang": "zh",
-#     "verse-ua": "a42b9928820764bfa432049923f874be",
-#     "version": "1.1.3",
-#     "isphysicaldevice": "true",
-#     "sktime": "1774854185592",
-#     "cid": "42661545",
-#     "sk": "d4f95a2f040337a73979f641138deb8f"
-# }
-
-
-
 
 def after_log(retry_state):
     """
@@ -95,7 +73,6 @@ def get_proxys(log):
         raise e
 
 
-
 @retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
 def make_request(log, method, url, params=None, data=None, headers=None, proxies=None, timeout=22, token=None):
     """
@@ -113,7 +90,7 @@ def make_request(log, method, url, params=None, data=None, headers=None, proxies
     """
     if headers is None:
         headers = HEADERS
-        if 'getHitCardReport' or 'getCardPublicly' or 'productDetailDynamics'  or 'productShowList' in url:
+        if 'getHitCardReport' or 'getCardPublicly' or 'productDetailDynamics' or 'productShowList' in url:
             if not token:
                 token = "a-87768ad775bf4513bec14f2c4fd2cd0c"
             headers["authori-zation"] = token

+ 7 - 7
yueka_spider/start_yueka_spider.py

@@ -27,17 +27,17 @@ def schedule_task():
     爬虫模块的启动文件
     """
     # 立即运行一次任务
-    # login_main()
-    # time.sleep(5)
-    
-    # run_threaded(yueka_main, log=logger)
-    
+    login_main()
+    time.sleep(5)
+
+    run_threaded(yueka_main, log=logger)
+
     # 每天  定时更新token
     schedule.every().day.at("00:01").do(run_threaded, login_main)
-    
+
     # 设置定时任务
     schedule.every().day.at("00:06").do(run_threaded, yueka_main, log=logger)
-    
+
     while True:
         schedule.run_pending()
         time.sleep(1)