Преглед на файлове

fix(spider): 修正代理获取重试并扩大任务查询范围

- 添加代理获取函数的重试装饰器,最多尝试5次,间隔1秒
- 更新任务查询SQL语句,筛选状态为0且auth_code大于18606718的数据
- 扩大单线程循环处理任务的auth_code范围,提高处理量
- 删除多余空行,优化代码格式和注释说明
charley преди 1 месец
родител
ревизия
8b6726beec
променени са 1 файла, в които са добавени 5 реда и са изтрити 3 реда
  1. 5 3
      bgs_spider/bgs_new_daily_spider.py

+ 5 - 3
bgs_spider/bgs_new_daily_spider.py

@@ -50,6 +50,7 @@ def get_proxys(log):
         log.error(f"Error getting proxy: {e}")
         raise e
 
+
 @retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
 def get_proxys1(log):
     # 已购买账户  北美
@@ -70,7 +71,6 @@ def get_proxys1(log):
         raise e
 
 
-
 def save_data(mysql_pool, info):
     """
     :param mysql_pool:
@@ -195,6 +195,7 @@ def bgs_main(log):
         2026/01/20  修改  +3000,-3000,
         2026/02/02  修改  +20000,-100000,
         2026/02/02 18:00  修改  大于1500w的  单线程 循环跑  
+        2026/03/26  修改为大于17990000的 扩大范围  18606719开始
         """
         # sql_bgs_id_list = sql_pool.select_all(
         #     "SELECT auth_code FROM bgs_task WHERE state != 1 AND auth_code BETWEEN %s AND %s LIMIT 120000",
@@ -202,8 +203,9 @@ def bgs_main(log):
 
         while True:
             sql_bgs_id_list = sql_pool.select_all(
-                "SELECT auth_code FROM bgs_task WHERE state!=1 AND auth_code > 15000000 LIMIT 5000")
-                # "SELECT auth_code FROM bgs_task WHERE state!=1 AND auth_code > 17990000 LIMIT 5000"
+                # "SELECT auth_code FROM bgs_task WHERE state!=1 AND auth_code > 15000000 LIMIT 5000"
+                "SELECT auth_code FROM bgs_task WHERE state = 0 AND auth_code > 18606718 LIMIT 50000"
+            )
             sql_bgs_id_list = [bid[0] for bid in sql_bgs_id_list]
 
             if not sql_bgs_id_list: