update bgs 10.30.1

bgs_spider/bgs_new_daily_spider.py

@@ -172,25 +172,25 @@ def bgs_main(log):
 
 
         # 20251029 往前2000  往后3000
-        while True:
-            # 倒序查 5000个
-            sql_bgs_id_list = sql_pool.select_all(
-                "SELECT auth_code FROM bgs_task WHERE state!=1 AND auth_code BETWEEN %s AND %s LIMIT 5000",
-                (max_bgs_id - 2000, max_bgs_id + 3000))
-
-            # sql_bgs_id_list = sql_pool.select_all(
-            #     "SELECT auth_code FROM bgs_task WHERE state!=1 AND auth_code > 18239485 LIMIT 5000")
-                # "SELECT auth_code FROM bgs_task WHERE state!=1 AND auth_code > 17990000 LIMIT 5000"
-            sql_bgs_id_list = [bid[0] for bid in sql_bgs_id_list]
-            # for bid in sql_bgs_id_list:
-            try:
-                process_urls(log, sql_bgs_id_list, sql_pool, batch_size=1000,
-                             max_workers=10)  # 根据需要调整batch_size和max_workers
-                # get_data(bid, mysql_pool)
-            except Exception as e:
-                log.error('process urls: ', e)
-
-            time.sleep(5)
+        # while True:
+        # 倒序查 5000个
+        sql_bgs_id_list = sql_pool.select_all(
+            "SELECT auth_code FROM bgs_task WHERE state!=1 AND auth_code BETWEEN %s AND %s LIMIT 5000",
+            (max_bgs_id - 2000, max_bgs_id + 3000))
+
+        # sql_bgs_id_list = sql_pool.select_all(
+        #     "SELECT auth_code FROM bgs_task WHERE state!=1 AND auth_code > 18239485 LIMIT 5000")
+            # "SELECT auth_code FROM bgs_task WHERE state!=1 AND auth_code > 17990000 LIMIT 5000"
+        sql_bgs_id_list = [bid[0] for bid in sql_bgs_id_list]
+        # for bid in sql_bgs_id_list:
+        try:
+            process_urls(log, sql_bgs_id_list, sql_pool, batch_size=1000,
+                         max_workers=10)  # 根据需要调整batch_size和max_workers
+            # get_data(bid, mysql_pool)
+        except Exception as e:
+            log.error('process urls: ', e)
+
+        # time.sleep(5)
 
     except Exception as e:
         log.error(e)