update bgs task modify 9.23.1

bgs_spider/bgs_new_daily_spider.py

@@ -172,17 +172,21 @@ def bgs_main(log):
         #
         # sql_pool.insert_all("INSERT INTO bgs_task(auth_code) VALUES (%s)", bgs_id_list)
 
-        # 查 5000个
-        sql_bgs_id_list = sql_pool.select_all(
-            "SELECT auth_code FROM bgs_task WHERE state!=1 AND auth_code > 17990000 LIMIT 5000")
-        sql_bgs_id_list = [bid[0] for bid in sql_bgs_id_list]
-        # for bid in sql_bgs_id_list:
-        try:
-            process_urls(log, sql_bgs_id_list, sql_pool, batch_size=1000,
-                         max_workers=10)  # 根据需要调整batch_size和max_workers
-            # get_data(bid, mysql_pool)
-        except Exception as e:
-            log.error('process urls: ', e)
+        # 20250923 查询到Friday, May 16, 2025  往后进行
+        while True:
+            # 倒序查 5000个
+            sql_bgs_id_list = sql_pool.select_all(
+                "SELECT auth_code FROM bgs_task WHERE state!=1 AND auth_code > 17990000 LIMIT 5000")
+            sql_bgs_id_list = [bid[0] for bid in sql_bgs_id_list]
+            # for bid in sql_bgs_id_list:
+            try:
+                process_urls(log, sql_bgs_id_list, sql_pool, batch_size=1000,
+                             max_workers=10)  # 根据需要调整batch_size和max_workers
+                # get_data(bid, mysql_pool)
+            except Exception as e:
+                log.error('process urls: ', e)
+
+            time.sleep(5)
 
     except Exception as e:
         log.error(e)