update 26.1.15.1

baocui_spider/bc_new_daily_spider.py

@@ -206,27 +206,28 @@ def baocui_main(log):
         raise Exception("MySQL数据库连接失败")
 
     try:
-        while True:
-            # keyword = 'E000011631'
-            sql_code_list = sql_pool.select_all("select keyword from baocui_task where state = 0 limit 10000")
-            sql_code_list = [i[0] for i in sql_code_list]
-            # if not sql_code_list:
-            #     log.debug(
-            #         f"{inspect.currentframe().f_code.co_name} -> No len sql_code_list, Wait 1 hours again query.........")
-            #     time.sleep(3600)
-            #     sql_code_list = sql_pool.select_all("select keyword from baocui_task where state = 0 limit 10000")
-            #     sql_code_list = [i[0] for i in sql_code_list]
-            if not sql_code_list:
-                log.debug(f"{inspect.currentframe().f_code.co_name} -> No len sql_code_list")
-                break
-
-            for keyword in sql_code_list:
-                try:
-                    get_bc_resp(log, keyword, sql_pool)
-                except Exception as e:
-                    log.error(f"Loop keyword:{keyword} 获取失败, error:{e}")
-                    sql_pool.update_one("update baocui_task set state = 3 where keyword = %s", (keyword,))
-                time.sleep(random.uniform(0.5, 1))
+        # while True:
+        # keyword = 'E000011631'
+        sql_code_list = sql_pool.select_all("select keyword from baocui_task where state != 1")
+        sql_code_list = [i[0] for i in sql_code_list]
+        # if not sql_code_list:
+        #     log.debug(
+        #         f"{inspect.currentframe().f_code.co_name} -> No len sql_code_list, Wait 1 hours again query.........")
+        #     time.sleep(3600)
+        #     sql_code_list = sql_pool.select_all("select keyword from baocui_task where state = 0 limit 10000")
+        #     sql_code_list = [i[0] for i in sql_code_list]
+        if not sql_code_list:
+            log.debug(f"{inspect.currentframe().f_code.co_name} -> No len sql_code_list")
+            # break
+            return
+
+        for keyword in sql_code_list:
+            try:
+                get_bc_resp(log, keyword, sql_pool)
+            except Exception as e:
+                log.error(f"Loop keyword:{keyword} 获取失败, error:{e}")
+                sql_pool.update_one("update baocui_task set state = 3 where keyword = %s", (keyword,))
+            time.sleep(random.uniform(0.5, 1))
     except Exception as e:
         log.error(f'{inspect.currentframe().f_code.co_name} error: {e}')
     finally:
@@ -241,7 +242,8 @@ def schedule_task():
     baocui_main(log=logger)
 
     # 设置定时任务
-    schedule.every(30).days.at("00:01").do(baocui_main, log=logger)
+    # schedule.every(30).days.at("00:01").do(baocui_main, log=logger)
+    schedule.every().days.at("05:00").do(baocui_main, log=logger)
 
     while True:
         schedule.run_pending()

zhongjian_spider/zj_new_daily_spider.py

@@ -165,17 +165,18 @@ def zhongjian_main(log):
         raise Exception("MySQL数据库连接失败")
 
     try:
-        while True:
-            sql_ra_no_list = sql_pool.select_all('select tag_no from zhongjian_task where state = 0 limit 10000')
-            sql_ra_no_list = [i[0] for i in sql_ra_no_list]
-            if not sql_ra_no_list:
-                log.info(f'没有需要处理的数据,等待下一轮处理........................................................')
-                break
-
-            try:
-                loop_rating_no(log, sql_pool, sql_ra_no_list)
-            except  Exception as e:
-                log.error(f'{inspect.currentframe().f_code.co_name} error: {e}')
+        # while True:
+        sql_ra_no_list = sql_pool.select_all('select tag_no from zhongjian_task where state = 0 limit 10000')
+        sql_ra_no_list = [i[0] for i in sql_ra_no_list]
+        if not sql_ra_no_list:
+            log.info(f'没有需要处理的数据,等待下一轮处理........................................................')
+            # break
+            return
+
+        try:
+            loop_rating_no(log, sql_pool, sql_ra_no_list)
+        except  Exception as e:
+            log.error(f'{inspect.currentframe().f_code.co_name} error: {e}')
     except Exception as e:
         log.error(f'{inspect.currentframe().f_code.co_name} error: {e}')
     finally:
@@ -190,7 +191,8 @@ def schedule_task():
     zhongjian_main(log=logger)
 
     # 设置定时任务
-    schedule.every(30).days.at("00:01").do(zhongjian_main, log=logger)
+    # schedule.every(30).days.at("00:01").do(zhongjian_main, log=logger)
+    schedule.every().days.at("05:00").do(zhongjian_main, log=logger)
 
     while True:
         schedule.run_pending()