update wait time sleep 6.13.3

kaogujia_spider/kgj_kapai_spider.py

@@ -23,7 +23,7 @@ from request_live_detail import get_live_detail
 urllib3.disable_warnings()
 
 logger.remove()
-logger.add("./kapai_logs/{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00",
+logger.add("./logs/{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00",
            format="[{time:YYYY-MM-DD HH:mm:ss.SSS}] {level} {message}",
            level="DEBUG", retention="14 day")
 
@@ -52,10 +52,6 @@ def get_proxys(log):
     获取代理
     :return: 代理
     """
-    # tunnel = "h991.kdltps.com:15818"
-    # kdl_username = "t12136177769785"
-    # kdl_password = "ety9bdi8"
-
     tunnel = "x371.kdltps.com:15818"
     kdl_username = "t13753103189895"
     kdl_password = "o0yefv6z"
@@ -515,52 +511,52 @@ def kgj_kapai_main(log):
 
     try:
         keyword = "卡牌"
-        # log.info("开始获取 product_list 数据............................................................")
-        # # sql_product_id_list = sql_pool.select_all("SELECT DISTINCT product_id FROM kgj_kapai_product_list_record")
-        # # sql_product_id_list = [item[0] for item in sql_product_id_list]
-        # try:
-        #     get_kgj_product_list(log, keyword, sql_pool, KGJ_HEADERS)
-        # except Exception as e:
-        #     log.error(f"Error main -> getting kgj product list: {e}")
-        #
-        # # sql_product_id_list.clear()
-        # log.info("product_list 数据获取完成............................................................")
-        #
-        # time.sleep(5)
-        #
-        # log.info("开始获取 product_overview 数据............................................................")
-        # product_id_list_for_product = sql_pool.select_all(
-        #     "SELECT product_id FROM kgj_kapai_product_list_record WHERE product_state = 0")
-        # product_id_list = [item[0] for item in product_id_list_for_product]
-        # for product_id in product_id_list:
-        #     try:
-        #         log.info(f"开始获取 product_id: {product_id} 的 product_overview 数据............................")
-        #         get_kgj_product_overview(log, sql_pool, product_id, KGJ_HEADERS)
-        #     except Exception as e:
-        #         log.error(f"Error main -> getting kgj product overview: {e}")
-        #         time.sleep(random.randint(4, 6))
-        #         continue
-        #     time.sleep(random.randint(4, 6))
-        # log.info("product_overview 数据获取完成............................................................")
-        #
-        # time.sleep(5)
-        #
-        # log.info("开始获取 linked_live 数据............................................................")
-        # product_id_list_for_live = sql_pool.select_all(
-        #     "SELECT product_id FROM kgj_kapai_product_list_record WHERE live_state = 0")
-        # product_id_list = [item[0] for item in product_id_list_for_live]
-        # for product_id in product_id_list:
-        #     try:
-        #         log.info(f"开始获取 product_id: {product_id} 的 linked_live 数据............................")
-        #         get_linked_live(log, sql_pool, product_id, KGJ_HEADERS)
-        #     except Exception as e:
-        #         log.error(f"Error main -> getting kgj linked_live: {e}")
-        #         time.sleep(random.randint(4, 6))
-        #         continue
-        #     time.sleep(random.randint(4, 6))
-        # log.info("linked_live 数据获取完成............................................................")
-        #
-        # time.sleep(5)
+        log.info("开始获取 product_list 数据............................................................")
+        # sql_product_id_list = sql_pool.select_all("SELECT DISTINCT product_id FROM kgj_kapai_product_list_record")
+        # sql_product_id_list = [item[0] for item in sql_product_id_list]
+        try:
+            get_kgj_product_list(log, keyword, sql_pool, KGJ_HEADERS)
+        except Exception as e:
+            log.error(f"Error main -> getting kgj product list: {e}")
+
+        # sql_product_id_list.clear()
+        log.info("product_list 数据获取完成............................................................")
+
+        time.sleep(5)
+
+        log.info("开始获取 product_overview 数据............................................................")
+        product_id_list_for_product = sql_pool.select_all(
+            "SELECT product_id FROM kgj_kapai_product_list_record WHERE product_state = 0")
+        product_id_list = [item[0] for item in product_id_list_for_product]
+        for product_id in product_id_list:
+            try:
+                log.info(f"开始获取 product_id: {product_id} 的 product_overview 数据............................")
+                get_kgj_product_overview(log, sql_pool, product_id, KGJ_HEADERS)
+            except Exception as e:
+                log.error(f"Error main -> getting kgj product overview: {e}")
+                time.sleep(random.randint(4, 6))
+                continue
+            time.sleep(random.randint(4, 6))
+        log.info("product_overview 数据获取完成............................................................")
+
+        time.sleep(5)
+
+        log.info("开始获取 linked_live 数据............................................................")
+        product_id_list_for_live = sql_pool.select_all(
+            "SELECT product_id FROM kgj_kapai_product_list_record WHERE live_state = 0")
+        product_id_list = [item[0] for item in product_id_list_for_live]
+        for product_id in product_id_list:
+            try:
+                log.info(f"开始获取 product_id: {product_id} 的 linked_live 数据............................")
+                get_linked_live(log, sql_pool, product_id, KGJ_HEADERS)
+            except Exception as e:
+                log.error(f"Error main -> getting kgj linked_live: {e}")
+                time.sleep(random.randint(4, 6))
+                continue
+            time.sleep(random.randint(4, 6))
+        log.info("linked_live 数据获取完成............................................................")
+
+        time.sleep(5)
 
         log.info("开始获取 live_detail 数据............................................................")
         sql_room_id_list = sql_pool.select_all(
@@ -592,7 +588,7 @@ def schedule_task():
     设置定时任务
     """
     # 立即运行一次任务
-    kgj_kapai_main(logger)
+    # kgj_kapai_main(logger)
 
     # 设置定时任务  考古加 -> 卡牌  一周一次  卡牌类的周三跑 抓取时间比较久 和其他几个类错开时间
     schedule.every().wednesday.at("01:01").do(kgj_kapai_main, logger)