refactor(spider): 统一主函数命名并新增调度启动文件

- 注释掉三个爬虫模块中的日志配置，避免重复初始化日志
- 将 en_one_piece_spider、jp_one_piece_spider 和 jz_one_piece_spider 的主函数名由 op_main 改为带前缀的名称以示区分
- 新增 start_onepiece_spider.py 启动文件，整合三个爬虫主函数的调用和调度
- 启动文件实现多线程运行爬虫任务，支持按周定时执行
- 启动文件中初始化日志，统一日志格式和目录管理

one_piece_spider/en_one_piece_spider.py

@@ -11,10 +11,10 @@ from parsel import Selector
 from mysql_pool import MySQLConnectionPool
 from tenacity import retry, stop_after_attempt, wait_fixed
 
-logger.remove()
-logger.add("./logs/{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00",
-           format="[{time:YYYY-MM-DD HH:mm:ss.SSS}] {level} {message}",
-           level="DEBUG", retention="7 day")
+# logger.remove()
+# logger.add("./logs/{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00",
+#            format="[{time:YYYY-MM-DD HH:mm:ss.SSS}] {level} {message}",
+#            level="DEBUG", retention="7 day")
 
 headers = {
     "accept": "application/json, text/plain, */*",
@@ -143,7 +143,7 @@ def get_single_page(log, card_type_id, card_type_name, sql_pool):
 
 
 @retry(stop=stop_after_attempt(100), wait=wait_fixed(3600), after=after_log)
-def op_main(log):
+def en_op_main(log):
     """
     主函数
     :param log: logger对象
@@ -172,4 +172,4 @@ def op_main(log):
 if __name__ == '__main__':
     # get_single_page(logger, 1, 569302)
     # get_cachelist(logger)
-    op_main(logger)
+    en_op_main(logger)

one_piece_spider/jp_one_piece_spider.py

@@ -11,10 +11,10 @@ from parsel import Selector
 from mysql_pool import MySQLConnectionPool
 from tenacity import retry, stop_after_attempt, wait_fixed
 
-logger.remove()
-logger.add("./logs/{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00",
-           format="[{time:YYYY-MM-DD HH:mm:ss.SSS}] {level} {message}",
-           level="DEBUG", retention="7 day")
+# logger.remove()
+# logger.add("./logs/{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00",
+#            format="[{time:YYYY-MM-DD HH:mm:ss.SSS}] {level} {message}",
+#            level="DEBUG", retention="7 day")
 
 headers = {
     "accept": "application/json, text/plain, */*",
@@ -146,7 +146,7 @@ def get_single_page(log, card_type_id, card_type_name, sql_pool):
 
 
 @retry(stop=stop_after_attempt(100), wait=wait_fixed(3600), after=after_log)
-def op_main(log):
+def jp_op_main(log):
     """
     主函数
     :param log: logger对象
@@ -176,4 +176,4 @@ def op_main(log):
 if __name__ == '__main__':
     # get_single_page(logger, 1, 569302)
     # get_cachelist(logger)
-    op_main(logger)
+    jp_op_main(logger)

one_piece_spider/jz_one_piece_spider.py

@@ -9,10 +9,10 @@ from loguru import logger
 from tenacity import retry, stop_after_attempt, wait_fixed
 from mysql_pool import MySQLConnectionPool
 
-logger.remove()
-logger.add("./logs/{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00",
-           format="[{time:YYYY-MM-DD HH:mm:ss.SSS}] {level} {message}",
-           level="DEBUG", retention="7 day")
+# logger.remove()
+# logger.add("./logs/{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00",
+#            format="[{time:YYYY-MM-DD HH:mm:ss.SSS}] {level} {message}",
+#            level="DEBUG", retention="7 day")
 
 headers = {
     "accept": "application/json, text/plain, */*",
@@ -224,7 +224,7 @@ def get_detail(log, card_type_id, card_type_name, card_id, sql_pool):
 
 
 @retry(stop=stop_after_attempt(100), wait=wait_fixed(3600), after=after_log)
-def op_main(log):
+def jz_op_main(log):
     """
     主函数
     :param log: logger对象
@@ -249,4 +249,4 @@ def op_main(log):
 if __name__ == '__main__':
     # get_single_page(logger, 1)
     # get_cachelist(logger)
-    op_main(logger)
+    jz_op_main(logger)

one_piece_spider/start_onepiece_spider.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+# Author : Charley
+# Python : 3.10.8
+# Date   : 2025/8/15 17:53
+import time
+import schedule
+import threading
+from loguru import logger
+
+from en_one_piece_spider import en_op_main
+from jp_one_piece_spider import jp_op_main
+from jz_one_piece_spider import jz_op_main
+
+logger.remove()
+logger.add("./logs/{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00",
+           format="[{time:YYYY-MM-DD HH:mm:ss.SSS}] {level} {message}",
+           level="DEBUG", retention="7 day")
+
+
+def run_threaded(job_func, *args, **kwargs):
+    """
+    在新线程中运行给定的函数，并传递参数。
+
+    :param job_func: 要运行的目标函数
+    :param args: 位置参数
+    :param kwargs: 关键字参数
+    """
+    job_thread = threading.Thread(target=job_func, args=args, kwargs=kwargs)
+    job_thread.start()
+
+
+def schedule_task():
+    """
+    两个爬虫模块的启动文件
+    bidding_main
+    weika_change_card_by_id_spider
+    change_card_main
+    """
+    # 立即运行一次任务
+    run_threaded(en_op_main(log=logger))
+    run_threaded(jp_op_main(log=logger))
+    run_threaded(jz_op_main(log=logger))
+
+    # 设置定时任务
+    schedule.every().monday.at("01:01").do(run_threaded, en_op_main, log=logger)
+    schedule.every().monday.at("02:01").do(run_threaded, jp_op_main, log=logger)
+    schedule.every().monday.at("03:01").do(run_threaded, jz_op_main, log=logger)
+
+    while True:
+        schedule.run_pending()
+        time.sleep(1)
+
+
+if __name__ == '__main__':
+    schedule_task()