update 12.22.1

courtyard_spider/YamlLoader.py

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+#
+import os, re
+import yaml
+
+regex = re.compile(r'^\$\{(?P<ENV>[A-Z_\-]+\:)?(?P<VAL>[\w\.]+)\}$')
+
+class YamlConfig:
+    def __init__(self, config):
+        self.config = config
+
+    def get(self, key:str):
+        return YamlConfig(self.config.get(key))
+    
+    def getValueAsString(self, key: str):
+        try:
+            match = regex.match(self.config[key])
+            group = match.groupdict()
+            if group['ENV'] != None:
+                env = group['ENV'][:-1]
+                return os.getenv(env, group['VAL'])
+            return None
+        except:
+            return self.config[key]
+    
+    def getValueAsInt(self, key: str):
+        try:
+            match = regex.match(self.config[key])
+            group = match.groupdict()
+            if group['ENV'] != None:
+                env = group['ENV'][:-1]
+                return int(os.getenv(env, group['VAL']))
+            return 0
+        except:
+            return int(self.config[key])
+        
+    def getValueAsBool(self, key: str, env: str = None):
+        try:
+            match = regex.match(self.config[key])
+            group = match.groupdict()
+            if group['ENV'] != None:
+                env = group['ENV'][:-1]
+                return bool(os.getenv(env, group['VAL']))
+            return False
+        except:
+            return bool(self.config[key])
+    
+def readYaml(path:str = 'application.yml', profile:str = None) -> YamlConfig:
+    if os.path.exists(path):
+        with open(path) as fd:
+            conf = yaml.load(fd, Loader=yaml.FullLoader)
+
+    if profile != None:
+        result = path.split('.')
+        profiledYaml = f'{result[0]}-{profile}.{result[1]}'
+        if os.path.exists(profiledYaml):
+            with open(profiledYaml) as fd:
+                conf.update(yaml.load(fd, Loader=yaml.FullLoader))
+
+    return YamlConfig(conf)
+
+# res = readYaml()
+# mysqlConf = res.get('mysql')
+# print(mysqlConf)
+
+# print(res.getValueAsString("host"))
+# mysqlYaml = mysqlConf.getValueAsString("host")
+# print(mysqlYaml)
+# host = mysqlYaml.get("host").split(':')[-1][:-1]
+# port = mysqlYaml.get("port").split(':')[-1][:-1]
+# username = mysqlYaml.get("username").split(':')[-1][:-1]
+# password = mysqlYaml.get("password").split(':')[-1][:-1]
+# mysql_db = mysqlYaml.get("db").split(':')[-1][:-1]
+# print(host,port,username,password)

courtyard_spider/application.yml

@@ -0,0 +1,6 @@
+mysql:
+  host: ${MYSQL_HOST:100.64.0.21}
+  port: ${MYSQL_PROT:3306}
+  username: ${MYSQL_USERNAME:crawler}
+  password: ${MYSQL_PASSWORD:Pass2022}
+  db: ${MYSQL_DATABASE:crawler}

courtyard_spider/courtyard_spider.py

@@ -0,0 +1,466 @@
+# -*- coding: utf-8 -*-
+# Author : Charley
+# Python : 3.10.8
+# Date   : 2025/12/2 14:08
+import threading
+import time
+import inspect
+import requests
+import schedule
+import user_agent
+from loguru import logger
+from parsel import Selector
+from datetime import datetime
+from mysql_pool import MySQLConnectionPool
+from DrissionPage import ChromiumPage, ChromiumOptions
+from tenacity import retry, stop_after_attempt, wait_fixed
+
+"""
+扣驾的
+"""
+
+logger.remove()
+logger.add("./logs/{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00",
+           format="[{time:YYYY-MM-DD HH:mm:ss.SSS}] {level} {message}",
+           level="DEBUG", retention="7 day")
+
+headers = {
+    "accept": "application/json",
+    "referer": "https://courtyard.io/",
+    "user-agent": user_agent.generate_user_agent()
+}
+
+
+
+# 全局变量标识首次运行是否完成
+detail_first_run_completed = False
+
+def after_log(retry_state):
+    """
+    retry 回调
+    :param retry_state: RetryCallState 对象
+    """
+    # 检查 args 是否存在且不为空
+    if retry_state.args and len(retry_state.args) > 0:
+        log = retry_state.args[0]  # 获取传入的 logger
+    else:
+        log = logger  # 使用全局 logger
+
+    if retry_state.outcome.failed:
+        log.warning(
+            f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} Times")
+    else:
+        log.info(f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} succeeded")
+
+
+@retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
+def get_proxys(log):
+    """
+    获取代理
+    :return: 代理
+    """
+    tunnel = "x371.kdltps.com:15818"
+    kdl_username = "t13753103189895"
+    kdl_password = "o0yefv6z"
+    try:
+        proxies = {
+            "http": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": kdl_username, "pwd": kdl_password, "proxy": tunnel},
+            "https": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": kdl_username, "pwd": kdl_password, "proxy": tunnel}
+        }
+        return proxies
+    except Exception as e:
+        log.error(f"Error getting proxy: {e}")
+        raise e
+
+
+@retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
+def get_goods_list(log, sql_pool):
+    """
+    获取商品列表
+    :param log: logger对象
+    :param sql_pool: MySQL连接池对象
+    :return:
+    """
+    log.info(f"========================== 开始获取商品列表 ==========================")
+    url = "https://api.courtyard.io/vending-machines"
+    response = requests.get(url, headers=headers, timeout=22)
+    # print(response.text)
+    response.raise_for_status()
+
+    vendingMachines = response.json().get("vendingMachines", [])
+
+    for item in vendingMachines:
+        bag_id = item.get("id")
+        bag_title = item.get("title")
+        # sealed_pack_animation = item.get("sealedPackAnimation")
+        # sealed_pack_image = item.get("sealedPackImage")
+        category_title = item.get("category", {}).get("title")
+        price = item.get("saleDetails", {}).get("salePriceUsd")
+
+        data_dict = {
+            "bag_id": bag_id,
+            "bag_title": bag_title,
+            "category": category_title,
+            "price":price
+        }
+        # log.info(f'get_goods_list: {data_dict}')
+
+        try:
+            get_goods_detail(log, data_dict, sql_pool)
+        except Exception as e:
+            log.error(f"Error processing item: {e}")
+
+    # 保存数据
+    # if info_list:
+    #     log.info(f"获取商品列表成功, 共 {len(info_list)} 条数据")
+    #     sql_pool.insert_many(table="courtyard_vending_machines_record", data_list=info_list, ignore= True)
+
+
+@retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
+def get_goods_detail(log, query_dict: dict, sql_pool=None):
+    """
+    获取商品详情
+    :param log: logger对象
+    :param query_dict: query_dict
+    :param sql_pool: MySQL连接池对象
+    :return:
+    """
+    log.info(f"========================== 获取商品详情 ==========================")
+    url = "https://api.courtyard.io/index/query/recent-pulls"
+    params = {
+        "limit": "250",
+        # "vendingMachineIds": "pkmn-basic-pack"
+        "vendingMachineIds": query_dict["bag_id"]
+    }
+    response = requests.get(url, headers=headers, params=params, timeout=22)
+    # print(response.text)
+    response.raise_for_status()
+
+    pulls = response.json().get("assets", [])
+    info_list = []
+    for item in pulls:
+        detail_title = item.get("title")
+        detail_id = item.get("proof_of_integrity")
+
+        if not detail_id:
+            log.error(f"信息异常, detail_id: {detail_id}")
+            continue
+
+        asset_pictures = item.get("asset_pictures", [])
+        img_front = asset_pictures[0] if len(asset_pictures) > 0 else None
+        img_back = asset_pictures[1] if len(asset_pictures) > 1 else None
+
+        # crawl_date = time.strftime("%Y-%m-%d", time.localtime())
+
+        data_dict = {
+            "bag_id": query_dict["bag_id"],
+            "bag_title": query_dict["bag_title"],
+            "category": query_dict["category"],
+            "price": query_dict["price"],
+            "detail_id": detail_id,
+            "detail_title": detail_title,
+            "img_front": img_front,
+            "img_back": img_back,
+            # "crawler_date": crawl_date
+        }
+        # log.info(f'data_dict:{data_dict}')
+        info_list.append(data_dict)
+
+    # 保存数据
+    if info_list:
+        log.info(f"获取商品详情成功, 共 {len(info_list)} 条数据")
+        sql_pool.insert_many(table="courtyard_list_record", data_list=info_list, ignore=True)
+
+
+def convert_time_format(time_str):
+    """
+    将时间字符串转换为标准格式
+    :param time_str: 原始时间字符串，如 "December 3, 2025 at 4:29 PM"
+    :return: 标准时间格式字符串，如 "2025-12-03 16:29:00"
+    """
+    if not time_str:
+        return None
+
+    try:
+        dt_obj = datetime.strptime(time_str, "%B %d, %Y at %I:%M %p")
+        return dt_obj.strftime("%Y-%m-%d %H:%M:%S")
+    except ValueError as e:
+        logger.warning(f"时间转换失败: {time_str}, 错误: {e}")
+        return None
+
+
+@retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
+def get_sale_detail_single_page(log, page, sql_id, detail_id, sql_pool=None):
+    """
+    获取商品详情
+    :param log: logger对象
+    :param page: page对象
+    :param sql_id: 数据库id
+    :param detail_id: 商品详情id
+    :param sql_pool: MySQL连接池对象
+    :return:
+    """
+    log.info(f"========================== 获取商品 <sale> 详情, sql_id: {sql_id} ==========================")
+    # page_url = "https://courtyard.io/asset/a4f0bbebd858370567f1779fddf0f55630810116d80965e33940fc8ff5ac94b4"
+    page_url = f"https://courtyard.io/asset/{detail_id}"
+    page.get(page_url)
+    page.wait.load_start()
+    log.debug(f'{inspect.currentframe().f_code.co_name} -> 页面加载成功, url: {page_url}')
+
+    html = page.html
+    if not html:
+        log.error(f'{inspect.currentframe().f_code.co_name} -> 页面加载失败...........')
+        raise Exception('页面加载失败, 重新加载........')  # 抛出异常以便重试
+
+    selector = Selector(text=html)
+
+    # 方法一：通过文本内容匹配（优先）
+    correlation_spans = selector.xpath('//span[contains(text(), ":")]/text()')
+    correlation_id = None
+
+    for text_selector in correlation_spans:
+        correlation_id = text_selector.get()  # ✅ 获取字符串
+
+        # match = re.search(r'[\w\s]+:\s*(\d+)', text)
+        # if match:
+        #     correlation_id = match.group(1)
+        #     break  # 获取第一个有效 ID
+    # correlation_spans = selector.xpath('//span[contains(text(), ":")]')
+    # correlation_text = None
+    #
+    # for span in correlation_spans:
+    #     text = span.get()
+    #     if text and ":" in text:
+    #         correlation_text = text
+    #         break
+
+    # 如果方法一失败，使用方法二：通过结构定位（备用）
+    if not correlation_id:
+        correlation_span = selector.xpath('//a[contains(@href, "cgccards.com")]/preceding-sibling::span[1]/text()')
+        correlation_id = correlation_span.get()
+
+    # 初始化所有可能的字段为None
+    data_dict = {"detail_id": detail_id, "correlation_id": correlation_id, "burn_from": None,
+                 "burn_to": None, "burn_time": None, "sale_price": None, "sale_from": None, "sale_to": None,
+                 "sale_time": None, "mint_price": None, "mint_from": None, "mint_to": None, "mint_time": None}
+
+    # 提取各个标签的数据
+    # tag_div_list = selector.xpath('//div[@class="MuiBox-root css-aylq9e"]/div')# class="MuiBox-root css-1b09fes"
+    # 获取 "Activity history" 后面的 div
+    activity_div = selector.xpath('//h6[text()="Activity history"]/following-sibling::div[1]/div')
+    for tag_div in activity_div:
+        tag_name = tag_div.xpath('./div[1]/div/span/text()').get()
+        if not tag_name:
+            continue
+
+        if tag_name == "Burn":
+            data_dict["burn_from"] = tag_div.xpath('./div[2]/div[1]//h6/text()').get()
+            data_dict["burn_to"] = tag_div.xpath('./div[2]/div[2]//h6/text()').get()
+            data_dict["burn_time"] = tag_div.xpath('./div[2]/div[3]/span/@aria-label').get()
+            # December 3, 2025 at 4:29 PM 转换时间格式
+            data_dict["burn_time"] = convert_time_format(data_dict["burn_time"])
+        elif tag_name == "Sale":
+            sale_price = tag_div.xpath('./div[2]/span/text()').get()
+            if sale_price:
+                sale_price = sale_price.replace("$", "").replace(",", "")
+            data_dict["sale_price"] = sale_price
+
+            data_dict["sale_from"] = tag_div.xpath('./div[3]/div[1]//h6/text()').get()
+            data_dict["sale_to"] = tag_div.xpath('./div[3]/div[2]//h6/text()').get()
+            data_dict["sale_time"] = tag_div.xpath('./div[3]/div[3]/span/@aria-label').get()
+            # December 3, 2025 at 4:29 PM 转换时间格式
+            data_dict["sale_time"] = convert_time_format(data_dict["sale_time"])
+        elif tag_name == "Mint":
+            mint_price = tag_div.xpath('./div[2]/span/text()').get()
+            if mint_price:
+                mint_price = mint_price.replace("$", "").replace(",", "")
+            data_dict["mint_price"] = mint_price
+
+            data_dict["mint_from"] = tag_div.xpath('./div[3]/div[1]//h6/text()').get()
+            data_dict["mint_to"] = tag_div.xpath('./div[3]/div[2]//h6/text()').get()
+            data_dict["mint_time"] = tag_div.xpath('./div[3]/div[3]/span/@aria-label').get()
+            # December 3, 2025 at 4:29 PM 转换时间格式
+            data_dict["mint_time"] = convert_time_format(data_dict["mint_time"])
+
+    # log.info(f'Sale detail data: {data_dict}')
+
+    # 保存数据
+    sql_pool.insert_one_or_dict(table="courtyard_detail_record", data=data_dict, ignore=True)
+
+
+@retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
+def get_sale_detail_list(log, sql_pool=None):
+    """
+    获取商品详情
+    :param log: logger对象
+    # :param detail_id_list: 详情id列表
+    :param sql_pool: MySQL连接池对象
+    :return:
+    """
+    log.info(f"========================== 获取商品 <sale> 详情  LIST ==========================")
+    options = ChromiumOptions()
+    options.set_paths(local_port=9138, user_data_path=r'D:\Drissionpage_temp\courtyard_port_9138')
+    # options.set_proxy("http://" + tunnel)
+    # options.auto_port(True)
+
+    options.no_imgs(True)
+    # 最大化
+    options.set_argument("--start-maximized")
+    options.set_argument("--disable-gpu")
+    options.set_argument("-accept-lang=en-US")
+    page = ChromiumPage(options)
+    try:
+        sql_detail_id_list = sql_pool.select_all("SELECT id, detail_id FROM courtyard_list_record WHERE state = 0")
+
+        for sql_detail_id in sql_detail_id_list:
+            sql_id = sql_detail_id[0]
+            detail_id = sql_detail_id[1]
+            try:
+                get_sale_detail_single_page(log, page, sql_id, detail_id, sql_pool)
+                sql_pool.update_one("UPDATE courtyard_list_record SET state = 1 WHERE id = %s", (sql_id,))
+            except Exception as e:
+                log.error(f'get_sale_detail_single_page error: {e}')
+                sql_pool.update_one("UPDATE courtyard_list_record SET state = 2 WHERE id = %s", (sql_id,))
+
+    except Exception as e:
+        log.error(f'get_response error: {e}')
+        raise 'get_response error'
+    finally:
+        page.quit()
+
+
+@retry(stop=stop_after_attempt(100), wait=wait_fixed(3600), after=after_log)
+def list_main(log):
+    """
+    主函数 自动售货机
+    :param log: logger对象
+    """
+    log.info(
+        f'开始运行 {inspect.currentframe().f_code.co_name} 爬虫任务....................................................')
+    start = time.time()
+
+    # 配置 MySQL 连接池
+    sql_pool = MySQLConnectionPool(log=log)
+    if not sql_pool.check_pool_health():
+        log.error("数据库连接池异常")
+        raise RuntimeError("数据库连接池异常")
+
+    try:
+        try:
+            log.debug('------------------- 开始获取商品列表 -------------------')
+            get_goods_list(log, sql_pool)
+        except Exception as e:
+            log.error(f'get_goods_list error: {e}')
+
+    except Exception as e:
+        log.error(f'{inspect.currentframe().f_code.co_name} error: {e}')
+    finally:
+        log.info(f'爬虫程序 {inspect.currentframe().f_code.co_name} 运行结束,等待下一轮的采集任务............')
+
+    end = time.time()
+    elapsed_time = end - start
+    log.info(f'============================== 本次爬虫运行时间:{elapsed_time:.2f} 秒 ===============================')
+
+    return elapsed_time
+
+
+@retry(stop=stop_after_attempt(100), wait=wait_fixed(3600), after=after_log)
+def detail_main(log):
+    """
+    主函数 自动售货机
+    :param log: logger对象
+    """
+    log.info(
+        f'开始运行 {inspect.currentframe().f_code.co_name} 爬虫任务....................................................')
+
+    # 配置 MySQL 连接池
+    sql_pool = MySQLConnectionPool(log=log)
+    if not sql_pool.check_pool_health():
+        log.error("数据库连接池异常")
+        raise RuntimeError("数据库连接池异常")
+
+    global detail_first_run_completed
+
+    try:
+        # 获取详情页信息
+        try:
+            log.debug('------------------- 获取商品 detail 数据 -------------------')
+            get_sale_detail_list(log, sql_pool)
+        except Exception as e:
+            log.error(f'get_sale_detail_list error: {e}')
+
+    except Exception as e:
+        log.error(f'{inspect.currentframe().f_code.co_name} error: {e}')
+    finally:
+        detail_first_run_completed = True
+        log.info(f'爬虫程序 {inspect.currentframe().f_code.co_name} 运行结束,等待下一轮的采集任务............')
+
+
+def control_list_mask(log):
+    """
+    控制列表爬虫任务  每10分钟运行
+    :param log: logger对象
+    """
+    while True:
+        log.info(
+            f'--------------------- 开始运行 {inspect.currentframe().f_code.co_name} 新一轮的爬虫任务 ---------------------')
+        elapsed_time = list_main(log)
+        # 计算剩余时间
+        wait_time = max(0, 300 - int(elapsed_time))
+        if wait_time > 0:
+            log.info(f"程序运行时间{elapsed_time:.2f}秒, 小于 5 分钟，等待 {wait_time:.2f} 秒后再开始下一轮任务")
+            time.sleep(wait_time)
+        else:
+            log.info("程序运行时间大于等于5分钟，直接开始下一轮任务")
+
+
+def scheduled_detail_main(log):
+    """定时任务调用的包装函数"""
+    global detail_first_run_completed
+    if detail_first_run_completed:
+        detail_main(log)
+    else:
+        log.info("Skipping scheduled task as first run is not completed yet")
+
+
+def run_threaded(job_func, *args, **kwargs):
+    """
+    在新线程中运行给定的函数，并传递参数。
+
+    :param job_func: 要运行的目标函数
+    :param args: 位置参数
+    :param kwargs: 关键字参数
+    """
+    job_thread = threading.Thread(target=job_func, args=args, kwargs=kwargs)
+    job_thread.start()
+
+
+
+def schedule_task():
+    """
+    设置定时任务
+    """
+    # 启动 control_list_mask 任务线程
+    list_thread = threading.Thread(target=control_list_mask, args=(logger,))
+    list_thread.daemon = True  # 设置为守护线程，主程序退出时自动结束
+    list_thread.start()
+
+    # 启动 detail_main 任务线程（首次运行）
+    detail_thread = threading.Thread(target=detail_main, args=(logger,))
+    detail_thread.daemon = True
+    detail_thread.start()
+
+    # 设置定时任务  每天
+    # schedule.every().day.at("00:01").do(run_threaded, detail_main, logger)
+    schedule.every().day.at("00:01").do(run_threaded, scheduled_detail_main, logger)
+
+    while True:
+        schedule.run_pending()
+        time.sleep(1)
+
+
+if __name__ == '__main__':
+    schedule_task()
+    # detail_main(log=logger)
+    # get_sale_detail_list(log, ((1, 'a4f0bbebd858370567f1779fddf0f55630810116d80965e33940fc8ff5ac94b4'),
+    #                            (2, 'a4f0bbebd858370567f1779fddf0f55630810116d80965e33940fc8ff5ac94b4')), sql_pool)

courtyard_spider/mysql_pool.py

@@ -0,0 +1,574 @@
+# -*- coding: utf-8 -*-
+# Author : Charley
+# Python : 3.10.8
+# Date   : 2025/3/25 14:14
+import re
+import pymysql
+import YamlLoader
+from loguru import logger
+from dbutils.pooled_db import PooledDB
+
+# 获取yaml配置
+yaml = YamlLoader.readYaml()
+mysqlYaml = yaml.get("mysql")
+sql_host = mysqlYaml.getValueAsString("host")
+sql_port = mysqlYaml.getValueAsInt("port")
+sql_user = mysqlYaml.getValueAsString("username")
+sql_password = mysqlYaml.getValueAsString("password")
+sql_db = mysqlYaml.getValueAsString("db")
+
+
+class MySQLConnectionPool:
+    """
+    MySQL连接池
+    """
+
+    def __init__(self, mincached=4, maxcached=5, maxconnections=10, log=None):
+        """
+        初始化连接池
+        :param mincached: 初始化时，链接池中至少创建的链接，0表示不创建
+        :param maxcached: 池中空闲连接的最大数目（0 或 None 表示池大小不受限制）
+        :param maxconnections: 允许的最大连接数（0 或 None 表示任意数量的连接）
+        :param log: 自定义日志记录器
+        """
+        # 使用 loguru 的 logger，如果传入了其他 logger，则使用传入的 logger
+        self.log = log or logger
+        self.pool = PooledDB(
+            creator=pymysql,
+            mincached=mincached,
+            maxcached=maxcached,
+            maxconnections=maxconnections,
+            blocking=True,  # 连接池中如果没有可用连接后，是否阻塞等待。True，等待；False，不等待然后报错
+            host=sql_host,
+            port=sql_port,
+            user=sql_user,
+            password=sql_password,
+            database=sql_db,
+            ping=0  # 每次连接使用时自动检查有效性（0=不检查,1=执行query前检查,2=每次执行前检查）
+        )
+
+    def _execute(self, query, args=None, commit=False):
+        """
+        执行SQL
+        :param query: SQL语句
+        :param args: SQL参数
+        :param commit: 是否提交事务
+        :return: 查询结果
+        """
+        try:
+            with self.pool.connection() as conn:
+                with conn.cursor() as cursor:
+                    cursor.execute(query, args)
+                    if commit:
+                        conn.commit()
+                    self.log.debug(f"sql _execute, Query: {query}, Rows: {cursor.rowcount}")
+                    return cursor
+        except Exception as e:
+            if commit:
+                conn.rollback()
+            self.log.exception(f"Error executing query: {e}, Query: {query}, Args: {args}")
+            raise e
+
+    def select_one(self, query, args=None):
+        """
+        执行查询，返回单个结果
+        :param query: 查询语句
+        :param args: 查询参数
+        :return: 查询结果
+        """
+        cursor = self._execute(query, args)
+        return cursor.fetchone()
+
+    def select_all(self, query, args=None):
+        """
+        执行查询，返回所有结果
+        :param query: 查询语句
+        :param args: 查询参数
+        :return: 查询结果
+        """
+        cursor = self._execute(query, args)
+        return cursor.fetchall()
+
+    def insert_one(self, query, args):
+        """
+        执行单条插入语句
+        :param query: 插入语句
+        :param args: 插入参数
+        """
+        self.log.info('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>data insert_one 入库中>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
+        cursor = self._execute(query, args, commit=True)
+        return cursor.lastrowid  # 返回插入的ID
+
+    def insert_all(self, query, args_list):
+        """
+        执行批量插入语句，如果失败则逐条插入
+        :param query: 插入语句
+        :param args_list: 插入参数列表
+        """
+        conn = None
+        cursor = None
+        try:
+            conn = self.pool.connection()
+            cursor = conn.cursor()
+            cursor.executemany(query, args_list)
+            conn.commit()
+            self.log.debug(f"sql insert_all, SQL: {query}, Rows: {len(args_list)}")
+            self.log.info('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>data insert_all 入库中>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
+        except Exception as e:
+            conn.rollback()
+            self.log.error(f"Batch insertion failed after 5 attempts. Trying single inserts. Error: {e}")
+            # 如果批量插入失败，则逐条插入
+            rowcount = 0
+            for args in args_list:
+                self.insert_one(query, args)
+                rowcount += 1
+            self.log.debug(f"Batch insertion failed. Inserted {rowcount} rows individually.")
+        finally:
+            if cursor:
+                cursor.close()
+            if conn:
+                conn.close()
+
+    def insert_one_or_dict(self, table=None, data=None, query=None, args=None, commit=True, ignore=False):
+        """
+        单条插入（支持字典或原始SQL）
+        :param table: 表名（字典插入时必需）
+        :param data: 字典数据 {列名: 值}
+        :param query: 直接SQL语句（与data二选一）
+        :param args: SQL参数（query使用时必需）
+        :param commit: 是否自动提交
+        :param ignore: 是否使用ignore
+        :return: 最后插入ID
+        """
+        if data is not None:
+            if not isinstance(data, dict):
+                raise ValueError("Data must be a dictionary")
+
+            keys = ', '.join([self._safe_identifier(k) for k in data.keys()])
+            values = ', '.join(['%s'] * len(data))
+
+            # query = f"INSERT INTO {self._safe_identifier(table)} ({keys}) VALUES ({values})"
+            # 构建 INSERT IGNORE 语句
+            ignore_clause = "IGNORE" if ignore else ""
+            # insert_sql = f"INSERT {ignore_clause} INTO {table} ({columns}) VALUES ({placeholders})"
+            query = f"INSERT {ignore_clause} INTO {self._safe_identifier(table)} ({keys}) VALUES ({values})"
+            args = tuple(data.values())
+        elif query is None:
+            raise ValueError("Either data or query must be provided")
+
+        # cursor = self._execute(query, args, commit)
+        # self.log.info(f"sql insert_one_or_dict, Table: {table}, Rows: {cursor.rowcount}")
+        # self.log.info('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>data insert_one_or_dict 入库中>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
+        # return cursor.lastrowid
+
+        try:
+            cursor = self._execute(query, args, commit)
+            self.log.info(f"sql insert_one_or_dict, Table: {table}, Rows: {cursor.rowcount}")
+            self.log.info('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>data insert_one_or_dict 入库中>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
+            return cursor.lastrowid
+        except pymysql.err.IntegrityError as e:
+            if "Duplicate entry" in str(e):
+                self.log.warning(f"插入失败：重复条目，已跳过。错误详情: {e}")
+                # print("插入失败：重复条目", e)
+                return -1  # 返回 -1 表示重复条目被跳过
+            else:
+                self.log.exception(f"数据库完整性错误: {e}")
+                # print("插入失败：完整性错误", e)
+                raise
+        except Exception as e:
+            # self.log.error(f"未知错误: {str(e)}", exc_info=True)
+            self.log.exception(f"未知错误: {e}")  # 记录完整异常信息
+            # print("插入失败：未知错误", e)
+            raise
+
+    def insert_many(self, table=None, data_list=None, query=None, args_list=None, batch_size=1000, commit=True, ignore=False):
+        """
+        批量插入（支持字典列表或原始SQL）
+        :param table: 表名（字典插入时必需）
+        :param data_list: 字典列表 [{列名: 值}]
+        :param query: 直接SQL语句（与data_list二选一）
+        :param args_list: SQL参数列表（query使用时必需）
+        :param batch_size: 分批大小
+        :param commit: 是否自动提交
+        :param ignore: 是否使用ignore
+        :return: 影响行数
+        """
+        if data_list is not None:
+            if not data_list or not isinstance(data_list[0], dict):
+                raise ValueError("Data_list must be a non-empty list of dictionaries")
+
+            keys = ', '.join([self._safe_identifier(k) for k in data_list[0].keys()])
+            values = ', '.join(['%s'] * len(data_list[0]))
+
+            # 构建 INSERT IGNORE 语句
+            ignore_clause = "IGNORE" if ignore else ""
+            # insert_sql = f"INSERT {ignore_clause} INTO {table} ({columns}) VALUES ({placeholders})"
+            query = f"INSERT {ignore_clause} INTO {self._safe_identifier(table)} ({keys}) VALUES ({values})"
+            args_list = [tuple(d.values()) for d in data_list]
+        elif query is None:
+            raise ValueError("Either data_list or query must be provided")
+
+        total = 0
+        for i in range(0, len(args_list), batch_size):
+            batch = args_list[i:i + batch_size]
+            try:
+                with self.pool.connection() as conn:
+                    with conn.cursor() as cursor:
+                        cursor.executemany(query, batch)
+                        if commit:
+                            conn.commit()
+                        total += cursor.rowcount
+            except pymysql.Error as e:
+                if "Duplicate entry" in str(e):
+                    # self.log.warning(f"检测到重复条目，开始逐条插入。错误详情: {e}")
+                    raise  e
+                    # rowcount = 0
+                    # for args in batch:
+                    #     try:
+                    #         self.insert_one_or_dict(table=table, data=dict(zip(data_list[0].keys(), args)),
+                    #                                 commit=commit)
+                    #         rowcount += 1
+                    #     except pymysql.err.IntegrityError as e2:
+                    #         if "Duplicate entry" in str(e2):
+                    #             self.log.warning(f"跳过重复条目: {args}")
+                    #         else:
+                    #             self.log.error(f"插入失败: {e2}, 参数: {args}")
+                    # total += rowcount
+                else:
+                    self.log.exception(f"数据库错误: {e}")
+                    if commit:
+                        conn.rollback()
+                    raise e
+                # 重新抛出异常，供外部捕获
+                # 降级为单条插入
+                # for args in batch:
+                #     try:
+                #         self.insert_one_or_dict(table=None, query=query, args=args, commit=commit)
+                #         total += 1
+                #     except Exception as e2:
+                #         self.log.error(f"Single insert failed: {e2}")
+                        # continue
+        self.log.info(f"sql insert_many, Table: {table}, Total Rows: {total}")
+        return total
+
+    def insert_many_two(self, table=None, data_list=None, query=None, args_list=None, batch_size=1000, commit=True):
+        """
+        批量插入（支持字典列表或原始SQL）
+        :param table: 表名（字典插入时必需）
+        :param data_list: 字典列表 [{列名: 值}]
+        :param query: 直接SQL语句（与data_list二选一）
+        :param args_list: SQL参数列表（query使用时必需）
+        :param batch_size: 分批大小
+        :param commit: 是否自动提交
+        :return: 影响行数
+        """
+        if data_list is not None:
+            if not data_list or not isinstance(data_list[0], dict):
+                raise ValueError("Data_list must be a non-empty list of dictionaries")
+            keys = ', '.join([self._safe_identifier(k) for k in data_list[0].keys()])
+            values = ', '.join(['%s'] * len(data_list[0]))
+            query = f"INSERT INTO {self._safe_identifier(table)} ({keys}) VALUES ({values})"
+            args_list = [tuple(d.values()) for d in data_list]
+        elif query is None:
+            raise ValueError("Either data_list or query must be provided")
+
+        total = 0
+        for i in range(0, len(args_list), batch_size):
+            batch = args_list[i:i + batch_size]
+            try:
+                with self.pool.connection() as conn:
+                    with conn.cursor() as cursor:
+                        # 添加调试日志：输出 SQL 和参数示例
+                        # self.log.debug(f"Batch insert SQL: {query}")
+                        # self.log.debug(f"Sample args: {batch[0] if batch else 'None'}")
+                        cursor.executemany(query, batch)
+                        if commit:
+                            conn.commit()
+                        total += cursor.rowcount
+                        # self.log.debug(f"Batch insert succeeded. Rows: {cursor.rowcount}")
+            except Exception as e:  # 明确捕获数据库异常
+                self.log.exception(f"Batch insert failed: {e}")  # 使用 exception 记录堆栈
+                self.log.error(f"Failed SQL: {query}, Args count: {len(batch)}")
+                if commit:
+                    conn.rollback()
+                # 降级为单条插入，并记录每个错误
+                rowcount = 0
+                for args in batch:
+                    try:
+                        self.insert_one(query, args)
+                        rowcount += 1
+                    except Exception as e2:
+                        self.log.error(f"Single insert failed: {e2}, Args: {args}")
+                total += rowcount
+                self.log.debug(f"Inserted {rowcount}/{len(batch)} rows individually.")
+        self.log.info(f"sql insert_many, Table: {table}, Total Rows: {total}")
+        return total
+
+    def insert_too_many(self, query, args_list, batch_size=1000):
+        """
+        执行批量插入语句，分片提交, 单次插入大于十万+时可用, 如果失败则降级为逐条插入
+        :param query: 插入语句
+        :param args_list: 插入参数列表
+        :param batch_size: 每次插入的条数
+        """
+        for i in range(0, len(args_list), batch_size):
+            batch = args_list[i:i + batch_size]
+            try:
+                with self.pool.connection() as conn:
+                    with conn.cursor() as cursor:
+                        cursor.executemany(query, batch)
+                        conn.commit()
+            except Exception as e:
+                self.log.error(f"insert_too_many error. Trying single insert. Error: {e}")
+                # 当前批次降级为单条插入
+                for args in batch:
+                    self.insert_one(query, args)
+
+    def update_one(self, query, args):
+        """
+        执行单条更新语句
+        :param query: 更新语句
+        :param args: 更新参数
+        """
+        self.log.info('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>data update_one 更新中>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
+        return self._execute(query, args, commit=True)
+
+    def update_all(self, query, args_list):
+        """
+        执行批量更新语句，如果失败则逐条更新
+        :param query: 更新语句
+        :param args_list: 更新参数列表
+        """
+        conn = None
+        cursor = None
+        try:
+            conn = self.pool.connection()
+            cursor = conn.cursor()
+            cursor.executemany(query, args_list)
+            conn.commit()
+            self.log.debug(f"sql update_all, SQL: {query}, Rows: {len(args_list)}")
+            self.log.info('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>data update_all 更新中>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
+        except Exception as e:
+            conn.rollback()
+            self.log.error(f"Error executing query: {e}")
+            # 如果批量更新失败，则逐条更新
+            rowcount = 0
+            for args in args_list:
+                self.update_one(query, args)
+                rowcount += 1
+            self.log.debug(f'Batch update failed. Updated {rowcount} rows individually.')
+        finally:
+            if cursor:
+                cursor.close()
+            if conn:
+                conn.close()
+
+    def update_one_or_dict(self, table=None, data=None, condition=None, query=None, args=None, commit=True):
+        """
+        单条更新（支持字典或原始SQL）
+        :param table: 表名（字典模式必需）
+        :param data: 字典数据 {列名: 值}（与 query 二选一）
+        :param condition: 更新条件，支持以下格式：
+            - 字典: {"id": 1} → "WHERE id = %s"
+            - 字符串: "id = 1" → "WHERE id = 1"（需自行确保安全）
+            - 元组: ("id = %s", [1]) → "WHERE id = %s"（参数化查询）
+        :param query: 直接SQL语句（与 data 二选一）
+        :param args: SQL参数（query 模式下必需）
+        :param commit: 是否自动提交
+        :return: 影响行数
+        :raises: ValueError 参数校验失败时抛出
+        """
+        # 参数校验
+        if data is not None:
+            if not isinstance(data, dict):
+                raise ValueError("Data must be a dictionary")
+            if table is None:
+                raise ValueError("Table name is required for dictionary update")
+            if condition is None:
+                raise ValueError("Condition is required for dictionary update")
+
+            # 构建 SET 子句
+            set_clause = ", ".join([f"{self._safe_identifier(k)} = %s" for k in data.keys()])
+            set_values = list(data.values())
+
+            # 解析条件
+            condition_clause, condition_args = self._parse_condition(condition)
+            query = f"UPDATE {self._safe_identifier(table)} SET {set_clause} WHERE {condition_clause}"
+            args = set_values + condition_args
+
+        elif query is None:
+            raise ValueError("Either data or query must be provided")
+
+        # 执行更新
+        cursor = self._execute(query, args, commit)
+        # self.log.debug(
+        #     f"Updated table={table}, rows={cursor.rowcount}, query={query[:100]}...",
+        #     extra={"table": table, "rows": cursor.rowcount}
+        # )
+        return cursor.rowcount
+
+    def _parse_condition(self, condition):
+        """
+        解析条件为 (clause, args) 格式
+        :param condition: 字典/字符串/元组
+        :return: (str, list) SQL 子句和参数列表
+        """
+        if isinstance(condition, dict):
+            clause = " AND ".join([f"{self._safe_identifier(k)} = %s" for k in condition.keys()])
+            args = list(condition.values())
+        elif isinstance(condition, str):
+            clause = condition  # 注意：需调用方确保安全
+            args = []
+        elif isinstance(condition, (tuple, list)) and len(condition) == 2:
+            clause, args = condition[0], condition[1]
+            if not isinstance(args, (list, tuple)):
+                args = [args]
+        else:
+            raise ValueError("Condition must be dict/str/(clause, args)")
+        return clause, args
+
+    def update_many(self, table=None, data_list=None, condition_list=None, query=None, args_list=None, batch_size=500,
+                    commit=True):
+        """
+        批量更新（支持字典列表或原始SQL）
+        :param table: 表名（字典插入时必需）
+        :param data_list: 字典列表 [{列名: 值}]
+        :param condition_list: 条件列表（必须为字典，与data_list等长）
+        :param query: 直接SQL语句（与data_list二选一）
+        :param args_list: SQL参数列表（query使用时必需）
+        :param batch_size: 分批大小
+        :param commit: 是否自动提交
+        :return: 影响行数
+        """
+        if data_list is not None:
+            if not data_list or not isinstance(data_list[0], dict):
+                raise ValueError("Data_list must be a non-empty list of dictionaries")
+            if condition_list is None or len(data_list) != len(condition_list):
+                raise ValueError("Condition_list must be provided and match the length of data_list")
+            if not all(isinstance(cond, dict) for cond in condition_list):
+                raise ValueError("All elements in condition_list must be dictionaries")
+
+            # 获取第一个数据项和条件项的键
+            first_data_keys = set(data_list[0].keys())
+            first_cond_keys = set(condition_list[0].keys())
+
+            # 构造基础SQL
+            set_clause = ', '.join([self._safe_identifier(k) + ' = %s' for k in data_list[0].keys()])
+            condition_clause = ' AND '.join([self._safe_identifier(k) + ' = %s' for k in condition_list[0].keys()])
+            base_query = f"UPDATE {self._safe_identifier(table)} SET {set_clause} WHERE {condition_clause}"
+            total = 0
+
+            # 分批次处理
+            for i in range(0, len(data_list), batch_size):
+                batch_data = data_list[i:i + batch_size]
+                batch_conds = condition_list[i:i + batch_size]
+                batch_args = []
+
+                # 检查当前批次的结构是否一致
+                can_batch = True
+                for data, cond in zip(batch_data, batch_conds):
+                    data_keys = set(data.keys())
+                    cond_keys = set(cond.keys())
+                    if data_keys != first_data_keys or cond_keys != first_cond_keys:
+                        can_batch = False
+                        break
+                    batch_args.append(tuple(data.values()) + tuple(cond.values()))
+
+                if not can_batch:
+                    # 结构不一致，转为单条更新
+                    for data, cond in zip(batch_data, batch_conds):
+                        self.update_one_or_dict(table=table, data=data, condition=cond, commit=commit)
+                        total += 1
+                    continue
+
+                # 执行批量更新
+                try:
+                    with self.pool.connection() as conn:
+                        with conn.cursor() as cursor:
+                            cursor.executemany(base_query, batch_args)
+                            if commit:
+                                conn.commit()
+                            total += cursor.rowcount
+                            self.log.debug(f"Batch update succeeded. Rows: {cursor.rowcount}")
+                except Exception as e:
+                    if commit:
+                        conn.rollback()
+                    self.log.error(f"Batch update failed: {e}")
+                    # 降级为单条更新
+                    for args, data, cond in zip(batch_args, batch_data, batch_conds):
+                        try:
+                            self._execute(base_query, args, commit=commit)
+                            total += 1
+                        except Exception as e2:
+                            self.log.error(f"Single update failed: {e2}, Data: {data}, Condition: {cond}")
+            self.log.info(f"Total updated rows: {total}")
+            return total
+        elif query is not None:
+            # 处理原始SQL和参数列表
+            if args_list is None:
+                raise ValueError("args_list must be provided when using query")
+
+            total = 0
+            for i in range(0, len(args_list), batch_size):
+                batch_args = args_list[i:i + batch_size]
+                try:
+                    with self.pool.connection() as conn:
+                        with conn.cursor() as cursor:
+                            cursor.executemany(query, batch_args)
+                            if commit:
+                                conn.commit()
+                            total += cursor.rowcount
+                            self.log.debug(f"Batch update succeeded. Rows: {cursor.rowcount}")
+                except Exception as e:
+                    if commit:
+                        conn.rollback()
+                    self.log.error(f"Batch update failed: {e}")
+                    # 降级为单条更新
+                    for args in batch_args:
+                        try:
+                            self._execute(query, args, commit=commit)
+                            total += 1
+                        except Exception as e2:
+                            self.log.error(f"Single update failed: {e2}, Args: {args}")
+            self.log.info(f"Total updated rows: {total}")
+            return total
+        else:
+            raise ValueError("Either data_list or query must be provided")
+
+    def check_pool_health(self):
+        """
+        检查连接池中有效连接数
+
+        # 使用示例
+        # 配置 MySQL 连接池
+        sql_pool = MySQLConnectionPool(log=log)
+        if not sql_pool.check_pool_health():
+            log.error("数据库连接池异常")
+            raise RuntimeError("数据库连接池异常")
+        """
+        try:
+            with self.pool.connection() as conn:
+                conn.ping(reconnect=True)
+                return True
+        except Exception as e:
+            self.log.error(f"Connection pool health check failed: {e}")
+            return False
+
+    @staticmethod
+    def _safe_identifier(name):
+        """SQL标识符安全校验"""
+        if not re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*$', name):
+            raise ValueError(f"Invalid SQL identifier: {name}")
+        return name
+
+
+if __name__ == '__main__':
+    sql_pool = MySQLConnectionPool()
+    data_dic = {'card_type_id': 111, 'card_type_name': '补充包 继承的意志【OPC-13】', 'card_type_position': 964,
+                'card_id': 5284, 'card_name': '蒙奇·D·路飞', 'card_number': 'OP13-001', 'card_rarity': 'L',
+                'card_img': 'https://source.windoent.com/OnePiecePc/Picture/1757929283612OP13-001.png',
+                'card_life': '4', 'card_attribute': '打', 'card_power': '5000', 'card_attack': '-',
+                'card_color': '红/绿', 'subscript': 4, 'card_features': '超新星/草帽一伙',
+                'card_text_desc': '【咚!!×1】【对方的攻击时】我方处于活跃状态的咚!!不多于5张的场合，可以将我方任意张数的咚!!转为休息状态。每有1张转为休息状态的咚!!，本次战斗中，此领袖或我方最多1张拥有《草帽一伙》特征的角色力量+2000。',
+                'card_offer_type': '补充包 继承的意志【OPC-13】', 'crawler_language': '简中'}
+    sql_pool.insert_one_or_dict(table="one_piece_record", data=data_dic)

courtyard_spider/package.json

@@ -0,0 +1,15 @@
+{
+  'detail_id': '9ada504150943eef632b31b44b630f674fb40d6b9f32c0b8bbb829d942ae31b5',
+  'correlation_id': 'CGC: 6001842147',
+  'burn_from': 'cy_buyback',
+  'burn_to': '0x0000...0000',
+  'burn_time': '2025-12-03 16:30:00',
+  'sale_price': '$8.10',
+  'sale_from': 'onet1me99',
+  'sale_to': 'cy_buyback',
+  'sale_time': '2025-12-03 16:29:00',
+  'mint_price': '$10.00',
+  'mint_from': '0x0000...0000',
+  'mint_to': 'onet1me99',
+  'mint_time': '2025-12-03 16:29:00'
+}

courtyard_spider/replenish_price.py

@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+# Author : Charley
+# Python : 3.10.8
+# Date   : 2025/12/11 13:26
+import requests
+import user_agent
+from loguru import logger
+from mysql_pool import MySQLConnectionPool
+from tenacity import retry, stop_after_attempt, wait_fixed
+
+headers = {
+    "accept": "application/json",
+    "referer": "https://courtyard.io/",
+    "user-agent": user_agent.generate_user_agent()
+}
+
+
+def after_log(retry_state):
+    """
+    retry 回调
+    :param retry_state: RetryCallState 对象
+    """
+    # 检查 args 是否存在且不为空
+    if retry_state.args and len(retry_state.args) > 0:
+        log = retry_state.args[0]  # 获取传入的 logger
+    else:
+        log = logger  # 使用全局 logger
+
+    if retry_state.outcome.failed:
+        log.warning(
+            f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} Times")
+    else:
+        log.info(f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} succeeded")
+
+
+@retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
+def replenish_price(log, bag_id, sql_pool=None):
+    """
+    补全价格
+    :param log:
+    :param bag_id:
+    :param sql_pool:
+    :return:
+    """
+    log.info(f"========================== 获取商品详情, bag_id: {bag_id} ==========================")
+    url = "https://api.courtyard.io/vending-machines"
+    response = requests.get(url, headers=headers, timeout=22)
+    # print(response.text)
+    response.raise_for_status()
+
+    vendingMachines = response.json().get("vendingMachines", [])
+
+    for item in vendingMachines:
+        # print(item)
+        bag_id = item.get("id")
+        price = item.get("saleDetails", {}).get("salePriceUsd")
+        log.debug(f"bag_id: {bag_id}, price: {price}")
+
+        # 根据bag_id更新price  如果在数据库中存在则更新，否则插入
+        sql_pool.update_one("UPDATE courtyard_list_record SET price = %s WHERE bag_id = %s", (price, bag_id))
+
+
+if __name__ == '__main__':
+    sql_pool = MySQLConnectionPool(log=logger)
+    replenish_price(logger, "pkmn-basic-pack",sql_pool)

courtyard_spider/update_price.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+# Author : Charley
+# Python : 3.10.8
+# Date   : 2025/12/10 16:14
+from mysql_pool import MySQLConnectionPool
+from loguru import logger
+
+
+def update_price_fields():
+    """
+    更新 courtyard_detail_record 表中 sale_price 和 mint_price 字段的值
+    去除美元符号($)和逗号(,)
+    """
+    # 配置 MySQL 连接池
+    sql_pool = MySQLConnectionPool(log=logger)
+    if not sql_pool.check_pool_health():
+        logger.error("数据库连接池异常")
+        raise RuntimeError("数据库连接池异常")
+
+    try:
+        # 查询所有需要更新的记录
+        query = "SELECT id, sale_price, mint_price FROM courtyard_detail_record WHERE sale_price IS NOT NULL OR mint_price IS NOT NULL"
+        records = sql_pool.select_all(query)
+
+        logger.info(f"共找到 {len(records)} 条记录需要更新")
+
+        updated_count = 0
+        for record in records:
+            record_id = record[0]
+            sale_price = record[1]
+            mint_price = record[2]
+
+            # 处理 sale_price
+            if sale_price and isinstance(sale_price, str):
+                cleaned_sale_price = sale_price.replace("$", "").replace(",", "")
+                # 如果能转换为数字则更新，否则保持原值
+                try:
+                    float(cleaned_sale_price)
+                except ValueError:
+                    cleaned_sale_price = sale_price  # 无法转换则保持原值
+
+            # 处理 mint_price
+            if mint_price and isinstance(mint_price, str):
+                cleaned_mint_price = mint_price.replace("$", "").replace(",", "")
+                # 如果能转换为数字则更新，否则保持原值
+                try:
+                    float(cleaned_mint_price)
+                except ValueError:
+                    cleaned_mint_price = mint_price  # 无法转换则保持原值
+
+            # 更新数据库记录
+            update_query = "UPDATE courtyard_detail_record SET sale_price = %s, mint_price = %s WHERE id = %s"
+            update_args = (cleaned_sale_price if 'cleaned_sale_price' in locals() else sale_price,
+                           cleaned_mint_price if 'cleaned_mint_price' in locals() else mint_price,
+                           record_id)
+
+            try:
+                sql_pool.update_one(update_query, update_args)
+                updated_count += 1
+            except Exception as e:
+                logger.error(f"更新记录 ID {record_id} 失败: {e}")
+
+        logger.info(f"成功更新 {updated_count} 条记录")
+
+    except Exception as e:
+        logger.error(f"更新价格字段时发生错误: {e}")
+        raise e
+
+
+if __name__ == '__main__':
+    update_price_fields()

rhyf_spider/YamlLoader.py

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+#
+import os, re
+import yaml
+
+regex = re.compile(r'^\$\{(?P<ENV>[A-Z_\-]+\:)?(?P<VAL>[\w\.]+)\}$')
+
+class YamlConfig:
+    def __init__(self, config):
+        self.config = config
+
+    def get(self, key:str):
+        return YamlConfig(self.config.get(key))
+    
+    def getValueAsString(self, key: str):
+        try:
+            match = regex.match(self.config[key])
+            group = match.groupdict()
+            if group['ENV'] != None:
+                env = group['ENV'][:-1]
+                return os.getenv(env, group['VAL'])
+            return None
+        except:
+            return self.config[key]
+    
+    def getValueAsInt(self, key: str):
+        try:
+            match = regex.match(self.config[key])
+            group = match.groupdict()
+            if group['ENV'] != None:
+                env = group['ENV'][:-1]
+                return int(os.getenv(env, group['VAL']))
+            return 0
+        except:
+            return int(self.config[key])
+        
+    def getValueAsBool(self, key: str, env: str = None):
+        try:
+            match = regex.match(self.config[key])
+            group = match.groupdict()
+            if group['ENV'] != None:
+                env = group['ENV'][:-1]
+                return bool(os.getenv(env, group['VAL']))
+            return False
+        except:
+            return bool(self.config[key])
+    
+def readYaml(path:str = 'application.yml', profile:str = None) -> YamlConfig:
+    if os.path.exists(path):
+        with open(path) as fd:
+            conf = yaml.load(fd, Loader=yaml.FullLoader)
+
+    if profile != None:
+        result = path.split('.')
+        profiledYaml = f'{result[0]}-{profile}.{result[1]}'
+        if os.path.exists(profiledYaml):
+            with open(profiledYaml) as fd:
+                conf.update(yaml.load(fd, Loader=yaml.FullLoader))
+
+    return YamlConfig(conf)
+
+# res = readYaml()
+# mysqlConf = res.get('mysql')
+# print(mysqlConf)
+
+# print(res.getValueAsString("host"))
+# mysqlYaml = mysqlConf.getValueAsString("host")
+# print(mysqlYaml)
+# host = mysqlYaml.get("host").split(':')[-1][:-1]
+# port = mysqlYaml.get("port").split(':')[-1][:-1]
+# username = mysqlYaml.get("username").split(':')[-1][:-1]
+# password = mysqlYaml.get("password").split(':')[-1][:-1]
+# mysql_db = mysqlYaml.get("db").split(':')[-1][:-1]
+# print(host,port,username,password)

rhyf_spider/application.yml

@@ -0,0 +1,6 @@
+mysql:
+  host: ${MYSQL_HOST:100.64.0.25}
+  port: ${MYSQL_PROT:3306}
+  username: ${MYSQL_USERNAME:crawler}
+  password: ${MYSQL_PASSWORD:Pass2022}
+  db: ${MYSQL_DATABASE:crawler}

rhyf_spider/mysql_pool.py

@@ -0,0 +1,574 @@
+# -*- coding: utf-8 -*-
+# Author : Charley
+# Python : 3.10.8
+# Date   : 2025/3/25 14:14
+import re
+import pymysql
+import YamlLoader
+from loguru import logger
+from dbutils.pooled_db import PooledDB
+
+# 获取yaml配置
+yaml = YamlLoader.readYaml()
+mysqlYaml = yaml.get("mysql")
+sql_host = mysqlYaml.getValueAsString("host")
+sql_port = mysqlYaml.getValueAsInt("port")
+sql_user = mysqlYaml.getValueAsString("username")
+sql_password = mysqlYaml.getValueAsString("password")
+sql_db = mysqlYaml.getValueAsString("db")
+
+
+class MySQLConnectionPool:
+    """
+    MySQL连接池
+    """
+
+    def __init__(self, mincached=4, maxcached=5, maxconnections=10, log=None):
+        """
+        初始化连接池
+        :param mincached: 初始化时，链接池中至少创建的链接，0表示不创建
+        :param maxcached: 池中空闲连接的最大数目（0 或 None 表示池大小不受限制）
+        :param maxconnections: 允许的最大连接数（0 或 None 表示任意数量的连接）
+        :param log: 自定义日志记录器
+        """
+        # 使用 loguru 的 logger，如果传入了其他 logger，则使用传入的 logger
+        self.log = log or logger
+        self.pool = PooledDB(
+            creator=pymysql,
+            mincached=mincached,
+            maxcached=maxcached,
+            maxconnections=maxconnections,
+            blocking=True,  # 连接池中如果没有可用连接后，是否阻塞等待。True，等待；False，不等待然后报错
+            host=sql_host,
+            port=sql_port,
+            user=sql_user,
+            password=sql_password,
+            database=sql_db,
+            ping=0  # 每次连接使用时自动检查有效性（0=不检查,1=执行query前检查,2=每次执行前检查）
+        )
+
+    def _execute(self, query, args=None, commit=False):
+        """
+        执行SQL
+        :param query: SQL语句
+        :param args: SQL参数
+        :param commit: 是否提交事务
+        :return: 查询结果
+        """
+        try:
+            with self.pool.connection() as conn:
+                with conn.cursor() as cursor:
+                    cursor.execute(query, args)
+                    if commit:
+                        conn.commit()
+                    self.log.debug(f"sql _execute, Query: {query}, Rows: {cursor.rowcount}")
+                    return cursor
+        except Exception as e:
+            if commit:
+                conn.rollback()
+            self.log.exception(f"Error executing query: {e}, Query: {query}, Args: {args}")
+            raise e
+
+    def select_one(self, query, args=None):
+        """
+        执行查询，返回单个结果
+        :param query: 查询语句
+        :param args: 查询参数
+        :return: 查询结果
+        """
+        cursor = self._execute(query, args)
+        return cursor.fetchone()
+
+    def select_all(self, query, args=None):
+        """
+        执行查询，返回所有结果
+        :param query: 查询语句
+        :param args: 查询参数
+        :return: 查询结果
+        """
+        cursor = self._execute(query, args)
+        return cursor.fetchall()
+
+    def insert_one(self, query, args):
+        """
+        执行单条插入语句
+        :param query: 插入语句
+        :param args: 插入参数
+        """
+        self.log.info('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>data insert_one 入库中>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
+        cursor = self._execute(query, args, commit=True)
+        return cursor.lastrowid  # 返回插入的ID
+
+    def insert_all(self, query, args_list):
+        """
+        执行批量插入语句，如果失败则逐条插入
+        :param query: 插入语句
+        :param args_list: 插入参数列表
+        """
+        conn = None
+        cursor = None
+        try:
+            conn = self.pool.connection()
+            cursor = conn.cursor()
+            cursor.executemany(query, args_list)
+            conn.commit()
+            self.log.debug(f"sql insert_all, SQL: {query}, Rows: {len(args_list)}")
+            self.log.info('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>data insert_all 入库中>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
+        except Exception as e:
+            conn.rollback()
+            self.log.error(f"Batch insertion failed after 5 attempts. Trying single inserts. Error: {e}")
+            # 如果批量插入失败，则逐条插入
+            rowcount = 0
+            for args in args_list:
+                self.insert_one(query, args)
+                rowcount += 1
+            self.log.debug(f"Batch insertion failed. Inserted {rowcount} rows individually.")
+        finally:
+            if cursor:
+                cursor.close()
+            if conn:
+                conn.close()
+
+    def insert_one_or_dict(self, table=None, data=None, query=None, args=None, commit=True, ignore=False):
+        """
+        单条插入（支持字典或原始SQL）
+        :param table: 表名（字典插入时必需）
+        :param data: 字典数据 {列名: 值}
+        :param query: 直接SQL语句（与data二选一）
+        :param args: SQL参数（query使用时必需）
+        :param commit: 是否自动提交
+        :param ignore: 是否使用ignore
+        :return: 最后插入ID
+        """
+        if data is not None:
+            if not isinstance(data, dict):
+                raise ValueError("Data must be a dictionary")
+
+            keys = ', '.join([self._safe_identifier(k) for k in data.keys()])
+            values = ', '.join(['%s'] * len(data))
+
+            # query = f"INSERT INTO {self._safe_identifier(table)} ({keys}) VALUES ({values})"
+            # 构建 INSERT IGNORE 语句
+            ignore_clause = "IGNORE" if ignore else ""
+            # insert_sql = f"INSERT {ignore_clause} INTO {table} ({columns}) VALUES ({placeholders})"
+            query = f"INSERT {ignore_clause} INTO {self._safe_identifier(table)} ({keys}) VALUES ({values})"
+            args = tuple(data.values())
+        elif query is None:
+            raise ValueError("Either data or query must be provided")
+
+        # cursor = self._execute(query, args, commit)
+        # self.log.info(f"sql insert_one_or_dict, Table: {table}, Rows: {cursor.rowcount}")
+        # self.log.info('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>data insert_one_or_dict 入库中>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
+        # return cursor.lastrowid
+
+        try:
+            cursor = self._execute(query, args, commit)
+            self.log.info(f"sql insert_one_or_dict, Table: {table}, Rows: {cursor.rowcount}")
+            self.log.info('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>data insert_one_or_dict 入库中>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
+            return cursor.lastrowid
+        except pymysql.err.IntegrityError as e:
+            if "Duplicate entry" in str(e):
+                self.log.warning(f"插入失败：重复条目，已跳过。错误详情: {e}")
+                # print("插入失败：重复条目", e)
+                return -1  # 返回 -1 表示重复条目被跳过
+            else:
+                self.log.exception(f"数据库完整性错误: {e}")
+                # print("插入失败：完整性错误", e)
+                raise
+        except Exception as e:
+            # self.log.error(f"未知错误: {str(e)}", exc_info=True)
+            self.log.exception(f"未知错误: {e}")  # 记录完整异常信息
+            # print("插入失败：未知错误", e)
+            raise
+
+    def insert_many(self, table=None, data_list=None, query=None, args_list=None, batch_size=1000, commit=True, ignore=False):
+        """
+        批量插入（支持字典列表或原始SQL）
+        :param table: 表名（字典插入时必需）
+        :param data_list: 字典列表 [{列名: 值}]
+        :param query: 直接SQL语句（与data_list二选一）
+        :param args_list: SQL参数列表（query使用时必需）
+        :param batch_size: 分批大小
+        :param commit: 是否自动提交
+        :param ignore: 是否使用ignore
+        :return: 影响行数
+        """
+        if data_list is not None:
+            if not data_list or not isinstance(data_list[0], dict):
+                raise ValueError("Data_list must be a non-empty list of dictionaries")
+
+            keys = ', '.join([self._safe_identifier(k) for k in data_list[0].keys()])
+            values = ', '.join(['%s'] * len(data_list[0]))
+
+            # 构建 INSERT IGNORE 语句
+            ignore_clause = "IGNORE" if ignore else ""
+            # insert_sql = f"INSERT {ignore_clause} INTO {table} ({columns}) VALUES ({placeholders})"
+            query = f"INSERT {ignore_clause} INTO {self._safe_identifier(table)} ({keys}) VALUES ({values})"
+            args_list = [tuple(d.values()) for d in data_list]
+        elif query is None:
+            raise ValueError("Either data_list or query must be provided")
+
+        total = 0
+        for i in range(0, len(args_list), batch_size):
+            batch = args_list[i:i + batch_size]
+            try:
+                with self.pool.connection() as conn:
+                    with conn.cursor() as cursor:
+                        cursor.executemany(query, batch)
+                        if commit:
+                            conn.commit()
+                        total += cursor.rowcount
+            except pymysql.Error as e:
+                if "Duplicate entry" in str(e):
+                    # self.log.warning(f"检测到重复条目，开始逐条插入。错误详情: {e}")
+                    raise  e
+                    # rowcount = 0
+                    # for args in batch:
+                    #     try:
+                    #         self.insert_one_or_dict(table=table, data=dict(zip(data_list[0].keys(), args)),
+                    #                                 commit=commit)
+                    #         rowcount += 1
+                    #     except pymysql.err.IntegrityError as e2:
+                    #         if "Duplicate entry" in str(e2):
+                    #             self.log.warning(f"跳过重复条目: {args}")
+                    #         else:
+                    #             self.log.error(f"插入失败: {e2}, 参数: {args}")
+                    # total += rowcount
+                else:
+                    self.log.exception(f"数据库错误: {e}")
+                    if commit:
+                        conn.rollback()
+                    raise e
+                # 重新抛出异常，供外部捕获
+                # 降级为单条插入
+                # for args in batch:
+                #     try:
+                #         self.insert_one_or_dict(table=None, query=query, args=args, commit=commit)
+                #         total += 1
+                #     except Exception as e2:
+                #         self.log.error(f"Single insert failed: {e2}")
+                        # continue
+        self.log.info(f"sql insert_many, Table: {table}, Total Rows: {total}")
+        return total
+
+    def insert_many_two(self, table=None, data_list=None, query=None, args_list=None, batch_size=1000, commit=True):
+        """
+        批量插入（支持字典列表或原始SQL）
+        :param table: 表名（字典插入时必需）
+        :param data_list: 字典列表 [{列名: 值}]
+        :param query: 直接SQL语句（与data_list二选一）
+        :param args_list: SQL参数列表（query使用时必需）
+        :param batch_size: 分批大小
+        :param commit: 是否自动提交
+        :return: 影响行数
+        """
+        if data_list is not None:
+            if not data_list or not isinstance(data_list[0], dict):
+                raise ValueError("Data_list must be a non-empty list of dictionaries")
+            keys = ', '.join([self._safe_identifier(k) for k in data_list[0].keys()])
+            values = ', '.join(['%s'] * len(data_list[0]))
+            query = f"INSERT INTO {self._safe_identifier(table)} ({keys}) VALUES ({values})"
+            args_list = [tuple(d.values()) for d in data_list]
+        elif query is None:
+            raise ValueError("Either data_list or query must be provided")
+
+        total = 0
+        for i in range(0, len(args_list), batch_size):
+            batch = args_list[i:i + batch_size]
+            try:
+                with self.pool.connection() as conn:
+                    with conn.cursor() as cursor:
+                        # 添加调试日志：输出 SQL 和参数示例
+                        # self.log.debug(f"Batch insert SQL: {query}")
+                        # self.log.debug(f"Sample args: {batch[0] if batch else 'None'}")
+                        cursor.executemany(query, batch)
+                        if commit:
+                            conn.commit()
+                        total += cursor.rowcount
+                        # self.log.debug(f"Batch insert succeeded. Rows: {cursor.rowcount}")
+            except Exception as e:  # 明确捕获数据库异常
+                self.log.exception(f"Batch insert failed: {e}")  # 使用 exception 记录堆栈
+                self.log.error(f"Failed SQL: {query}, Args count: {len(batch)}")
+                if commit:
+                    conn.rollback()
+                # 降级为单条插入，并记录每个错误
+                rowcount = 0
+                for args in batch:
+                    try:
+                        self.insert_one(query, args)
+                        rowcount += 1
+                    except Exception as e2:
+                        self.log.error(f"Single insert failed: {e2}, Args: {args}")
+                total += rowcount
+                self.log.debug(f"Inserted {rowcount}/{len(batch)} rows individually.")
+        self.log.info(f"sql insert_many, Table: {table}, Total Rows: {total}")
+        return total
+
+    def insert_too_many(self, query, args_list, batch_size=1000):
+        """
+        执行批量插入语句，分片提交, 单次插入大于十万+时可用, 如果失败则降级为逐条插入
+        :param query: 插入语句
+        :param args_list: 插入参数列表
+        :param batch_size: 每次插入的条数
+        """
+        for i in range(0, len(args_list), batch_size):
+            batch = args_list[i:i + batch_size]
+            try:
+                with self.pool.connection() as conn:
+                    with conn.cursor() as cursor:
+                        cursor.executemany(query, batch)
+                        conn.commit()
+            except Exception as e:
+                self.log.error(f"insert_too_many error. Trying single insert. Error: {e}")
+                # 当前批次降级为单条插入
+                for args in batch:
+                    self.insert_one(query, args)
+
+    def update_one(self, query, args):
+        """
+        执行单条更新语句
+        :param query: 更新语句
+        :param args: 更新参数
+        """
+        self.log.info('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>data update_one 更新中>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
+        return self._execute(query, args, commit=True)
+
+    def update_all(self, query, args_list):
+        """
+        执行批量更新语句，如果失败则逐条更新
+        :param query: 更新语句
+        :param args_list: 更新参数列表
+        """
+        conn = None
+        cursor = None
+        try:
+            conn = self.pool.connection()
+            cursor = conn.cursor()
+            cursor.executemany(query, args_list)
+            conn.commit()
+            self.log.debug(f"sql update_all, SQL: {query}, Rows: {len(args_list)}")
+            self.log.info('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>data update_all 更新中>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
+        except Exception as e:
+            conn.rollback()
+            self.log.error(f"Error executing query: {e}")
+            # 如果批量更新失败，则逐条更新
+            rowcount = 0
+            for args in args_list:
+                self.update_one(query, args)
+                rowcount += 1
+            self.log.debug(f'Batch update failed. Updated {rowcount} rows individually.')
+        finally:
+            if cursor:
+                cursor.close()
+            if conn:
+                conn.close()
+
+    def update_one_or_dict(self, table=None, data=None, condition=None, query=None, args=None, commit=True):
+        """
+        单条更新（支持字典或原始SQL）
+        :param table: 表名（字典模式必需）
+        :param data: 字典数据 {列名: 值}（与 query 二选一）
+        :param condition: 更新条件，支持以下格式：
+            - 字典: {"id": 1} → "WHERE id = %s"
+            - 字符串: "id = 1" → "WHERE id = 1"（需自行确保安全）
+            - 元组: ("id = %s", [1]) → "WHERE id = %s"（参数化查询）
+        :param query: 直接SQL语句（与 data 二选一）
+        :param args: SQL参数（query 模式下必需）
+        :param commit: 是否自动提交
+        :return: 影响行数
+        :raises: ValueError 参数校验失败时抛出
+        """
+        # 参数校验
+        if data is not None:
+            if not isinstance(data, dict):
+                raise ValueError("Data must be a dictionary")
+            if table is None:
+                raise ValueError("Table name is required for dictionary update")
+            if condition is None:
+                raise ValueError("Condition is required for dictionary update")
+
+            # 构建 SET 子句
+            set_clause = ", ".join([f"{self._safe_identifier(k)} = %s" for k in data.keys()])
+            set_values = list(data.values())
+
+            # 解析条件
+            condition_clause, condition_args = self._parse_condition(condition)
+            query = f"UPDATE {self._safe_identifier(table)} SET {set_clause} WHERE {condition_clause}"
+            args = set_values + condition_args
+
+        elif query is None:
+            raise ValueError("Either data or query must be provided")
+
+        # 执行更新
+        cursor = self._execute(query, args, commit)
+        # self.log.debug(
+        #     f"Updated table={table}, rows={cursor.rowcount}, query={query[:100]}...",
+        #     extra={"table": table, "rows": cursor.rowcount}
+        # )
+        return cursor.rowcount
+
+    def _parse_condition(self, condition):
+        """
+        解析条件为 (clause, args) 格式
+        :param condition: 字典/字符串/元组
+        :return: (str, list) SQL 子句和参数列表
+        """
+        if isinstance(condition, dict):
+            clause = " AND ".join([f"{self._safe_identifier(k)} = %s" for k in condition.keys()])
+            args = list(condition.values())
+        elif isinstance(condition, str):
+            clause = condition  # 注意：需调用方确保安全
+            args = []
+        elif isinstance(condition, (tuple, list)) and len(condition) == 2:
+            clause, args = condition[0], condition[1]
+            if not isinstance(args, (list, tuple)):
+                args = [args]
+        else:
+            raise ValueError("Condition must be dict/str/(clause, args)")
+        return clause, args
+
+    def update_many(self, table=None, data_list=None, condition_list=None, query=None, args_list=None, batch_size=500,
+                    commit=True):
+        """
+        批量更新（支持字典列表或原始SQL）
+        :param table: 表名（字典插入时必需）
+        :param data_list: 字典列表 [{列名: 值}]
+        :param condition_list: 条件列表（必须为字典，与data_list等长）
+        :param query: 直接SQL语句（与data_list二选一）
+        :param args_list: SQL参数列表（query使用时必需）
+        :param batch_size: 分批大小
+        :param commit: 是否自动提交
+        :return: 影响行数
+        """
+        if data_list is not None:
+            if not data_list or not isinstance(data_list[0], dict):
+                raise ValueError("Data_list must be a non-empty list of dictionaries")
+            if condition_list is None or len(data_list) != len(condition_list):
+                raise ValueError("Condition_list must be provided and match the length of data_list")
+            if not all(isinstance(cond, dict) for cond in condition_list):
+                raise ValueError("All elements in condition_list must be dictionaries")
+
+            # 获取第一个数据项和条件项的键
+            first_data_keys = set(data_list[0].keys())
+            first_cond_keys = set(condition_list[0].keys())
+
+            # 构造基础SQL
+            set_clause = ', '.join([self._safe_identifier(k) + ' = %s' for k in data_list[0].keys()])
+            condition_clause = ' AND '.join([self._safe_identifier(k) + ' = %s' for k in condition_list[0].keys()])
+            base_query = f"UPDATE {self._safe_identifier(table)} SET {set_clause} WHERE {condition_clause}"
+            total = 0
+
+            # 分批次处理
+            for i in range(0, len(data_list), batch_size):
+                batch_data = data_list[i:i + batch_size]
+                batch_conds = condition_list[i:i + batch_size]
+                batch_args = []
+
+                # 检查当前批次的结构是否一致
+                can_batch = True
+                for data, cond in zip(batch_data, batch_conds):
+                    data_keys = set(data.keys())
+                    cond_keys = set(cond.keys())
+                    if data_keys != first_data_keys or cond_keys != first_cond_keys:
+                        can_batch = False
+                        break
+                    batch_args.append(tuple(data.values()) + tuple(cond.values()))
+
+                if not can_batch:
+                    # 结构不一致，转为单条更新
+                    for data, cond in zip(batch_data, batch_conds):
+                        self.update_one_or_dict(table=table, data=data, condition=cond, commit=commit)
+                        total += 1
+                    continue
+
+                # 执行批量更新
+                try:
+                    with self.pool.connection() as conn:
+                        with conn.cursor() as cursor:
+                            cursor.executemany(base_query, batch_args)
+                            if commit:
+                                conn.commit()
+                            total += cursor.rowcount
+                            self.log.debug(f"Batch update succeeded. Rows: {cursor.rowcount}")
+                except Exception as e:
+                    if commit:
+                        conn.rollback()
+                    self.log.error(f"Batch update failed: {e}")
+                    # 降级为单条更新
+                    for args, data, cond in zip(batch_args, batch_data, batch_conds):
+                        try:
+                            self._execute(base_query, args, commit=commit)
+                            total += 1
+                        except Exception as e2:
+                            self.log.error(f"Single update failed: {e2}, Data: {data}, Condition: {cond}")
+            self.log.info(f"Total updated rows: {total}")
+            return total
+        elif query is not None:
+            # 处理原始SQL和参数列表
+            if args_list is None:
+                raise ValueError("args_list must be provided when using query")
+
+            total = 0
+            for i in range(0, len(args_list), batch_size):
+                batch_args = args_list[i:i + batch_size]
+                try:
+                    with self.pool.connection() as conn:
+                        with conn.cursor() as cursor:
+                            cursor.executemany(query, batch_args)
+                            if commit:
+                                conn.commit()
+                            total += cursor.rowcount
+                            self.log.debug(f"Batch update succeeded. Rows: {cursor.rowcount}")
+                except Exception as e:
+                    if commit:
+                        conn.rollback()
+                    self.log.error(f"Batch update failed: {e}")
+                    # 降级为单条更新
+                    for args in batch_args:
+                        try:
+                            self._execute(query, args, commit=commit)
+                            total += 1
+                        except Exception as e2:
+                            self.log.error(f"Single update failed: {e2}, Args: {args}")
+            self.log.info(f"Total updated rows: {total}")
+            return total
+        else:
+            raise ValueError("Either data_list or query must be provided")
+
+    def check_pool_health(self):
+        """
+        检查连接池中有效连接数
+
+        # 使用示例
+        # 配置 MySQL 连接池
+        sql_pool = MySQLConnectionPool(log=log)
+        if not sql_pool.check_pool_health():
+            log.error("数据库连接池异常")
+            raise RuntimeError("数据库连接池异常")
+        """
+        try:
+            with self.pool.connection() as conn:
+                conn.ping(reconnect=True)
+                return True
+        except Exception as e:
+            self.log.error(f"Connection pool health check failed: {e}")
+            return False
+
+    @staticmethod
+    def _safe_identifier(name):
+        """SQL标识符安全校验"""
+        if not re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*$', name):
+            raise ValueError(f"Invalid SQL identifier: {name}")
+        return name
+
+
+if __name__ == '__main__':
+    sql_pool = MySQLConnectionPool()
+    data_dic = {'card_type_id': 111, 'card_type_name': '补充包 继承的意志【OPC-13】', 'card_type_position': 964,
+                'card_id': 5284, 'card_name': '蒙奇·D·路飞', 'card_number': 'OP13-001', 'card_rarity': 'L',
+                'card_img': 'https://source.windoent.com/OnePiecePc/Picture/1757929283612OP13-001.png',
+                'card_life': '4', 'card_attribute': '打', 'card_power': '5000', 'card_attack': '-',
+                'card_color': '红/绿', 'subscript': 4, 'card_features': '超新星/草帽一伙',
+                'card_text_desc': '【咚!!×1】【对方的攻击时】我方处于活跃状态的咚!!不多于5张的场合，可以将我方任意张数的咚!!转为休息状态。每有1张转为休息状态的咚!!，本次战斗中，此领袖或我方最多1张拥有《草帽一伙》特征的角色力量+2000。',
+                'card_offer_type': '补充包 继承的意志【OPC-13】', 'crawler_language': '简中'}
+    sql_pool.insert_one_or_dict(table="one_piece_record", data=data_dic)

rhyf_spider/package-lock.json

@@ -0,0 +1,363 @@
+{
+  "error": 200,
+  "message": "操作成功",
+  "data": {
+    "areaType": 0,
+    "boxNo": "20251212988239",
+    "boxStatus": 0,
+    "createTimes": 0,
+    "enabled": 1,
+    "endTime": "2025-12-12 23:22:02",
+    "goodsId": 0,
+    "id": 84343,
+    "orderNo": 39,
+    "price": 69,
+    "queueMode": 1,
+    "saleStock": 0,
+    "saleTotal": 0,
+    "setActiveData": "",
+    "setId": 9882,
+    "setType": 1,
+    "spPrize": "#1",
+    "startTime": "2025-12-12 23:22:02",
+    "status": 1,
+    "stock": 11,
+    "total": 80,
+    "updateTimes": 0,
+    "setContent": "",
+    "setDescr": "龙珠DRAGON BALL 40th 第一弹",
+    "setHeadUrl": "/images/1765250524051n3474_800x800.jpg",
+    "setName": "龙珠DRAGON BALL 40th 第一弹",
+    "setImg": "",
+    "prizeType": 0,
+    "setActiveDesc": "",
+    "setAreaType": 1,
+    "boxTotal": 17,
+    "setBoxTotal": 56,
+    "setEffect": "",
+    "takeAllLimit": 0,
+    "scoresTimes": 1,
+    "setSaleStatus": 1,
+    "topWord": "平台发货不设门槛！赏柜内提交发货申请后3-5个工作日安排发货。非偏远地区每单满5件包邮，不满5件需支付运费10元。",
+    "setTopWord": "平台发货不设门槛！赏柜内提交发货申请后3-5个工作日安排发货。包邮地区每单满5件包邮，不满5件需支付运费10元，不支持7天无理由退换货。",
+    "awardList": [
+      {
+        "avPrice": 0,
+        "bIcon": "/images/yfs_bicon_a_333x209.png",
+        "cardId": 0,
+        "content": "",
+        "createTimes": 0,
+        "enabled": 0,
+        "fromNo": 0,
+        "fromTypeStr": "",
+        "goodsId": 21374,
+        "headUrl": "/images/1765247221165n8111_800x800.jpg",
+        "icon": "/images/yfs_icon_a_99x66.png",
+        "id": 328141,
+        "name": "龙珠40周年 第一弹 A赏 第一卷 DRAGON BALL手办(每抽获赠一件商品，官方零售价69元(已售罄))",
+        "price": 69,
+        "rare": 9,
+        "realStock": -1,
+        "saleTime": "2025-12",
+        "setId": 9882,
+        "setStatus": 0,
+        "status": 1,
+        "total": 2,
+        "tradable": 0,
+        "type": 1,
+        "typeStr": "A",
+        "updateTimes": 0,
+        "baseName": "龙珠40周年 第一弹 A赏 第一卷 DRAGON BALL手办",
+        "stock": 0,
+        "chance": "0",
+        "rareImg": "https://static.ichibankuji.cn/images/yfs_rare_ssr.png"
+      },
+      {
+        "avPrice": 0,
+        "bIcon": "/images/yfs_bicon_b_333x209.png",
+        "cardId": 0,
+        "content": "",
+        "createTimes": 0,
+        "enabled": 0,
+        "fromNo": 0,
+        "fromTypeStr": "",
+        "goodsId": 21373,
+        "headUrl": "/images/1765247167736n9189_800x800.jpg",
+        "icon": "/images/yfs_icon_b_99x66.png",
+        "id": 328142,
+        "name": "龙珠40周年 第一弹 B赏 第四十二卷 DRAGON BALL手办(每抽获赠一件商品，官方零售价69元(已售罄))",
+        "price": 69,
+        "rare": 9,
+        "realStock": -1,
+        "saleTime": "2025-12",
+        "setId": 9882,
+        "setStatus": 0,
+        "status": 1,
+        "total": 2,
+        "tradable": 0,
+        "type": 1,
+        "typeStr": "B",
+        "updateTimes": 0,
+        "baseName": "龙珠40周年 第一弹 B赏 第四十二卷 DRAGON BALL手办",
+        "stock": 0,
+        "chance": "0",
+        "rareImg": "https://static.ichibankuji.cn/images/yfs_rare_ssr.png"
+      },
+      {
+        "avPrice": 0,
+        "bIcon": "/images/yfs_bicon_c_333x209.png",
+        "cardId": 0,
+        "content": "",
+        "createTimes": 0,
+        "enabled": 0,
+        "fromNo": 0,
+        "fromTypeStr": "",
+        "goodsId": 21372,
+        "headUrl": "/images/1765247127041n9812_800x800.jpg",
+        "icon": "/images/yfs_icon_c_99x66.png",
+        "id": 328143,
+        "name": "龙珠40周年 第一弹 C赏 40周年纪念 视觉插画(每抽获赠一件商品，官方零售价69元(已售罄))",
+        "price": 69,
+        "rare": 7,
+        "realStock": -1,
+        "saleTime": "2025-12",
+        "setId": 9882,
+        "setStatus": 0,
+        "status": 1,
+        "total": 2,
+        "tradable": 0,
+        "type": 1,
+        "typeStr": "C",
+        "updateTimes": 0,
+        "baseName": "龙珠40周年 第一弹 C赏 40周年纪念 视觉插画",
+        "stock": 0,
+        "chance": "0",
+        "rareImg": "https://static.ichibankuji.cn/images/yfs_rare_sr.png"
+      },
+      {
+        "avPrice": 0,
+        "bIcon": "/images/yfs_bicon_d_333x209.png",
+        "cardId": 0,
+        "content": "",
+        "createTimes": 0,
+        "enabled": 0,
+        "fromNo": 0,
+        "fromTypeStr": "",
+        "goodsId": 21371,
+        "headUrl": "/images/1765247090025n5138_800x800.jpg",
+        "icon": "/images/yfs_icon_d_99x66.png",
+        "id": 328144,
+        "name": "龙珠40周年 第一弹 D赏 陶瓷杯(每抽获赠一件商品，官方零售价69元(已售罄))",
+        "price": 69,
+        "rare": 7,
+        "realStock": -1,
+        "saleTime": "2025-12",
+        "setId": 9882,
+        "setStatus": 0,
+        "status": 1,
+        "total": 2,
+        "tradable": 0,
+        "type": 1,
+        "typeStr": "D",
+        "updateTimes": 0,
+        "baseName": "龙珠40周年 第一弹 D赏 陶瓷杯",
+        "stock": 0,
+        "chance": "0",
+        "rareImg": "https://static.ichibankuji.cn/images/yfs_rare_sr.png"
+      },
+      {
+        "avPrice": 0,
+        "bIcon": "/images/yfs_bicon_e_333x209.png",
+        "cardId": 0,
+        "content": "",
+        "createTimes": 0,
+        "enabled": 0,
+        "fromNo": 0,
+        "fromTypeStr": "",
+        "goodsId": 21370,
+        "headUrl": "/images/1765247014313n4336_800x800.jpg",
+        "icon": "/images/yfs_icon_e_99x66.png",
+        "id": 328145,
+        "name": "龙珠40周年 第一弹 E赏 MANGA STRUCTURE 手办(每抽获赠一件商品，官方零售价69元(已售罄))",
+        "price": 69,
+        "rare": 7,
+        "realStock": -1,
+        "saleTime": "2025-12",
+        "setId": 9882,
+        "setStatus": 0,
+        "status": 1,
+        "total": 5,
+        "tradable": 0,
+        "type": 1,
+        "typeStr": "E",
+        "updateTimes": 0,
+        "baseName": "龙珠40周年 第一弹 E赏 MANGA STRUCTURE 手办",
+        "stock": 0,
+        "chance": "0",
+        "rareImg": "https://static.ichibankuji.cn/images/yfs_rare_sr.png"
+      },
+      {
+        "avPrice": 0,
+        "bIcon": "/images/yfs_bicon_f_333x209.png",
+        "cardId": 0,
+        "content": "",
+        "createTimes": 0,
+        "enabled": 0,
+        "fromNo": 0,
+        "fromTypeStr": "",
+        "goodsId": 21369,
+        "headUrl": "/images/1765246965729n1423_800x800.jpg",
+        "icon": "/images/yfs_icon_f_99x66.png",
+        "id": 328146,
+        "name": "龙珠40周年 第一弹 F赏 龙珠 鸟山明的世界 亚克力立牌(每抽获赠一件商品，官方零售价69元，获得概率约为18.18%)",
+        "price": 69,
+        "rare": 5,
+        "realStock": -1,
+        "saleTime": "2025-12",
+        "setId": 9882,
+        "setStatus": 0,
+        "status": 1,
+        "total": 20,
+        "tradable": 0,
+        "type": 1,
+        "typeStr": "F",
+        "updateTimes": 0,
+        "baseName": "龙珠40周年 第一弹 F赏 龙珠 鸟山明的世界 亚克力立牌",
+        "stock": 2,
+        "chance": "18.18",
+        "rareImg": "https://static.ichibankuji.cn/images/yfs_rare_r.png"
+      },
+      {
+        "avPrice": 0,
+        "bIcon": "/images/yfs_bicon_g_333x209.png",
+        "cardId": 0,
+        "content": "",
+        "createTimes": 0,
+        "enabled": 0,
+        "fromNo": 0,
+        "fromTypeStr": "",
+        "goodsId": 21368,
+        "headUrl": "/images/1765246929693n4523_800x800.jpg",
+        "icon": "/images/yfs_icon_g_99x66.png",
+        "id": 328147,
+        "name": "龙珠40周年 第一弹 G赏 漫画文件夹套装(每抽获赠一件商品，官方零售价69元，获得概率约为36.37%)",
+        "price": 69,
+        "rare": 5,
+        "realStock": -1,
+        "saleTime": "2025-12",
+        "setId": 9882,
+        "setStatus": 0,
+        "status": 1,
+        "total": 21,
+        "tradable": 0,
+        "type": 2,
+        "typeStr": "G",
+        "updateTimes": 0,
+        "baseName": "龙珠40周年 第一弹 G赏 漫画文件夹套装",
+        "stock": 4,
+        "chance": "36.37",
+        "rareImg": "https://static.ichibankuji.cn/images/yfs_rare_r.png"
+      },
+      {
+        "avPrice": 0,
+        "bIcon": "/images/yfs_bicon_h_333x209.png",
+        "cardId": 0,
+        "content": "",
+        "createTimes": 0,
+        "enabled": 0,
+        "fromNo": 0,
+        "fromTypeStr": "",
+        "goodsId": 21367,
+        "headUrl": "/images/1765246750682n5253_800x800.jpg",
+        "icon": "/images/yfs_icon_h_99x66.png",
+        "id": 328148,
+        "name": "龙珠40周年 第一弹 H赏 鸟山明原创插画 塑胶彩画(每抽获赠一件商品，官方零售价69元，获得概率约为9.09%)",
+        "price": 69,
+        "rare": 5,
+        "realStock": -1,
+        "saleTime": "2025-12",
+        "setId": 9882,
+        "setStatus": 0,
+        "status": 1,
+        "total": 13,
+        "tradable": 0,
+        "type": 2,
+        "typeStr": "H",
+        "updateTimes": 0,
+        "baseName": "龙珠40周年 第一弹 H赏 鸟山明原创插画 塑胶彩画",
+        "stock": 1,
+        "chance": "9.09",
+        "rareImg": "https://static.ichibankuji.cn/images/yfs_rare_r.png"
+      },
+      {
+        "avPrice": 0,
+        "bIcon": "/images/yfs_bicon_i_333x209.png",
+        "cardId": 0,
+        "content": "",
+        "createTimes": 0,
+        "enabled": 0,
+        "fromNo": 0,
+        "fromTypeStr": "",
+        "goodsId": 21366,
+        "headUrl": "/images/1765246699825n2184_800x800.jpg",
+        "icon": "/images/yfs_icon_i_99x66.png",
+        "id": 328149,
+        "name": "龙珠40周年 第一弹 I赏 贴纸套装(每抽获赠一件商品，官方零售价69元，获得概率约为36.36%)",
+        "price": 69,
+        "rare": 5,
+        "realStock": -1,
+        "saleTime": "2025-12",
+        "setId": 9882,
+        "setStatus": 0,
+        "status": 1,
+        "total": 13,
+        "tradable": 0,
+        "type": 2,
+        "typeStr": "I",
+        "updateTimes": 0,
+        "baseName": "龙珠40周年 第一弹 I赏 贴纸套装",
+        "stock": 4,
+        "chance": "36.36",
+        "rareImg": "https://static.ichibankuji.cn/images/yfs_rare_r.png"
+      },
+      {
+        "avPrice": 0,
+        "bIcon": "/images/yfs_bicon_last_333x209.png",
+        "cardId": 0,
+        "content": "",
+        "createTimes": 0,
+        "enabled": 0,
+        "fromNo": 0,
+        "fromTypeStr": "",
+        "goodsId": 21365,
+        "headUrl": "/images/1765246456166n2929_800x800.jpg",
+        "icon": "/images/yfs_icon_last_99x66.png",
+        "id": 328150,
+        "name": "龙珠40周年 第一弹 LAST赏 超级赛亚人孙悟空 GIGA手办(非卖品，随最后一抽赠送)",
+        "price": 69,
+        "rare": 10,
+        "realStock": -1,
+        "saleTime": "2025-12",
+        "setId": 9882,
+        "setStatus": 0,
+        "status": 1,
+        "total": 1,
+        "tradable": 0,
+        "type": 1,
+        "typeStr": "LAST",
+        "updateTimes": 0,
+        "baseName": "龙珠40周年 第一弹 LAST赏 超级赛亚人孙悟空 GIGA手办(非卖品，随最后一抽赠送)",
+        "stock": 1,
+        "rareImg": "https://static.ichibankuji.cn/images/yfs_rare_sp.png"
+      }
+    ],
+    "queuerList": [],
+    "queuerNum": 0,
+    "cardNum": 0,
+    "usableCouponNum": 0,
+    "isQueuer": 1,
+    "nowTime": "2025-12-17 17:03:53",
+    "queueMode1": 17,
+    "queueMode2": 19
+  }
+}

rhyf_spider/package.json

@@ -0,0 +1,362 @@
+{
+  "error": 200,
+  "message": "操作成功",
+  "data": {
+    "areaType": 0,
+    "boxNo": "20251212988239",
+    "boxStatus": 0,
+    "createTimes": 0,
+    "enabled": 1,
+    "endTime": "2025-12-12 23:22:02",
+    "goodsId": 0,
+    "id": 84343,
+    "orderNo": 39,
+    "price": 69,
+    "queueMode": 1,
+    "saleStock": 0,
+    "saleTotal": 0,
+    "setActiveData": "",
+    "setId": 9882,
+    "setType": 1,
+    "spPrize": "#1",
+    "startTime": "2025-12-12 23:22:02",
+    "status": 1,
+    "stock": 11,
+    "total": 80,
+    "updateTimes": 0,
+    "setContent": "",
+    "setDescr": "龙珠DRAGON BALL 40th 第一弹",
+    "setHeadUrl": "/images/1765250524051n3474_800x800.jpg",
+    "setName": "龙珠DRAGON BALL 40th 第一弹",
+    "setImg": "",
+    "prizeType": 0,
+    "setActiveDesc": "",
+    "setAreaType": 1,
+    "boxTotal": 17,
+    "setBoxTotal": 56,
+    "setEffect": "",
+    "takeAllLimit": 0,
+    "scoresTimes": 1,
+    "setSaleStatus": 1,
+    "topWord": "平台发货不设门槛！赏柜内提交发货申请后3-5个工作日安排发货。非偏远地区每单满5件包邮，不满5件需支付运费10元。",
+    "setTopWord": "平台发货不设门槛！赏柜内提交发货申请后3-5个工作日安排发货。包邮地区每单满5件包邮，不满5件需支付运费10元，不支持7天无理由退换货。",
+    "awardList": [
+      {
+        "avPrice": 0,
+        "bIcon": "/images/yfs_bicon_a_333x209.png",
+        "cardId": 0,
+        "content": "",
+        "createTimes": 0,
+        "enabled": 0,
+        "fromNo": 0,
+        "fromTypeStr": "",
+        "goodsId": 21374,
+        "headUrl": "/images/1765247221165n8111_800x800.jpg",
+        "icon": "/images/yfs_icon_a_99x66.png",
+        "id": 328141,
+        "name": "龙珠40周年 第一弹 A赏 第一卷 DRAGON BALL手办(每抽获赠一件商品，官方零售价69元(已售罄))",
+        "price": 69,
+        "rare": 9,
+        "realStock": -1,
+        "saleTime": "2025-12",
+        "setId": 9882,
+        "setStatus": 0,
+        "status": 1,
+        "total": 2,
+        "tradable": 0,
+        "type": 1,
+        "typeStr": "A",
+        "updateTimes": 0,
+        "baseName": "龙珠40周年 第一弹 A赏 第一卷 DRAGON BALL手办",
+        "stock": 0,
+        "chance": "0",
+        "rareImg": "https://static.ichibankuji.cn/images/yfs_rare_ssr.png"
+      },
+      {
+        "avPrice": 0,
+        "bIcon": "/images/yfs_bicon_b_333x209.png",
+        "cardId": 0,
+        "content": "",
+        "createTimes": 0,
+        "enabled": 0,
+        "fromNo": 0,
+        "fromTypeStr": "",
+        "goodsId": 21373,
+        "headUrl": "/images/1765247167736n9189_800x800.jpg",
+        "icon": "/images/yfs_icon_b_99x66.png",
+        "id": 328142,
+        "name": "龙珠40周年 第一弹 B赏 第四十二卷 DRAGON BALL手办(每抽获赠一件商品，官方零售价69元(已售罄))",
+        "price": 69,
+        "rare": 9,
+        "realStock": -1,
+        "saleTime": "2025-12",
+        "setId": 9882,
+        "setStatus": 0,
+        "status": 1,
+        "total": 2,
+        "tradable": 0,
+        "type": 1,
+        "typeStr": "B",
+        "updateTimes": 0,
+        "baseName": "龙珠40周年 第一弹 B赏 第四十二卷 DRAGON BALL手办",
+        "stock": 0,
+        "chance": "0",
+        "rareImg": "https://static.ichibankuji.cn/images/yfs_rare_ssr.png"
+      },
+      {
+        "avPrice": 0,
+        "bIcon": "/images/yfs_bicon_c_333x209.png",
+        "cardId": 0,
+        "content": "",
+        "createTimes": 0,
+        "enabled": 0,
+        "fromNo": 0,
+        "fromTypeStr": "",
+        "goodsId": 21372,
+        "headUrl": "/images/1765247127041n9812_800x800.jpg",
+        "icon": "/images/yfs_icon_c_99x66.png",
+        "id": 328143,
+        "name": "龙珠40周年 第一弹 C赏 40周年纪念 视觉插画(每抽获赠一件商品，官方零售价69元(已售罄))",
+        "price": 69,
+        "rare": 7,
+        "realStock": -1,
+        "saleTime": "2025-12",
+        "setId": 9882,
+        "setStatus": 0,
+        "status": 1,
+        "total": 2,
+        "tradable": 0,
+        "type": 1,
+        "typeStr": "C",
+        "updateTimes": 0,
+        "baseName": "龙珠40周年 第一弹 C赏 40周年纪念 视觉插画",
+        "stock": 0,
+        "chance": "0",
+        "rareImg": "https://static.ichibankuji.cn/images/yfs_rare_sr.png"
+      },
+      {
+        "avPrice": 0,
+        "bIcon": "/images/yfs_bicon_d_333x209.png",
+        "cardId": 0,
+        "content": "",
+        "createTimes": 0,
+        "enabled": 0,
+        "fromNo": 0,
+        "fromTypeStr": "",
+        "goodsId": 21371,
+        "headUrl": "/images/1765247090025n5138_800x800.jpg",
+        "icon": "/images/yfs_icon_d_99x66.png",
+        "id": 328144,
+        "name": "龙珠40周年 第一弹 D赏 陶瓷杯(每抽获赠一件商品，官方零售价69元(已售罄))",
+        "price": 69,
+        "rare": 7,
+        "realStock": -1,
+        "saleTime": "2025-12",
+        "setId": 9882,
+        "setStatus": 0,
+        "status": 1,
+        "total": 2,
+        "tradable": 0,
+        "type": 1,
+        "typeStr": "D",
+        "updateTimes": 0,
+        "baseName": "龙珠40周年 第一弹 D赏 陶瓷杯",
+        "stock": 0,
+        "chance": "0",
+        "rareImg": "https://static.ichibankuji.cn/images/yfs_rare_sr.png"
+      },
+      {
+        "avPrice": 0,
+        "bIcon": "/images/yfs_bicon_e_333x209.png",
+        "cardId": 0,
+        "content": "",
+        "createTimes": 0,
+        "enabled": 0,
+        "fromNo": 0,
+        "fromTypeStr": "",
+        "goodsId": 21370,
+        "headUrl": "/images/1765247014313n4336_800x800.jpg",
+        "icon": "/images/yfs_icon_e_99x66.png",
+        "id": 328145,
+        "name": "龙珠40周年 第一弹 E赏 MANGA STRUCTURE 手办(每抽获赠一件商品，官方零售价69元(已售罄))",
+        "price": 69,
+        "rare": 7,
+        "realStock": -1,
+        "saleTime": "2025-12",
+        "setId": 9882,
+        "setStatus": 0,
+        "status": 1,
+        "total": 5,
+        "tradable": 0,
+        "type": 1,
+        "typeStr": "E",
+        "updateTimes": 0,
+        "baseName": "龙珠40周年 第一弹 E赏 MANGA STRUCTURE 手办",
+        "stock": 0,
+        "chance": "0",
+        "rareImg": "https://static.ichibankuji.cn/images/yfs_rare_sr.png"
+      },
+      {
+        "avPrice": 0,
+        "bIcon": "/images/yfs_bicon_f_333x209.png",
+        "cardId": 0,
+        "content": "",
+        "createTimes": 0,
+        "enabled": 0,
+        "fromNo": 0,
+        "fromTypeStr": "",
+        "goodsId": 21369,
+        "headUrl": "/images/1765246965729n1423_800x800.jpg",
+        "icon": "/images/yfs_icon_f_99x66.png",
+        "id": 328146,
+        "name": "龙珠40周年 第一弹 F赏 龙珠 鸟山明的世界 亚克力立牌(每抽获赠一件商品，官方零售价69元，获得概率约为18.18%)",
+        "price": 69,
+        "rare": 5,
+        "realStock": -1,
+        "saleTime": "2025-12",
+        "setId": 9882,
+        "setStatus": 0,
+        "status": 1,
+        "total": 20,
+        "tradable": 0,
+        "type": 1,
+        "typeStr": "F",
+        "updateTimes": 0,
+        "baseName": "龙珠40周年 第一弹 F赏 龙珠 鸟山明的世界 亚克力立牌",
+        "stock": 2,
+        "chance": "18.18",
+        "rareImg": "https://static.ichibankuji.cn/images/yfs_rare_r.png"
+      },
+      {
+        "avPrice": 0,
+        "bIcon": "/images/yfs_bicon_g_333x209.png",
+        "cardId": 0,
+        "content": "",
+        "createTimes": 0,
+        "enabled": 0,
+        "fromNo": 0,
+        "fromTypeStr": "",
+        "goodsId": 21368,
+        "headUrl": "/images/1765246929693n4523_800x800.jpg",
+        "icon": "/images/yfs_icon_g_99x66.png",
+        "id": 328147,
+        "name": "龙珠40周年 第一弹 G赏 漫画文件夹套装(每抽获赠一件商品，官方零售价69元，获得概率约为36.37%)",
+        "price": 69,
+        "rare": 5,
+        "realStock": -1,
+        "saleTime": "2025-12",
+        "setId": 9882,
+        "setStatus": 0,
+        "status": 1,
+        "total": 21,
+        "tradable": 0,
+        "type": 2,
+        "typeStr": "G",
+        "updateTimes": 0,
+        "baseName": "龙珠40周年 第一弹 G赏 漫画文件夹套装",
+        "stock": 4,
+        "chance": "36.37",
+        "rareImg": "https://static.ichibankuji.cn/images/yfs_rare_r.png"
+      },
+      {
+        "avPrice": 0,
+        "bIcon": "/images/yfs_bicon_h_333x209.png",
+        "cardId": 0,
+        "content": "",
+        "createTimes": 0,
+        "enabled": 0,
+        "fromNo": 0,
+        "fromTypeStr": "",
+        "goodsId": 21367,
+        "headUrl": "/images/1765246750682n5253_800x800.jpg",
+        "icon": "/images/yfs_icon_h_99x66.png",
+        "id": 328148,
+        "name": "龙珠40周年 第一弹 H赏 鸟山明原创插画 塑胶彩画(每抽获赠一件商品，官方零售价69元，获得概率约为9.09%)",
+        "price": 69,
+        "rare": 5,
+        "realStock": -1,
+        "saleTime": "2025-12",
+        "setId": 9882,
+        "setStatus": 0,
+        "status": 1,
+        "total": 13,
+        "tradable": 0,
+        "type": 2,
+        "typeStr": "H",
+        "updateTimes": 0,
+        "baseName": "龙珠40周年 第一弹 H赏 鸟山明原创插画 塑胶彩画",
+        "stock": 1,
+        "chance": "9.09",
+        "rareImg": "https://static.ichibankuji.cn/images/yfs_rare_r.png"
+      },
+      {
+        "avPrice": 0,
+        "bIcon": "/images/yfs_bicon_i_333x209.png",
+        "cardId": 0,
+        "content": "",
+        "createTimes": 0,
+        "enabled": 0,
+        "fromNo": 0,
+        "fromTypeStr": "",
+        "goodsId": 21366,
+        "headUrl": "/images/1765246699825n2184_800x800.jpg",
+        "icon": "/images/yfs_icon_i_99x66.png",
+        "id": 328149,
+        "name": "龙珠40周年 第一弹 I赏 贴纸套装(每抽获赠一件商品，官方零售价69元，获得概率约为36.36%)",
+        "price": 69,
+        "rare": 5,
+        "realStock": -1,
+        "saleTime": "2025-12",
+        "setId": 9882,
+        "setStatus": 0,
+        "status": 1,
+        "total": 13,
+        "tradable": 0,
+        "type": 2,
+        "typeStr": "I",
+        "updateTimes": 0,
+        "baseName": "龙珠40周年 第一弹 I赏 贴纸套装",
+        "stock": 4,
+        "chance": "36.36",
+        "rareImg": "https://static.ichibankuji.cn/images/yfs_rare_r.png"
+      },
+      {
+        "avPrice": 0,
+        "bIcon": "/images/yfs_bicon_last_333x209.png",
+        "cardId": 0,
+        "content": "",
+        "createTimes": 0,
+        "enabled": 0,
+        "fromNo": 0,
+        "fromTypeStr": "",
+        "goodsId": 21365,
+        "headUrl": "/images/1765246456166n2929_800x800.jpg",
+        "icon": "/images/yfs_icon_last_99x66.png",
+        "id": 328150,
+        "name": "龙珠40周年 第一弹 LAST赏 超级赛亚人孙悟空 GIGA手办(非卖品，随最后一抽赠送)",
+        "price": 69,
+        "rare": 10,
+        "realStock": -1,
+        "saleTime": "2025-12",
+        "setId": 9882,
+        "setStatus": 0,
+        "status": 1,
+        "total": 1,
+        "tradable": 0,
+        "type": 1,
+        "typeStr": "LAST",
+        "updateTimes": 0,
+        "baseName": "龙珠40周年 第一弹 LAST赏 超级赛亚人孙悟空 GIGA手办(非卖品，随最后一抽赠送)",
+        "stock": 1,
+        "rareImg": "https://static.ichibankuji.cn/images/yfs_rare_sp.png"
+      }
+    ],
+    "queuerList": [],
+    "queuerNum": 0,
+    "cardNum": 0,
+    "usableCouponNum": 0,
+    "isQueuer": 1,
+    "nowTime": "2025-12-17 17:03:59",
+    "commentList": []
+  }
+}

rhyf_spider/rhyf_login.py

@@ -0,0 +1,97 @@
+# -*- coding: utf-8 -*-
+# Author : Charley
+# Python : 3.10.8
+# Date   : 2025/12/16 13:26
+import requests
+import json
+
+from utils import create_request_data, decrypt_request_data
+
+headers = {
+    "authority": "wechatapp.ichibankuji.cn",
+    "accept": "*/*",
+    "accept-language": "zh-CN,zh;q=0.9",
+    "authorization": "7d04b7ZDRB9Vb65wzF4G6!1xyVJssI4IR!1zre6RpycuTj2sZm3Fi8UAH6MWYSwNXxKdjcSYJ3sHeZm!1bVUMVa2A295TA==",
+    "content-type": "application/json",
+    "referer": "https://servicewechat.com/wxd21e3190b2a44f73/21/page-frame.html",
+    "sec-fetch-dest": "empty",
+    "sec-fetch-mode": "cors",
+    "sec-fetch-site": "cross-site",
+    "terminalos": "YFSXZF",
+    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 MicroMessenger/7.0.20.1781(0x6700143B) NetType/WIFI MiniProgramEnv/Windows WindowsWechat/WMPF WindowsWechat(0x63090c33) XWEB/9129",
+    "xweb_xhr": "1"
+}
+url = "https://wechatapp.ichibankuji.cn/wechat/mini/login"
+data = {
+    "da": "f8+9FfRjJzuNMgzOSZ4M8Ah9d1K0curzOi3v3Dan+vztCnUhoUyXFKcbVDqHybWO2Rol61gMJZ/xswwXFNr1UI3gKQiqCBxBETjGhmMEj9kvgtNKff4WruOb8avpxtFidtY3XYqsIBex6K9bYWI7RkiCYktIgqK6FGiInHPwikNwacgsiG7w/JxsJgceKiU+wArK2H3R9OiKueKyCUq8SbaiDweYMELEuYF7SSxj1eTwWuU/sAautT2H5yGe/B/Aou"
+}
+# 解密得到明文
+plaintext = decrypt_request_data(data["da"])
+print(plaintext)
+
+# {'clickId': '', 'gdtVid': '', 'jsCode': '0c3hwaGa1SKTQK0vfHHa17Wg241hwaGK', 'miniName': 'YFSXZF', 'nonceStr': 'hGcdaRiXN4G4YpDXBG7mnrNpYnQfnZsY', 'qzGdt': '', 'time': 1765862748744, 'weixinadinfo': ''}
+original_data = {'clickId': '', 'gdtVid': '', 'jsCode': '0c3hwaGa1SKTQK0vfHHa17Wg241hwaGK', 'qzGdt': '', 'weixinadinfo': ''}
+# original_data = {'clickId': '', 'gdtVid': '', 'jsCode': '0c3hwaGa1SKTQK0vfHHa17Wg241hwaGK', 'qzGdt': '',
+#                  'weixinadinfo': ''}
+request_data = create_request_data(original_data)
+
+payload = {
+    "da": request_data
+}
+response = requests.post(url, headers=headers, json=payload)
+
+print(response.text)
+print(response)
+
+"""
+{"error":200,"message":"操作成功","data":
+{"nickName":"游客(171905678)","baseNickName":"游客(171905678)",
+"avatarUrl":"","authorized":1,"amount":0,"scores":0,"totalScores":0,"cashOut":500,"cashOutNum":10,
+"createTime":"2025-12-17 14:15:27","checked":0,"uid":171905678,"userNo":30245478,
+"registerTime":15609600,"userNoStr":"p30245478","subscribed":1,"maxPackNum":150,"packNum":100,"maxTradeNum":15,
+"tradeNum":10,"tradeAwardLimit":24,"alipay":"","realName":"","inviteNum":0,"inviteScores":0,"freeShippingCoupon":0}}
+
+
+if ("the code is a mock one" != o.code) {
+if (wx.getStorageSync("channelId") && wx.getStorageSync("inviteId"))
+    var r = {
+        jsCode: o.code,
+        gdtVid: "",
+        clickId: "",
+        qzGdt: "",
+        weixinadinfo: "",
+        xhsClickId: "",
+        inviteId: wx.getStorageSync("inviteId"),
+        channelId: wx.getStorageSync("channelId")
+    };
+else if (wx.getStorageSync("inviteId"))
+    r = {
+        jsCode: o.code,
+        gdtVid: "",
+        clickId: "",
+        qzGdt: "",
+        weixinadinfo: "",
+        xhsClickId: "",
+        inviteId: wx.getStorageSync("inviteId")
+    };
+else if (wx.getStorageSync("channelId"))
+    r = {
+        jsCode: o.code,
+        gdtVid: "",
+        clickId: "",
+        qzGdt: "",
+        weixinadinfo: "",
+        xhsClickId: "",
+        channelId: wx.getStorageSync("channelId")
+    };
+else
+    r = {
+        jsCode: o.code,
+        gdtVid: "",
+        qzGdt: "",
+        clickId: "",
+        weixinadinfo: ""
+    };
+                                    
+                                    
+"""

rhyf_spider/rhyf_spider.py

@@ -0,0 +1,387 @@
+# -*- coding: utf-8 -*-
+# Author : Charley
+# Python : 3.10.8
+# Date   : 2025/12/8 13:44
+import time
+import random
+import inspect
+import requests
+import user_agent
+from loguru import logger
+from mysql_pool import MySQLConnectionPool
+from tenacity import retry, stop_after_attempt, wait_fixed
+from utils import create_request_data
+
+logger.remove()
+logger.add("./logs/{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00",
+           format="[{time:YYYY-MM-DD HH:mm:ss.SSS}] {level} {message}",
+           level="DEBUG", retention="7 day")
+
+auth = "266577ZDRB9Vb65wzF4G6!1xyVJv7BGKi7fcI757ILRQW!1sI9jAV5AOWQhXJjA!2oIrMsaLY!2WU7AnCtCshdk4EC3dvjQ=="
+headers = {
+    'authority': 'wechatapp.ichibankuji.cn',
+    'content-type': 'application/json',
+    'referer': 'https://servicewechat.com/wxd21e3190b2a44f73/21/page-frame.html',
+    "terminalos": "YFSXZF",
+    'user-agent': user_agent.generate_user_agent(),
+    'authorization': auth
+}
+
+
+def after_log(retry_state):
+    """
+    retry 回调
+    :param retry_state: RetryCallState 对象
+    """
+    # 检查 args 是否存在且不为空
+    if retry_state.args and len(retry_state.args) > 0:
+        log = retry_state.args[0]  # 获取传入的 logger
+    else:
+        log = logger  # 使用全局 logger
+
+    if retry_state.outcome.failed:
+        log.warning(
+            f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} Times")
+    else:
+        log.info(f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} succeeded")
+
+
+@retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
+def get_proxys(log):
+    """
+    获取代理
+    :return: 代理
+    """
+    tunnel = "x371.kdltps.com:15818"
+    kdl_username = "t13753103189895"
+    kdl_password = "o0yefv6z"
+    try:
+        proxies = {
+            "http": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": kdl_username, "pwd": kdl_password, "proxy": tunnel},
+            "https": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": kdl_username, "pwd": kdl_password, "proxy": tunnel}
+        }
+        return proxies
+    except Exception as e:
+        log.error(f"Error getting proxy: {e}")
+        raise e
+
+
+@retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
+def get_set_single_page(log, page=1, sql_pool=None):
+    """
+    获取单个页面数据
+    :param log: 日志对象
+    :param page: 页码
+    :param sql_pool: 数据库连接池对象
+    :return: 数据长度
+    """
+    log.debug(f"{inspect.currentframe().f_code.co_name}, 正在获取第{page}页数据........")
+    original_data = {
+        "orderType": 0,
+        "pageNum": page,
+        "pageSize": 10,
+        "status": 0,
+        "type": 1
+    }
+    url = "https://wechatapp.ichibankuji.cn/wechat/yfs/getSetList"
+    request_data = create_request_data(original_data)
+
+    payload = {
+        "da": request_data
+    }
+
+    response = requests.post(
+        url,
+        headers=headers,
+        json=payload,
+        timeout=22
+    )
+    # print("响应内容:", response.text)
+
+    response.raise_for_status()
+
+    resp_json = response.json().get("data", {}).get("list", [])
+    if not resp_json:
+        log.debug("Not resp_json, 没有数据........")
+        return 0
+
+    info_list = []
+    for item in resp_json:
+        box_id = item.get("id")
+        name = item.get("name")
+        price = item.get("price")
+        onsaleTime = item.get("onsaleTime")
+        headUrl = item.get("headUrl")
+        if headUrl:
+            headUrl = f"https://static.ichibankuji.cn{headUrl}"
+        scoresDeduct = item.get("scoresDeduct")  # 积分抵扣
+        status = item.get("status")
+        themeId = item.get("themeId")
+        total = item.get("total")
+        box_type = item.get("type")
+
+        data_dict = {
+            "set_id": box_id,
+            "set_name": name,
+            "price": price,
+            "onsale_time": onsaleTime,
+            "head_url": headUrl,
+            "scores_deduct": scoresDeduct,
+            "status": status,
+            "theme_id": themeId,
+            "set_total": total,
+            "set_type": box_type
+        }
+        # print(data_dict)
+        info_list.append(data_dict)
+
+    if info_list:
+        sql_pool.insert_many(table="rhyf_product_list_record", data_list=info_list, ignore=True)
+
+    return len(info_list)
+
+
+def get_setlist_all_page(log, sql_pool=None):
+    """
+    获取所有页面数据
+    :param log: 日志对象
+    :param sql_pool: 数据库连接池对象
+    :return:
+    """
+    page = 1
+    while True:
+        try:
+            len_data = get_set_single_page(log, page, sql_pool)
+
+            if len_data < 10:
+                log.debug(f"当前页为{page}, 数据长度为{len_data} ,数据已经获取完毕........")
+                break
+
+            page += 1
+        except Exception as e:
+            log.error(f"Error getting set single page: {e}")
+            break
+
+
+# ----------------------------------------------------------------------------------------------------------------------
+@retry(stop=stop_after_attempt(5), wait=wait_fixed(5), after=after_log)
+def get_box_detail(log, set_id: int, sql_pool=None):
+    """
+    获取箱子详情
+    :param log: 日志对象
+    :param set_id: 箱子ID
+    :param sql_pool: 数据库连接池对象
+    :return: 数据长度
+    """
+    log.debug(f"{inspect.currentframe().f_code.co_name}, set_id:{set_id}, 获取 <详情> 数据........")
+    url = "https://wechatapp.ichibankuji.cn/wechat/yfs/firstBoxDetail"
+    original_data = {
+        "queueMode": 2,
+        "setId": f"{set_id}"
+    }
+    request_data = create_request_data(original_data)
+
+    payload = {
+        "da": request_data
+    }
+    response = requests.post(url, headers=headers, json=payload, timeout=22)
+    # print(response.text)
+    response.raise_for_status()
+
+    resp_json = response.json().get("data", {})
+    if not resp_json:
+        log.debug("Not resp_json, 没有数据........")
+        return 0
+
+    boxNo = resp_json.get("boxNo")  # 箱号
+    box_id = resp_json.get("id")
+    orderNo = resp_json.get("orderNo")
+    # price = resp_json.get("price")
+    # status = resp_json.get("status")
+    stock = resp_json.get("stock")  # 库存
+    total = resp_json.get("total")  # 总抽数
+
+    data_dict = {
+        "set_id": set_id,
+        "box_no": boxNo,
+        "box_id": box_id,
+        "order_no": orderNo,
+        "stock": stock,
+        "total": total
+    }
+    # print(data_dict)
+    sql_pool.insert_one_or_dict(table="rhyf_product_detail_record", data=data_dict)
+
+
+@retry(stop=stop_after_attempt(5), wait=wait_fixed(5), after=after_log)
+def get_draw_single_page(log, box_id: int, page=1, sql_pool=None):
+    """
+    获取箱子抽奖数据
+    :param log: 日志对象
+    :param box_id: 箱子ID
+    :param page: 页码
+    :param sql_pool: 数据库连接池对象
+    :return: 数据长度
+    """
+    log.debug(f"{inspect.currentframe().f_code.co_name}, box_id:{box_id}, 获取 <抽奖> 数据........")
+    url = "https://wechatapp.ichibankuji.cn/wechat/yfs/getBoxDrawLog"
+    original_data = {
+        # "boxId": 84633,
+        "boxId": box_id,
+        "pageSize": 30,
+        "isMine": 0,
+        "pageNum": page
+    }
+    request_data = create_request_data(original_data)
+
+    payload = {
+        "da": request_data
+    }
+    response = requests.post(url, headers=headers, json=payload, timeout=22)
+    # print(response.text)
+    response.raise_for_status()
+
+    resp_json = response.json().get("data", {}).get("list", [])
+    if not resp_json:
+        log.debug("Not resp_json, 没有数据........")
+        return 0
+
+    info_list = []
+    for item in resp_json:
+        set_id = item.get("setId")
+        draw_id = item.get("id")
+        nickName = item.get("nickName")
+        createTime = item.get("createTime")
+
+        awardInfo = item.get("awardInfo", [])
+        if not awardInfo:
+            log.debug(f"box_id:{box_id}, Not awardInfo List, 没有数据........")
+            continue
+
+        for award in awardInfo:
+            award_type = award.get("typeStr")  # 奖品等级
+            award_total = award.get("total")  # 奖品数量
+            data_dict = {
+                "set_id": set_id,
+                "box_id": box_id,
+                "draw_id": draw_id,
+                "nick_name": nickName,
+                "create_time": createTime,
+                "award_type": award_type,
+                "award_total": award_total
+            }
+            # print(data_dict)
+            info_list.append(data_dict)
+
+    # 写入数据库
+    if info_list:
+        sql_pool.insert_many(table="rhyf_product_draw_record", data_list=info_list, ignore=True)
+
+    return len(info_list)
+
+
+def get_draw_list(log, box_id: int, sql_pool=None):
+    """
+    获取箱子所有抽奖数据
+    :param log: 日志对象
+    :param box_id: 箱子ID
+    :param sql_pool: 数据库连接池对象
+    :return:
+    """
+    page = 1
+    while True:
+        try:
+            len_data = get_draw_single_page(log, box_id, page, sql_pool)
+
+            if len_data < 30:
+                log.debug(f"当前页为{page}, 数据长度为{len_data} ,数据已经获取完毕........")
+                break
+
+            page += 1
+        except Exception as e:
+            log.error(f"Error getting draw single page: {e}")
+            break
+
+
+# ----------------------------------------------------------------------------------------------------------------------
+@retry(stop=stop_after_attempt(100), wait=wait_fixed(3600), after=after_log)
+def rhyf_main(log):
+    """
+    主函数 自动售货机
+    :param log: logger对象
+    """
+    log.info(
+        f'开始运行 {inspect.currentframe().f_code.co_name} 爬虫任务....................................................')
+
+    # 配置 MySQL 连接池
+    sql_pool = MySQLConnectionPool(log=log)
+    if not sql_pool.check_pool_health():
+        log.error("数据库连接池异常")
+        raise RuntimeError("数据库连接池异常")
+
+    try:
+        try:
+            log.debug('------------------- 获取所有页码商品 -------------------')
+            get_setlist_all_page(log, sql_pool)
+        except Exception as e:
+            log.error(f'get_all_page error: {e}')
+
+        # --------------------------------------------------------------------
+        try:
+            log.debug('------------------- 获取所有商品详情 -------------------')
+            sql_set_id_list = sql_pool.select_all("select set_id from rhyf_product_list_record where detail_state = 0")
+            sql_set_id_list = [item[0] for item in sql_set_id_list]
+            for set_id in sql_set_id_list:
+                try:
+                    get_box_detail(log, set_id, sql_pool)
+                    # 更改状态
+                    sql_pool.update_one_or_dict(
+                        table="rhyf_product_list_record",
+                        data={"detail_state": 1},
+                        condition={"set_id": set_id}
+                    )
+                except Exception as e:
+                    log.error(f'get_box_detail error: {e}')
+                    sql_pool.update_one_or_dict(
+                        table="rhyf_product_list_record",
+                        data={"detail_state": 2},
+                        condition={"set_id": set_id}
+                    )
+        except Exception as e:
+            log.error(f'get_box_detail error: {e}')
+
+        # ---------------------------------------------------------------------
+        try:
+            log.debug('------------------- 获取所有商品抽奖数据 -------------------')
+            sql_box_id_list = sql_pool.select_all("select box_id from rhyf_product_detail_record where draw_state = 0")
+            sql_box_id_list = [item[0] for item in sql_box_id_list]
+            for box_id in sql_box_id_list:
+                try:
+                    get_draw_list(log, box_id, sql_pool)
+                    # 更改状态
+                    sql_pool.update_one_or_dict(
+                        table="rhyf_product_detail_record",
+                        data={"draw_state": 1},
+                        condition={"box_id": box_id}
+                    )
+                except Exception as e:
+                    log.error(f'get_draw_list error: {e}')
+                    sql_pool.update_one_or_dict(
+                        table="rhyf_product_detail_record",
+                        data={"draw_state": 2},
+                        condition={"box_id": box_id}
+                    )
+        except Exception as e:
+            log.error(f'get_draw_list error: {e}')
+
+    except Exception as e:
+        log.error(f'{inspect.currentframe().f_code.co_name} error: {e}')
+    finally:
+        log.info(f'爬虫程序 {inspect.currentframe().f_code.co_name} 运行结束,等待下一轮的采集任务............')
+
+
+if __name__ == '__main__':
+    # get_set_single_page(logger)
+    # get_box_detail(logger, 9936)
+    # get_draw_single_page(logger, 84633)
+    rhyf_main(logger)

rhyf_spider/utils.py

@@ -0,0 +1,323 @@
+# -*- coding: utf-8 -*-
+# Author : Charley
+# Python : 3.10.8
+# Date   : 2025/12/12 16:00
+import time
+import requests
+import json
+import random
+from loguru import logger
+from Crypto.Cipher import AES
+from Crypto.Util.Padding import pad, unpad
+import base64
+
+
+def generate_nonce_str(length=32):
+    """生成随机字符串，对应JS中的s()函数"""
+    chars = "ABCDEFGHJKMNPQRSTWXYZabcdefhijkmnprstwxyz2345678"
+    return ''.join(random.choice(chars) for _ in range(length))
+
+
+def generate_random_char():
+    """生成单个随机字符，对应JS中的c()函数"""
+    chars = "abcdefghijklmnopqrstuvwxyz0123456789"
+    return random.choice(chars)
+
+
+def obfuscate_encrypted_data(encrypted_data):
+    """对加密数据进行混淆处理，对应JS中的d()函数"""
+    data_list = list(encrypted_data)
+    # 在索引3位置插入随机字符
+    data_list.insert(3, generate_random_char())
+    # 在索引5位置插入随机字符
+    data_list.insert(5, generate_random_char())
+    return ''.join(data_list)
+
+
+def sort_dict_by_key(data_dict):
+    """对字典按键排序，对应JS中的a()函数"""
+    sorted_keys = sorted(data_dict.keys())
+    sorted_dict = {}
+    for key in sorted_keys:
+        sorted_dict[key] = data_dict[key]
+    return sorted_dict
+
+
+def aes_encrypt(plaintext: str) -> str:
+    """
+    AES ECB 模式加密，PKCS7 填充
+    """
+    key = "pSlJACAvYnAgOIlluJgK2F=="
+
+    # 将密钥转换为字节（UTF-8编码）
+    key_bytes = key.encode('utf-8')
+
+    # AES 要求密钥长度为 16/24/32 字节
+    # 如果密钥长度不符合要求，需要调整
+    # 这里密钥是24字节，符合AES-192的要求
+
+    # 创建 AES 加密器（ECB 模式）
+    cipher = AES.new(key_bytes, AES.MODE_ECB)
+
+    # 将明文转换为字节并进行 PKCS7 填充
+    plaintext_bytes = plaintext.encode('utf-8')
+    padded_data = pad(plaintext_bytes, AES.block_size)
+
+    # 加密
+    encrypted_bytes = cipher.encrypt(padded_data)
+
+    # 返回 Base64 编码的密文
+    return base64.b64encode(encrypted_bytes).decode('utf-8')
+
+
+def create_request_data(original_data,nonce_field_name='nonceStr'):
+    """创建完整的请求数据"""
+    # 添加必要字段
+    data_with_fields = original_data.copy()
+    data_with_fields['time'] = int(time.time() * 1000)  # 时间戳
+    # data_with_fields['nonceStr'] = generate_nonce_str()  # 随机字符串
+    data_with_fields[nonce_field_name] = generate_nonce_str()  # 随机字符串
+    data_with_fields['miniName'] = 'YFSXZF'  # 根据环境确定
+    # data_with_fields = {"miniName":"YFSXZF","nonceStr":"iCxy35dfFNJtQSKcrB8yZsRd7B4p3c2h","orderType":0,"pageNum":6,"pageSize":10,"status":0,"time":1765792372017,"type":1}
+
+    # 按键排序
+    sorted_data = sort_dict_by_key(data_with_fields)
+    # print(f'sorted_data: {sorted_data}')
+
+    # 转换为JSON字符串
+    json_str = json.dumps(sorted_data, separators=(',', ':'))
+    # print(f'json_str: {json_str}')
+
+    # AES加密
+    encrypted_data = aes_encrypt(json_str)
+    # "ssB0tJIFaPpO+oDa/FVfaEedp/7js3I0kaRkoTVblr7ssvn71/Y5zDsys/XxOXoNXP3fNdhJdAymwswqYNCmd/7Fk3GT1+5xowXLd64Z7BErz3v1AZuI7yithKjcFlYbafeUkOXI9LPHdUWc7/m4YWpZI3EhJhhGs322ScFsr/31Lu5TN6M/SQ9mZf8IaaNl2xlI+wr+ADV5zC2nEFxg1A=="
+    #  ssB0tJIFaPpO+oDa/FVfaEedp/7js3I0kaRkoTVblr7ssvn71/Y5zDsys/XxOXoNXP3fNdhJdAymwswqYNCmd/7Fk3GT1+5xowXLd64Z7BErz3v1AZuI7yithKjcFlYbafeUkOXI9LPHdUWc7/m4YWpZI3EhJhhGs322ScFsr/31Lu5TN6M/SQ9mZf8IaaNl2xlI+wr+ADV5zC2nEFxg1A==
+    # print(f'encrypted_data: {encrypted_data}')
+
+    # 混淆处理
+    final_data = obfuscate_encrypted_data(encrypted_data)
+    # print(f'final_data: {final_data}')
+
+    return final_data
+
+# -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+
+def deobfuscate_encrypted_data(obfuscated_data):
+    """反混淆处理，移除d()函数插入的随机字符"""
+    data_list = list(obfuscated_data)
+    # 先移除索引5位置的字符（后插入的先移除）
+    del data_list[5]
+    # 再移除索引3位置的字符
+    del data_list[3]
+    return ''.join(data_list)
+
+
+def aes_decrypt(ciphertext: str) -> str:
+    """
+    AES ECB 模式解密，PKCS7 填充
+    """
+    key = "pSlJACAvYnAgOIlluJgK2F=="
+    key_bytes = key.encode('utf-8')
+
+    cipher = AES.new(key_bytes, AES.MODE_ECB)
+
+    # Base64解码
+    encrypted_bytes = base64.b64decode(ciphertext)
+
+    # 解密
+    decrypted_padded = cipher.decrypt(encrypted_bytes)
+
+    # 移除PKCS7填充
+    decrypted_data = unpad(decrypted_padded, AES.block_size)
+
+    return decrypted_data.decode('utf-8')
+
+
+def decrypt_request_data(obfuscated_data: str) -> dict:
+    """解密请求数据，返回明文字典"""
+    # 1. 反混淆
+    encrypted_data = deobfuscate_encrypted_data(obfuscated_data)
+    # 2. AES解密
+    json_str = aes_decrypt(encrypted_data)
+    # 3. 解析JSON
+    return json.loads(json_str)
+
+
+# ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+
+def get_req():
+    # 原始数据对象
+    original_data = {
+        # "miniName": "YFSXZF",
+        # "nonceStr": nonce_str,
+        "orderType": 0,
+        "pageNum": 1,
+        "pageSize": 10,
+        "status": 0,
+        # "time": timestamp,
+        "type": 1,
+    }
+    # raw_data={"miniName":"YFSXZF","nonceStr":"iCxy35dfFNJtQSKcrB8yZsRd7B4p3c2h","orderType":0,"pageNum":6,"pageSize":10,"status":0,"time":1765792372017,"type":1}
+    # "{"areaType":1,"awardType":2,"miniName":"YFSXZF","nonceStr":"iXbhcQ53iEJbdcwaC5mjpdbK43GsfxCP","pageNum":1,"pageSize":100,"time":1765949383885}"
+    request_data = create_request_data(original_data)
+
+    headers = {
+        'authority': 'wechatapp.ichibankuji.cn',
+        'accept': '*/*',
+        'accept-language': 'zh-CN,zh;q=0.9',
+        'content-type': 'application/json',
+        'referer': 'https://servicewechat.com/wxd21e3190b2a44f73/21/page-frame.html',
+        # 'sec-fetch-dest': 'empty',
+        # 'sec-fetch-mode': 'cors',
+        # 'sec-fetch-site': 'cross-site',
+        # 'terminalos': 'YFSXZF',
+        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 MicroMessenger/7.0.20.1781(0x6700143B) NetType/WIFI MiniProgramEnv/Windows WindowsWechat/WMPF WindowsWechat(0x63090c33) XWEB/9129',
+        'authorization': 'c3e87jrY!1q!2QwQclRyFVXzctNU6lVSUiF9WkPqIzNPL5YN5P3pxQbmzGDyajASc5BcFFM!2MeZG39B2EPdxesjVngQbw=='
+    }
+
+    payload = {
+        "da": request_data
+    }
+
+    # 发送POST请求
+    response = requests.post(
+        'https://wechatapp.ichibankuji.cn/wechat/yfs/getSetList',
+        headers=headers,
+        json=payload
+    )
+
+    print("响应状态码:", response.status_code)
+    print("响应内容:", response.text)
+
+
+def test_getsetlist():
+    headers = {
+        "authority": "wechatapp.ichibankuji.cn",
+        "accept": "*/*",
+        "accept-language": "zh-CN,zh;q=0.9",
+        # "authorization": "28380RBWRJQZN9UkYGYXeOZrd2GjFh9eteBx7Rz5rDxG3Tq!1m6vSq7EQTr!2ln2jtcinFZ2D8AuMyCjxckvBV2sRazTA==",
+        "authorization": "6974cjrY!1q!2QwQclRyFVXzctNU5!2B7lMuGi5lR70PeegsV0vYbrPmb8JQT!2oQDnJG7J0qCJlt3CZC!29Fz8JNmlv3X1Q==",
+        "content-type": "application/json",
+        "referer": "https://servicewechat.com/wx57f3ee376518bba9/164/page-frame.html",
+        "sec-fetch-dest": "empty",
+        "sec-fetch-mode": "cors",
+        "sec-fetch-site": "cross-site",
+        "terminalos": "YFSJJ",
+        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 MicroMessenger/7.0.20.1781(0x6700143B) NetType/WIFI MiniProgramEnv/Windows WindowsWechat/WMPF WindowsWechat(0x63090c33) XWEB/9129",
+        "xweb_xhr": "1"
+    }
+    url = "https://wechatapp.ichibankuji.cn/wechat/yfs/getSetList"
+    raw_data = {
+        # "miniName": "YFSXZF",
+        # "nonceStr": nonce_str,
+        "orderType": 0,
+        "pageNum": 2,
+        "pageSize": 10,
+        "areaType": 2,
+        # "time": timestamp,
+        "type": 1,
+    }
+    resp = requests.post(url, headers=headers, json=raw_data)
+    print(resp.text)
+    print(resp)
+
+
+def test_boxdetail():
+    headers = {
+        "authority": "wechatapp.ichibankuji.cn",
+        "accept": "*/*",
+        "accept-language": "zh-CN,zh;q=0.9",
+        "authorization": "7d04b7ZDRB9Vb65wzF4G6!1xyVJssI4IR!1zre6RpycuTj2sZm3Fi8UAH6MWYSwNXxKdjcSYJ3sHeZm!1bVUMVa2A295TA==",
+        "content-type": "application/json",
+        "referer": "https://servicewechat.com/wxd21e3190b2a44f73/21/page-frame.html",
+        "sec-fetch-dest": "empty",
+        "sec-fetch-mode": "cors",
+        "sec-fetch-site": "cross-site",
+        "terminalos": "YFSXZF",
+        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 MicroMessenger/7.0.20.1781(0x6700143B) NetType/WIFI MiniProgramEnv/Windows WindowsWechat/WMPF WindowsWechat(0x63090c33) XWEB/9129",
+        "xweb_xhr": "1"
+    }
+    url = "https://wechatapp.ichibankuji.cn/wechat/yfs/firstBoxDetail"
+    # data = {
+    #     "da": "ssB20vtJIFaPpO+oDa/FVfaEedp/7js3I0kaRkoTVblr7XIZ7YGVs8AcEOmSfhlz/+5BiZQXv6PS329MWxHrSbuhLwwyqZfHGNqP2CKzweRzmqwL/nj2D3zd8Su96eyrCx+0+4Q/6jBhNNiRg8eaW3E9Y+5PmRdyItuydMbDWc5uA="
+    # }
+
+    """
+    "{"miniName":"YFSXZF","nonceStr":"eNjXXEYNNHiji3BCy6m8PFs7eBneZ6fB","queueMode":2,"setId":"9882","time":1765963408861}"
+    
+    "{"boxId":84301,"commentId":0,"miniName":"YFSXZF","nonceStr":"64PZEyTKQsC6cX8FWNKcAZ2Zr8QAEk7T","startTime":"","time":1765963457147}"
+    
+    "{"buyNum":80,"miniName":"YFSXZF","nonceStr":"EkjQQkSYarsWdjykFjAraQ2SXMmmXEPJ","pageNum":1,"pageSize":100,"productId":9882,"time":1765963504672,"type":1}"
+    
+    """
+
+    original_data = {
+        "queueMode": 2,
+        "setId": "9882"
+    }
+    request_data = create_request_data(original_data)
+
+    payload = {
+        "da": request_data
+    }
+    response = requests.post(url, headers=headers, json=payload)
+
+    print(response.text)
+    print(response)
+
+
+def test_draw():
+    headers = {
+        "authority": "wechatapp.ichibankuji.cn",
+        "accept": "*/*",
+        "accept-language": "zh-CN,zh;q=0.9",
+        # "authorization": "7d04b7ZDRB9Vb65wzF4G6!1xyVJssI4IR!1zre6RpycuTj2sZm3Fi8UAH6MWYSwNXxKdjcSYJ3sHeZm!1bVUMVa2A295TA==",
+        "content-type": "application/json",
+        "referer": "https://servicewechat.com/wxd21e3190b2a44f73/21/page-frame.html",
+        "sec-fetch-dest": "empty",
+        "sec-fetch-mode": "cors",
+        "sec-fetch-site": "cross-site",
+        "terminalos": "YFSXZF",
+        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 MicroMessenger/7.0.20.1781(0x6700143B) NetType/WIFI MiniProgramEnv/Windows WindowsWechat/WMPF WindowsWechat(0x63090c33) XWEB/9129",
+        "xweb_xhr": "1"
+    }
+    url = "https://wechatapp.ichibankuji.cn/wechat/yfs/getBoxDrawLog"
+    # data = {
+    #     "da": "uO9f53IZgBHpi/l7iN2eu3MfwMsBfYtK0eJTXuKAii4+3zCldZE24R/vWUbRFLfvwdm5QCsWJ5pSEq7f48nPOIfhaST5NZn2H3uJSbj/tcC/cXI5He79UNx2uULpFXxLhP6iVhvKmUuxt9bpZ8TzSIw/bAZNaSXafp5GgWORhDTMx/YBnhZGrFp345Y32X7Qpw"
+    # }
+    original_data = {
+        "boxId": 84301,
+        "pageSize": 30,
+        "isMine": 0,
+        "pageNum": 1
+    }
+    """
+    "{"boxId":84301,"commentId":0,"miniName":"YFSXZF","nonceStr":"RRxbFkS66F4saYPMwJPDn5pswCfCfHeY","startTime":"2025-12-17 17:25:04","time":1765963725767}"
+    
+    "{"boxId":84301,"isMine":0,"miniName":"YFSXZF","nonceStr":"7sfHGRhdwp23bndHKzbBR67RG2EP4pxR","pageNum":1,"pageSize":30,"time":1765963927009}"
+    
+    """
+    request_data = create_request_data(original_data)
+
+    payload = {
+        "da": request_data
+    }
+    response = requests.post(url, headers=headers, json=payload)
+
+    print(response.text)
+    print(response)
+
+
+# 示例使用
+if __name__ == "__main__":
+    # get_req()
+    # test_boxdetail()
+    test_draw()
+    # test_getsetlist()
+    # en_data = {"miniName": "YFSJJ", "nonceStr": "Rmn4Ee8WajfDimtfCTPhRfGmGrp7pXrB", "orderType": 0, "pageNum": 2, "pageSize": 10, "status": 0, "time": 1765524424213, "type": 1}
+    # generate_sign(en_data)
+#     404a70c1316dd2b0b234d6cb5d25d8c3
+#     y = "ssB0tJIFaPpO+oDa/FVfaEedp/7js3I0kaRkoTVblr58IvkiE/orkJbZG+zg/g+fqP+J6wOBxsxo3+xBbc3J10pvq0el2y2mnxWMDL8TlBnNRnLu5l2SsmKM/C8QFg16"
+#     print(d(y))
+# "ssBp0ntJIFaPpO+oDa/FVfaEedp/7js3I0kaRkoTVblr58IvkiE/orkJbZG+zg/g+fqP+J6wOBxsxo3+xBbc3J10pvq0el2y2mnxWMDL8TlBnNRnLu5l2SsmKM/C8QFg16"
+# ssBf0tnJIFaPpO+oDa/FVfaEedp/7js3I0kaRkoTVblr58IvkiE/orkJbZG+zg/g+fqP+J6wOBxsxo3+xBbc3J10pvq0el2y2mnxWMDL8TlBnNRnLu5l2SsmKM/C8QFg1

veriswap_spider/YamlLoader.py

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+#
+import os, re
+import yaml
+
+regex = re.compile(r'^\$\{(?P<ENV>[A-Z_\-]+\:)?(?P<VAL>[\w\.]+)\}$')
+
+class YamlConfig:
+    def __init__(self, config):
+        self.config = config
+
+    def get(self, key:str):
+        return YamlConfig(self.config.get(key))
+    
+    def getValueAsString(self, key: str):
+        try:
+            match = regex.match(self.config[key])
+            group = match.groupdict()
+            if group['ENV'] != None:
+                env = group['ENV'][:-1]
+                return os.getenv(env, group['VAL'])
+            return None
+        except:
+            return self.config[key]
+    
+    def getValueAsInt(self, key: str):
+        try:
+            match = regex.match(self.config[key])
+            group = match.groupdict()
+            if group['ENV'] != None:
+                env = group['ENV'][:-1]
+                return int(os.getenv(env, group['VAL']))
+            return 0
+        except:
+            return int(self.config[key])
+        
+    def getValueAsBool(self, key: str, env: str = None):
+        try:
+            match = regex.match(self.config[key])
+            group = match.groupdict()
+            if group['ENV'] != None:
+                env = group['ENV'][:-1]
+                return bool(os.getenv(env, group['VAL']))
+            return False
+        except:
+            return bool(self.config[key])
+    
+def readYaml(path:str = 'application.yml', profile:str = None) -> YamlConfig:
+    if os.path.exists(path):
+        with open(path) as fd:
+            conf = yaml.load(fd, Loader=yaml.FullLoader)
+
+    if profile != None:
+        result = path.split('.')
+        profiledYaml = f'{result[0]}-{profile}.{result[1]}'
+        if os.path.exists(profiledYaml):
+            with open(profiledYaml) as fd:
+                conf.update(yaml.load(fd, Loader=yaml.FullLoader))
+
+    return YamlConfig(conf)
+
+# res = readYaml()
+# mysqlConf = res.get('mysql')
+# print(mysqlConf)
+
+# print(res.getValueAsString("host"))
+# mysqlYaml = mysqlConf.getValueAsString("host")
+# print(mysqlYaml)
+# host = mysqlYaml.get("host").split(':')[-1][:-1]
+# port = mysqlYaml.get("port").split(':')[-1][:-1]
+# username = mysqlYaml.get("username").split(':')[-1][:-1]
+# password = mysqlYaml.get("password").split(':')[-1][:-1]
+# mysql_db = mysqlYaml.get("db").split(':')[-1][:-1]
+# print(host,port,username,password)

veriswap_spider/application.yml

@@ -0,0 +1,6 @@
+mysql:
+  host: ${MYSQL_HOST:100.64.0.21}
+  port: ${MYSQL_PROT:3306}
+  username: ${MYSQL_USERNAME:crawler}
+  password: ${MYSQL_PASSWORD:Pass2022}
+  db: ${MYSQL_DATABASE:crawler}

veriswap_spider/mysql_pool.py

@@ -0,0 +1,574 @@
+# -*- coding: utf-8 -*-
+# Author : Charley
+# Python : 3.10.8
+# Date   : 2025/3/25 14:14
+import re
+import pymysql
+import YamlLoader
+from loguru import logger
+from dbutils.pooled_db import PooledDB
+
+# 获取yaml配置
+yaml = YamlLoader.readYaml()
+mysqlYaml = yaml.get("mysql")
+sql_host = mysqlYaml.getValueAsString("host")
+sql_port = mysqlYaml.getValueAsInt("port")
+sql_user = mysqlYaml.getValueAsString("username")
+sql_password = mysqlYaml.getValueAsString("password")
+sql_db = mysqlYaml.getValueAsString("db")
+
+
+class MySQLConnectionPool:
+    """
+    MySQL连接池
+    """
+
+    def __init__(self, mincached=4, maxcached=5, maxconnections=10, log=None):
+        """
+        初始化连接池
+        :param mincached: 初始化时，链接池中至少创建的链接，0表示不创建
+        :param maxcached: 池中空闲连接的最大数目（0 或 None 表示池大小不受限制）
+        :param maxconnections: 允许的最大连接数（0 或 None 表示任意数量的连接）
+        :param log: 自定义日志记录器
+        """
+        # 使用 loguru 的 logger，如果传入了其他 logger，则使用传入的 logger
+        self.log = log or logger
+        self.pool = PooledDB(
+            creator=pymysql,
+            mincached=mincached,
+            maxcached=maxcached,
+            maxconnections=maxconnections,
+            blocking=True,  # 连接池中如果没有可用连接后，是否阻塞等待。True，等待；False，不等待然后报错
+            host=sql_host,
+            port=sql_port,
+            user=sql_user,
+            password=sql_password,
+            database=sql_db,
+            ping=0  # 每次连接使用时自动检查有效性（0=不检查,1=执行query前检查,2=每次执行前检查）
+        )
+
+    def _execute(self, query, args=None, commit=False):
+        """
+        执行SQL
+        :param query: SQL语句
+        :param args: SQL参数
+        :param commit: 是否提交事务
+        :return: 查询结果
+        """
+        try:
+            with self.pool.connection() as conn:
+                with conn.cursor() as cursor:
+                    cursor.execute(query, args)
+                    if commit:
+                        conn.commit()
+                    self.log.debug(f"sql _execute, Query: {query}, Rows: {cursor.rowcount}")
+                    return cursor
+        except Exception as e:
+            if commit:
+                conn.rollback()
+            self.log.exception(f"Error executing query: {e}, Query: {query}, Args: {args}")
+            raise e
+
+    def select_one(self, query, args=None):
+        """
+        执行查询，返回单个结果
+        :param query: 查询语句
+        :param args: 查询参数
+        :return: 查询结果
+        """
+        cursor = self._execute(query, args)
+        return cursor.fetchone()
+
+    def select_all(self, query, args=None):
+        """
+        执行查询，返回所有结果
+        :param query: 查询语句
+        :param args: 查询参数
+        :return: 查询结果
+        """
+        cursor = self._execute(query, args)
+        return cursor.fetchall()
+
+    def insert_one(self, query, args):
+        """
+        执行单条插入语句
+        :param query: 插入语句
+        :param args: 插入参数
+        """
+        self.log.info('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>data insert_one 入库中>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
+        cursor = self._execute(query, args, commit=True)
+        return cursor.lastrowid  # 返回插入的ID
+
+    def insert_all(self, query, args_list):
+        """
+        执行批量插入语句，如果失败则逐条插入
+        :param query: 插入语句
+        :param args_list: 插入参数列表
+        """
+        conn = None
+        cursor = None
+        try:
+            conn = self.pool.connection()
+            cursor = conn.cursor()
+            cursor.executemany(query, args_list)
+            conn.commit()
+            self.log.debug(f"sql insert_all, SQL: {query}, Rows: {len(args_list)}")
+            self.log.info('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>data insert_all 入库中>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
+        except Exception as e:
+            conn.rollback()
+            self.log.error(f"Batch insertion failed after 5 attempts. Trying single inserts. Error: {e}")
+            # 如果批量插入失败，则逐条插入
+            rowcount = 0
+            for args in args_list:
+                self.insert_one(query, args)
+                rowcount += 1
+            self.log.debug(f"Batch insertion failed. Inserted {rowcount} rows individually.")
+        finally:
+            if cursor:
+                cursor.close()
+            if conn:
+                conn.close()
+
+    def insert_one_or_dict(self, table=None, data=None, query=None, args=None, commit=True, ignore=False):
+        """
+        单条插入（支持字典或原始SQL）
+        :param table: 表名（字典插入时必需）
+        :param data: 字典数据 {列名: 值}
+        :param query: 直接SQL语句（与data二选一）
+        :param args: SQL参数（query使用时必需）
+        :param commit: 是否自动提交
+        :param ignore: 是否使用ignore
+        :return: 最后插入ID
+        """
+        if data is not None:
+            if not isinstance(data, dict):
+                raise ValueError("Data must be a dictionary")
+
+            keys = ', '.join([self._safe_identifier(k) for k in data.keys()])
+            values = ', '.join(['%s'] * len(data))
+
+            # query = f"INSERT INTO {self._safe_identifier(table)} ({keys}) VALUES ({values})"
+            # 构建 INSERT IGNORE 语句
+            ignore_clause = "IGNORE" if ignore else ""
+            # insert_sql = f"INSERT {ignore_clause} INTO {table} ({columns}) VALUES ({placeholders})"
+            query = f"INSERT {ignore_clause} INTO {self._safe_identifier(table)} ({keys}) VALUES ({values})"
+            args = tuple(data.values())
+        elif query is None:
+            raise ValueError("Either data or query must be provided")
+
+        # cursor = self._execute(query, args, commit)
+        # self.log.info(f"sql insert_one_or_dict, Table: {table}, Rows: {cursor.rowcount}")
+        # self.log.info('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>data insert_one_or_dict 入库中>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
+        # return cursor.lastrowid
+
+        try:
+            cursor = self._execute(query, args, commit)
+            self.log.info(f"sql insert_one_or_dict, Table: {table}, Rows: {cursor.rowcount}")
+            self.log.info('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>data insert_one_or_dict 入库中>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
+            return cursor.lastrowid
+        except pymysql.err.IntegrityError as e:
+            if "Duplicate entry" in str(e):
+                self.log.warning(f"插入失败：重复条目，已跳过。错误详情: {e}")
+                # print("插入失败：重复条目", e)
+                return -1  # 返回 -1 表示重复条目被跳过
+            else:
+                self.log.exception(f"数据库完整性错误: {e}")
+                # print("插入失败：完整性错误", e)
+                raise
+        except Exception as e:
+            # self.log.error(f"未知错误: {str(e)}", exc_info=True)
+            self.log.exception(f"未知错误: {e}")  # 记录完整异常信息
+            # print("插入失败：未知错误", e)
+            raise
+
+    def insert_many(self, table=None, data_list=None, query=None, args_list=None, batch_size=1000, commit=True, ignore=False):
+        """
+        批量插入（支持字典列表或原始SQL）
+        :param table: 表名（字典插入时必需）
+        :param data_list: 字典列表 [{列名: 值}]
+        :param query: 直接SQL语句（与data_list二选一）
+        :param args_list: SQL参数列表（query使用时必需）
+        :param batch_size: 分批大小
+        :param commit: 是否自动提交
+        :param ignore: 是否使用ignore
+        :return: 影响行数
+        """
+        if data_list is not None:
+            if not data_list or not isinstance(data_list[0], dict):
+                raise ValueError("Data_list must be a non-empty list of dictionaries")
+
+            keys = ', '.join([self._safe_identifier(k) for k in data_list[0].keys()])
+            values = ', '.join(['%s'] * len(data_list[0]))
+
+            # 构建 INSERT IGNORE 语句
+            ignore_clause = "IGNORE" if ignore else ""
+            # insert_sql = f"INSERT {ignore_clause} INTO {table} ({columns}) VALUES ({placeholders})"
+            query = f"INSERT {ignore_clause} INTO {self._safe_identifier(table)} ({keys}) VALUES ({values})"
+            args_list = [tuple(d.values()) for d in data_list]
+        elif query is None:
+            raise ValueError("Either data_list or query must be provided")
+
+        total = 0
+        for i in range(0, len(args_list), batch_size):
+            batch = args_list[i:i + batch_size]
+            try:
+                with self.pool.connection() as conn:
+                    with conn.cursor() as cursor:
+                        cursor.executemany(query, batch)
+                        if commit:
+                            conn.commit()
+                        total += cursor.rowcount
+            except pymysql.Error as e:
+                if "Duplicate entry" in str(e):
+                    # self.log.warning(f"检测到重复条目，开始逐条插入。错误详情: {e}")
+                    raise  e
+                    # rowcount = 0
+                    # for args in batch:
+                    #     try:
+                    #         self.insert_one_or_dict(table=table, data=dict(zip(data_list[0].keys(), args)),
+                    #                                 commit=commit)
+                    #         rowcount += 1
+                    #     except pymysql.err.IntegrityError as e2:
+                    #         if "Duplicate entry" in str(e2):
+                    #             self.log.warning(f"跳过重复条目: {args}")
+                    #         else:
+                    #             self.log.error(f"插入失败: {e2}, 参数: {args}")
+                    # total += rowcount
+                else:
+                    self.log.exception(f"数据库错误: {e}")
+                    if commit:
+                        conn.rollback()
+                    raise e
+                # 重新抛出异常，供外部捕获
+                # 降级为单条插入
+                # for args in batch:
+                #     try:
+                #         self.insert_one_or_dict(table=None, query=query, args=args, commit=commit)
+                #         total += 1
+                #     except Exception as e2:
+                #         self.log.error(f"Single insert failed: {e2}")
+                        # continue
+        self.log.info(f"sql insert_many, Table: {table}, Total Rows: {total}")
+        return total
+
+    def insert_many_two(self, table=None, data_list=None, query=None, args_list=None, batch_size=1000, commit=True):
+        """
+        批量插入（支持字典列表或原始SQL）
+        :param table: 表名（字典插入时必需）
+        :param data_list: 字典列表 [{列名: 值}]
+        :param query: 直接SQL语句（与data_list二选一）
+        :param args_list: SQL参数列表（query使用时必需）
+        :param batch_size: 分批大小
+        :param commit: 是否自动提交
+        :return: 影响行数
+        """
+        if data_list is not None:
+            if not data_list or not isinstance(data_list[0], dict):
+                raise ValueError("Data_list must be a non-empty list of dictionaries")
+            keys = ', '.join([self._safe_identifier(k) for k in data_list[0].keys()])
+            values = ', '.join(['%s'] * len(data_list[0]))
+            query = f"INSERT INTO {self._safe_identifier(table)} ({keys}) VALUES ({values})"
+            args_list = [tuple(d.values()) for d in data_list]
+        elif query is None:
+            raise ValueError("Either data_list or query must be provided")
+
+        total = 0
+        for i in range(0, len(args_list), batch_size):
+            batch = args_list[i:i + batch_size]
+            try:
+                with self.pool.connection() as conn:
+                    with conn.cursor() as cursor:
+                        # 添加调试日志：输出 SQL 和参数示例
+                        # self.log.debug(f"Batch insert SQL: {query}")
+                        # self.log.debug(f"Sample args: {batch[0] if batch else 'None'}")
+                        cursor.executemany(query, batch)
+                        if commit:
+                            conn.commit()
+                        total += cursor.rowcount
+                        # self.log.debug(f"Batch insert succeeded. Rows: {cursor.rowcount}")
+            except Exception as e:  # 明确捕获数据库异常
+                self.log.exception(f"Batch insert failed: {e}")  # 使用 exception 记录堆栈
+                self.log.error(f"Failed SQL: {query}, Args count: {len(batch)}")
+                if commit:
+                    conn.rollback()
+                # 降级为单条插入，并记录每个错误
+                rowcount = 0
+                for args in batch:
+                    try:
+                        self.insert_one(query, args)
+                        rowcount += 1
+                    except Exception as e2:
+                        self.log.error(f"Single insert failed: {e2}, Args: {args}")
+                total += rowcount
+                self.log.debug(f"Inserted {rowcount}/{len(batch)} rows individually.")
+        self.log.info(f"sql insert_many, Table: {table}, Total Rows: {total}")
+        return total
+
+    def insert_too_many(self, query, args_list, batch_size=1000):
+        """
+        执行批量插入语句，分片提交, 单次插入大于十万+时可用, 如果失败则降级为逐条插入
+        :param query: 插入语句
+        :param args_list: 插入参数列表
+        :param batch_size: 每次插入的条数
+        """
+        for i in range(0, len(args_list), batch_size):
+            batch = args_list[i:i + batch_size]
+            try:
+                with self.pool.connection() as conn:
+                    with conn.cursor() as cursor:
+                        cursor.executemany(query, batch)
+                        conn.commit()
+            except Exception as e:
+                self.log.error(f"insert_too_many error. Trying single insert. Error: {e}")
+                # 当前批次降级为单条插入
+                for args in batch:
+                    self.insert_one(query, args)
+
+    def update_one(self, query, args):
+        """
+        执行单条更新语句
+        :param query: 更新语句
+        :param args: 更新参数
+        """
+        self.log.info('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>data update_one 更新中>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
+        return self._execute(query, args, commit=True)
+
+    def update_all(self, query, args_list):
+        """
+        执行批量更新语句，如果失败则逐条更新
+        :param query: 更新语句
+        :param args_list: 更新参数列表
+        """
+        conn = None
+        cursor = None
+        try:
+            conn = self.pool.connection()
+            cursor = conn.cursor()
+            cursor.executemany(query, args_list)
+            conn.commit()
+            self.log.debug(f"sql update_all, SQL: {query}, Rows: {len(args_list)}")
+            self.log.info('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>data update_all 更新中>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
+        except Exception as e:
+            conn.rollback()
+            self.log.error(f"Error executing query: {e}")
+            # 如果批量更新失败，则逐条更新
+            rowcount = 0
+            for args in args_list:
+                self.update_one(query, args)
+                rowcount += 1
+            self.log.debug(f'Batch update failed. Updated {rowcount} rows individually.')
+        finally:
+            if cursor:
+                cursor.close()
+            if conn:
+                conn.close()
+
+    def update_one_or_dict(self, table=None, data=None, condition=None, query=None, args=None, commit=True):
+        """
+        单条更新（支持字典或原始SQL）
+        :param table: 表名（字典模式必需）
+        :param data: 字典数据 {列名: 值}（与 query 二选一）
+        :param condition: 更新条件，支持以下格式：
+            - 字典: {"id": 1} → "WHERE id = %s"
+            - 字符串: "id = 1" → "WHERE id = 1"（需自行确保安全）
+            - 元组: ("id = %s", [1]) → "WHERE id = %s"（参数化查询）
+        :param query: 直接SQL语句（与 data 二选一）
+        :param args: SQL参数（query 模式下必需）
+        :param commit: 是否自动提交
+        :return: 影响行数
+        :raises: ValueError 参数校验失败时抛出
+        """
+        # 参数校验
+        if data is not None:
+            if not isinstance(data, dict):
+                raise ValueError("Data must be a dictionary")
+            if table is None:
+                raise ValueError("Table name is required for dictionary update")
+            if condition is None:
+                raise ValueError("Condition is required for dictionary update")
+
+            # 构建 SET 子句
+            set_clause = ", ".join([f"{self._safe_identifier(k)} = %s" for k in data.keys()])
+            set_values = list(data.values())
+
+            # 解析条件
+            condition_clause, condition_args = self._parse_condition(condition)
+            query = f"UPDATE {self._safe_identifier(table)} SET {set_clause} WHERE {condition_clause}"
+            args = set_values + condition_args
+
+        elif query is None:
+            raise ValueError("Either data or query must be provided")
+
+        # 执行更新
+        cursor = self._execute(query, args, commit)
+        # self.log.debug(
+        #     f"Updated table={table}, rows={cursor.rowcount}, query={query[:100]}...",
+        #     extra={"table": table, "rows": cursor.rowcount}
+        # )
+        return cursor.rowcount
+
+    def _parse_condition(self, condition):
+        """
+        解析条件为 (clause, args) 格式
+        :param condition: 字典/字符串/元组
+        :return: (str, list) SQL 子句和参数列表
+        """
+        if isinstance(condition, dict):
+            clause = " AND ".join([f"{self._safe_identifier(k)} = %s" for k in condition.keys()])
+            args = list(condition.values())
+        elif isinstance(condition, str):
+            clause = condition  # 注意：需调用方确保安全
+            args = []
+        elif isinstance(condition, (tuple, list)) and len(condition) == 2:
+            clause, args = condition[0], condition[1]
+            if not isinstance(args, (list, tuple)):
+                args = [args]
+        else:
+            raise ValueError("Condition must be dict/str/(clause, args)")
+        return clause, args
+
+    def update_many(self, table=None, data_list=None, condition_list=None, query=None, args_list=None, batch_size=500,
+                    commit=True):
+        """
+        批量更新（支持字典列表或原始SQL）
+        :param table: 表名（字典插入时必需）
+        :param data_list: 字典列表 [{列名: 值}]
+        :param condition_list: 条件列表（必须为字典，与data_list等长）
+        :param query: 直接SQL语句（与data_list二选一）
+        :param args_list: SQL参数列表（query使用时必需）
+        :param batch_size: 分批大小
+        :param commit: 是否自动提交
+        :return: 影响行数
+        """
+        if data_list is not None:
+            if not data_list or not isinstance(data_list[0], dict):
+                raise ValueError("Data_list must be a non-empty list of dictionaries")
+            if condition_list is None or len(data_list) != len(condition_list):
+                raise ValueError("Condition_list must be provided and match the length of data_list")
+            if not all(isinstance(cond, dict) for cond in condition_list):
+                raise ValueError("All elements in condition_list must be dictionaries")
+
+            # 获取第一个数据项和条件项的键
+            first_data_keys = set(data_list[0].keys())
+            first_cond_keys = set(condition_list[0].keys())
+
+            # 构造基础SQL
+            set_clause = ', '.join([self._safe_identifier(k) + ' = %s' for k in data_list[0].keys()])
+            condition_clause = ' AND '.join([self._safe_identifier(k) + ' = %s' for k in condition_list[0].keys()])
+            base_query = f"UPDATE {self._safe_identifier(table)} SET {set_clause} WHERE {condition_clause}"
+            total = 0
+
+            # 分批次处理
+            for i in range(0, len(data_list), batch_size):
+                batch_data = data_list[i:i + batch_size]
+                batch_conds = condition_list[i:i + batch_size]
+                batch_args = []
+
+                # 检查当前批次的结构是否一致
+                can_batch = True
+                for data, cond in zip(batch_data, batch_conds):
+                    data_keys = set(data.keys())
+                    cond_keys = set(cond.keys())
+                    if data_keys != first_data_keys or cond_keys != first_cond_keys:
+                        can_batch = False
+                        break
+                    batch_args.append(tuple(data.values()) + tuple(cond.values()))
+
+                if not can_batch:
+                    # 结构不一致，转为单条更新
+                    for data, cond in zip(batch_data, batch_conds):
+                        self.update_one_or_dict(table=table, data=data, condition=cond, commit=commit)
+                        total += 1
+                    continue
+
+                # 执行批量更新
+                try:
+                    with self.pool.connection() as conn:
+                        with conn.cursor() as cursor:
+                            cursor.executemany(base_query, batch_args)
+                            if commit:
+                                conn.commit()
+                            total += cursor.rowcount
+                            self.log.debug(f"Batch update succeeded. Rows: {cursor.rowcount}")
+                except Exception as e:
+                    if commit:
+                        conn.rollback()
+                    self.log.error(f"Batch update failed: {e}")
+                    # 降级为单条更新
+                    for args, data, cond in zip(batch_args, batch_data, batch_conds):
+                        try:
+                            self._execute(base_query, args, commit=commit)
+                            total += 1
+                        except Exception as e2:
+                            self.log.error(f"Single update failed: {e2}, Data: {data}, Condition: {cond}")
+            self.log.info(f"Total updated rows: {total}")
+            return total
+        elif query is not None:
+            # 处理原始SQL和参数列表
+            if args_list is None:
+                raise ValueError("args_list must be provided when using query")
+
+            total = 0
+            for i in range(0, len(args_list), batch_size):
+                batch_args = args_list[i:i + batch_size]
+                try:
+                    with self.pool.connection() as conn:
+                        with conn.cursor() as cursor:
+                            cursor.executemany(query, batch_args)
+                            if commit:
+                                conn.commit()
+                            total += cursor.rowcount
+                            self.log.debug(f"Batch update succeeded. Rows: {cursor.rowcount}")
+                except Exception as e:
+                    if commit:
+                        conn.rollback()
+                    self.log.error(f"Batch update failed: {e}")
+                    # 降级为单条更新
+                    for args in batch_args:
+                        try:
+                            self._execute(query, args, commit=commit)
+                            total += 1
+                        except Exception as e2:
+                            self.log.error(f"Single update failed: {e2}, Args: {args}")
+            self.log.info(f"Total updated rows: {total}")
+            return total
+        else:
+            raise ValueError("Either data_list or query must be provided")
+
+    def check_pool_health(self):
+        """
+        检查连接池中有效连接数
+
+        # 使用示例
+        # 配置 MySQL 连接池
+        sql_pool = MySQLConnectionPool(log=log)
+        if not sql_pool.check_pool_health():
+            log.error("数据库连接池异常")
+            raise RuntimeError("数据库连接池异常")
+        """
+        try:
+            with self.pool.connection() as conn:
+                conn.ping(reconnect=True)
+                return True
+        except Exception as e:
+            self.log.error(f"Connection pool health check failed: {e}")
+            return False
+
+    @staticmethod
+    def _safe_identifier(name):
+        """SQL标识符安全校验"""
+        if not re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*$', name):
+            raise ValueError(f"Invalid SQL identifier: {name}")
+        return name
+
+
+if __name__ == '__main__':
+    sql_pool = MySQLConnectionPool()
+    data_dic = {'card_type_id': 111, 'card_type_name': '补充包 继承的意志【OPC-13】', 'card_type_position': 964,
+                'card_id': 5284, 'card_name': '蒙奇·D·路飞', 'card_number': 'OP13-001', 'card_rarity': 'L',
+                'card_img': 'https://source.windoent.com/OnePiecePc/Picture/1757929283612OP13-001.png',
+                'card_life': '4', 'card_attribute': '打', 'card_power': '5000', 'card_attack': '-',
+                'card_color': '红/绿', 'subscript': 4, 'card_features': '超新星/草帽一伙',
+                'card_text_desc': '【咚!!×1】【对方的攻击时】我方处于活跃状态的咚!!不多于5张的场合，可以将我方任意张数的咚!!转为休息状态。每有1张转为休息状态的咚!!，本次战斗中，此领袖或我方最多1张拥有《草帽一伙》特征的角色力量+2000。',
+                'card_offer_type': '补充包 继承的意志【OPC-13】', 'crawler_language': '简中'}
+    sql_pool.insert_one_or_dict(table="one_piece_record", data=data_dic)

veriswap_spider/veriswap_spider.py

@@ -0,0 +1,225 @@
+# -*- coding: utf-8 -*-
+# Author : Charley
+# Python : 3.10.8
+# Date   : 2025/12/8 19:23
+import random
+import time
+import inspect
+import requests
+import schedule
+import user_agent
+from loguru import logger
+from datetime import datetime
+from mysql_pool import MySQLConnectionPool
+from tenacity import retry, stop_after_attempt, wait_fixed
+
+"""
+沃瑞斯外部
+"""
+
+logger.remove()
+logger.add("./logs/{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00",
+           format="[{time:YYYY-MM-DD HH:mm:ss.SSS}] {level} {message}",
+           level="DEBUG", retention="7 day")
+
+
+def after_log(retry_state):
+    """
+    retry 回调
+    :param retry_state: RetryCallState 对象
+    """
+    # 检查 args 是否存在且不为空
+    if retry_state.args and len(retry_state.args) > 0:
+        log = retry_state.args[0]  # 获取传入的 logger
+    else:
+        log = logger  # 使用全局 logger
+
+    if retry_state.outcome.failed:
+        log.warning(
+            f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} Times")
+    else:
+        log.info(f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} succeeded")
+
+
+@retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
+def get_proxys(log):
+    """
+    获取代理
+    :return: 代理
+    """
+    tunnel = "x371.kdltps.com:15818"
+    kdl_username = "t13753103189895"
+    kdl_password = "o0yefv6z"
+    try:
+        proxies = {
+            "http": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": kdl_username, "pwd": kdl_password, "proxy": tunnel},
+            "https": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": kdl_username, "pwd": kdl_password, "proxy": tunnel}
+        }
+        return proxies
+    except Exception as e:
+        log.error(f"Error getting proxy: {e}")
+        raise e
+
+
+@retry(stop=stop_after_attempt(5), wait=wait_fixed(5), after=after_log)
+def get_single_page(log, page, sql_pool):
+    """
+    获取单页商品
+    :param log: logger对象
+    :param page: 页码
+    :param sql_pool: MySQL连接池对象
+    :return: ietms 长度
+    """
+    log.debug(f"............... 开始获取第 {page} 页数据 ...............")
+    headers = {
+        "content-type": "application/json",
+        "referer": "https://veriswap.com/",
+        "user-agent": user_agent.generate_user_agent()
+    }
+    url = "https://veriswap-backend-689107296832.us-central1.run.app/cards/get/all"
+    data = {
+        "query": "",
+        "page": page,
+        "filterBy": "isHidden:=false",
+        "sortBy": "updatedAt:desc"
+    }
+    response = requests.post(url, headers=headers, json=data, timeout=22)
+    # print(response.json())
+    response.raise_for_status()
+
+    resp_json = response.json().get("items", [])
+    if not resp_json:
+        log.debug("没有数据........")
+        log.debug(response.json())
+        return 0
+
+    info_list = []
+    for item in resp_json:
+        card_id = item.get("cardId")
+        card_name = item.get("cardName")
+        condition = item.get("condition")
+
+        created_at = item.get("createdAt")  # 1747107593
+        created_at = datetime.fromtimestamp(created_at).strftime('%Y-%m-%d %H:%M:%S')
+        updated_at = item.get("updatedAt")
+        updated_at = datetime.fromtimestamp(updated_at).strftime('%Y-%m-%d %H:%M:%S')
+
+        era = item.get("era")
+        for_sale = item.get("forSale")
+
+        images = item.get("images", {})
+        img_front = images.get("front")
+        img_back = images.get("back")
+
+        is_vaulted = item.get("isVaulted")
+        parallel = item.get("parallel")
+        player_name = item.get("playerName")
+        price = item.get("price")
+        print_run = item.get("printRun")
+        set_name = item.get("set")
+        sport = item.get("sport")
+        card_type = item.get("type")
+        user_id = item.get("userId")
+        year = item.get("year")
+
+        data_dict = {
+            "card_id": card_id,
+            "card_name": card_name,
+            "card_condition": condition,
+            "created_at": created_at,
+            "updated_at": updated_at,
+            "era": era,
+            "for_sale": for_sale,
+            "img_front": img_front,
+            "img_back": img_back,
+            "is_vaulted": is_vaulted,
+            "parallel": parallel,
+            "player_name": player_name,
+            "price": price,
+            "print_run": print_run,
+            "set_name": set_name,
+            "sport": sport,
+            "card_type": card_type,
+            "user_id": user_id,
+            "year": year
+        }
+        # log.debug(data_dict)
+        info_list.append(data_dict)
+
+    # 保存数据
+    if info_list:
+        sql_pool.insert_many(table="veriswap_card_record", data_list=info_list, ignore=True)
+
+    return len(resp_json)
+
+
+def get_all_page(log, sql_pool):
+    """
+    获取所有页码商品  翻页
+    :param log: logger对象
+    :param sql_pool: MySQL连接池对象
+    """
+    # page = 1
+    page = 16903
+    max_page = 20000
+    while page <= max_page:
+        try:
+            len_list = get_single_page(log, page, sql_pool)
+        except Exception as e:
+            log.error(f'Request get_single_page error: {e}')
+            len_list = 0
+
+        if len_list < 20:
+            log.debug(f'当前页码为: {page}, 数据长度为: {len_list}, 停止爬取 !!!')
+            break
+
+        page += 1
+        # time.sleep(random.uniform(1, 2))
+
+
+@retry(stop=stop_after_attempt(100), wait=wait_fixed(3600), after=after_log)
+def wap_main(log):
+    """
+    主函数 自动售货机
+    :param log: logger对象
+    """
+    log.info(
+        f'开始运行 {inspect.currentframe().f_code.co_name} 爬虫任务....................................................')
+
+    # 配置 MySQL 连接池
+    sql_pool = MySQLConnectionPool(log=log)
+    if not sql_pool.check_pool_health():
+        log.error("数据库连接池异常")
+        raise RuntimeError("数据库连接池异常")
+
+    try:
+        try:
+            log.debug('------------------- 获取所有页码商品 -------------------')
+            get_all_page(log, sql_pool)
+        except Exception as e:
+            log.error(f'get_all_page error: {e}')
+
+    except Exception as e:
+        log.error(f'{inspect.currentframe().f_code.co_name} error: {e}')
+    finally:
+        log.info(f'爬虫程序 {inspect.currentframe().f_code.co_name} 运行结束,等待下一轮的采集任务............')
+
+
+def schedule_task():
+    """
+    爬虫模块 定时任务 的启动文件
+    """
+    # 立即运行一次任务
+    # wap_main(log=logger)
+
+    # 设置定时任务
+    schedule.every().day.at("00:01").do(wap_main, log=logger)
+
+    while True:
+        schedule.run_pending()
+        time.sleep(1)
+
+
+if __name__ == '__main__':
+    wap_main(logger)
+    # get_all_page(logger, None)