spider_work/chaogujie_spider/cgj_history_spider.py

# -*- coding: utf-8 -*-
# Author : Charley
# Python : 3.12.10
# Date   : 2026/07/06
"""潮谷街历史全量采集脚本（一次性）。

与 cgj_daily_spider（每天增量）的区别：本脚本对每个商户的历史成交「全量深翻」所有页
（调 get_sold_list(incremental=False)，不走停止线早停），把历史商品 / 玩家一次性灌满。
跑一次即止，不带 schedule。

公共函数（请求 / 翻页 / 详情 / 玩家）全部从 cgj_daily_spider 复用，避免重复维护。

运行：python cgj_history_spider.py
"""
import sys

from loguru import logger
from mysql_pool import MySQLConnectionPool

from cgj_daily_spider import (
    get_shop_list,
    get_sold_list,
    get_player_list,
    refill_replays,
)


def history_main(log):
    """历史全量采集主函数：商户发现 → 全量深翻商品(+详情) → 全量采玩家。

    Args:
        log: 日志对象。

    Raises:
        RuntimeError: 数据库连接池异常时抛出。
    """
    log.info(f"开始运行 {sys._getframe().f_code.co_name} 潮谷街历史全量采集" + "." * 40)

    sql_pool = MySQLConnectionPool(log=log)
    if not sql_pool.check_pool_health():
        log.error("数据库连接池异常")
        raise RuntimeError("数据库连接池异常")

    try:
        # 1) 商户发现：先用 shop 列表页把当前商户补进 cgj_shop_record（历史脚本也要完整商户清单）
        try:
            n = get_shop_list(log, sql_pool)
            log.info(f"商户发现完成，去重商户 {n} 个")
        except Exception as e:
            log.error(f"get_shop_list error: {e}")

        # 2) 商品全量：遍历所有商户，深翻历史成交全部页（incremental=False，不早停）
        try:
            shop_rows = sql_pool.select_all("SELECT shop_id, shop_name FROM cgj_shop_record WHERE is_deleted = 0")
            log.info(f"待全量采集商户 {len(shop_rows)} 个")
            for shop_id, shop_name in shop_rows:
                try:
                    cnt = get_sold_list(log, shop_id, shop_name, sql_pool, incremental=False)
                    log.info(f"商户 {shop_id} {shop_name} 全量商品采集完成，写入 {cnt} 个")
                except Exception as e:
                    log.error(f"get_sold_list error（商户 {shop_id}）: {e}")
        except Exception as e:
            log.error(f"iterate_shop_list error: {e}")

        # 3) 玩家全量：遍历尚未成功采集玩家的商品（player_state != 1）
        try:
            prod_rows = sql_pool.select_all("SELECT pid FROM cgj_product_record WHERE player_state != 1")
            pids = [row[0] for row in prod_rows] if prod_rows else []
            log.info(f"待采集玩家的商品 {len(pids)} 个")
            for pid in pids:
                try:
                    # 先置 1 表示开始采集
                    sql_pool.update_one("UPDATE cgj_product_record SET player_state = 1 WHERE pid = %s", (pid,))
                    has_data = get_player_list(log, pid, sql_pool)
                    if not has_data:
                        # 无玩家置 2，下次仍会重试
                        sql_pool.update_one("UPDATE cgj_product_record SET player_state = 2 WHERE pid = %s", (pid,))
                except Exception as pid_error:
                    log.error(f"商品 {pid} 玩家采集失败: {pid_error}")
                    try:
                        sql_pool.update_one("UPDATE cgj_product_record SET player_state = 3 WHERE pid = %s", (pid,))
                    except Exception as update_error:
                        log.error(f"更新商品 {pid} 状态失败: {update_error}")
        except Exception as e:
            log.error(f"iterate_player_list error: {e}")

        # 4) 回放补采：对已采玩家(player_state=1)但 replay_url 仍空的商品重取回放
        try:
            n = refill_replays(log, sql_pool)
            log.info(f"回放补采完成，本轮 {n} 条")
        except Exception as e:
            log.error(f"refill_replays error: {e}")
    except Exception as e:
        log.error(f"{sys._getframe().f_code.co_name} error: {e}")
    finally:
        log.info(f"潮谷街历史全量采集 {sys._getframe().f_code.co_name} 运行结束")


if __name__ == "__main__":
    history_main(logger)