# -*- coding: utf-8 -*- # Author : Charley # Python : 3.12.10 # Date : 2026/07/06 """潮谷街历史全量采集脚本(一次性)。 与 cgj_daily_spider(每天增量)的区别:本脚本对每个商户的历史成交「全量深翻」所有页 (调 get_sold_list(incremental=False),不走停止线早停),把历史商品 / 玩家一次性灌满。 跑一次即止,不带 schedule。 公共函数(请求 / 翻页 / 详情 / 玩家)全部从 cgj_daily_spider 复用,避免重复维护。 运行:python cgj_history_spider.py """ import sys from loguru import logger from mysql_pool import MySQLConnectionPool from cgj_daily_spider import ( get_shop_list, get_sold_list, get_player_list, refill_replays, ) def history_main(log): """历史全量采集主函数:商户发现 → 全量深翻商品(+详情) → 全量采玩家。 Args: log: 日志对象。 Raises: RuntimeError: 数据库连接池异常时抛出。 """ log.info(f"开始运行 {sys._getframe().f_code.co_name} 潮谷街历史全量采集" + "." * 40) sql_pool = MySQLConnectionPool(log=log) if not sql_pool.check_pool_health(): log.error("数据库连接池异常") raise RuntimeError("数据库连接池异常") try: # 1) 商户发现:先用 shop 列表页把当前商户补进 cgj_shop_record(历史脚本也要完整商户清单) try: n = get_shop_list(log, sql_pool) log.info(f"商户发现完成,去重商户 {n} 个") except Exception as e: log.error(f"get_shop_list error: {e}") # 2) 商品全量:遍历所有商户,深翻历史成交全部页(incremental=False,不早停) try: shop_rows = sql_pool.select_all("SELECT shop_id, shop_name FROM cgj_shop_record WHERE is_deleted = 0") log.info(f"待全量采集商户 {len(shop_rows)} 个") for shop_id, shop_name in shop_rows: try: cnt = get_sold_list(log, shop_id, shop_name, sql_pool, incremental=False) log.info(f"商户 {shop_id} {shop_name} 全量商品采集完成,写入 {cnt} 个") except Exception as e: log.error(f"get_sold_list error(商户 {shop_id}): {e}") except Exception as e: log.error(f"iterate_shop_list error: {e}") # 3) 玩家全量:遍历尚未成功采集玩家的商品(player_state != 1) try: prod_rows = sql_pool.select_all("SELECT pid FROM cgj_product_record WHERE player_state != 1") pids = [row[0] for row in prod_rows] if prod_rows else [] log.info(f"待采集玩家的商品 {len(pids)} 个") for pid in pids: try: # 先置 1 表示开始采集 sql_pool.update_one("UPDATE cgj_product_record SET player_state = 1 WHERE pid = %s", (pid,)) has_data = get_player_list(log, pid, sql_pool) if not has_data: # 无玩家置 2,下次仍会重试 sql_pool.update_one("UPDATE cgj_product_record SET player_state = 2 WHERE pid = %s", (pid,)) except Exception as pid_error: log.error(f"商品 {pid} 玩家采集失败: {pid_error}") try: sql_pool.update_one("UPDATE cgj_product_record SET player_state = 3 WHERE pid = %s", (pid,)) except Exception as update_error: log.error(f"更新商品 {pid} 状态失败: {update_error}") except Exception as e: log.error(f"iterate_player_list error: {e}") # 4) 回放补采:对已采玩家(player_state=1)但 replay_url 仍空的商品重取回放 try: n = refill_replays(log, sql_pool) log.info(f"回放补采完成,本轮 {n} 条") except Exception as e: log.error(f"refill_replays error: {e}") except Exception as e: log.error(f"{sys._getframe().f_code.co_name} error: {e}") finally: log.info(f"潮谷街历史全量采集 {sys._getframe().f_code.co_name} 运行结束") if __name__ == "__main__": history_main(logger)