spider_work/one_piece_spider/jz_one_piece_spider.py

# -*- coding: utf-8 -*-
# Author : Charley
# Python : 3.10.8
# Date   : 2025/10/24 10:20
import user_agent
import requests
import inspect
from loguru import logger
from tenacity import retry, stop_after_attempt, wait_fixed
from mysql_pool import MySQLConnectionPool

logger.remove()
logger.add("./logs/{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00",
           format="[{time:YYYY-MM-DD HH:mm:ss.SSS}] {level} {message}",
           level="DEBUG", retention="7 day")

headers = {
    "accept": "application/json, text/plain, */*",
    "user-agent": user_agent.generate_user_agent()
}

crawler_language = '简中'


def after_log(retry_state):
    """
    retry 回调
    :param retry_state: RetryCallState 对象
    """
    # 检查 args 是否存在且不为空
    if retry_state.args and len(retry_state.args) > 0:
        log = retry_state.args[0]  # 获取传入的 logger
    else:
        log = logger  # 使用全局 logger

    if retry_state.outcome.failed:
        log.warning(
            f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} Times")
    else:
        log.info(f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} succeeded")


@retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
def get_proxys(log):
    """
    获取代理
    :return: 代理
    """
    tunnel = "x371.kdltps.com:15818"
    kdl_username = "t13753103189895"
    kdl_password = "o0yefv6z"
    try:
        proxies = {
            "http": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": kdl_username, "pwd": kdl_password, "proxy": tunnel},
            "https": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": kdl_username, "pwd": kdl_password, "proxy": tunnel}
        }
        return proxies
    except Exception as e:
        log.error(f"Error getting proxy: {e}")
        raise e


@retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
def get_cachelist(log, sql_pool):
    """
    获取 cachelist 选项数据
    :param log: 日志对象
    :param sql_pool: MySQL连接池对象
    """
    log.debug("Getting cachelist ..........................")
    url = "https://onepieceserve.windoent.com/cardType/cardofferingtype/cachelist"
    response = requests.get(url, headers=headers)
    response.raise_for_status()

    resp_json = response.json()
    if resp_json["code"] == 0:
        resp_json_list = resp_json["list"]
        for item in resp_json_list:
            try:
                log.debug(f"Getting cachelist: {item['name']}")
                card_type_name = item.get("name")
                card_type_id = item.get("id")
                # card_type_position = item.get("position")
                get_all_page(log, card_type_id, card_type_name, sql_pool)
            except Exception as e:
                log.error(f"Error getting cachelist: {e}")
    else:
        log.error(f"Error getting cachelist: {resp_json['message']}")
        raise Exception(resp_json["message"])


@retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
def get_single_page(log, page, card_type_name):
    """
    获取单页数据
    :param log: 日志对象
    :param page: 页码
    :param card_type_name: 卡牌类型
    :return: 响应JSON数据
    """
    url = "https://onepieceserve.windoent.com/cardList/cardlist/weblist"
    params = {
        "cardOfferType": card_type_name,
        "cardColor": "",
        "cardType": "",
        "cardCartograph": "",
        "subscript": "",
        "limit": "20",
        "page": str(page)
    }
    response = requests.get(url, headers=headers, params=params)
    response.raise_for_status()

    resp_json = response.json()
    if resp_json["code"] == 0:
        return resp_json
    else:
        log.error(f"Error getting page {page}: {resp_json['message']}")
        raise Exception(resp_json["message"])


def get_all_page(log, card_type_id, card_type_name, sql_pool):
    """
    获取所有页面数据
    :param log: 日志对象
    :param card_type_id: 卡牌id
    :param card_type_name: 卡牌类型
    :param sql_pool: MySQL连接池对象
    """
    log.debug("Getting all single page ..........................")
    page = 1
    total_page = 1

    while page <= total_page:
        log.debug(f"Getting single page: {page}")
        try:
            resp_json = get_single_page(log, page, card_type_name)

            # 更新总页数
            total_page = resp_json["page"]["totalPage"]
            log.debug(f"Total pages: {total_page}, Current page: {page}")

            resp_json_list = resp_json.get("page", {}).get("list", [])

            for item in resp_json_list:
                card_id = item.get("id")
                # card_img = item.get("cardImg")
                try:
                    get_detail(log, card_type_id, card_type_name, card_id, sql_pool)
                except Exception as e:
                    log.error(f"Error getting detail: {e}")

            if len(resp_json_list) < 20:
                log.debug(
                    f"No more data, total pages: {total_page}, current page: {page}, len_resp_json_list:{len(resp_json_list)}, break !!!")
                break

            page += 1
        except Exception as e:
            log.error(f"Error getting page {page}: {e}")
            raise e


@retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
def get_detail(log, card_type_id, card_type_name, card_id, sql_pool):
    """
    获取卡片详情信息
    :param log: 日志对象
    :param card_type_id: 卡片类型 id
    :param card_type_name: 卡片类型 名称
    :param card_id: 卡片 id
    :param sql_pool: MySQL连接池对象
    """
    log.debug(f"Getting detail: {card_id} >>>>>>>>>>>>>>>>>>>>")
    url = f"https://onepieceserve.windoent.com/cardList/cardlist/webInfo/{card_id}"
    response = requests.get(url, headers=headers)
    response.raise_for_status()

    resp_json = response.json()
    if resp_json["code"] == 0:
        resp_json_info = resp_json.get("info")
        card_name = resp_json_info.get("cardName")  # 名称
        card_number = resp_json_info.get("cardNumber")  # 编号
        card_rarity = resp_json_info.get("cardRarity")  # 稀有度
        detail_card_type = resp_json_info.get("cardType")  # 类型
        card_img = resp_json_info.get("cardImg")  # 图片
        card_life = resp_json_info.get("cardLife")  # 费用
        cardAttribute = resp_json_info.get("cardAttribute", [])  # 属性
        card_attribute = "|".join(cardAttribute) if cardAttribute else ""
        card_power = resp_json_info.get("cardPower")  # 力量
        card_attack = resp_json_info.get("cardAttack")  # 反击值
        card_color = resp_json_info.get("cardColor")  # 颜色
        subscript = resp_json_info.get("subscript")  # 角标
        card_features = resp_json_info.get("cardFeatures")  # 特征
        card_text_desc = resp_json_info.get("cardTextDesc")  # 效果
        card_offer_type = resp_json_info.get("cardOfferType")  # 获取方式

        data_dict = {
            "card_type_id": card_type_id,
            "card_type_name": card_type_name,
            "card_id": card_id,
            "card_name": card_name,
            "card_number": card_number,
            "card_rarity": card_rarity,
            "detail_card_type": detail_card_type,
            "card_img": card_img,
            "card_life": card_life,
            "card_attribute": card_attribute,
            "card_power": card_power,
            "card_attack": card_attack,
            "card_color": card_color,
            "subscript": subscript,
            "card_features": card_features,
            "card_text_desc": card_text_desc,
            "card_offer_type": card_offer_type,
            "crawler_language": crawler_language
        }
        # print(data_dict)
        sql_pool.insert_one_or_dict(table="one_piece_record", data=data_dict)

    else:
        log.error(f"Error getting detail: {resp_json['message']}")
        raise Exception(resp_json["message"])


@retry(stop=stop_after_attempt(100), wait=wait_fixed(3600), after=after_log)
def op_main(log):
    """
    主函数
    :param log: logger对象
    """
    log.info(
        f'开始运行 {inspect.currentframe().f_code.co_name} 爬虫任务....................................................')

    # 配置 MySQL 连接池
    sql_pool = MySQLConnectionPool(log=log)
    if not sql_pool.check_pool_health():
        log.error("数据库连接池异常")
        raise RuntimeError("数据库连接池异常")

    try:
        get_cachelist(log, sql_pool)
    except Exception as e:
        log.error(f'{inspect.currentframe().f_code.co_name} error: {e}')
    finally:
        log.info(f'爬虫程序 {inspect.currentframe().f_code.co_name} 运行结束,等待下一轮的采集任务............')


if __name__ == '__main__':
    # get_single_page(logger, 1)
    # get_cachelist(logger)
    op_main(logger)