# -*- coding: utf-8 -*- # Author : Charley # Python : 3.10.8 # Date : 2025/10/24 10:20 import user_agent import requests import inspect from loguru import logger from tenacity import retry, stop_after_attempt, wait_fixed from mysql_pool import MySQLConnectionPool logger.remove() logger.add("./logs/{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00", format="[{time:YYYY-MM-DD HH:mm:ss.SSS}] {level} {message}", level="DEBUG", retention="7 day") headers = { "accept": "application/json, text/plain, */*", "user-agent": user_agent.generate_user_agent() } crawler_language = '简中' def after_log(retry_state): """ retry 回调 :param retry_state: RetryCallState 对象 """ # 检查 args 是否存在且不为空 if retry_state.args and len(retry_state.args) > 0: log = retry_state.args[0] # 获取传入的 logger else: log = logger # 使用全局 logger if retry_state.outcome.failed: log.warning( f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} Times") else: log.info(f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} succeeded") @retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log) def get_proxys(log): """ 获取代理 :return: 代理 """ tunnel = "x371.kdltps.com:15818" kdl_username = "t13753103189895" kdl_password = "o0yefv6z" try: proxies = { "http": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": kdl_username, "pwd": kdl_password, "proxy": tunnel}, "https": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": kdl_username, "pwd": kdl_password, "proxy": tunnel} } return proxies except Exception as e: log.error(f"Error getting proxy: {e}") raise e @retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log) def get_cachelist(log, sql_pool): """ 获取 cachelist 选项数据 :param log: 日志对象 :param sql_pool: MySQL连接池对象 """ log.debug("Getting cachelist ..........................") url = "https://onepieceserve.windoent.com/cardType/cardofferingtype/cachelist" response = requests.get(url, headers=headers) response.raise_for_status() resp_json = response.json() if resp_json["code"] == 0: resp_json_list = resp_json["list"] for item in resp_json_list: try: log.debug(f"Getting cachelist: {item['name']}") card_type_name = item.get("name") card_type_id = item.get("id") # card_type_position = item.get("position") get_all_page(log, card_type_id, card_type_name, sql_pool) except Exception as e: log.error(f"Error getting cachelist: {e}") else: log.error(f"Error getting cachelist: {resp_json['message']}") raise Exception(resp_json["message"]) @retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log) def get_single_page(log, page, card_type_name): """ 获取单页数据 :param log: 日志对象 :param page: 页码 :param card_type_name: 卡牌类型 :return: 响应JSON数据 """ url = "https://onepieceserve.windoent.com/cardList/cardlist/weblist" params = { "cardOfferType": card_type_name, "cardColor": "", "cardType": "", "cardCartograph": "", "subscript": "", "limit": "20", "page": str(page) } response = requests.get(url, headers=headers, params=params) response.raise_for_status() resp_json = response.json() if resp_json["code"] == 0: return resp_json else: log.error(f"Error getting page {page}: {resp_json['message']}") raise Exception(resp_json["message"]) def get_all_page(log, card_type_id, card_type_name, sql_pool): """ 获取所有页面数据 :param log: 日志对象 :param card_type_id: 卡牌id :param card_type_name: 卡牌类型 :param sql_pool: MySQL连接池对象 """ log.debug("Getting all single page ..........................") page = 1 total_page = 1 while page <= total_page: log.debug(f"Getting single page: {page}") try: resp_json = get_single_page(log, page, card_type_name) # 更新总页数 total_page = resp_json["page"]["totalPage"] log.debug(f"Total pages: {total_page}, Current page: {page}") resp_json_list = resp_json.get("page", {}).get("list", []) for item in resp_json_list: card_id = item.get("id") # card_img = item.get("cardImg") try: get_detail(log, card_type_id, card_type_name, card_id, sql_pool) except Exception as e: log.error(f"Error getting detail: {e}") if len(resp_json_list) < 20: log.debug( f"No more data, total pages: {total_page}, current page: {page}, len_resp_json_list:{len(resp_json_list)}, break !!!") break page += 1 except Exception as e: log.error(f"Error getting page {page}: {e}") raise e @retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log) def get_detail(log, card_type_id, card_type_name, card_id, sql_pool): """ 获取卡片详情信息 :param log: 日志对象 :param card_type_id: 卡片类型 id :param card_type_name: 卡片类型 名称 :param card_id: 卡片 id :param sql_pool: MySQL连接池对象 """ log.debug(f"Getting detail: {card_id} >>>>>>>>>>>>>>>>>>>>") url = f"https://onepieceserve.windoent.com/cardList/cardlist/webInfo/{card_id}" response = requests.get(url, headers=headers) response.raise_for_status() resp_json = response.json() if resp_json["code"] == 0: resp_json_info = resp_json.get("info") card_name = resp_json_info.get("cardName") # 名称 card_number = resp_json_info.get("cardNumber") # 编号 card_rarity = resp_json_info.get("cardRarity") # 稀有度 detail_card_type = resp_json_info.get("cardType") # 类型 card_img = resp_json_info.get("cardImg") # 图片 card_life = resp_json_info.get("cardLife") # 费用 cardAttribute = resp_json_info.get("cardAttribute", []) # 属性 card_attribute = "|".join(cardAttribute) if cardAttribute else "" card_power = resp_json_info.get("cardPower") # 力量 card_attack = resp_json_info.get("cardAttack") # 反击值 card_color = resp_json_info.get("cardColor") # 颜色 subscript = resp_json_info.get("subscript") # 角标 card_features = resp_json_info.get("cardFeatures") # 特征 card_text_desc = resp_json_info.get("cardTextDesc") # 效果 card_offer_type = resp_json_info.get("cardOfferType") # 获取方式 data_dict = { "card_type_id": card_type_id, "card_type_name": card_type_name, "card_id": card_id, "card_name": card_name, "card_number": card_number, "card_rarity": card_rarity, "detail_card_type": detail_card_type, "card_img": card_img, "card_life": card_life, "card_attribute": card_attribute, "card_power": card_power, "card_attack": card_attack, "card_color": card_color, "subscript": subscript, "card_features": card_features, "card_text_desc": card_text_desc, "card_offer_type": card_offer_type, "crawler_language": crawler_language } # print(data_dict) sql_pool.insert_one_or_dict(table="one_piece_record", data=data_dict) else: log.error(f"Error getting detail: {resp_json['message']}") raise Exception(resp_json["message"]) @retry(stop=stop_after_attempt(100), wait=wait_fixed(3600), after=after_log) def op_main(log): """ 主函数 :param log: logger对象 """ log.info( f'开始运行 {inspect.currentframe().f_code.co_name} 爬虫任务....................................................') # 配置 MySQL 连接池 sql_pool = MySQLConnectionPool(log=log) if not sql_pool.check_pool_health(): log.error("数据库连接池异常") raise RuntimeError("数据库连接池异常") try: get_cachelist(log, sql_pool) except Exception as e: log.error(f'{inspect.currentframe().f_code.co_name} error: {e}') finally: log.info(f'爬虫程序 {inspect.currentframe().f_code.co_name} 运行结束,等待下一轮的采集任务............') if __name__ == '__main__': # get_single_page(logger, 1) # get_cachelist(logger) op_main(logger)