refactor(spider): 重构数据报告获取逻辑适配网站改版

- 更新Python版本信息为3.12.10
- 调整请求接口为新版API，改为JSON数据解析
- 移除原有的HTML解析和XPath选择器代码
- 合并数据获取为单一函数获取卡组数据报告信息列表
- 优化重试机制，增加重试等待时间
- 批量插入数据库，提升性能和稳定性
- 注释调整，删除冗余代码和未使用导入
- 调整定时任务代码，注释掉立即运行调用

ags_spider/ags_pop_spider.py

@@ -1,18 +1,18 @@
 # -*- coding: utf-8 -*-
 # Author : Charley
-# Python : 3.10.8
-# Date   : 2025/11/6 10:48
+# Python : 3.12.10
+# Date   : 2026/5/7 13:48
 import inspect
 import time
 import requests
 import schedule
 import user_agent
 from loguru import logger
-from parsel import Selector
 from mysql_pool import MySQLConnectionPool
 from tenacity import retry, stop_after_attempt, wait_fixed
-
-
+"""
+2026/5/7 网站改版  代码重构
+"""
 logger.remove()
 logger.add("./logs/{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00",
            format="[{time:YYYY-MM-DD HH:mm:ss.SSS}] {level} {message}",
@@ -57,50 +57,6 @@ def get_proxys(log):
         raise e
 
 
-@retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
-def get_detail_data(log, sql_pool, category_name, category_link, crawl_date):
-    """
-    获取卡组 数据报告 信息详情
-    :param log: logger对象
-    :param sql_pool: MySQLConnectionPool对象
-    :param category_name: 分类名称
-    :param category_link: 分类链接
-    :param crawl_date: 抓取日期
-    """
-    log.debug(f"开始获取 {category_name} 数据报告 信息详情, 链接为: {category_link}")
-    headers = {
-        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
-        "referer": "https://agscard.com/pop",
-        "user-agent": user_agent.generate_user_agent()
-    }
-    response = requests.get(category_link, headers=headers, timeout=10)
-    response.raise_for_status()
-    selector = Selector(text=response.text)
-
-    tag_h2_list = selector.xpath('//h2[@class="pop-hero__text-subheading pop-hero__text-stats"]')
-    category_sets = tag_h2_list.xpath('./span[1]/span[1]/text()').get()
-    category_sets = category_sets.strip().replace(",", "") if category_sets else None
-
-    category_cards = tag_h2_list.xpath('./span[2]/span[1]/text()').get()
-    category_cards = category_cards.strip().replace(",", "") if category_cards else None
-
-    category_graded = tag_h2_list.xpath('./span[3]/span[1]/text()').get()
-    category_graded = category_graded.strip().replace(",", "") if category_graded else None
-
-    data_dict = {
-        'category_name': category_name,
-        'category_link': category_link,
-        'category_sets': category_sets,
-        'category_cards': category_cards,
-        'category_graded': category_graded,
-        'crawl_date': crawl_date
-    }
-    # print(data_dict)
-
-    # 保存数据
-    sql_pool.insert_one_or_dict(table="ags_pop_record", data=data_dict, ignore=True)
-
-
 @retry(stop=stop_after_attempt(5), wait=wait_fixed(2), after=after_log)
 def get_ags_pop_list(log, sql_pool):
     """
@@ -110,26 +66,43 @@ def get_ags_pop_list(log, sql_pool):
     """
     crawl_date = time.strftime("%Y-%m-%d", time.localtime())
     headers = {
-        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+        "accept": "application/json",
         "user-agent": user_agent.generate_user_agent()
     }
-    url = "https://agscard.com/pop"
+    url = "https://app.agscard.com/api/v2/pop/categories"
     response = requests.get(url, headers=headers, timeout=10)
     # print(response.text)
     # print(response)
     response.raise_for_status()
 
-    selector = Selector(text=response.text)
-    tag_a_list = selector.xpath('//div[@class="row"]/a')
+    resp_json = response.json()
+    resp_data = resp_json.get("data", [])
+    if not resp_data:
+        log.error("No data found in the response")
+        return
 
     info_list = []
-    for tag_a in tag_a_list:
-        category_name = tag_a.xpath('./div[1]/img/@alt').get()
-        category_link = tag_a.xpath('./@href').get()
-        # category_sets = tag_a.xpath('./div[2]/div[1]/div[1]/text()').get()
-        # category_cards = tag_a.xpath('./div[2]/div[2]/div[1]/text()').get()
-        # category_graded = tag_a.xpath('./div[2]/div[3]/div[1]/text()').get()
-        get_detail_data(log, sql_pool, category_name, category_link, crawl_date)
+    for category_ in resp_data:
+        category_id = category_.get("id")
+        category_name = category_.get("name")
+        category_link = f"https://agscard.com/pop/{category_id}"
+        category_sets = category_.get("card_sets_count")
+        category_cards = category_.get("card_products_count")
+        category_graded = category_.get("total_population")
+
+        data_dict = {
+            'category_name': category_name,
+            'category_link': category_link,
+            'category_sets': category_sets,
+            'category_cards': category_cards,
+            'category_graded': category_graded,
+            'crawl_date': crawl_date
+        }
+        # print(data_dict)
+        info_list.append(data_dict)
+
+    # 保存数据
+    sql_pool.insert_many(table="ags_pop_record", data_list=info_list, ignore=True)
 
 
 @retry(stop=stop_after_attempt(100), wait=wait_fixed(3600), after=after_log)
@@ -162,7 +135,7 @@ def schedule_task():
     爬虫模块 定时任务 的启动文件
     """
     # 立即运行一次任务
-    ags_pop_main(log=logger)
+    # ags_pop_main(log=logger)
 
     # 设置定时任务
     schedule.every().day.at("00:01").do(ags_pop_main, log=logger)