update ags pop 3.5.6

ags_spider/ags_history_spider.py

@@ -16,7 +16,7 @@ from datetime import datetime
 from mysq_pool import MySQLConnectionPool
 
 logger.remove()
-logger.add("logs/{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00",
+logger.add("logs/his_{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00",
            format="[{time:YYYY-MM-DD HH:mm:ss.SSS}] {level} {message}",
            level="DEBUG", retention="30 day")
 
@@ -43,9 +43,12 @@ def get_proxys_(log):
 @retry(stop_max_attempt_number=5, wait_fixed=1000)
 def get_proxys():
     # 已购买账户  北美
+    # http_proxy = "http://u1952150085001297:sJMHl4qc4bM0@proxy.123proxy.cn:36927"
+    # https_proxy = "http://u1952150085001297:sJMHl4qc4bM0@proxy.123proxy.cn:36927"
     http_proxy = "http://u1952150085001297:sJMHl4qc4bM0@proxy.123proxy.cn:36928"
     https_proxy = "http://u1952150085001297:sJMHl4qc4bM0@proxy.123proxy.cn:36928"
 
+    # url = "https://ifconfig.me"
     try:
         proxySettings = {
             "http": http_proxy,

ags_spider/ags_pop_spider.py

@@ -0,0 +1,177 @@
+# -*- coding: utf-8 -*-
+# Author : Charley
+# Python : 3.10.8
+# Date   : 2025/11/6 10:48
+import inspect
+import time
+import requests
+import schedule
+import user_agent
+from loguru import logger
+from parsel import Selector
+from mysql_pool import MySQLConnectionPool
+from tenacity import retry, stop_after_attempt, wait_fixed
+
+
+logger.remove()
+logger.add("./logs/{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00",
+           format="[{time:YYYY-MM-DD HH:mm:ss.SSS}] {level} {message}",
+           level="DEBUG", retention="7 day")
+
+
+def after_log(retry_state):
+    """
+    retry 回调
+    :param retry_state: RetryCallState 对象
+    """
+    # 检查 args 是否存在且不为空
+    if retry_state.args and len(retry_state.args) > 0:
+        log = retry_state.args[0]  # 获取传入的 logger
+    else:
+        log = logger  # 使用全局 logger
+
+    if retry_state.outcome.failed:
+        log.warning(
+            f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} Times")
+    else:
+        log.info(f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} succeeded")
+
+
+@retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
+def get_proxys(log):
+    """
+    获取代理
+    :return: 代理
+    """
+    tunnel = "x371.kdltps.com:15818"
+    kdl_username = "t13753103189895"
+    kdl_password = "o0yefv6z"
+    try:
+        proxies = {
+            "http": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": kdl_username, "pwd": kdl_password, "proxy": tunnel},
+            "https": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": kdl_username, "pwd": kdl_password, "proxy": tunnel}
+        }
+        return proxies
+    except Exception as e:
+        log.error(f"Error getting proxy: {e}")
+        raise e
+
+
+@retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
+def get_detail_data(log, sql_pool, category_name, category_link, crawl_date):
+    """
+    获取卡组 数据报告 信息详情
+    :param log: logger对象
+    :param sql_pool: MySQLConnectionPool对象
+    :param category_name: 分类名称
+    :param category_link: 分类链接
+    :param crawl_date: 抓取日期
+    """
+    log.debug(f"开始获取 {category_name} 数据报告 信息详情, 链接为: {category_link}")
+    headers = {
+        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+        "referer": "https://agscard.com/pop",
+        "user-agent": user_agent.generate_user_agent()
+    }
+    response = requests.get(category_link, headers=headers, timeout=10)
+    response.raise_for_status()
+    selector = Selector(text=response.text)
+
+    tag_h2_list = selector.xpath('//h2[@class="pop-hero__text-subheading pop-hero__text-stats"]')
+    category_sets = tag_h2_list.xpath('./span[1]/span[1]/text()').get()
+    category_sets = category_sets.strip().replace(",", "") if category_sets else None
+
+    category_cards = tag_h2_list.xpath('./span[2]/span[1]/text()').get()
+    category_cards = category_cards.strip().replace(",", "") if category_cards else None
+
+    category_graded = tag_h2_list.xpath('./span[3]/span[1]/text()').get()
+    category_graded = category_graded.strip().replace(",", "") if category_graded else None
+
+    data_dict = {
+        'category_name': category_name,
+        'category_link': category_link,
+        'category_sets': category_sets,
+        'category_cards': category_cards,
+        'category_graded': category_graded,
+        'crawl_date': crawl_date
+    }
+    # print(data_dict)
+
+    # 保存数据
+    sql_pool.insert_one_or_dict(table="ags_pop_record", data=data_dict, ignore=True)
+
+
+@retry(stop=stop_after_attempt(5), wait=wait_fixed(2), after=after_log)
+def get_ags_pop_list(log, sql_pool):
+    """
+    获取卡组 数据报告 信息
+    :param log: logger对象
+    :param sql_pool: MySQLConnectionPool对象
+    """
+    crawl_date = time.strftime("%Y-%m-%d", time.localtime())
+    headers = {
+        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+        "user-agent": user_agent.generate_user_agent()
+    }
+    url = "https://agscard.com/pop"
+    response = requests.get(url, headers=headers, timeout=10)
+    # print(response.text)
+    # print(response)
+    response.raise_for_status()
+
+    selector = Selector(text=response.text)
+    tag_a_list = selector.xpath('//div[@class="row"]/a')
+
+    info_list = []
+    for tag_a in tag_a_list:
+        category_name = tag_a.xpath('./div[1]/img/@alt').get()
+        category_link = tag_a.xpath('./@href').get()
+        # category_sets = tag_a.xpath('./div[2]/div[1]/div[1]/text()').get()
+        # category_cards = tag_a.xpath('./div[2]/div[2]/div[1]/text()').get()
+        # category_graded = tag_a.xpath('./div[2]/div[3]/div[1]/text()').get()
+        get_detail_data(log, sql_pool, category_name, category_link, crawl_date)
+
+
+@retry(stop=stop_after_attempt(100), wait=wait_fixed(3600), after=after_log)
+def ags_pop_main(log):
+    """
+    主函数
+    :param log: logger对象
+    """
+    log.info(
+        f'开始运行 {inspect.currentframe().f_code.co_name} 爬虫任务....................................................')
+
+    # 配置 MySQL 连接池
+    sql_pool = MySQLConnectionPool(log=log)
+    if not sql_pool.check_pool_health():
+        log.error("数据库连接池异常")
+        raise RuntimeError("数据库连接池异常")
+
+    try:
+        # 获取所有卡组中的卡牌信息
+        get_ags_pop_list(log, sql_pool)
+
+    except Exception as e:
+        log.error(f'{inspect.currentframe().f_code.co_name} error: {e}')
+    finally:
+        log.info(f'爬虫程序 {inspect.currentframe().f_code.co_name} 运行结束,等待下一轮的采集任务............')
+
+
+def schedule_task():
+    """
+    爬虫模块 定时任务 的启动文件
+    """
+    # 立即运行一次任务
+    # ags_pop_main(log=logger)
+
+    # 设置定时任务
+    schedule.every().day.at("00:01").do(ags_pop_main, log=logger)
+
+    while True:
+        schedule.run_pending()
+        time.sleep(1)
+
+
+if __name__ == '__main__':
+    schedule_task()
+    # ags_pop_main(log=logger)

ags_spider/requirements.txt

@@ -1,10 +1,11 @@
 -i https://mirrors.aliyun.com/pypi/simple/
-DBUtils==3.1.0
+DBUtils==3.1.2
 loguru==0.7.3
 parsel==1.10.0
-PyMySQL==1.1.1
-PyYAML==6.0.2
-requests==2.32.3
-retrying==1.3.4
+PyMySQL==1.1.2
+PyYAML==6.0.1
+requests==2.32.5
+retrying==1.4.2
 schedule==1.2.2
-user_agent==0.1.10
+tenacity==9.1.2
+user_agent==0.1.14