|
|
@@ -0,0 +1,177 @@
|
|
|
+# -*- coding: utf-8 -*-
|
|
|
+# Author : Charley
|
|
|
+# Python : 3.10.8
|
|
|
+# Date : 2025/11/6 10:48
|
|
|
+import inspect
|
|
|
+import time
|
|
|
+import requests
|
|
|
+import schedule
|
|
|
+import user_agent
|
|
|
+from loguru import logger
|
|
|
+from parsel import Selector
|
|
|
+from mysql_pool import MySQLConnectionPool
|
|
|
+from tenacity import retry, stop_after_attempt, wait_fixed
|
|
|
+
|
|
|
+
|
|
|
+logger.remove()
|
|
|
+logger.add("./logs/{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00",
|
|
|
+ format="[{time:YYYY-MM-DD HH:mm:ss.SSS}] {level} {message}",
|
|
|
+ level="DEBUG", retention="7 day")
|
|
|
+
|
|
|
+
|
|
|
+def after_log(retry_state):
|
|
|
+ """
|
|
|
+ retry 回调
|
|
|
+ :param retry_state: RetryCallState 对象
|
|
|
+ """
|
|
|
+ # 检查 args 是否存在且不为空
|
|
|
+ if retry_state.args and len(retry_state.args) > 0:
|
|
|
+ log = retry_state.args[0] # 获取传入的 logger
|
|
|
+ else:
|
|
|
+ log = logger # 使用全局 logger
|
|
|
+
|
|
|
+ if retry_state.outcome.failed:
|
|
|
+ log.warning(
|
|
|
+ f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} Times")
|
|
|
+ else:
|
|
|
+ log.info(f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} succeeded")
|
|
|
+
|
|
|
+
|
|
|
+@retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
|
|
|
+def get_proxys(log):
|
|
|
+ """
|
|
|
+ 获取代理
|
|
|
+ :return: 代理
|
|
|
+ """
|
|
|
+ tunnel = "x371.kdltps.com:15818"
|
|
|
+ kdl_username = "t13753103189895"
|
|
|
+ kdl_password = "o0yefv6z"
|
|
|
+ try:
|
|
|
+ proxies = {
|
|
|
+ "http": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": kdl_username, "pwd": kdl_password, "proxy": tunnel},
|
|
|
+ "https": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": kdl_username, "pwd": kdl_password, "proxy": tunnel}
|
|
|
+ }
|
|
|
+ return proxies
|
|
|
+ except Exception as e:
|
|
|
+ log.error(f"Error getting proxy: {e}")
|
|
|
+ raise e
|
|
|
+
|
|
|
+
|
|
|
+@retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
|
|
|
+def get_detail_data(log, sql_pool, category_name, category_link, crawl_date):
|
|
|
+ """
|
|
|
+ 获取卡组 数据报告 信息详情
|
|
|
+ :param log: logger对象
|
|
|
+ :param sql_pool: MySQLConnectionPool对象
|
|
|
+ :param category_name: 分类名称
|
|
|
+ :param category_link: 分类链接
|
|
|
+ :param crawl_date: 抓取日期
|
|
|
+ """
|
|
|
+ log.debug(f"开始获取 {category_name} 数据报告 信息详情, 链接为: {category_link}")
|
|
|
+ headers = {
|
|
|
+ "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
|
|
+ "referer": "https://agscard.com/pop",
|
|
|
+ "user-agent": user_agent.generate_user_agent()
|
|
|
+ }
|
|
|
+ response = requests.get(category_link, headers=headers, timeout=10)
|
|
|
+ response.raise_for_status()
|
|
|
+ selector = Selector(text=response.text)
|
|
|
+
|
|
|
+ tag_h2_list = selector.xpath('//h2[@class="pop-hero__text-subheading pop-hero__text-stats"]')
|
|
|
+ category_sets = tag_h2_list.xpath('./span[1]/span[1]/text()').get()
|
|
|
+ category_sets = category_sets.strip().replace(",", "") if category_sets else None
|
|
|
+
|
|
|
+ category_cards = tag_h2_list.xpath('./span[2]/span[1]/text()').get()
|
|
|
+ category_cards = category_cards.strip().replace(",", "") if category_cards else None
|
|
|
+
|
|
|
+ category_graded = tag_h2_list.xpath('./span[3]/span[1]/text()').get()
|
|
|
+ category_graded = category_graded.strip().replace(",", "") if category_graded else None
|
|
|
+
|
|
|
+ data_dict = {
|
|
|
+ 'category_name': category_name,
|
|
|
+ 'category_link': category_link,
|
|
|
+ 'category_sets': category_sets,
|
|
|
+ 'category_cards': category_cards,
|
|
|
+ 'category_graded': category_graded,
|
|
|
+ 'crawl_date': crawl_date
|
|
|
+ }
|
|
|
+ # print(data_dict)
|
|
|
+
|
|
|
+ # 保存数据
|
|
|
+ sql_pool.insert_one_or_dict(table="ags_pop_record", data=data_dict, ignore=True)
|
|
|
+
|
|
|
+
|
|
|
+@retry(stop=stop_after_attempt(5), wait=wait_fixed(2), after=after_log)
|
|
|
+def get_ags_pop_list(log, sql_pool):
|
|
|
+ """
|
|
|
+ 获取卡组 数据报告 信息
|
|
|
+ :param log: logger对象
|
|
|
+ :param sql_pool: MySQLConnectionPool对象
|
|
|
+ """
|
|
|
+ crawl_date = time.strftime("%Y-%m-%d", time.localtime())
|
|
|
+ headers = {
|
|
|
+ "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
|
|
+ "user-agent": user_agent.generate_user_agent()
|
|
|
+ }
|
|
|
+ url = "https://agscard.com/pop"
|
|
|
+ response = requests.get(url, headers=headers, timeout=10)
|
|
|
+ # print(response.text)
|
|
|
+ # print(response)
|
|
|
+ response.raise_for_status()
|
|
|
+
|
|
|
+ selector = Selector(text=response.text)
|
|
|
+ tag_a_list = selector.xpath('//div[@class="row"]/a')
|
|
|
+
|
|
|
+ info_list = []
|
|
|
+ for tag_a in tag_a_list:
|
|
|
+ category_name = tag_a.xpath('./div[1]/img/@alt').get()
|
|
|
+ category_link = tag_a.xpath('./@href').get()
|
|
|
+ # category_sets = tag_a.xpath('./div[2]/div[1]/div[1]/text()').get()
|
|
|
+ # category_cards = tag_a.xpath('./div[2]/div[2]/div[1]/text()').get()
|
|
|
+ # category_graded = tag_a.xpath('./div[2]/div[3]/div[1]/text()').get()
|
|
|
+ get_detail_data(log, sql_pool, category_name, category_link, crawl_date)
|
|
|
+
|
|
|
+
|
|
|
+@retry(stop=stop_after_attempt(100), wait=wait_fixed(3600), after=after_log)
|
|
|
+def ags_pop_main(log):
|
|
|
+ """
|
|
|
+ 主函数
|
|
|
+ :param log: logger对象
|
|
|
+ """
|
|
|
+ log.info(
|
|
|
+ f'开始运行 {inspect.currentframe().f_code.co_name} 爬虫任务....................................................')
|
|
|
+
|
|
|
+ # 配置 MySQL 连接池
|
|
|
+ sql_pool = MySQLConnectionPool(log=log)
|
|
|
+ if not sql_pool.check_pool_health():
|
|
|
+ log.error("数据库连接池异常")
|
|
|
+ raise RuntimeError("数据库连接池异常")
|
|
|
+
|
|
|
+ try:
|
|
|
+ # 获取所有卡组中的卡牌信息
|
|
|
+ get_ags_pop_list(log, sql_pool)
|
|
|
+
|
|
|
+ except Exception as e:
|
|
|
+ log.error(f'{inspect.currentframe().f_code.co_name} error: {e}')
|
|
|
+ finally:
|
|
|
+ log.info(f'爬虫程序 {inspect.currentframe().f_code.co_name} 运行结束,等待下一轮的采集任务............')
|
|
|
+
|
|
|
+
|
|
|
+def schedule_task():
|
|
|
+ """
|
|
|
+ 爬虫模块 定时任务 的启动文件
|
|
|
+ """
|
|
|
+ # 立即运行一次任务
|
|
|
+ # ags_pop_main(log=logger)
|
|
|
+
|
|
|
+ # 设置定时任务
|
|
|
+ schedule.every().day.at("00:01").do(ags_pop_main, log=logger)
|
|
|
+
|
|
|
+ while True:
|
|
|
+ schedule.run_pending()
|
|
|
+ time.sleep(1)
|
|
|
+
|
|
|
+
|
|
|
+if __name__ == '__main__':
|
|
|
+ schedule_task()
|
|
|
+ # ags_pop_main(log=logger)
|
