spider_work/psa_spider/psa_pop_spider.py

# -*- coding: utf-8 -*-
# Author : Charley
# Python : 3.10.8
# Date   : 2025/9/15 14:53
import inspect
import random
import time
import schedule
import user_agent
from curl_cffi import requests
from loguru import logger
from parsel import Selector
from mysql_pool import MySQLConnectionPool
from tenacity import retry, stop_after_attempt, wait_fixed

logger.remove()
logger.add("logs/{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00",
           format="[{time:YYYY-MM-DD HH:mm:ss.SSS}] {level} {message}",
           level="DEBUG", retention="3 day")

client_identifier_list = [
    "edge99", "edge101", "chrome99", "chrome100", "chrome101", "chrome104", "chrome107",
    "chrome110", "chrome116", "chrome119", "chrome120", "chrome123", "chrome124",
    "chrome99_android", "safari15_3", "safari15_5", "safari17_0", "safari17_2_ios"
]

BASE_URL = 'https://www.psacard.com'

headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
    'User-Agent': user_agent.generate_user_agent(),
    "Cookie": "QuantumMetricUserID=22c3803bf07dbc25f851ff507866df96; psa-locale=en-US; osano_consentmanager_uuid=42a9bb07-9acc-4ef6-9d02-bc32ffdf1fff; osano_consentmanager=lGHfgSS0ydA-ImkuudYFlb5aEWtBTq5wJAH3DT-hDQsVV1_nHBaT0m6s9cx83OXlJnvHbPIBSg838CBSaf3vgGEYWKfxt2i39N64C_s__RcE8ytvj42bK586lLGEDewsKh9Y1y4IVfgoOrAM0YtmMWDcGZ2KHUypn3K4g7mL57USueK19vGEDz7nUfcVo_mFofK3Jmw9BnNrTTbD4mKI85Dz-nCiC1DlzRn9-_88x0U3gb2RsWmIID076x7TJ8VgHICQQ8zj3VdML4yLpeUkfN6lfGEbTiFqvmzTFYqo8A4rfpHKfDZd18FjTtngIDnzMK3uAtf3ABA=; _tt_enable_cookie=1; _ttp=01KE8YM1RMWGA3H8W289ASMXY1_.tt.1; _scid=xSwAZLOM2z1UVzbeMSa7SdoxzOdRpCXC; sa-user-id=s%253A0-d546cca6-494b-5861-5610-008c7a982faf.C1TY4QSpzfnyfWTcMQHcwLs%252BsKPZ%252F7EjyWkQQC5esLU; sa-user-id-v2=s%253A1UbMpklLWGFWEACMepgvr8uvDiA.X7BM3pvlugPl4Ibhtnz4Wfz1gE0%252FBH%252FS0QcjBVi3gzU; sa-user-id-v3=s%253AAQAKIJBI0vf1ZJkqIoEdlaHb5fDvZhDpA19TppSKMX8vvYMcEAEYAyDozPLKBjABOgRi0YmEQgSnRVhY.YOIWenkOAJAaMUIYT0ChxjNb7XdupE9HuIDDE5Qh%252Bt8; psa-apr-recentsearch=%7B%22recentSearches%22%3A%5B%7B%22searchId%22%3A%220-Pokemon-Coin%20Promo%22%2C%22catId%22%3A%220%22%2C%22catName%22%3A%22%22%2C%22search%22%3A%22Pokemon%20Coin%20Promo%22%7D%5D%7D; _gcl_au=1.1.1354962831.1781235727; _uutmz=source=google|campaign=(organic)|medium=organic|keyword=(not%20provided); _uutmzh=0.4~2026-01-06T14%3A06%3A21.822%2B-8~2026-06-12T11%3A42%3A07.359%2B-8~%3Bnone%3Adirect%3Bo%3Ag%3B; sa-user-id-v4=s%253A.o6W7wkJsHSTU4%252BLlDruZ%252FwNjVcUZZMvakQpSatDoAgo; _gid=GA1.2.15976690.1781235897; sa-r-source=app.collectors.com; _ga_HD10604KCZ=GS2.1.s1781254798$o2$g1$t1781254913$j22$l0$h0; _hp2_id.1106280392=%7B%22userId%22%3A%224865765352100683%22%2C%22pageviewId%22%3A%227549583361365791%22%2C%22sessionId%22%3A%225840239242373286%22%2C%22identity%22%3A%22103751284%22%2C%22trackerVersion%22%3A%224.0%22%2C%22identityField%22%3Anull%2C%22isIdentified%22%3A1%7D; QuantumMetricSessionID=7bd235423a4a9cb3c9c4c7137935da54; env=prod; accessToken=eyJhbGciOiJSUzI1NiIsImtpZCI6IlNLMnhqZ1lQVERhSkVPV2FGNmtKd2tiZzJsWkVKIiwidHlwIjoiSldUIn0.eyJDb2xsZWN0b3JzSWQiOjEwMzc1MTI4NCwiTmF2aXNpb25JZCI6IjUzODQ1MTkiLCJhbXIiOlsicHdkIl0sImF1ZCI6WyJVREo0YW1kWlNqQlpkakJVUlZkSlMyaENZV2wyV1cxWmJIRmpVVHBVVUVFek4wWjZRbmRaWlU5bFJFc3hVVU4xZDI1RlpFRTJWMmhEVFZVPSIsIlAyeGpnWUowWXYwVEVXSUtoQmFpdlltWWxxY1EiXSwiYXpwIjoiVURKNGFtZFpTakJaZGpCVVJWZEpTMmhDWVdsMldXMVpiSEZqVVRwVVVFRXpOMFo2UW5kWlpVOWxSRXN4VVVOMWQyNUZaRUUyVjJoRFRWVT0iLCJjaWQiOiJVREo0YW1kWlNqQlpkakJVUlZkSlMyaENZV2wyV1cxWmJIRmpVVHBVVUVFek4wWjZRbmRaWlU5bFJFc3hVVU4xZDI1RlpFRTJWMmhEVFZVPSIsImRjaSI6IjNGMUh3TTZqellxUklFcTYxamZaZUYxRmlacCIsImRzdWIiOiJVMzVjUDdYZjZibFhvaTlCRXZMQmhwbjdSOVN6IiwiZXhwIjoxNzgxMzQ2MTY3LCJpYXQiOjE3ODEyNTk3NjcsImlzcyI6Imh0dHBzOi8vYXV0aC5jb2xsZWN0b3JzLmNvbS92MS9hcHBzL1AyeGpnWUowWXYwVEVXSUtoQmFpdlltWWxxY1EiLCJqdGkiOiJMSjNGMjRLaGUzeDFpcDlHbVQzZWFYOXltaVRBUyIsInJleHAiOiIyMDM2LTA2LTExVDEwOjIyOjQ3WiIsInNjb3BlIjoiY3VzdG9tZXIgb2ZmbGluZV9hY2Nlc3MiLCJzY3AiOlsib2ZmbGluZV9hY2Nlc3MiLCJjdXN0b21lciIsIm9wZW5pZCJdLCJzdWIiOiJjaGFybGV5X2xlb0AxNjMuY29tIiwidG9rZW5fdHlwZSI6ImFjY2Vzc190b2tlbiJ9.R1y5QrMPDBarJbnA6feNxf30EZCM9heoRDOXof3I35jN2lULtcdtr9DU1il9nOiwaI2LI_vupXyXe8ppKbIV9-LczVS9OSCgFCUCgLdXOUtq_4xNbfDhgO44mNjG5MxmNNae86NxhgcrwrtM3Y_9ck_n7-Fq2Es5AA16xrHY2Fk3gTX5_zlt70A0q7BWhx0wkdNZzUxjSOdS4iIAJIDyOkY4lGjsSZT38k-dFsWGqJ2uIhw81N-5Dym7q3WDUwp-3ekacBAwJcYhkUjKuy0eFaVFlQFbkycqZcYEw9UVdKFLj1ItgkATIwmcUTOrQVY8PGUn9uKH1OpmKo7kMiAVTQ; refreshToken=eyJhbGciOiJSUzI1NiIsImtpZCI6IlNLMnhqZ1lQVERhSkVPV2FGNmtKd2tiZzJsWkVKIiwidHlwIjoiSldUIn0.eyJDb2xsZWN0b3JzSWQiOjEwMzc1MTI4NCwiTmF2aXNpb25JZCI6IjUzODQ1MTkiLCJhbXIiOlsicHdkIl0sImF1ZCI6WyJVREo0YW1kWlNqQlpkakJVUlZkSlMyaENZV2wyV1cxWmJIRmpVVHBVVUVFek4wWjZRbmRaWlU5bFJFc3hVVU4xZDI1RlpFRTJWMmhEVFZVPSIsIlAyeGpnWUowWXYwVEVXSUtoQmFpdlltWWxxY1EiXSwiYXpwIjoiVURKNGFtZFpTakJaZGpCVVJWZEpTMmhDWVdsMldXMVpiSEZqVVRwVVVFRXpOMFo2UW5kWlpVOWxSRXN4VVVOMWQyNUZaRUUyVjJoRFRWVT0iLCJjaWQiOiJVREo0YW1kWlNqQlpkakJVUlZkSlMyaENZV2wyV1cxWmJIRmpVVHBVVUVFek4wWjZRbmRaWlU5bFJFc3hVVU4xZDI1RlpFRTJWMmhEVFZVPSIsImRjaSI6IjNGMUh3TTZqellxUklFcTYxamZaZUYxRmlacCIsImRjbCI6WyJDb2xsZWN0b3JzSWQiLCJOYXZpc2lvbklkIiwic2NwIiwic3ViIl0sImRzdWIiOiJVMzVjUDdYZjZibFhvaTlCRXZMQmhwbjdSOVN6IiwiZHYiOjEsImV4cCI6MjA5Njc5MjU2NywiaWF0IjoxNzgxMjU5NzY3LCJpc3MiOiJodHRwczovL2F1dGguY29sbGVjdG9ycy5jb20vdjEvYXBwcy9QMnhqZ1lKMFl2MFRFV0lLaEJhaXZZbVlscWNRIiwianRpIjoiTEozRjI0S2o1V3BmaVpDaWJRSkR3TVFYeFhFcjYiLCJzY29wZSI6ImN1c3RvbWVyIG9mZmxpbmVfYWNjZXNzIiwic2NwIjpbIm9mZmxpbmVfYWNjZXNzIiwiY3VzdG9tZXIiLCJvcGVuaWQiXSwic3ViIjoiY2hhcmxleV9sZW9AMTYzLmNvbSIsInRva2VuX3R5cGUiOiJyZWZyZXNoX3Rva2VuIn0.dV3QGtrD_vqADjAETDlMnOchwaMSh8gtwPKsFkYJSVt0xapHZiLI_VcDlCbx_nnw2JsKA0iH8rx6vNNLIcrmhKwnQ-UNiqU1IBuNE28aCnngx4HDXxLMbm5-3ZRwKWFbSSaP5LEl8sZqBxigbCzdwrHnuVKAwgWmv6Pw-AmDWWiMKXwEqNT_tRwsFrK7WnxORv_fzaaJIZ8bVQc4JqbdbZBcKcGU1nnHS6o7OUxBKjg0YyeH3ssMbjsncBV2cxZzQVtRwFwPWiGz_316S9fnljmI1jJsiy5Cu9jeZekPWG6IPX1ZqQ3A7H2NFb6ONG7Vm4MhfxUa3BIuEJ3y2U5JIg; CMGUID=645f97f3-6d98-47db-8bff-baaf8415d51e; cf_clearance=cVaqwdjxY3nGeU0OzOhGwgyYkeETvcDsS407_vcnlDU-1781259770-1.2.1.1-JsTu2GnznqiUjoNKsZ0rqFKbBGBtkWA2CM5ybV1.8XNxtkFFtaGkC6kGHnJnMtKddQFyknAGmuX6aOgsgWXEWi0ibmKP5N6vRUCqprmuitNS4hdfrPTihnG_12EEReqVgsVcGjL5gkvbCUbepbbMmDELyrQQnuj66itIFFvpzwnoOo3V8Nw2HaQ09kvbtAyP7EMjLMMgfYkugvaB4VopFzip4xEnkNvcXrO8oL9U7W0aX8qXx.e68BaxApriDtdefKRUDoMnhFsmHvKUQytveD8R7U_a6K2wjmjvC.7ornBRn5D0mrYqmL1vcdsxxzzQBb8el809ktpts4W8mAmPt8J6vkXY4ZIBY0FfqF928LLjKr2L.5gEOh3nvY_WMw1WfRlzrNpftLAAc30pxHXpYgZYAFVIAUl64OCreS4ZgA8; AWSALBTG=NdS+ZrDWCMJQfszb9918wgArOZZFDAxMNsjJ81v7H+hLWa4ryC8KKiNpYLQamQgXi6+eF2ymYxtam3f0Xt6dBpTJrA8oHd2wMfvM9jkWT2NBaHeYqZztLKzg8nRv40jDy5BMW4mC91vcSqv/TkS9Nr3OzcsekoW/ZA+d6YYjfXBY; AWSALBTGCORS=NdS+ZrDWCMJQfszb9918wgArOZZFDAxMNsjJ81v7H+hLWa4ryC8KKiNpYLQamQgXi6+eF2ymYxtam3f0Xt6dBpTJrA8oHd2wMfvM9jkWT2NBaHeYqZztLKzg8nRv40jDy5BMW4mC91vcSqv/TkS9Nr3OzcsekoW/ZA+d6YYjfXBY; AWSALB=L1FiNF7SKbJoBNGHNI19ReDFH8Ji/8/pnei/a/JYVJTZAeUZZv99lK0VEtwpMlmj/BHZYbikaEVSro06u142T+lx8X3/FyUELIeDHbG27JT0PpcwJPlRBQH+j4ti; AWSALBCORS=L1FiNF7SKbJoBNGHNI19ReDFH8Ji/8/pnei/a/JYVJTZAeUZZv99lK0VEtwpMlmj/BHZYbikaEVSro06u142T+lx8X3/FyUELIeDHbG27JT0PpcwJPlRBQH+j4ti; _ga_5YHEXX56Q3=GS2.1.s1781259769$o16$g0$t1781259774$j55$l0$h0; __cf_bm=8OcNvWr3asBXt26hgXf46a6w2UsvoaUHbC2rwoJfZN0-1781259775.5736341-1.0.1.1-M7tQFsK0OURujJxMk8S33cnsJta2pG39ZK5aO2h3upBKHc_QKRsWn64lCgYNI4tDY68eyyBiEvLMqveVCWY0mnCkMDsFGK6.2dF0IAss9d6UH_tHaKG9z7ADzhBEDN4kFA.XAhja0o2VRlg7vP.Y1w; _rdt_uuid=1767679591675.3ca088ed-2201-4dd0-960d-54ce76c1c454; _dc_gtm_UA-3791435-10=1; _ga=GA1.1.1620777947.1767679582; _ga_GGS8NWPYE2=GS2.1.s1781259751$o27$g1$t1781259774$j37$l0$h0; _uetsid=afafc940661011f1a11f296e502f7168; _uetvid=d91d21f0eac511f0b640d36deb73db38; _ga_1QVXQ1V575=GS2.1.s1781235728$o14$g1$t1781259774$j37$l1$h501104539; _ga_LBYZ0MN5NP=GS2.1.s1781259751$o27$g1$t1781259774$j37$l0$h0; _scid_r=xKwAZLOM2z1UVzbeMSa7SdoxzOdRpCXC_B7I6A; _ga_8C5B608QLW=GS2.2.s1781259775$o15$g0$t1781259775$j60$l0$h1025460194"

}

category_link_list = {'Baseball Cards': 'https://www.psacard.com/pop/baseball-cards/20003',
                      'Baseball Coins': 'https://www.psacard.com/pop/baseball-coins/82797',
                      'Basketball Cards': 'https://www.psacard.com/pop/basketball-cards/20019',
                      'Basketball Coins': 'https://www.psacard.com/pop/basketball-coins/83007',
                      'Boxing / Wrestling Cards / MMA': 'https://www.psacard.com/pop/boxing-wrestling-cards-mma/20021',
                      'Football Cards': 'https://www.psacard.com/pop/football-cards/20014',
                      'Football Coins': 'https://www.psacard.com/pop/football-coins/83011',
                      'Golf Cards': 'https://www.psacard.com/pop/golf-cards/20023',
                      'Hockey Cards': 'https://www.psacard.com/pop/hockey-cards/20020',
                      'Hockey Coins': 'https://www.psacard.com/pop/hockey-coins/83012',
                      'Minor League Cards': 'https://www.psacard.com/pop/minor-league-cards/20031',
                      'Misc Cards': 'https://www.psacard.com/pop/misc-cards/20033',
                      'Multi-Sport Cards': 'https://www.psacard.com/pop/multi-sport-cards/20006',
                      'Multi-Sport Coins': 'https://www.psacard.com/pop/multi-sport-coins/102825',
                      'Non-Sport Cards': 'https://www.psacard.com/pop/non-sport-cards/20032',
                      'Non-Sport Coins': 'https://www.psacard.com/pop/non-sport-coins/82981',
                      'Packs': 'https://www.psacard.com/pop/packs/20017',
                      'Pins': 'https://www.psacard.com/pop/pins/20013',
                      'Soccer Cards': 'https://www.psacard.com/pop/soccer-cards/20004',
                      'TCG Cards': 'https://www.psacard.com/pop/tcg-cards/156940',
                      'Tickets': 'https://www.psacard.com/pop/tickets/20022',
                      'Game-Used Bats': 'https://www.psacard.com/pop/bats'}


def after_log(retry_state):
    """
    retry 回调
    :param retry_state: RetryCallState 对象
    """
    # 检查 args 是否存在且不为空
    if retry_state.args and len(retry_state.args) > 0:
        log = retry_state.args[0]  # 获取传入的 logger
    else:
        log = logger  # 使用全局 logger

    if retry_state.outcome.failed:
        log.warning(
            f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} Times")
    else:
        log.info(f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} succeeded")


@retry(stop=stop_after_attempt(5), wait=wait_fixed(2), after=after_log)
def get_proxys(log):
    # 已购买账户  北美
    # http_proxy = "http://u1952150085001297:sJMHl4qc4bM0@proxy.123proxy.cn:36927"
    # https_proxy = "http://u1952150085001297:sJMHl4qc4bM0@proxy.123proxy.cn:36927"
    http_proxy = "http://u1952150085001297:sJMHl4qc4bM0@proxy.123proxy.cn:36931"
    https_proxy = "http://u1952150085001297:sJMHl4qc4bM0@proxy.123proxy.cn:36931"

    try:
        proxySettings = {
            "http": http_proxy,
            "https": https_proxy,
        }
        return proxySettings
    except Exception as e:
        log.error(f"Error getting proxy: {e}")
        raise e


@retry(stop=stop_after_attempt(10), wait=wait_fixed(3), after=after_log)
def get_detail_data(log, category, link, sql_pool):
    """
    获取详情数据
    :param log:
    :param category:
    :param link:
    :param sql_pool:
    """
    try:
        with requests.Session() as session:
            resp = session.get(link, impersonate=random.choice(client_identifier_list), headers=headers,
                               proxies=get_proxys(log), timeout=22, allow_redirects=False)
            # log.debug(resp.text)

        if 'Just a moment' in resp.text:
            log.debug('Just a moment , retrying.....')
            raise Exception('Just a moment')

        resp_selector = Selector(text=resp.text)
        tag_td_list = resp_selector.xpath(
            '//*[@id="tableCategory"]/thead/tr/td[@class="text-right"]/text() | //*[@id="tableBats"]/thead/tr/td[@class="text-right"]/text()').getall()

        if tag_td_list:
            # number_of_sets = tag_td_list[0]
            # total_items = tag_td_list[1]
            total_graded = tag_td_list[-1]
            if total_graded:
                total_graded = total_graded.replace(',', '')
            log.debug(f"Total Graded: {total_graded}")
            data_dict = {
                "category": category,
                "category_link": link,
                "total_graded": total_graded,
                "crawl_date": time.strftime("%Y-%m-%d", time.localtime())
            }
            # print(data_dict)
            try:
                sql_pool.insert_one_or_dict(table="psa_pop_record", data=data_dict, ignore=True)
            except Exception as e1:
                log.error(f"Error inserting data: {e1}")

    except Exception as e:
        log.error(f"Error getting detail data: {e}")
        raise e


@retry(stop=stop_after_attempt(10), wait=wait_fixed(3), after=after_log)
def get_pop_data(log, sql_pool):
    """
    获取 pop 列表页数据
    :param log:
    :param sql_pool:
    """
    url = "https://www.psacard.com/pop"
    try:
        with requests.Session() as session:
            resp = session.get(url, impersonate=random.choice(client_identifier_list), headers=headers,
                               proxies=get_proxys(log), timeout=22, allow_redirects=False)
            # log.debug(resp.text)

        if 'Just a moment' in resp.text:
            log.debug('Just a moment , retrying.....')
            raise Exception('Just a moment')

        resp_selector = Selector(text=resp.text)

        tag_a_list = resp_selector.xpath('//*[@id="mainContent"]/div[2]/div/a')
        for tag_a in tag_a_list:
            category = tag_a.xpath('./div/div/text()').get()
            category_link = tag_a.xpath('./@href').get()
            if 'https://' not in category_link:
                category_link = BASE_URL + category_link

            log.debug(f"Category: {category}, Link: {category_link}")

            try:
                get_detail_data(log, category, category_link, sql_pool)
            except Exception as e1:
                log.error(f"Error getting detail data: {e1}")

    except Exception as e:
        log.error(f"Error getting pop data: {e}")
        raise e


@retry(stop=stop_after_attempt(100), wait=wait_fixed(3600), after=after_log)
def pop_main(log):
    """
    主函数
    """
    log.info(
        f'开始运行 {inspect.currentframe().f_code.co_name} 爬虫任务....................................................')

    # 配置 MySQL 连接池
    sql_pool = MySQLConnectionPool(log=log)
    if not sql_pool.check_pool_health():
        log.error("数据库连接池异常")
        raise RuntimeError("数据库连接池异常")

    try:
        log.debug(".......... 开始获取数据报告 ..........")
        # get_pop_data(log, sql_pool)
        for _ in range(2):
            for category, category_link in category_link_list.items():
                log.debug(f"{category}第一次查询, 开始获取数据.......")
                try:
                    get_detail_data(log, category, category_link, sql_pool)
                except Exception as e1:
                    log.error(f"Error getting detail data: {e1}")

            time.sleep(5)

    except Exception as e:
        log.error(f'{inspect.currentframe().f_code.co_name} error: {e}')
    finally:
        log.info(f'爬虫程序 {inspect.currentframe().f_code.co_name} 运行结束,等待下一轮的采集任务............')


def schedule_task():
    """
    两个爬虫模块的启动文件
    bidding_main
    weika_change_card_by_id_spider
    change_card_main
    """
    # 立即运行一次任务
    pop_main(log=logger)

    # 设置定时任务
    schedule.every().day.at("05:00").do(pop_main, log=logger)

    while True:
        schedule.run_pending()
        time.sleep(1)


if __name__ == '__main__':
    pop_main(logger)
    # schedule_task()
    # get_detail_data(logger, '','https://www.psacard.com/pop/bats',None)