# # -*- coding: utf-8 -*- # # Author : Charley # # Python : 3.10.8 # # Date : 2025/9/15 14:53 # import inspect # import random # import time # import schedule # import user_agent # from curl_cffi import requests # from loguru import logger # from parsel import Selector # from mysql_pool import MySQLConnectionPool # from tenacity import retry, stop_after_attempt, wait_fixed # # logger.remove() # logger.add("logs/{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00", # format="[{time:YYYY-MM-DD HH:mm:ss.SSS}] {level} {message}", # level="DEBUG", retention="3 day") # # client_identifier_list = [ # "edge99", "edge101", "chrome99", "chrome100", "chrome101", "chrome104", "chrome107", # "chrome110", "chrome116", "chrome119", "chrome120", "chrome123", "chrome124", # "chrome99_android", "safari15_3", "safari15_5", "safari17_0", "safari17_2_ios" # ] # # BASE_URL = 'https://www.psacard.com' # # headers = { # "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", # 'User-Agent': user_agent.generate_user_agent(), # "Cookie": "QuantumMetricUserID=22c3803bf07dbc25f851ff507866df96; psa-locale=en-US; osano_consentmanager_uuid=42a9bb07-9acc-4ef6-9d02-bc32ffdf1fff; osano_consentmanager=lGHfgSS0ydA-ImkuudYFlb5aEWtBTq5wJAH3DT-hDQsVV1_nHBaT0m6s9cx83OXlJnvHbPIBSg838CBSaf3vgGEYWKfxt2i39N64C_s__RcE8ytvj42bK586lLGEDewsKh9Y1y4IVfgoOrAM0YtmMWDcGZ2KHUypn3K4g7mL57USueK19vGEDz7nUfcVo_mFofK3Jmw9BnNrTTbD4mKI85Dz-nCiC1DlzRn9-_88x0U3gb2RsWmIID076x7TJ8VgHICQQ8zj3VdML4yLpeUkfN6lfGEbTiFqvmzTFYqo8A4rfpHKfDZd18FjTtngIDnzMK3uAtf3ABA=; _tt_enable_cookie=1; _ttp=01KE8YM1RMWGA3H8W289ASMXY1_.tt.1; _scid=xSwAZLOM2z1UVzbeMSa7SdoxzOdRpCXC; sa-user-id=s%253A0-d546cca6-494b-5861-5610-008c7a982faf.C1TY4QSpzfnyfWTcMQHcwLs%252BsKPZ%252F7EjyWkQQC5esLU; sa-user-id-v2=s%253A1UbMpklLWGFWEACMepgvr8uvDiA.X7BM3pvlugPl4Ibhtnz4Wfz1gE0%252FBH%252FS0QcjBVi3gzU; sa-user-id-v3=s%253AAQAKIJBI0vf1ZJkqIoEdlaHb5fDvZhDpA19TppSKMX8vvYMcEAEYAyDozPLKBjABOgRi0YmEQgSnRVhY.YOIWenkOAJAaMUIYT0ChxjNb7XdupE9HuIDDE5Qh%252Bt8; psa-apr-recentsearch=%7B%22recentSearches%22%3A%5B%7B%22searchId%22%3A%220-Pokemon-Coin%20Promo%22%2C%22catId%22%3A%220%22%2C%22catName%22%3A%22%22%2C%22search%22%3A%22Pokemon%20Coin%20Promo%22%7D%5D%7D; _gcl_au=1.1.1354962831.1781235727; _uutmz=source=google|campaign=(organic)|medium=organic|keyword=(not%20provided); _uutmzh=0.4~2026-01-06T14%3A06%3A21.822%2B-8~2026-06-12T11%3A42%3A07.359%2B-8~%3Bnone%3Adirect%3Bo%3Ag%3B; sa-user-id-v4=s%253A.o6W7wkJsHSTU4%252BLlDruZ%252FwNjVcUZZMvakQpSatDoAgo; _gid=GA1.2.15976690.1781235897; sa-r-source=app.collectors.com; _ga_HD10604KCZ=GS2.1.s1781254798$o2$g1$t1781254913$j22$l0$h0; _hp2_id.1106280392=%7B%22userId%22%3A%224865765352100683%22%2C%22pageviewId%22%3A%227549583361365791%22%2C%22sessionId%22%3A%225840239242373286%22%2C%22identity%22%3A%22103751284%22%2C%22trackerVersion%22%3A%224.0%22%2C%22identityField%22%3Anull%2C%22isIdentified%22%3A1%7D; QuantumMetricSessionID=7bd235423a4a9cb3c9c4c7137935da54; env=prod; accessToken=eyJhbGciOiJSUzI1NiIsImtpZCI6IlNLMnhqZ1lQVERhSkVPV2FGNmtKd2tiZzJsWkVKIiwidHlwIjoiSldUIn0.eyJDb2xsZWN0b3JzSWQiOjEwMzc1MTI4NCwiTmF2aXNpb25JZCI6IjUzODQ1MTkiLCJhbXIiOlsicHdkIl0sImF1ZCI6WyJVREo0YW1kWlNqQlpkakJVUlZkSlMyaENZV2wyV1cxWmJIRmpVVHBVVUVFek4wWjZRbmRaWlU5bFJFc3hVVU4xZDI1RlpFRTJWMmhEVFZVPSIsIlAyeGpnWUowWXYwVEVXSUtoQmFpdlltWWxxY1EiXSwiYXpwIjoiVURKNGFtZFpTakJaZGpCVVJWZEpTMmhDWVdsMldXMVpiSEZqVVRwVVVFRXpOMFo2UW5kWlpVOWxSRXN4VVVOMWQyNUZaRUUyVjJoRFRWVT0iLCJjaWQiOiJVREo0YW1kWlNqQlpkakJVUlZkSlMyaENZV2wyV1cxWmJIRmpVVHBVVUVFek4wWjZRbmRaWlU5bFJFc3hVVU4xZDI1RlpFRTJWMmhEVFZVPSIsImRjaSI6IjNGMUh3TTZqellxUklFcTYxamZaZUYxRmlacCIsImRzdWIiOiJVMzVjUDdYZjZibFhvaTlCRXZMQmhwbjdSOVN6IiwiZXhwIjoxNzgxMzQ2MTY3LCJpYXQiOjE3ODEyNTk3NjcsImlzcyI6Imh0dHBzOi8vYXV0aC5jb2xsZWN0b3JzLmNvbS92MS9hcHBzL1AyeGpnWUowWXYwVEVXSUtoQmFpdlltWWxxY1EiLCJqdGkiOiJMSjNGMjRLaGUzeDFpcDlHbVQzZWFYOXltaVRBUyIsInJleHAiOiIyMDM2LTA2LTExVDEwOjIyOjQ3WiIsInNjb3BlIjoiY3VzdG9tZXIgb2ZmbGluZV9hY2Nlc3MiLCJzY3AiOlsib2ZmbGluZV9hY2Nlc3MiLCJjdXN0b21lciIsIm9wZW5pZCJdLCJzdWIiOiJjaGFybGV5X2xlb0AxNjMuY29tIiwidG9rZW5fdHlwZSI6ImFjY2Vzc190b2tlbiJ9.R1y5QrMPDBarJbnA6feNxf30EZCM9heoRDOXof3I35jN2lULtcdtr9DU1il9nOiwaI2LI_vupXyXe8ppKbIV9-LczVS9OSCgFCUCgLdXOUtq_4xNbfDhgO44mNjG5MxmNNae86NxhgcrwrtM3Y_9ck_n7-Fq2Es5AA16xrHY2Fk3gTX5_zlt70A0q7BWhx0wkdNZzUxjSOdS4iIAJIDyOkY4lGjsSZT38k-dFsWGqJ2uIhw81N-5Dym7q3WDUwp-3ekacBAwJcYhkUjKuy0eFaVFlQFbkycqZcYEw9UVdKFLj1ItgkATIwmcUTOrQVY8PGUn9uKH1OpmKo7kMiAVTQ; refreshToken=eyJhbGciOiJSUzI1NiIsImtpZCI6IlNLMnhqZ1lQVERhSkVPV2FGNmtKd2tiZzJsWkVKIiwidHlwIjoiSldUIn0.eyJDb2xsZWN0b3JzSWQiOjEwMzc1MTI4NCwiTmF2aXNpb25JZCI6IjUzODQ1MTkiLCJhbXIiOlsicHdkIl0sImF1ZCI6WyJVREo0YW1kWlNqQlpkakJVUlZkSlMyaENZV2wyV1cxWmJIRmpVVHBVVUVFek4wWjZRbmRaWlU5bFJFc3hVVU4xZDI1RlpFRTJWMmhEVFZVPSIsIlAyeGpnWUowWXYwVEVXSUtoQmFpdlltWWxxY1EiXSwiYXpwIjoiVURKNGFtZFpTakJaZGpCVVJWZEpTMmhDWVdsMldXMVpiSEZqVVRwVVVFRXpOMFo2UW5kWlpVOWxSRXN4VVVOMWQyNUZaRUUyVjJoRFRWVT0iLCJjaWQiOiJVREo0YW1kWlNqQlpkakJVUlZkSlMyaENZV2wyV1cxWmJIRmpVVHBVVUVFek4wWjZRbmRaWlU5bFJFc3hVVU4xZDI1RlpFRTJWMmhEVFZVPSIsImRjaSI6IjNGMUh3TTZqellxUklFcTYxamZaZUYxRmlacCIsImRjbCI6WyJDb2xsZWN0b3JzSWQiLCJOYXZpc2lvbklkIiwic2NwIiwic3ViIl0sImRzdWIiOiJVMzVjUDdYZjZibFhvaTlCRXZMQmhwbjdSOVN6IiwiZHYiOjEsImV4cCI6MjA5Njc5MjU2NywiaWF0IjoxNzgxMjU5NzY3LCJpc3MiOiJodHRwczovL2F1dGguY29sbGVjdG9ycy5jb20vdjEvYXBwcy9QMnhqZ1lKMFl2MFRFV0lLaEJhaXZZbVlscWNRIiwianRpIjoiTEozRjI0S2o1V3BmaVpDaWJRSkR3TVFYeFhFcjYiLCJzY29wZSI6ImN1c3RvbWVyIG9mZmxpbmVfYWNjZXNzIiwic2NwIjpbIm9mZmxpbmVfYWNjZXNzIiwiY3VzdG9tZXIiLCJvcGVuaWQiXSwic3ViIjoiY2hhcmxleV9sZW9AMTYzLmNvbSIsInRva2VuX3R5cGUiOiJyZWZyZXNoX3Rva2VuIn0.dV3QGtrD_vqADjAETDlMnOchwaMSh8gtwPKsFkYJSVt0xapHZiLI_VcDlCbx_nnw2JsKA0iH8rx6vNNLIcrmhKwnQ-UNiqU1IBuNE28aCnngx4HDXxLMbm5-3ZRwKWFbSSaP5LEl8sZqBxigbCzdwrHnuVKAwgWmv6Pw-AmDWWiMKXwEqNT_tRwsFrK7WnxORv_fzaaJIZ8bVQc4JqbdbZBcKcGU1nnHS6o7OUxBKjg0YyeH3ssMbjsncBV2cxZzQVtRwFwPWiGz_316S9fnljmI1jJsiy5Cu9jeZekPWG6IPX1ZqQ3A7H2NFb6ONG7Vm4MhfxUa3BIuEJ3y2U5JIg; CMGUID=645f97f3-6d98-47db-8bff-baaf8415d51e; cf_clearance=cVaqwdjxY3nGeU0OzOhGwgyYkeETvcDsS407_vcnlDU-1781259770-1.2.1.1-JsTu2GnznqiUjoNKsZ0rqFKbBGBtkWA2CM5ybV1.8XNxtkFFtaGkC6kGHnJnMtKddQFyknAGmuX6aOgsgWXEWi0ibmKP5N6vRUCqprmuitNS4hdfrPTihnG_12EEReqVgsVcGjL5gkvbCUbepbbMmDELyrQQnuj66itIFFvpzwnoOo3V8Nw2HaQ09kvbtAyP7EMjLMMgfYkugvaB4VopFzip4xEnkNvcXrO8oL9U7W0aX8qXx.e68BaxApriDtdefKRUDoMnhFsmHvKUQytveD8R7U_a6K2wjmjvC.7ornBRn5D0mrYqmL1vcdsxxzzQBb8el809ktpts4W8mAmPt8J6vkXY4ZIBY0FfqF928LLjKr2L.5gEOh3nvY_WMw1WfRlzrNpftLAAc30pxHXpYgZYAFVIAUl64OCreS4ZgA8; AWSALBTG=NdS+ZrDWCMJQfszb9918wgArOZZFDAxMNsjJ81v7H+hLWa4ryC8KKiNpYLQamQgXi6+eF2ymYxtam3f0Xt6dBpTJrA8oHd2wMfvM9jkWT2NBaHeYqZztLKzg8nRv40jDy5BMW4mC91vcSqv/TkS9Nr3OzcsekoW/ZA+d6YYjfXBY; AWSALBTGCORS=NdS+ZrDWCMJQfszb9918wgArOZZFDAxMNsjJ81v7H+hLWa4ryC8KKiNpYLQamQgXi6+eF2ymYxtam3f0Xt6dBpTJrA8oHd2wMfvM9jkWT2NBaHeYqZztLKzg8nRv40jDy5BMW4mC91vcSqv/TkS9Nr3OzcsekoW/ZA+d6YYjfXBY; AWSALB=L1FiNF7SKbJoBNGHNI19ReDFH8Ji/8/pnei/a/JYVJTZAeUZZv99lK0VEtwpMlmj/BHZYbikaEVSro06u142T+lx8X3/FyUELIeDHbG27JT0PpcwJPlRBQH+j4ti; AWSALBCORS=L1FiNF7SKbJoBNGHNI19ReDFH8Ji/8/pnei/a/JYVJTZAeUZZv99lK0VEtwpMlmj/BHZYbikaEVSro06u142T+lx8X3/FyUELIeDHbG27JT0PpcwJPlRBQH+j4ti; _ga_5YHEXX56Q3=GS2.1.s1781259769$o16$g0$t1781259774$j55$l0$h0; __cf_bm=8OcNvWr3asBXt26hgXf46a6w2UsvoaUHbC2rwoJfZN0-1781259775.5736341-1.0.1.1-M7tQFsK0OURujJxMk8S33cnsJta2pG39ZK5aO2h3upBKHc_QKRsWn64lCgYNI4tDY68eyyBiEvLMqveVCWY0mnCkMDsFGK6.2dF0IAss9d6UH_tHaKG9z7ADzhBEDN4kFA.XAhja0o2VRlg7vP.Y1w; _rdt_uuid=1767679591675.3ca088ed-2201-4dd0-960d-54ce76c1c454; _dc_gtm_UA-3791435-10=1; _ga=GA1.1.1620777947.1767679582; _ga_GGS8NWPYE2=GS2.1.s1781259751$o27$g1$t1781259774$j37$l0$h0; _uetsid=afafc940661011f1a11f296e502f7168; _uetvid=d91d21f0eac511f0b640d36deb73db38; _ga_1QVXQ1V575=GS2.1.s1781235728$o14$g1$t1781259774$j37$l1$h501104539; _ga_LBYZ0MN5NP=GS2.1.s1781259751$o27$g1$t1781259774$j37$l0$h0; _scid_r=xKwAZLOM2z1UVzbeMSa7SdoxzOdRpCXC_B7I6A; _ga_8C5B608QLW=GS2.2.s1781259775$o15$g0$t1781259775$j60$l0$h1025460194" # # } # # category_link_list = {'Baseball Cards': 'https://www.psacard.com/pop/baseball-cards/20003', # 'Baseball Coins': 'https://www.psacard.com/pop/baseball-coins/82797', # 'Basketball Cards': 'https://www.psacard.com/pop/basketball-cards/20019', # 'Basketball Coins': 'https://www.psacard.com/pop/basketball-coins/83007', # 'Boxing / Wrestling Cards / MMA': 'https://www.psacard.com/pop/boxing-wrestling-cards-mma/20021', # 'Football Cards': 'https://www.psacard.com/pop/football-cards/20014', # 'Football Coins': 'https://www.psacard.com/pop/football-coins/83011', # 'Golf Cards': 'https://www.psacard.com/pop/golf-cards/20023', # 'Hockey Cards': 'https://www.psacard.com/pop/hockey-cards/20020', # 'Hockey Coins': 'https://www.psacard.com/pop/hockey-coins/83012', # 'Minor League Cards': 'https://www.psacard.com/pop/minor-league-cards/20031', # 'Misc Cards': 'https://www.psacard.com/pop/misc-cards/20033', # 'Multi-Sport Cards': 'https://www.psacard.com/pop/multi-sport-cards/20006', # 'Multi-Sport Coins': 'https://www.psacard.com/pop/multi-sport-coins/102825', # 'Non-Sport Cards': 'https://www.psacard.com/pop/non-sport-cards/20032', # 'Non-Sport Coins': 'https://www.psacard.com/pop/non-sport-coins/82981', # 'Packs': 'https://www.psacard.com/pop/packs/20017', # 'Pins': 'https://www.psacard.com/pop/pins/20013', # 'Soccer Cards': 'https://www.psacard.com/pop/soccer-cards/20004', # 'TCG Cards': 'https://www.psacard.com/pop/tcg-cards/156940', # 'Tickets': 'https://www.psacard.com/pop/tickets/20022', # 'Game-Used Bats': 'https://www.psacard.com/pop/bats'} # # # def after_log(retry_state): # """ # retry 回调 # :param retry_state: RetryCallState 对象 # """ # # 检查 args 是否存在且不为空 # if retry_state.args and len(retry_state.args) > 0: # log = retry_state.args[0] # 获取传入的 logger # else: # log = logger # 使用全局 logger # # if retry_state.outcome.failed: # log.warning( # f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} Times") # else: # log.info(f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} succeeded") # # # @retry(stop=stop_after_attempt(5), wait=wait_fixed(2), after=after_log) # def get_proxys(log): # # 已购买账户 北美 # # http_proxy = "http://u1952150085001297:sJMHl4qc4bM0@proxy.123proxy.cn:36927" # # https_proxy = "http://u1952150085001297:sJMHl4qc4bM0@proxy.123proxy.cn:36927" # http_proxy = "http://u1952150085001297:sJMHl4qc4bM0@proxy.123proxy.cn:36931" # https_proxy = "http://u1952150085001297:sJMHl4qc4bM0@proxy.123proxy.cn:36931" # # try: # proxySettings = { # "http": http_proxy, # "https": https_proxy, # } # return proxySettings # except Exception as e: # log.error(f"Error getting proxy: {e}") # raise e # # # @retry(stop=stop_after_attempt(10), wait=wait_fixed(3), after=after_log) # def get_detail_data(log, category, link, sql_pool): # """ # 获取详情数据 # :param log: # :param category: # :param link: # :param sql_pool: # """ # try: # with requests.Session() as session: # resp = session.get(link, impersonate=random.choice(client_identifier_list), headers=headers, # proxies=get_proxys(log), timeout=22, allow_redirects=False) # # log.debug(resp.text) # # if 'Just a moment' in resp.text: # log.debug('Just a moment , retrying.....') # raise Exception('Just a moment') # # resp_selector = Selector(text=resp.text) # tag_td_list = resp_selector.xpath( # '//*[@id="tableCategory"]/thead/tr/td[@class="text-right"]/text() | //*[@id="tableBats"]/thead/tr/td[@class="text-right"]/text()').getall() # # if tag_td_list: # # number_of_sets = tag_td_list[0] # # total_items = tag_td_list[1] # total_graded = tag_td_list[-1] # if total_graded: # total_graded = total_graded.replace(',', '') # log.debug(f"Total Graded: {total_graded}") # data_dict = { # "category": category, # "category_link": link, # "total_graded": total_graded, # "crawl_date": time.strftime("%Y-%m-%d", time.localtime()) # } # # print(data_dict) # try: # sql_pool.insert_one_or_dict(table="psa_pop_record", data=data_dict, ignore=True) # except Exception as e1: # log.error(f"Error inserting data: {e1}") # # except Exception as e: # log.error(f"Error getting detail data: {e}") # raise e # # # @retry(stop=stop_after_attempt(10), wait=wait_fixed(3), after=after_log) # def get_pop_data(log, sql_pool): # """ # 获取 pop 列表页数据 # :param log: # :param sql_pool: # """ # url = "https://www.psacard.com/pop" # try: # with requests.Session() as session: # resp = session.get(url, impersonate=random.choice(client_identifier_list), headers=headers, # proxies=get_proxys(log), timeout=22, allow_redirects=False) # # log.debug(resp.text) # # if 'Just a moment' in resp.text: # log.debug('Just a moment , retrying.....') # raise Exception('Just a moment') # # resp_selector = Selector(text=resp.text) # # tag_a_list = resp_selector.xpath('//*[@id="mainContent"]/div[2]/div/a') # for tag_a in tag_a_list: # category = tag_a.xpath('./div/div/text()').get() # category_link = tag_a.xpath('./@href').get() # if 'https://' not in category_link: # category_link = BASE_URL + category_link # # log.debug(f"Category: {category}, Link: {category_link}") # # try: # get_detail_data(log, category, category_link, sql_pool) # except Exception as e1: # log.error(f"Error getting detail data: {e1}") # # except Exception as e: # log.error(f"Error getting pop data: {e}") # raise e # # # @retry(stop=stop_after_attempt(100), wait=wait_fixed(3600), after=after_log) # def pop_main(log): # """ # 主函数 # """ # log.info( # f'开始运行 {inspect.currentframe().f_code.co_name} 爬虫任务....................................................') # # # 配置 MySQL 连接池 # sql_pool = MySQLConnectionPool(log=log) # if not sql_pool.check_pool_health(): # log.error("数据库连接池异常") # raise RuntimeError("数据库连接池异常") # # try: # log.debug(".......... 开始获取数据报告 ..........") # # get_pop_data(log, sql_pool) # for _ in range(2): # for category, category_link in category_link_list.items(): # log.debug(f"{category}第一次查询, 开始获取数据.......") # try: # get_detail_data(log, category, category_link, sql_pool) # except Exception as e1: # log.error(f"Error getting detail data: {e1}") # # time.sleep(5) # # except Exception as e: # log.error(f'{inspect.currentframe().f_code.co_name} error: {e}') # finally: # log.info(f'爬虫程序 {inspect.currentframe().f_code.co_name} 运行结束,等待下一轮的采集任务............') # # # def schedule_task(): # """ # 两个爬虫模块的启动文件 # bidding_main # weika_change_card_by_id_spider # change_card_main # """ # # 立即运行一次任务 # pop_main(log=logger) # # # 设置定时任务 # schedule.every().day.at("05:00").do(pop_main, log=logger) # # while True: # schedule.run_pending() # time.sleep(1) # # # if __name__ == '__main__': # pop_main(logger) # # schedule_task() # # get_detail_data(logger, '','https://www.psacard.com/pop/bats',None) """ psa 网站变化 pop任务 改为本地运行 """