Startsida Utforska Hjälp
Logga in
python_spider
/
spider_work
3
0
Fork 0
Filer Ärenden 0 Pull-förfrågningar 0 Wiki
Träd: dfb499fdc1
Grenar Taggar
spider_work
/
psa_spider
/
psa_pop_spider.py

psa_pop_spider.py 16 KB
Historik Rå

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240 
  1. # # -*- coding: utf-8 -*-
    2. 

  2. # # Author : Charley
    3. 

  3. # # Python : 3.10.8
    4. 

  4. # # Date   : 2025/9/15 14:53
    5. 

  5. # import inspect
    6. 

  6. # import random
    7. 

  7. # import time
    8. 

  8. # import schedule
    9. 

  9. # import user_agent
    10. 

  10. # from curl_cffi import requests
    11. 

  11. # from loguru import logger
    12. 

  12. # from parsel import Selector
    13. 

  13. # from mysql_pool import MySQLConnectionPool
    14. 

  14. # from tenacity import retry, stop_after_attempt, wait_fixed
    15. 

  15. #
    16. 

  16. # logger.remove()
    17. 

  17. # logger.add("logs/{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00",
    18. 

  18. #            format="[{time:YYYY-MM-DD HH:mm:ss.SSS}] {level} {message}",
    19. 

  19. #            level="DEBUG", retention="3 day")
    20. 

  20. #
    21. 

  21. # client_identifier_list = [
    22. 

  22. #     "edge99", "edge101", "chrome99", "chrome100", "chrome101", "chrome104", "chrome107",
    23. 

  23. #     "chrome110", "chrome116", "chrome119", "chrome120", "chrome123", "chrome124",
    24. 

  24. #     "chrome99_android", "safari15_3", "safari15_5", "safari17_0", "safari17_2_ios"
    25. 

  25. # ]
    26. 

  26. #
    27. 

  27. # BASE_URL = 'https://www.psacard.com'
    28. 

  28. #
    29. 

  29. # headers = {
    30. 

  30. #     "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
    31. 

  31. #     'User-Agent': user_agent.generate_user_agent(),
    32. 

  32. #     "Cookie": "QuantumMetricUserID=22c3803bf07dbc25f851ff507866df96; psa-locale=en-US; osano_consentmanager_uuid=42a9bb07-9acc-4ef6-9d02-bc32ffdf1fff; osano_consentmanager=lGHfgSS0ydA-ImkuudYFlb5aEWtBTq5wJAH3DT-hDQsVV1_nHBaT0m6s9cx83OXlJnvHbPIBSg838CBSaf3vgGEYWKfxt2i39N64C_s__RcE8ytvj42bK586lLGEDewsKh9Y1y4IVfgoOrAM0YtmMWDcGZ2KHUypn3K4g7mL57USueK19vGEDz7nUfcVo_mFofK3Jmw9BnNrTTbD4mKI85Dz-nCiC1DlzRn9-_88x0U3gb2RsWmIID076x7TJ8VgHICQQ8zj3VdML4yLpeUkfN6lfGEbTiFqvmzTFYqo8A4rfpHKfDZd18FjTtngIDnzMK3uAtf3ABA=; _tt_enable_cookie=1; _ttp=01KE8YM1RMWGA3H8W289ASMXY1_.tt.1; _scid=xSwAZLOM2z1UVzbeMSa7SdoxzOdRpCXC; sa-user-id=s%253A0-d546cca6-494b-5861-5610-008c7a982faf.C1TY4QSpzfnyfWTcMQHcwLs%252BsKPZ%252F7EjyWkQQC5esLU; sa-user-id-v2=s%253A1UbMpklLWGFWEACMepgvr8uvDiA.X7BM3pvlugPl4Ibhtnz4Wfz1gE0%252FBH%252FS0QcjBVi3gzU; sa-user-id-v3=s%253AAQAKIJBI0vf1ZJkqIoEdlaHb5fDvZhDpA19TppSKMX8vvYMcEAEYAyDozPLKBjABOgRi0YmEQgSnRVhY.YOIWenkOAJAaMUIYT0ChxjNb7XdupE9HuIDDE5Qh%252Bt8; psa-apr-recentsearch=%7B%22recentSearches%22%3A%5B%7B%22searchId%22%3A%220-Pokemon-Coin%20Promo%22%2C%22catId%22%3A%220%22%2C%22catName%22%3A%22%22%2C%22search%22%3A%22Pokemon%20Coin%20Promo%22%7D%5D%7D; _gcl_au=1.1.1354962831.1781235727; _uutmz=source=google|campaign=(organic)|medium=organic|keyword=(not%20provided); _uutmzh=0.4~2026-01-06T14%3A06%3A21.822%2B-8~2026-06-12T11%3A42%3A07.359%2B-8~%3Bnone%3Adirect%3Bo%3Ag%3B; sa-user-id-v4=s%253A.o6W7wkJsHSTU4%252BLlDruZ%252FwNjVcUZZMvakQpSatDoAgo; _gid=GA1.2.15976690.1781235897; sa-r-source=app.collectors.com; _ga_HD10604KCZ=GS2.1.s1781254798$o2$g1$t1781254913$j22$l0$h0; _hp2_id.1106280392=%7B%22userId%22%3A%224865765352100683%22%2C%22pageviewId%22%3A%227549583361365791%22%2C%22sessionId%22%3A%225840239242373286%22%2C%22identity%22%3A%22103751284%22%2C%22trackerVersion%22%3A%224.0%22%2C%22identityField%22%3Anull%2C%22isIdentified%22%3A1%7D; QuantumMetricSessionID=7bd235423a4a9cb3c9c4c7137935da54; env=prod; accessToken=eyJhbGciOiJSUzI1NiIsImtpZCI6IlNLMnhqZ1lQVERhSkVPV2FGNmtKd2tiZzJsWkVKIiwidHlwIjoiSldUIn0.eyJDb2xsZWN0b3JzSWQiOjEwMzc1MTI4NCwiTmF2aXNpb25JZCI6IjUzODQ1MTkiLCJhbXIiOlsicHdkIl0sImF1ZCI6WyJVREo0YW1kWlNqQlpkakJVUlZkSlMyaENZV2wyV1cxWmJIRmpVVHBVVUVFek4wWjZRbmRaWlU5bFJFc3hVVU4xZDI1RlpFRTJWMmhEVFZVPSIsIlAyeGpnWUowWXYwVEVXSUtoQmFpdlltWWxxY1EiXSwiYXpwIjoiVURKNGFtZFpTakJaZGpCVVJWZEpTMmhDWVdsMldXMVpiSEZqVVRwVVVFRXpOMFo2UW5kWlpVOWxSRXN4VVVOMWQyNUZaRUUyVjJoRFRWVT0iLCJjaWQiOiJVREo0YW1kWlNqQlpkakJVUlZkSlMyaENZV2wyV1cxWmJIRmpVVHBVVUVFek4wWjZRbmRaWlU5bFJFc3hVVU4xZDI1RlpFRTJWMmhEVFZVPSIsImRjaSI6IjNGMUh3TTZqellxUklFcTYxamZaZUYxRmlacCIsImRzdWIiOiJVMzVjUDdYZjZibFhvaTlCRXZMQmhwbjdSOVN6IiwiZXhwIjoxNzgxMzQ2MTY3LCJpYXQiOjE3ODEyNTk3NjcsImlzcyI6Imh0dHBzOi8vYXV0aC5jb2xsZWN0b3JzLmNvbS92MS9hcHBzL1AyeGpnWUowWXYwVEVXSUtoQmFpdlltWWxxY1EiLCJqdGkiOiJMSjNGMjRLaGUzeDFpcDlHbVQzZWFYOXltaVRBUyIsInJleHAiOiIyMDM2LTA2LTExVDEwOjIyOjQ3WiIsInNjb3BlIjoiY3VzdG9tZXIgb2ZmbGluZV9hY2Nlc3MiLCJzY3AiOlsib2ZmbGluZV9hY2Nlc3MiLCJjdXN0b21lciIsIm9wZW5pZCJdLCJzdWIiOiJjaGFybGV5X2xlb0AxNjMuY29tIiwidG9rZW5fdHlwZSI6ImFjY2Vzc190b2tlbiJ9.R1y5QrMPDBarJbnA6feNxf30EZCM9heoRDOXof3I35jN2lULtcdtr9DU1il9nOiwaI2LI_vupXyXe8ppKbIV9-LczVS9OSCgFCUCgLdXOUtq_4xNbfDhgO44mNjG5MxmNNae86NxhgcrwrtM3Y_9ck_n7-Fq2Es5AA16xrHY2Fk3gTX5_zlt70A0q7BWhx0wkdNZzUxjSOdS4iIAJIDyOkY4lGjsSZT38k-dFsWGqJ2uIhw81N-5Dym7q3WDUwp-3ekacBAwJcYhkUjKuy0eFaVFlQFbkycqZcYEw9UVdKFLj1ItgkATIwmcUTOrQVY8PGUn9uKH1OpmKo7kMiAVTQ; refreshToken=eyJhbGciOiJSUzI1NiIsImtpZCI6IlNLMnhqZ1lQVERhSkVPV2FGNmtKd2tiZzJsWkVKIiwidHlwIjoiSldUIn0.eyJDb2xsZWN0b3JzSWQiOjEwMzc1MTI4NCwiTmF2aXNpb25JZCI6IjUzODQ1MTkiLCJhbXIiOlsicHdkIl0sImF1ZCI6WyJVREo0YW1kWlNqQlpkakJVUlZkSlMyaENZV2wyV1cxWmJIRmpVVHBVVUVFek4wWjZRbmRaWlU5bFJFc3hVVU4xZDI1RlpFRTJWMmhEVFZVPSIsIlAyeGpnWUowWXYwVEVXSUtoQmFpdlltWWxxY1EiXSwiYXpwIjoiVURKNGFtZFpTakJaZGpCVVJWZEpTMmhDWVdsMldXMVpiSEZqVVRwVVVFRXpOMFo2UW5kWlpVOWxSRXN4VVVOMWQyNUZaRUUyVjJoRFRWVT0iLCJjaWQiOiJVREo0YW1kWlNqQlpkakJVUlZkSlMyaENZV2wyV1cxWmJIRmpVVHBVVUVFek4wWjZRbmRaWlU5bFJFc3hVVU4xZDI1RlpFRTJWMmhEVFZVPSIsImRjaSI6IjNGMUh3TTZqellxUklFcTYxamZaZUYxRmlacCIsImRjbCI6WyJDb2xsZWN0b3JzSWQiLCJOYXZpc2lvbklkIiwic2NwIiwic3ViIl0sImRzdWIiOiJVMzVjUDdYZjZibFhvaTlCRXZMQmhwbjdSOVN6IiwiZHYiOjEsImV4cCI6MjA5Njc5MjU2NywiaWF0IjoxNzgxMjU5NzY3LCJpc3MiOiJodHRwczovL2F1dGguY29sbGVjdG9ycy5jb20vdjEvYXBwcy9QMnhqZ1lKMFl2MFRFV0lLaEJhaXZZbVlscWNRIiwianRpIjoiTEozRjI0S2o1V3BmaVpDaWJRSkR3TVFYeFhFcjYiLCJzY29wZSI6ImN1c3RvbWVyIG9mZmxpbmVfYWNjZXNzIiwic2NwIjpbIm9mZmxpbmVfYWNjZXNzIiwiY3VzdG9tZXIiLCJvcGVuaWQiXSwic3ViIjoiY2hhcmxleV9sZW9AMTYzLmNvbSIsInRva2VuX3R5cGUiOiJyZWZyZXNoX3Rva2VuIn0.dV3QGtrD_vqADjAETDlMnOchwaMSh8gtwPKsFkYJSVt0xapHZiLI_VcDlCbx_nnw2JsKA0iH8rx6vNNLIcrmhKwnQ-UNiqU1IBuNE28aCnngx4HDXxLMbm5-3ZRwKWFbSSaP5LEl8sZqBxigbCzdwrHnuVKAwgWmv6Pw-AmDWWiMKXwEqNT_tRwsFrK7WnxORv_fzaaJIZ8bVQc4JqbdbZBcKcGU1nnHS6o7OUxBKjg0YyeH3ssMbjsncBV2cxZzQVtRwFwPWiGz_316S9fnljmI1jJsiy5Cu9jeZekPWG6IPX1ZqQ3A7H2NFb6ONG7Vm4MhfxUa3BIuEJ3y2U5JIg; CMGUID=645f97f3-6d98-47db-8bff-baaf8415d51e; cf_clearance=cVaqwdjxY3nGeU0OzOhGwgyYkeETvcDsS407_vcnlDU-1781259770-1.2.1.1-JsTu2GnznqiUjoNKsZ0rqFKbBGBtkWA2CM5ybV1.8XNxtkFFtaGkC6kGHnJnMtKddQFyknAGmuX6aOgsgWXEWi0ibmKP5N6vRUCqprmuitNS4hdfrPTihnG_12EEReqVgsVcGjL5gkvbCUbepbbMmDELyrQQnuj66itIFFvpzwnoOo3V8Nw2HaQ09kvbtAyP7EMjLMMgfYkugvaB4VopFzip4xEnkNvcXrO8oL9U7W0aX8qXx.e68BaxApriDtdefKRUDoMnhFsmHvKUQytveD8R7U_a6K2wjmjvC.7ornBRn5D0mrYqmL1vcdsxxzzQBb8el809ktpts4W8mAmPt8J6vkXY4ZIBY0FfqF928LLjKr2L.5gEOh3nvY_WMw1WfRlzrNpftLAAc30pxHXpYgZYAFVIAUl64OCreS4ZgA8; AWSALBTG=NdS+ZrDWCMJQfszb9918wgArOZZFDAxMNsjJ81v7H+hLWa4ryC8KKiNpYLQamQgXi6+eF2ymYxtam3f0Xt6dBpTJrA8oHd2wMfvM9jkWT2NBaHeYqZztLKzg8nRv40jDy5BMW4mC91vcSqv/TkS9Nr3OzcsekoW/ZA+d6YYjfXBY; AWSALBTGCORS=NdS+ZrDWCMJQfszb9918wgArOZZFDAxMNsjJ81v7H+hLWa4ryC8KKiNpYLQamQgXi6+eF2ymYxtam3f0Xt6dBpTJrA8oHd2wMfvM9jkWT2NBaHeYqZztLKzg8nRv40jDy5BMW4mC91vcSqv/TkS9Nr3OzcsekoW/ZA+d6YYjfXBY; AWSALB=L1FiNF7SKbJoBNGHNI19ReDFH8Ji/8/pnei/a/JYVJTZAeUZZv99lK0VEtwpMlmj/BHZYbikaEVSro06u142T+lx8X3/FyUELIeDHbG27JT0PpcwJPlRBQH+j4ti; AWSALBCORS=L1FiNF7SKbJoBNGHNI19ReDFH8Ji/8/pnei/a/JYVJTZAeUZZv99lK0VEtwpMlmj/BHZYbikaEVSro06u142T+lx8X3/FyUELIeDHbG27JT0PpcwJPlRBQH+j4ti; _ga_5YHEXX56Q3=GS2.1.s1781259769$o16$g0$t1781259774$j55$l0$h0; __cf_bm=8OcNvWr3asBXt26hgXf46a6w2UsvoaUHbC2rwoJfZN0-1781259775.5736341-1.0.1.1-M7tQFsK0OURujJxMk8S33cnsJta2pG39ZK5aO2h3upBKHc_QKRsWn64lCgYNI4tDY68eyyBiEvLMqveVCWY0mnCkMDsFGK6.2dF0IAss9d6UH_tHaKG9z7ADzhBEDN4kFA.XAhja0o2VRlg7vP.Y1w; _rdt_uuid=1767679591675.3ca088ed-2201-4dd0-960d-54ce76c1c454; _dc_gtm_UA-3791435-10=1; _ga=GA1.1.1620777947.1767679582; _ga_GGS8NWPYE2=GS2.1.s1781259751$o27$g1$t1781259774$j37$l0$h0; _uetsid=afafc940661011f1a11f296e502f7168; _uetvid=d91d21f0eac511f0b640d36deb73db38; _ga_1QVXQ1V575=GS2.1.s1781235728$o14$g1$t1781259774$j37$l1$h501104539; _ga_LBYZ0MN5NP=GS2.1.s1781259751$o27$g1$t1781259774$j37$l0$h0; _scid_r=xKwAZLOM2z1UVzbeMSa7SdoxzOdRpCXC_B7I6A; _ga_8C5B608QLW=GS2.2.s1781259775$o15$g0$t1781259775$j60$l0$h1025460194"
    33. 

  33. #
    34. 

  34. # }
    35. 

  35. #
    36. 

  36. # category_link_list = {'Baseball Cards': 'https://www.psacard.com/pop/baseball-cards/20003',
    37. 

  37. #                       'Baseball Coins': 'https://www.psacard.com/pop/baseball-coins/82797',
    38. 

  38. #                       'Basketball Cards': 'https://www.psacard.com/pop/basketball-cards/20019',
    39. 

  39. #                       'Basketball Coins': 'https://www.psacard.com/pop/basketball-coins/83007',
    40. 

  40. #                       'Boxing / Wrestling Cards / MMA': 'https://www.psacard.com/pop/boxing-wrestling-cards-mma/20021',
    41. 

  41. #                       'Football Cards': 'https://www.psacard.com/pop/football-cards/20014',
    42. 

  42. #                       'Football Coins': 'https://www.psacard.com/pop/football-coins/83011',
    43. 

  43. #                       'Golf Cards': 'https://www.psacard.com/pop/golf-cards/20023',
    44. 

  44. #                       'Hockey Cards': 'https://www.psacard.com/pop/hockey-cards/20020',
    45. 

  45. #                       'Hockey Coins': 'https://www.psacard.com/pop/hockey-coins/83012',
    46. 

  46. #                       'Minor League Cards': 'https://www.psacard.com/pop/minor-league-cards/20031',
    47. 

  47. #                       'Misc Cards': 'https://www.psacard.com/pop/misc-cards/20033',
    48. 

  48. #                       'Multi-Sport Cards': 'https://www.psacard.com/pop/multi-sport-cards/20006',
    49. 

  49. #                       'Multi-Sport Coins': 'https://www.psacard.com/pop/multi-sport-coins/102825',
    50. 

  50. #                       'Non-Sport Cards': 'https://www.psacard.com/pop/non-sport-cards/20032',
    51. 

  51. #                       'Non-Sport Coins': 'https://www.psacard.com/pop/non-sport-coins/82981',
    52. 

  52. #                       'Packs': 'https://www.psacard.com/pop/packs/20017',
    53. 

  53. #                       'Pins': 'https://www.psacard.com/pop/pins/20013',
    54. 

  54. #                       'Soccer Cards': 'https://www.psacard.com/pop/soccer-cards/20004',
    55. 

  55. #                       'TCG Cards': 'https://www.psacard.com/pop/tcg-cards/156940',
    56. 

  56. #                       'Tickets': 'https://www.psacard.com/pop/tickets/20022',
    57. 

  57. #                       'Game-Used Bats': 'https://www.psacard.com/pop/bats'}
    58. 

  58. #
    59. 

  59. #
    60. 

  60. # def after_log(retry_state):
    61. 

  61. #     """
    62. 

  62. #     retry 回调
    63. 

  63. #     :param retry_state: RetryCallState 对象
    64. 

  64. #     """
    65. 

  65. #     # 检查 args 是否存在且不为空
    66. 

  66. #     if retry_state.args and len(retry_state.args) > 0:
    67. 

  67. #         log = retry_state.args[0]  # 获取传入的 logger
    68. 

  68. #     else:
    69. 

  69. #         log = logger  # 使用全局 logger
    70. 

  70. #
    71. 

  71. #     if retry_state.outcome.failed:
    72. 

  72. #         log.warning(
    73. 

  73. #             f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} Times")
    74. 

  74. #     else:
    75. 

  75. #         log.info(f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} succeeded")
    76. 

  76. #
    77. 

  77. #
    78. 

  78. # @retry(stop=stop_after_attempt(5), wait=wait_fixed(2), after=after_log)
    79. 

  79. # def get_proxys(log):
    80. 

  80. #     # 已购买账户  北美
    81. 

  81. #     # http_proxy = "http://u1952150085001297:sJMHl4qc4bM0@proxy.123proxy.cn:36927"
    82. 

  82. #     # https_proxy = "http://u1952150085001297:sJMHl4qc4bM0@proxy.123proxy.cn:36927"
    83. 

  83. #     http_proxy = "http://u1952150085001297:sJMHl4qc4bM0@proxy.123proxy.cn:36931"
    84. 

  84. #     https_proxy = "http://u1952150085001297:sJMHl4qc4bM0@proxy.123proxy.cn:36931"
    85. 

  85. #
    86. 

  86. #     try:
    87. 

  87. #         proxySettings = {
    88. 

  88. #             "http": http_proxy,
    89. 

  89. #             "https": https_proxy,
    90. 

  90. #         }
    91. 

  91. #         return proxySettings
    92. 

  92. #     except Exception as e:
    93. 

  93. #         log.error(f"Error getting proxy: {e}")
    94. 

  94. #         raise e
    95. 

  95. #
    96. 

  96. #
    97. 

  97. # @retry(stop=stop_after_attempt(10), wait=wait_fixed(3), after=after_log)
    98. 

  98. # def get_detail_data(log, category, link, sql_pool):
    99. 

  99. #     """
    100. 

  100. #     获取详情数据
    101. 

  101. #     :param log:
    102. 

  102. #     :param category:
    103. 

  103. #     :param link:
    104. 

  104. #     :param sql_pool:
    105. 

  105. #     """
    106. 

  106. #     try:
    107. 

  107. #         with requests.Session() as session:
    108. 

  108. #             resp = session.get(link, impersonate=random.choice(client_identifier_list), headers=headers,
    109. 

  109. #                                proxies=get_proxys(log), timeout=22, allow_redirects=False)
    110. 

  110. #             # log.debug(resp.text)
    111. 

  111. #
    112. 

  112. #         if 'Just a moment' in resp.text:
    113. 

  113. #             log.debug('Just a moment , retrying.....')
    114. 

  114. #             raise Exception('Just a moment')
    115. 

  115. #
    116. 

  116. #         resp_selector = Selector(text=resp.text)
    117. 

  117. #         tag_td_list = resp_selector.xpath(
    118. 

  118. #             '//*[@id="tableCategory"]/thead/tr/td[@class="text-right"]/text() | //*[@id="tableBats"]/thead/tr/td[@class="text-right"]/text()').getall()
    119. 

  119. #
    120. 

  120. #         if tag_td_list:
    121. 

  121. #             # number_of_sets = tag_td_list[0]
    122. 

  122. #             # total_items = tag_td_list[1]
    123. 

  123. #             total_graded = tag_td_list[-1]
    124. 

  124. #             if total_graded:
    125. 

  125. #                 total_graded = total_graded.replace(',', '')
    126. 

  126. #             log.debug(f"Total Graded: {total_graded}")
    127. 

  127. #             data_dict = {
    128. 

  128. #                 "category": category,
    129. 

  129. #                 "category_link": link,
    130. 

  130. #                 "total_graded": total_graded,
    131. 

  131. #                 "crawl_date": time.strftime("%Y-%m-%d", time.localtime())
    132. 

  132. #             }
    133. 

  133. #             # print(data_dict)
    134. 

  134. #             try:
    135. 

  135. #                 sql_pool.insert_one_or_dict(table="psa_pop_record", data=data_dict, ignore=True)
    136. 

  136. #             except Exception as e1:
    137. 

  137. #                 log.error(f"Error inserting data: {e1}")
    138. 

  138. #
    139. 

  139. #     except Exception as e:
    140. 

  140. #         log.error(f"Error getting detail data: {e}")
    141. 

  141. #         raise e
    142. 

  142. #
    143. 

  143. #
    144. 

  144. # @retry(stop=stop_after_attempt(10), wait=wait_fixed(3), after=after_log)
    145. 

  145. # def get_pop_data(log, sql_pool):
    146. 

  146. #     """
    147. 

  147. #     获取 pop 列表页数据
    148. 

  148. #     :param log:
    149. 

  149. #     :param sql_pool:
    150. 

  150. #     """
    151. 

  151. #     url = "https://www.psacard.com/pop"
    152. 

  152. #     try:
    153. 

  153. #         with requests.Session() as session:
    154. 

  154. #             resp = session.get(url, impersonate=random.choice(client_identifier_list), headers=headers,
    155. 

  155. #                                proxies=get_proxys(log), timeout=22, allow_redirects=False)
    156. 

  156. #             # log.debug(resp.text)
    157. 

  157. #
    158. 

  158. #         if 'Just a moment' in resp.text:
    159. 

  159. #             log.debug('Just a moment , retrying.....')
    160. 

  160. #             raise Exception('Just a moment')
    161. 

  161. #
    162. 

  162. #         resp_selector = Selector(text=resp.text)
    163. 

  163. #
    164. 

  164. #         tag_a_list = resp_selector.xpath('//*[@id="mainContent"]/div[2]/div/a')
    165. 

  165. #         for tag_a in tag_a_list:
    166. 

  166. #             category = tag_a.xpath('./div/div/text()').get()
    167. 

  167. #             category_link = tag_a.xpath('./@href').get()
    168. 

  168. #             if 'https://' not in category_link:
    169. 

  169. #                 category_link = BASE_URL + category_link
    170. 

  170. #
    171. 

  171. #             log.debug(f"Category: {category}, Link: {category_link}")
    172. 

  172. #
    173. 

  173. #             try:
    174. 

  174. #                 get_detail_data(log, category, category_link, sql_pool)
    175. 

  175. #             except Exception as e1:
    176. 

  176. #                 log.error(f"Error getting detail data: {e1}")
    177. 

  177. #
    178. 

  178. #     except Exception as e:
    179. 

  179. #         log.error(f"Error getting pop data: {e}")
    180. 

  180. #         raise e
    181. 

  181. #
    182. 

  182. #
    183. 

  183. # @retry(stop=stop_after_attempt(100), wait=wait_fixed(3600), after=after_log)
    184. 

  184. # def pop_main(log):
    185. 

  185. #     """
    186. 

  186. #     主函数
    187. 

  187. #     """
    188. 

  188. #     log.info(
    189. 

  189. #         f'开始运行 {inspect.currentframe().f_code.co_name} 爬虫任务....................................................')
    190. 

  190. #
    191. 

  191. #     # 配置 MySQL 连接池
    192. 

  192. #     sql_pool = MySQLConnectionPool(log=log)
    193. 

  193. #     if not sql_pool.check_pool_health():
    194. 

  194. #         log.error("数据库连接池异常")
    195. 

  195. #         raise RuntimeError("数据库连接池异常")
    196. 

  196. #
    197. 

  197. #     try:
    198. 

  198. #         log.debug(".......... 开始获取数据报告 ..........")
    199. 

  199. #         # get_pop_data(log, sql_pool)
    200. 

  200. #         for _ in range(2):
    201. 

  201. #             for category, category_link in category_link_list.items():
    202. 

  202. #                 log.debug(f"{category}第一次查询, 开始获取数据.......")
    203. 

  203. #                 try:
    204. 

  204. #                     get_detail_data(log, category, category_link, sql_pool)
    205. 

  205. #                 except Exception as e1:
    206. 

  206. #                     log.error(f"Error getting detail data: {e1}")
    207. 

  207. #
    208. 

  208. #             time.sleep(5)
    209. 

  209. #
    210. 

  210. #     except Exception as e:
    211. 

  211. #         log.error(f'{inspect.currentframe().f_code.co_name} error: {e}')
    212. 

  212. #     finally:
    213. 

  213. #         log.info(f'爬虫程序 {inspect.currentframe().f_code.co_name} 运行结束,等待下一轮的采集任务............')
    214. 

  214. #
    215. 

  215. #
    216. 

  216. # def schedule_task():
    217. 

  217. #     """
    218. 

  218. #     两个爬虫模块的启动文件
    219. 

  219. #     bidding_main
    220. 

  220. #     weika_change_card_by_id_spider
    221. 

  221. #     change_card_main
    222. 

  222. #     """
    223. 

  223. #     # 立即运行一次任务
    224. 

  224. #     pop_main(log=logger)
    225. 

  225. #
    226. 

  226. #     # 设置定时任务
    227. 

  227. #     schedule.every().day.at("05:00").do(pop_main, log=logger)
    228. 

  228. #
    229. 

  229. #     while True:
    230. 

  230. #         schedule.run_pending()
    231. 

  231. #         time.sleep(1)
    232. 

  232. #
    233. 

  233. #
    234. 

  234. # if __name__ == '__main__':
    235. 

  235. #     pop_main(logger)
    236. 

  236. #     # schedule_task()
    237. 

  237. #     # get_detail_data(logger, '','https://www.psacard.com/pop/bats',None)
    238. 

  238. """
    239. 

  239. psa 网站变化  pop任务  改为本地运行
    240. 

  240. """
    241.