zc_new_daily_spider.py 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697
  1. # -*- coding: utf-8 -*-
  2. # Author : Charley
  3. # Python : 3.10.8
  4. # Date : 2026/2/27 11:22
  5. import random
  6. import time
  7. import inspect
  8. import requests
  9. import schedule
  10. import user_agent
  11. from loguru import logger
  12. from crypto_utils import CryptoHelper
  13. from mysql_pool import MySQLConnectionPool
  14. # 2026/08/06 停用板块发现:get_board_shop_list 的旧 cateId 已随版本升级失效,
  15. # 改用「全部」接口(get_all_shop_list)覆盖全部在售店铺,故移除该导入(文件保留备用)。
  16. from tenacity import retry, stop_after_attempt, wait_fixed
  17. logger.remove()
  18. logger.add("./logs/{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00",
  19. format="[{time:YYYY-MM-DD HH:mm:ss.SSS}] {level} {message}",
  20. level="DEBUG", retention="7 day")
  21. # 基础配置
  22. BASE_URL = "https://cashier.yqszpay.com"
  23. PAGE_SIZE = 10
  24. headers = {
  25. "User-Agent": user_agent.generate_user_agent(os="android"), # 设置为安卓模拟器
  26. "Connection": "Keep-Alive",
  27. "Accept-Encoding": "gzip",
  28. "Content-Type": "application/json",
  29. "channelNo": "88888888",
  30. "pageSize": str(PAGE_SIZE),
  31. # "pageNum": 1,
  32. # 2026/08/06 版本失效修复:服务端按 version 头做灰度校验,旧值 1.9.9.82537
  33. # 会被静默降级为返回空 rows(HTTP 200 不报错、数据悄悄变空)。
  34. # 2026/10/19 实测:校验已改为硬拦截 + 精确等值匹配——非当前最新版直接返回
  35. # {"code":"500","message":"检测到您使用的版本过旧..."},且填更高的版本号(如
  36. # 1.3.20 / 9.9.9)同样被拒,无法靠填大版本一劳永逸。App 每次发版都要按抓包同步此值。
  37. "version": "1.3.19"
  38. }
  39. def after_log(retry_state):
  40. """
  41. retry 回调
  42. :param retry_state: RetryCallState 对象
  43. """
  44. # 检查 args 是否存在且不为空
  45. if retry_state.args and len(retry_state.args) > 0:
  46. log = retry_state.args[0] # 获取传入的 logger
  47. else:
  48. log = logger # 使用全局 logger
  49. if retry_state.outcome.failed:
  50. log.warning(
  51. f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} Times")
  52. else:
  53. log.info(f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} succeeded")
  54. @retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
  55. def get_proxys(log):
  56. """
  57. 获取代理配置
  58. :param log: 日志对象
  59. :return: 代理字典
  60. """
  61. tunnel = "x371.kdltps.com:15818"
  62. kdl_username = "t13753103189895"
  63. kdl_password = "o0yefv6z"
  64. try:
  65. proxies = {
  66. "http": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": kdl_username, "pwd": kdl_password, "proxy": tunnel},
  67. "https": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": kdl_username, "pwd": kdl_password, "proxy": tunnel}
  68. }
  69. return proxies
  70. except Exception as e:
  71. log.error(f"Error getting proxy: {e}")
  72. raise e
  73. @retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
  74. def make_encrypted_post_request(log, url: str, request_data: dict, extra_headers: dict = None):
  75. """
  76. 通用加密POST请求函数(带重试机制)
  77. :param log: 日志对象
  78. :param url: 请求URL
  79. :param request_data: 请求数据字典(会被加密)
  80. :param extra_headers: 额外的请求头
  81. :return: 解密后的响应数据,失败返回None
  82. """
  83. request_headers = headers.copy()
  84. if extra_headers:
  85. request_headers.update(extra_headers)
  86. log.debug(f"Request URL: {url}, Data: {request_data}")
  87. encrypted_body = CryptoHelper.encrypt_request_data(request_data)
  88. # print(request_headers)
  89. # response = requests.post(url, headers=request_headers, json=encrypted_body, timeout=22, proxies=get_proxys(log))
  90. response = requests.post(url, headers=request_headers, json=encrypted_body, timeout=(5, 30),
  91. proxies=get_proxys(log))
  92. # response = requests.post(url, headers=request_headers, json=encrypted_body, timeout=(5, 30))
  93. # response.raise_for_status()
  94. if response.status_code == 200:
  95. response_json = response.json()
  96. # log.debug(f"Raw response: {response_json}")
  97. # print(response_json)
  98. if 'data' in response_json:
  99. decrypted = CryptoHelper.decrypt_response_data(response_json)
  100. # log.debug(f"Decrypted response: {decrypted}")
  101. # print(decrypted)
  102. return decrypted
  103. return response_json
  104. else:
  105. log.error(f"请求失败: {response.status_code}, Response: {response.text}")
  106. return None
  107. def get_shop_single_page(log, page_num, page_size=PAGE_SIZE):
  108. """
  109. 获取商户列表(支持翻页)
  110. :param log: 日志对象
  111. :param page_num: 页码
  112. :param page_size: 每页条数
  113. """
  114. log.debug(f"Getting shop list, page: {page_num}")
  115. url = f"{BASE_URL}/zc-api/merchant/getMerMyList"
  116. request_data = {'pageNum': page_num, 'pageSize': page_size}
  117. try:
  118. resp = make_encrypted_post_request(log, url, request_data, extra_headers={"pageNum": str(page_num)})
  119. except Exception as e:
  120. log.error(f"Error getting shop list: {e}")
  121. resp = None
  122. return resp
  123. # 「全部」板块的分类ID。来源:抓包数据.txt 里「全部」两条 getActList 请求体(AES 解密后得到),
  124. # 2026/08/06 版本升级后该值为 "0,6,3"。此接口为公开接口,不校验 Authorization,无需 token。
  125. ALL_SHOP_CATE_ID = "0,6,3"
  126. def get_all_shop_single_page(log, page_num, page_size=PAGE_SIZE):
  127. """获取「全部」板块的活动列表单页(用于发现全部在售店铺)。
  128. 走 getActList「全部」板块(cateId=ALL_SHOP_CATE_ID),从每条活动里解析出
  129. merNo / merName,覆盖当前所有有进行中活动的店铺。为公开接口,无需 token。
  130. Args:
  131. log: 日志对象,从调用方透传。
  132. page_num (int): 页码,从 1 开始。
  133. page_size (int, optional): 每页条数。Defaults to PAGE_SIZE。
  134. Returns:
  135. dict | None: 解密后的响应字典(含 rows);请求失败时返回 None。
  136. """
  137. log.debug(f"Getting all-shop list, page: {page_num}")
  138. url = f"{BASE_URL}/zc-api/act/actProduct/getActList"
  139. # 请求体字段与抓包解密结果一一对应,仅 cateId 固定为「全部」板块:
  140. # finish / loading -> 前端分页状态位,抓包固定 false / true,照搬即可
  141. # actStatus=1 -> 只取进行中的活动
  142. request_data = {
  143. 'cateId': ALL_SHOP_CATE_ID,
  144. 'pageSize': page_size,
  145. 'pageNum': page_num,
  146. 'finish': False,
  147. 'loading': True,
  148. 'actStatus': 1
  149. }
  150. try:
  151. resp = make_encrypted_post_request(log, url, request_data, extra_headers={"pageNum": str(page_num)})
  152. except Exception as e:
  153. log.error(f"Error getting all-shop list, page {page_num}: {e}")
  154. resp = None
  155. return resp
  156. def parse_all_shop_data(log, items, sql_pool, seen):
  157. """解析「全部」接口活动里的店铺并写库(INSERT IGNORE,不覆盖存量)。
  158. 「全部」接口只返回 merNo / merName,拿不到 sold_number(销量) / fans(粉丝)。因此对
  159. 新发现的店铺 sold_number 写死哨兵值 1,让其能被每日任务「WHERE sold_number != 0」
  160. 选中进入商品采集管道;用 INSERT IGNORE 跳过已存在的店铺,避免覆盖 get_shop_list
  161. (getMerMyList)回填的真实 sold_number / fans。
  162. Args:
  163. log: 日志对象,从调用方透传。
  164. items (list[dict]): getActList「全部」板块返回的活动列表(每条含 merNo / merName)。
  165. sql_pool (MySQLConnectionPool): MySQL 连接池。
  166. seen (set): 跨页去重的 shop_id 集合,避免同一店铺重复写库。
  167. Returns:
  168. int: 本次实际写入的店铺数量(已去重)。
  169. """
  170. info_list = []
  171. for item in items:
  172. shop_id = item.get('merNo')
  173. shop_name = item.get('merName')
  174. # 同一店铺会出现在多条活动里,跨页去重以减少无谓写库
  175. if shop_id in seen:
  176. continue
  177. seen.add(shop_id)
  178. # sold_number 写死哨兵值 1:仅作「需采集」标记(板块拿不到真实销量),配合 INSERT IGNORE
  179. # 只对新插入店铺生效,已存在店铺整行被跳过;后续 get_shop_list 会回填真实值
  180. info_list.append({'shop_id': shop_id, 'shop_name': shop_name, 'sold_number': 1})
  181. if not info_list:
  182. return 0
  183. sql_pool.insert_many(table='zc_shop_record', data_list=info_list, ignore=True)
  184. return len(info_list)
  185. def get_all_shop_list(log, sql_pool):
  186. """「全部」接口店铺发现入口:翻页遍历全部在售店铺并写库。
  187. 店铺主来源,替代原来覆盖不全的 getMerMyList「我的商户列表」。翻页采用
  188. 「空页或不足整页即停」的可靠判断,受 MAX 页数保护。
  189. Args:
  190. log: 日志对象,从调用方透传。
  191. sql_pool (MySQLConnectionPool): MySQL 连接池。
  192. Returns:
  193. int: 去重后合计写入的店铺数量。
  194. """
  195. log.info("开始「全部」接口店铺发现......")
  196. page_num = 1
  197. max_pages = 100
  198. seen = set()
  199. while page_num <= max_pages:
  200. result = get_all_shop_single_page(log, page_num, PAGE_SIZE)
  201. if result is None:
  202. log.error(f"「全部」第 {page_num} 页请求失败,停止翻页")
  203. break
  204. data_list = result.get('rows', [])
  205. if len(data_list) == 0:
  206. log.info(f"「全部」第 {page_num} 页无数据,停止翻页")
  207. break
  208. parse_all_shop_data(log, data_list, sql_pool, seen)
  209. log.info(f"「全部」第 {page_num} 页完成,本页活动数: {len(data_list)},累计去重店铺: {len(seen)}")
  210. # 不足整页说明已到末页,停止翻页
  211. if len(data_list) < PAGE_SIZE:
  212. log.info(f"「全部」第 {page_num} 页不足整页,已到末页,停止翻页")
  213. break
  214. page_num += 1
  215. log.info(f"「全部」接口店铺发现结束,去重店铺数: {len(seen)}")
  216. return len(seen)
  217. def get_sold_single_page(log, mer_no, page_num):
  218. """
  219. 获取商品列表(支持翻页)
  220. :param log: 日志对象
  221. :param mer_no: 商户编号
  222. :param page_num: 页码
  223. """
  224. log.info(f"Getting sold items for mer_no: {mer_no}, page: {page_num}")
  225. # url = f"{BASE_URL}/zc-api/act/actProduct/getActList"
  226. # url = "https://cashier.yqszpay.com/zc-api/act/actProduct/getActEndList" 20260407接口变化 修改
  227. url = f"{BASE_URL}/zc-api/act/actProduct/getActEndList"
  228. request_data = {
  229. 'merNo': mer_no,
  230. 'pageNum': page_num,
  231. 'pageSize': PAGE_SIZE,
  232. 'queryType': 1
  233. }
  234. return make_encrypted_post_request(log, url, request_data, extra_headers={"pageNum": str(page_num)})
  235. def get_player_single_page(log, act_id, token, page_num, page_size=PAGE_SIZE):
  236. """
  237. 获取玩家列表(支持翻页)
  238. :param log: 日志对象
  239. :param act_id: 活动ID
  240. :param token: Authorization token
  241. :param page_num: 页码
  242. :param page_size: 每页条数
  243. """
  244. log.debug(f"Getting player list for act_id: {act_id}, page: {page_num}")
  245. # 2026/10/26 接口切换:getActOrderPublicDetails 返回行里没有 giveNumber / userName
  246. # (它为 id/orderId/secretName/addTime/userId/user_name 那套字段),导致
  247. # zc_player_record 的 give_number / user_name 长期写入 NULL。
  248. # 实测 getUserDetailsCount 返回 rows 含 actId/giveNumber/userId/userName/userLogo/isAnonym,
  249. # 与 parse_player_data 的取值字段完全对齐,故切换到此接口。
  250. url = f"{BASE_URL}/zc-api/act/actOrder/getUserDetailsCount"
  251. request_data = {'actId': act_id, 'pageNum': page_num, 'pageSize': page_size}
  252. return make_encrypted_post_request(
  253. log, url, request_data,
  254. extra_headers={"Authorization": token, "pageNum": str(page_num)}
  255. )
  256. def parse_shop_data(log, items, sql_pool):
  257. """
  258. 解析商户数据
  259. :param log: 日志对象
  260. :param items: 商户列表
  261. :param sql_pool: MySQL连接池
  262. :return: 解析后的数据列表
  263. """
  264. log.debug(f"Parsing shop data...........")
  265. info_list = []
  266. for item in items:
  267. # log.debug(f"Processing shop item: {item}")
  268. shop_id = item.get('merNo')
  269. shop_name = item.get('merName')
  270. sold_number = item.get('spell_number')
  271. # link_man = item.get('linkMan')
  272. # user_id = item.get('userId')
  273. fans = item.get('attentionNumber')
  274. data_dict = {
  275. 'shop_id': shop_id,
  276. 'shop_name': shop_name,
  277. 'sold_number': sold_number,
  278. 'fans': fans
  279. }
  280. log.debug(f"Parsed shop data: {data_dict}")
  281. info_list.append(data_dict)
  282. # 保存/更新 根据shop_id判断 是否存在,存在则更新,不存在则插入
  283. sql = "INSERT INTO zc_shop_record (shop_id, shop_name, sold_number, fans) VALUES (%s, %s, %s, %s) ON DUPLICATE KEY UPDATE shop_name=VALUES(shop_name), sold_number=VALUES(sold_number), fans=VALUES(fans)"
  284. # 将字典列表转换为元组列表
  285. args_list = [tuple(d.values()) for d in info_list]
  286. sql_pool.insert_many(query=sql, args_list=args_list)
  287. @retry(stop=stop_after_attempt(3), wait=wait_fixed(1), after=after_log)
  288. def get_video(log, token, pid):
  289. """
  290. 获取活动视频信息
  291. :param log: 日志对象
  292. :param token: Authorization token
  293. :param pid: 活动ID
  294. :return: (live_id, open_time, close_time, video_url)
  295. """
  296. url = "https://cashier.yqszpay.com/zc-api/live/actLive/getMerLiveInfo"
  297. request_data = {'actId': pid}
  298. log.debug(f"获取视频信息,actId: {pid}")
  299. resp_data = make_encrypted_post_request(
  300. log, url, request_data,
  301. extra_headers={"Authorization": token}
  302. )
  303. # log.debug(f"视频响应: {resp_data}")
  304. live_id = resp_data.get('live', {}).get('liveId')
  305. live_open_time = resp_data.get('live', {}).get('openTime')
  306. live_close_time = resp_data.get('live', {}).get('closeTime')
  307. video_url = resp_data.get('live', {}).get('videoUrl')
  308. return live_id, live_open_time, live_close_time, video_url
  309. def parse_sold_data(log, token, items, sql_pool, shop_name):
  310. """
  311. 解析商品数据
  312. :param log: 日志对象
  313. :param token: Authorization token
  314. :param items: 商品列表
  315. :param sql_pool: MySQL连接池
  316. :param shop_name: 商户名称
  317. :return: 解析后的数据列表
  318. """
  319. info_list = []
  320. for item in items:
  321. # log.debug(f"Processing sold item: {item}")
  322. shop_id = item.get('merNo') # 商户编号
  323. pid = item.get('id')
  324. act_day = item.get('actDay') # 活动天数
  325. act_logo = item.get('actLogo')
  326. act_name = item.get('actName') # 活动名称
  327. act_no = item.get('actNo') # 活动编号
  328. act_status = item.get('actStatus') # 活动状态
  329. startDate = item.get('startDate') # 开始时间
  330. endDate = item.get('complete_date') # 结束时间
  331. storageId = item.get('storageId') # 存储ID
  332. storageName = item.get('storageName') # 存储名称
  333. unitPrice = item.get('unitPrice') # 单价
  334. sumPrice = item.get('sumPrice') # 总价
  335. reality_price = item.get('realityPrice') # 实际价格
  336. packageNumber = item.get('packageNumber') # 包配置
  337. schedule_ = item.get('schedule') # 库存
  338. try:
  339. live_id, live_open_time, live_close_time, video_url = get_video(log, token, pid)
  340. except Exception as e:
  341. log.error(f"Error getting video info for pid {pid}: {e}")
  342. live_id, live_open_time, live_close_time, video_url = None, None, None, None
  343. data_dict = {
  344. 'shop_id': shop_id,
  345. 'shop_name': shop_name,
  346. 'pid': pid,
  347. 'act_day': act_day,
  348. 'act_img': act_logo,
  349. 'act_name': act_name,
  350. 'act_no': act_no,
  351. 'act_status': act_status,
  352. 'start_date': startDate,
  353. 'end_date': endDate,
  354. 'storage_id': storageId,
  355. 'storage_name': storageName,
  356. 'unit_price': unitPrice,
  357. 'sum_price': sumPrice,
  358. 'reality_price': reality_price,
  359. 'package_number': packageNumber,
  360. 'schedule': schedule_,
  361. 'live_id': live_id,
  362. 'live_open_time': live_open_time,
  363. 'live_close_time': live_close_time,
  364. 'video_url': video_url
  365. }
  366. # log.debug(f"Parsed sold data: {data_dict}")
  367. # { 'live_close_time': None, 'video_url': None}
  368. info_list.append(data_dict)
  369. # 保存数据
  370. sql_pool.insert_many(table='zc_product_record', data_list=info_list, ignore=True)
  371. def parse_player_data(log, items, sql_pool):
  372. """
  373. 解析玩家数据
  374. :param log: 日志对象
  375. :param items: 玩家列表
  376. :param sql_pool: MySQL连接池
  377. :return: 解析后的数据列表
  378. """
  379. log.debug(f"Parsing player data...........")
  380. info_list = []
  381. for item in items:
  382. # log.debug(f"Processing player item: {item}")
  383. pid = item.get('actId')
  384. give_number = item.get('giveNumber') # 份数
  385. user_id = item.get('userId')
  386. user_name = item.get('userName')
  387. data_dict = {
  388. 'pid': pid,
  389. 'give_number': give_number,
  390. 'user_id': user_id,
  391. 'user_name': user_name
  392. }
  393. # log.debug(f"Parsed player data: {data_dict}")
  394. info_list.append(data_dict)
  395. # 保存数据
  396. sql_pool.insert_many(table='zc_player_record', data_list=info_list, ignore=True)
  397. def get_shop_list(log, sql_pool):
  398. """
  399. 商户列表翻页生成器
  400. :param log: 日志对象
  401. :param sql_pool: MySQL连接池
  402. """
  403. page_num = 1
  404. max_pages = 100
  405. # 2026/08/06 翻页 bug 修复:原逻辑 total 初值为 0,却用「total is None」判断是否取总数,
  406. # 导致 total 永远停在 0,停止条件「(page_num-1)*PAGE_SIZE >= total」在第 1 页跑完后即
  407. # 0 >= 0 成立、立刻 break,只能采到第 1 页 10 个店铺。改为「空页 / 不足整页即停」的可靠判断。
  408. while page_num <= max_pages:
  409. result = get_shop_single_page(log, page_num, PAGE_SIZE)
  410. if result is None:
  411. log.error(f"第 {page_num} 页请求失败,停止翻页")
  412. break
  413. data_list = result.get('rows', [])
  414. # 空页直接停止,避免用空列表调用 insert_many 触发 ValueError
  415. if len(data_list) == 0:
  416. log.info(f"第 {page_num} 页无数据,停止翻页")
  417. break
  418. # getMerMyList 含真实 sold_number / fans,走 ON DUPLICATE KEY UPDATE 回填覆盖哨兵值
  419. parse_shop_data(log, data_list, sql_pool)
  420. log.info(f"第 {page_num} 页查询完成,本页条数: {len(data_list)}")
  421. # 不足整页说明已到末页,停止翻页
  422. if len(data_list) < PAGE_SIZE:
  423. log.info(f"第 {page_num} 页不足整页,已到末页,停止翻页")
  424. break
  425. page_num += 1
  426. def get_sold_list(log, shop_id, token, sql_pool, shop_name):
  427. """
  428. 商品列表翻页生成器
  429. :param log: 日志对象
  430. :param shop_id: shop_id
  431. :param token: Authorization token
  432. :param sql_pool: MySQL连接池
  433. :param shop_name: 商户名称
  434. """
  435. page_num = 1
  436. max_pages = 10
  437. while page_num <= max_pages:
  438. result = get_sold_single_page(log, shop_id, page_num)
  439. time.sleep(random.uniform(0.5, 1)) # 添加随机延迟,防止对目标服务器造成过大负载
  440. # print(result)
  441. if result is None:
  442. log.error(f"第 {page_num} 页请求失败,停止翻页")
  443. break
  444. data_list = result.get('rows', [])
  445. if not data_list:
  446. log.info(f"第 {page_num} 页无数据,停止翻页")
  447. # log.info(f'该店铺{shop_name}无数据, 修改为店铺注销状态..........')
  448. # # 更新店铺状态为注销状态
  449. # sql_pool.update_one("UPDATE zc_shop_record SET is_deleted = 1 WHERE shop_id = %s", (shop_id,))
  450. break
  451. parse_sold_data(log, token, data_list, sql_pool, shop_name)
  452. # 检查是否有数据
  453. if len(data_list) < 10:
  454. log.info(f"第 {page_num} 页无数据,停止翻页")
  455. break
  456. log.info(f"第 {page_num} 页查询完成,本页条数: {len(data_list)}")
  457. page_num += 1
  458. def get_player_list(log, act_id, token, sql_pool):
  459. """
  460. 玩家列表翻页生成器
  461. :param log: 日志对象
  462. :param act_id: 活动ID
  463. :param token: Authorization token
  464. :param sql_pool: MySQL连接池
  465. :return: has_data (True: 有数据, False: 无数据)
  466. """
  467. page_num = 1
  468. max_pages = 100
  469. has_data = False
  470. while page_num <= max_pages:
  471. result = get_player_single_page(log, act_id, token, page_num)
  472. if result is None:
  473. log.error(f"第 {page_num} 页请求失败,停止翻页")
  474. break
  475. data_list = result.get('rows', [])
  476. # 如果有数据才解析
  477. if len(data_list) > 0:
  478. has_data = True
  479. parse_player_data(log, data_list, sql_pool)
  480. # 检查是否有数据
  481. if len(data_list) < 10:
  482. log.info(f"第 {page_num} 页无数据,停止翻页")
  483. break
  484. log.info(f"第 {page_num} 页查询完成,本页条数: {len(data_list)}")
  485. page_num += 1
  486. return has_data
  487. @retry(stop=stop_after_attempt(100), wait=wait_fixed(3600), after=after_log)
  488. def zc_main(log):
  489. """
  490. 主函数
  491. :param log: logger对象
  492. """
  493. log.info(
  494. f'开始运行 {inspect.currentframe().f_code.co_name} 爬虫任务....................................................')
  495. # 配置 MySQL 连接池
  496. sql_pool = MySQLConnectionPool(log=log)
  497. if not sql_pool.check_pool_health():
  498. log.error("数据库连接池异常")
  499. raise RuntimeError("数据库连接池异常")
  500. try:
  501. # 获取 token
  502. token_row = sql_pool.select_one("SELECT token FROM zc_token WHERE id = 1")
  503. if not token_row:
  504. log.error("未查询到 token")
  505. return
  506. token = token_row[0]
  507. # player test
  508. # has_data = get_player_list(log, 1800, token, sql_pool)
  509. # 店铺主来源:「全部」接口发现全部在售店铺(替代覆盖不全的 getMerMyList)
  510. # 2026/08/06 停用板块发现 get_board_shop_list:其 5 个旧 cateId 已随版本升级失效,
  511. # 且「全部」接口(cateId=0,6,3)已覆盖所有在售店铺,板块发现冗余。
  512. try:
  513. get_all_shop_list(log, sql_pool)
  514. except Exception as e:
  515. log.error(f'get_all_shop_list error: {e}')
  516. # 补充:getMerMyList 回填「我的店铺」真实 sold_number / fans(「全部」接口拿不到这两个字段)
  517. try:
  518. get_shop_list(log, sql_pool)
  519. except Exception as e:
  520. log.error(f'iterate_shop_list error: {e}')
  521. time.sleep(5)
  522. # 获取sold data - 遍历所有商户
  523. try:
  524. # 从 shop 表查询所有 merNo 2026/5/9增加is_deleted判断是否注销店铺
  525. mer_no_rows = sql_pool.select_all(
  526. # "SELECT shop_id, shop_name FROM zc_shop_record WHERE is_deleted = 0 AND sold_number != 0"
  527. "SELECT shop_id, shop_name FROM zc_shop_record WHERE sold_number != 0"
  528. )
  529. log.info(f"查询到 {len(mer_no_rows)} 个商户编号: {mer_no_rows}")
  530. for shop_id, shop_name in mer_no_rows:
  531. log.info(f"开始爬取商户 {shop_id}, {shop_name} 的商品数据")
  532. # get_sold_list(log, shop_id, shop_name, token, sql_pool)
  533. get_sold_list(log, shop_id, token, sql_pool, shop_name)
  534. except Exception as e:
  535. log.error(f'get_sold_list error: {e}')
  536. time.sleep(5)
  537. # 获取player data - 遍历所有活动
  538. # try:
  539. # # 从 sold 表查询所有 actId
  540. # act_id_rows = sql_pool.select_all("SELECT pid FROM zc_product_record WHERE player_state != 1")
  541. # act_id_list = [row[0] for row in act_id_rows] if act_id_rows else []
  542. # log.info(f"查询到 {len(act_id_list)} 个活动ID")
  543. #
  544. # for act_id in act_id_list:
  545. # try:
  546. # # 先将当前 pid 的状态改为 1,表示开始查询
  547. # sql_pool.update_one("UPDATE zc_product_record SET player_state = 1 WHERE pid = %s", (act_id,))
  548. # log.info(f"将 pid: {act_id} 的状态更新为 1(开始查询)")
  549. #
  550. # log.info(f"开始爬取pid: {act_id} 的玩家数据")
  551. # has_data = get_player_list(log, act_id, token, sql_pool)
  552. #
  553. # # 根据是否有数据更新状态
  554. # if has_data:
  555. # log.info(f"pid: {act_id} 查询到数据,状态保持为 1")
  556. # else:
  557. # log.info(f"pid: {act_id} 没有数据,状态更新为 2")
  558. # sql_pool.update_one("UPDATE zc_product_record SET player_state = 2 WHERE pid = %s", (act_id,))
  559. #
  560. # except Exception as pid_error:
  561. # # 如果查询失败,将状态改为 3
  562. # log.error(f"pid: {act_id} 查询失败,错误: {pid_error}")
  563. # try:
  564. # sql_pool.update_one("UPDATE zc_product_record SET player_state = 3 WHERE pid = %s", (act_id,))
  565. # log.info(f"已将 pid: {act_id} 的状态更新为 3(查询异常)")
  566. # except Exception as update_error:
  567. # log.error(f"更新 pid: {act_id} 状态失败: {update_error}")
  568. # except Exception as e:
  569. # log.error(f'iterate_player_list error: {e}')
  570. except Exception as e:
  571. log.error(f'{inspect.currentframe().f_code.co_name} error: {e}')
  572. finally:
  573. log.info(f'爬虫程序 {inspect.currentframe().f_code.co_name} 运行结束,等待下一轮的采集任务............')
  574. def schedule_task():
  575. """
  576. 爬虫模块 定时任务 的启动文件
  577. """
  578. # 立即运行一次任务
  579. # zc_main(log=logger)
  580. # 设置定时任务
  581. schedule.every().day.at("06:01").do(zc_main, log=logger)
  582. while True:
  583. schedule.run_pending()
  584. time.sleep(1)
  585. if __name__ == '__main__':
  586. # zc_main(logger)
  587. schedule_task()