spider_work/feishezhang_spider/fsz_gu_spider.py

# -*- coding: utf-8 -*-
# Author : Charley
# Python : 3.10.8
# Date   : 2025/11/11 11:54
import time
import random
import inspect
import requests
from loguru import logger
from datetime import datetime
from mysql_pool import MySQLConnectionPool
from tenacity import retry, stop_after_attempt, wait_fixed

"""
谷赏  只有  feishezhang_reward_list_record 和 feishezhang_reward_order_record 两个表
"""
logger.remove()
logger.add("./logs/{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00",
           format="[{time:YYYY-MM-DD HH:mm:ss.SSS}] {level} {message}",
           level="DEBUG", retention="7 day")

headers = {
    "authority": "kurabu.feishezhang.com",
    "accept": "*/*",
    "accept-language": "zh-CN,zh;q=0.9",
    "content-type": "application/json",
    "referer": "https://servicewechat.com/wxa5880b2d8e8a0f37/17/page-frame.html",
    "sec-fetch-dest": "empty",
    "sec-fetch-mode": "cors",
    "sec-fetch-site": "cross-site",
    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 MicroMessenger/7.0.20.1781(0x6700143B) NetType/WIFI MiniProgramEnv/Windows WindowsWechat/WMPF WindowsWechat(0x63090c33) XWEB/9129",
    "xweb_xhr": "1"
}

token = "fb899abaca52caa9932dc4d7efc1dec8"

crawl_category = "谷赏"


def after_log(retry_state):
    """
    retry 回调
    :param retry_state: RetryCallState 对象
    """
    # 检查 args 是否存在且不为空
    if retry_state.args and len(retry_state.args) > 0:
        log = retry_state.args[0]  # 获取传入的 logger
    else:
        log = logger  # 使用全局 logger

    if retry_state.outcome.failed:
        log.warning(
            f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} Times")
    else:
        log.info(f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} succeeded")


@retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
def get_proxys(log):
    """
    获取代理
    :return: 代理
    """
    tunnel = "x371.kdltps.com:15818"
    kdl_username = "t13753103189895"
    kdl_password = "o0yefv6z"
    try:
        proxies = {
            "http": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": kdl_username, "pwd": kdl_password, "proxy": tunnel},
            "https": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": kdl_username, "pwd": kdl_password, "proxy": tunnel}
        }
        return proxies
    except Exception as e:
        log.error(f"Error getting proxy: {e}")
        raise e


@retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
def get_reward_single_page(log, page, sql_pool):
    """
    获取 赏品 单个页面数据
    :param log: 日志对象
    :param page: 页码
    :param sql_pool: 数据库连接池对象
    :return: len(data_list) -> 获取数据长度
    """
    log.debug(f"{inspect.currentframe().f_code.co_name} 开始获取第{page}页数据")
    url = "https://kurabu.feishezhang.com/api.php"
    params = {
        "s": "prize/search",
        "system_type": "default",
        "application": "app",
        "application_client_type": "weixin",
        "token": token,
        "uuid": "7601421a-dc03-4370-b7c7-c17ac84d72eb",
        "ajax": "ajax"
    }
    data = {
        "categoryIds": "",
        "keyword": None,
        "page": page
    }
    # data = json.dumps(data, separators=(',', ':'))
    response = requests.post(url, headers=headers, params=params, json=data)
    response.raise_for_status()

    resp_json = response.json()
    if resp_json['code'] == 0:
        data_list = resp_json.get('data', {}).get('data', [])

        info_list = []
        for cdata in data_list:
            reward_pool_id = cdata.get('id')  # 赏池id
            title = cdata.get('title')
            simple_desc = cdata.get('simple_desc')
            show_image = cdata.get('show_image')
            total_sessions = cdata.get('spec')  # 总箱数
            price = cdata.get('price')
            # status = cdata.get('status')
            # reward_type = cdata.get('type')
            sales_count = cdata.get('sales_count')

            sale_time_int = cdata.get('add_time')  # 开售时间
            sale_time = datetime.fromtimestamp(sale_time_int).strftime('%Y-%m-%d %H:%M:%S') if sale_time_int != 0 else 0

            order_time_int = cdata.get('upd_time')  # 下单时间
            order_time = datetime.fromtimestamp(order_time_int).strftime(
                '%Y-%m-%d %H:%M:%S') if order_time_int != 0 else 0

            category_id = cdata.get('category_id')

            data_dict = {
                "reward_pool_id": reward_pool_id,
                "title": title,
                "simple_desc": simple_desc,
                "show_image": show_image,
                "total_sessions": total_sessions,
                "price": price,
                # "status": status,
                # "reward_type": reward_type,
                "sales_count": sales_count,
                "sale_time": sale_time,
                "order_time": order_time,
                "category_id": category_id,
                "crawl_category": crawl_category
            }
            # print(data_dict)
            info_list.append(data_dict)
        sql_pool.insert_many(table="feishezhang_reward_list_record", data_list=info_list, ignore=True)

        return len(data_list)
    else:
        log.warning(resp_json['msg'])
        return 0


def get_reward_list(log, sql_pool):
    """
    获取 赏品 列表
    :param log: 日志对象
    :param sql_pool: 数据库连接池对象
    """
    log.debug(f"{inspect.currentframe().f_code.co_name} 开始获取列表数据")
    page = 1
    max_page = 1000
    while page <= max_page:
        len_data_list = get_reward_single_page(log, page, sql_pool)

        if len_data_list < 20:
            log.debug(f"当前页数据不足20条，已结束获取数据，当前页数：{page}")
            break

        page += 1

        time.sleep(random.uniform(1, 2))


@retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
def get_reward_order_single_page(log, reward_pool_id, sql_pool, page=1):
    """
    获取 赏池 订单列表
    :param log: 日志对象
    :param reward_pool_id: 赏池id
    :param page: 页码
    :param sql_pool: 数据库连接池对象
    :return: len(data_list) -> 获取数据长度
    """
    log.debug(f"{inspect.currentframe().f_code.co_name} 开始获取第 1 页数据")
    url = "https://kurabu.feishezhang.com/api.php"
    params = {
        "s": "prize/GetLotteryRecord",
        "system_type": "default",
        "application": "app",
        "application_client_type": "weixin",
        "token": token,
        "uuid": "7601421a-dc03-4370-b7c7-c17ac84d72eb",
        "ajax": "ajax",
        "page": str(page),
        # "page": '1',
        "prizePoolId": reward_pool_id
    }
    response = requests.get(url, headers=headers, params=params)
    response.raise_for_status()

    resp_json = response.json()
    if resp_json['code'] == 0:
        data_list = resp_json.get('data', {}).get('data', [])
        # print(data_list)
        if not data_list:
            log.debug(f"当前页数据为空，已结束获取数据，当前页数：{page}")
            return 0

        info_list = []
        for cdata in data_list:
            # order_id = cdata.get('id')
            order_id = cdata.get('order_id')
            goods_id = cdata.get('goods_id')
            draw_time = cdata.get('add_time')
            # level_nickname = cdata.get('level_nickname')
            user_id = cdata.get('user_id')
            images = cdata.get('images')
            title = cdata.get('title')
            nickname = cdata.get('nickname')
            price = cdata.get('price')
            buy_number = cdata.get('buy_number')

            data_dict = {
                "reward_pool_id": reward_pool_id,
                "order_id": order_id,
                "goods_id": goods_id,
                "draw_time": draw_time,
                # "level_nickname": level_nickname,
                "user_id": user_id,
                "images": images,
                "title": title,
                "nickname": nickname,
                "price": price,
                "buy_number": buy_number,
                "crawl_category": crawl_category
            }
            # print(data_dict)
            info_list.append(data_dict)

        sql_pool.insert_many(table="feishezhang_reward_order_record", data_list=info_list, ignore=True)

        return len(data_list)
    else:
        log.warning(resp_json['msg'])
        return 0


@retry(stop=stop_after_attempt(100), wait=wait_fixed(3600), after=after_log)
def fsz_gu_main(log):
    """
    主函数
    :param log: logger对象
    """
    log.info(
        f'开始运行 {inspect.currentframe().f_code.co_name} 爬虫任务....................................................')

    # 配置 MySQL 连接池
    sql_pool = MySQLConnectionPool(log=log)
    if not sql_pool.check_pool_health():
        log.error("数据库连接池异常")
        raise RuntimeError("数据库连接池异常")

    try:
        # 获取赏池列表数据
        # try:
        #     get_reward_list(log, sql_pool)
        # except Exception as e:
        #     log.error(f'get_reward_list -> error: {e}')

        # 获取订单列表数据
        sql_order_list = sql_pool.select_all(
            f"select id, reward_pool_id from feishezhang_reward_list_record where order_state != 1 and crawl_category = '{crawl_category}'")
        # f"select id, reward_pool_id from feishezhang_reward_list_record where order_state = 0 and crawl_category = '{crawl_category}'")
        for sql_order in sql_order_list:
            sql_id = sql_order[0]
            reward_pool_id = sql_order[1]
            log.info(f"开始处理数据：{reward_pool_id}")
            try:
                get_reward_order_single_page(log, reward_pool_id, sql_pool)
                sql_pool.update_one(
                    "update feishezhang_reward_list_record set order_state = 1 where id = %s", (sql_id,)
                )
            except Exception as e:
                log.error(f'get_reward_order_single_page -> error: {e}')
                sql_pool.update_one(
                    "update feishezhang_reward_list_record set order_state = 2 where id = %s", (sql_id,)
                )

    except Exception as e:
        log.error(f'{inspect.currentframe().f_code.co_name} error: {e}')
    finally:
        log.info(f'爬虫程序 {inspect.currentframe().f_code.co_name} 运行结束,等待下一轮的采集任务............')


if __name__ == '__main__':
    # get_reward_list(logger, None)
    # get_reward_order_list(logger, "2876", None)
    # get_reward_order_single_page(logger, "2373", None)
    fsz_gu_main(logger)