spider_work/kawan_spider/settings.py

# -*- coding: utf-8 -*-
# Author : Charley
# Python : 3.8.10
# Date   : 2025/4/14 11:58
import inspect
import requests
from loguru import logger
from tenacity import retry, stop_after_attempt, wait_fixed

logger.remove()
logger.add("./logs/{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00",
           format="[{time:YYYY-MM-DD HH:mm:ss.SSS}] {level} {message}",
           level="DEBUG", retention="7 day")

HEADERS = {
    "User-Agent": "Dart/3.6 (dart:io)"
}


def after_log(retry_state):
    """
    retry 回调
    :param retry_state: RetryCallState 对象
    """
    # 检查 args 是否存在且不为空
    if retry_state.args and len(retry_state.args) > 0:
        log = retry_state.args[0]  # 获取传入的 logger
    else:
        log = logger  # 使用全局 logger

    if retry_state.outcome.failed:
        log.warning(
            f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} Times")
    else:
        log.info(f"Function '{retry_state.fn.__name__}', Attempt {retry_state.attempt_number} succeeded")


@retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
def get_proxys(log):
    """
    获取代理
    :return: 代理
    """
    tunnel = "x371.kdltps.com:15818"
    kdl_username = "t13753103189895"
    kdl_password = "o0yefv6z"
    try:
        proxies = {
            "http": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": kdl_username, "pwd": kdl_password, "proxy": tunnel},
            "https": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": kdl_username, "pwd": kdl_password, "proxy": tunnel}
        }
        return proxies
    except Exception as e:
        log.error(f"Error getting proxy: {e}")
        raise e


@retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
def make_request(log, method, url, params=None, data=None, headers=None, proxies=None, timeout=22, token=None):
    """
    通用请求函数
    :param log: logger对象
    :param method: 请求方法 ('GET' 或 'POST')
    :param url: 请求的URL
    :param params: GET请求的查询参数
    :param data: POST请求的数据
    :param headers: 请求头
    :param proxies: 代理
    :param timeout: 请求超时时间
    :param token: token
    :return: 响应的JSON数据
    """
    if not log:
        log = logger
    if headers is None:
        headers = HEADERS
        if not token:
            token = "a-2df1084f01cb48f2ab67806867fea37f"
        headers["authorization"] = token

    if proxies is None:
        proxies = get_proxys(log)

    try:
        with requests.Session() as session:
            if method.upper() == 'GET':
                response = session.get(url, headers=headers, params=params, proxies=proxies, timeout=timeout)
            elif method.upper() == 'POST':
                response = session.post(url, headers=headers, json=data, proxies=proxies, timeout=timeout)
            else:
                log.error(f"Unsupported request method: {method}")
                return {}

            response.raise_for_status()
            data = response.json()
            if data["code"] == 200:
                log.info(f"Successfully fetched {method} request to {url}")
                return data
            else:
                log.warning(f"Warning {inspect.currentframe().f_code.co_name}: {data['msg']}")
                return {}
    except requests.exceptions.RequestException as e:
        log.error(f"Error making {method} request to {url}: {e}")
        raise e
    except ValueError as e:
        log.error(f"Error parsing JSON for {method} request to {url}: {e}")
        raise e
    except Exception as e:
        log.error(f"Error making {method} request to {url}: {e}")
        raise e