spider_work/baocui_spider/add_task.py

# -*- coding: utf-8 -*-
# Author : Charley
# Python : 3.10.8
# Date   : 2025/6/11 18:59
from loguru import logger

from mysql_pool import MySQLConnectionPool

# logger.remove()
# logger.add("./logs/add_{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00",
#            format="[{time:YYYY-MM-DD HH:mm:ss.SSS}] {level} {message}",
#            level="DEBUG", retention="7 day")

"""
C001000000，C开头到100万
D000220000，D开头到22万
E000050000，E开头到5万
H000500000，H开头到50万
L000400000，L开头到40万
"""

start = 1
sql_pool = MySQLConnectionPool(log=logger)
# sql_kwd_list = sql_pool.select_all("select keyword from baocui_record")
# sql_kwd_list = [i[0] for i in sql_kwd_list]

# for prefix, end in [('C', 600001), ('D', 200001), ('E', 40001)]:
for prefix, end in [('L', 400001)]:
    # if prefix == 'H':
    #     start = 40001
    info_list = []
    for i in range(start, end):
        kwd = f"{prefix}{i:09d}"
        # if kwd in sql_kwd_list:
        #     logger.info(f"{kwd}已存在")
        #     continue
        info_list.append((kwd,))  # 添加元组格式以匹配参数化查询
    # print(info_list)

    #     # 每10000条保存一次
    #     if len(info_list) >= 10000:
    #         sql_pool.insert_all("insert into baocui_task(keyword) values (%s)", info_list)
    #         info_list.clear()
    #
    # # 处理剩余不足10000条的部分
    if info_list:
        sql_pool.insert_many(query="INSERT IGNORE INTO baocui_task(keyword) values (%s)", args_list=info_list, ignore=True)
        info_list.clear()