|
@@ -185,7 +185,7 @@ def bgs_main(log):
|
|
|
# for bid in sql_bgs_id_list:
|
|
# for bid in sql_bgs_id_list:
|
|
|
try:
|
|
try:
|
|
|
process_urls(log, sql_bgs_id_list, sql_pool, batch_size=1000,
|
|
process_urls(log, sql_bgs_id_list, sql_pool, batch_size=1000,
|
|
|
- max_workers=10) # 根据需要调整batch_size和max_workers
|
|
|
|
|
|
|
+ max_workers=3) # 根据需要调整batch_size和max_workers
|
|
|
# get_data(bid, mysql_pool)
|
|
# get_data(bid, mysql_pool)
|
|
|
except Exception as e:
|
|
except Exception as e:
|
|
|
log.error('process urls: ', e)
|
|
log.error('process urls: ', e)
|
|
@@ -203,7 +203,7 @@ def schedule_task():
|
|
|
设置定时任务
|
|
设置定时任务
|
|
|
"""
|
|
"""
|
|
|
# 立即运行一次任务
|
|
# 立即运行一次任务
|
|
|
- bgs_main(logger)
|
|
|
|
|
|
|
+ # bgs_main(logger)
|
|
|
|
|
|
|
|
# 设置定时任务
|
|
# 设置定时任务
|
|
|
schedule.every().day.at("03:01").do(bgs_main, logger)
|
|
schedule.every().day.at("03:01").do(bgs_main, logger)
|
