|
@@ -14,7 +14,7 @@ from mysql_pool import MySQLConnectionPool
|
|
|
from tenacity import retry, stop_after_attempt, wait_fixed
|
|
from tenacity import retry, stop_after_attempt, wait_fixed
|
|
|
|
|
|
|
|
logger.remove()
|
|
logger.remove()
|
|
|
-logger.add("logs/{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00",
|
|
|
|
|
|
|
+logger.add("logs/pop_{time:YYYYMMDD}.log", encoding='utf-8', rotation="00:00",
|
|
|
format="[{time:YYYY-MM-DD HH:mm:ss.SSS}] {level} {message}",
|
|
format="[{time:YYYY-MM-DD HH:mm:ss.SSS}] {level} {message}",
|
|
|
level="DEBUG", retention="3 day")
|
|
level="DEBUG", retention="3 day")
|
|
|
|
|
|
|
@@ -92,7 +92,7 @@ def get_proxys(log):
|
|
|
raise e
|
|
raise e
|
|
|
|
|
|
|
|
|
|
|
|
|
-@retry(stop=stop_after_attempt(5), wait=wait_fixed(1), after=after_log)
|
|
|
|
|
|
|
+@retry(stop=stop_after_attempt(10), wait=wait_fixed(3), after=after_log)
|
|
|
def get_detail_data(log, category, link, sql_pool):
|
|
def get_detail_data(log, category, link, sql_pool):
|
|
|
"""
|
|
"""
|
|
|
获取详情数据
|
|
获取详情数据
|
|
@@ -139,6 +139,7 @@ def get_detail_data(log, category, link, sql_pool):
|
|
|
raise e
|
|
raise e
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
+@retry(stop=stop_after_attempt(10), wait=wait_fixed(3), after=after_log)
|
|
|
def get_pop_data(log, sql_pool):
|
|
def get_pop_data(log, sql_pool):
|
|
|
"""
|
|
"""
|
|
|
获取 pop 列表页数据
|
|
获取 pop 列表页数据
|
|
@@ -218,7 +219,7 @@ def schedule_task():
|
|
|
change_card_main
|
|
change_card_main
|
|
|
"""
|
|
"""
|
|
|
# 立即运行一次任务
|
|
# 立即运行一次任务
|
|
|
- # pop_main(log=logger)
|
|
|
|
|
|
|
+ pop_main(log=logger)
|
|
|
|
|
|
|
|
# 设置定时任务
|
|
# 设置定时任务
|
|
|
schedule.every().day.at("08:00").do(pop_main, log=logger)
|
|
schedule.every().day.at("08:00").do(pop_main, log=logger)
|