update qd 内存优化 26.1.19.2

qiandao_spider/qd_all_sg_spider/qd_auction_spider.py

@@ -388,7 +388,7 @@ def get_bid_buyer_one_page(log, auction_id, page, sql_pool):
 
         deduplicated_list = list(user_best_record.values())
         if deduplicated_list:
-            sql_pool.insert_many(table="qiandao_auction_bid_record", data_list=deduplicated_list)
+            sql_pool.insert_many(table="qiandao_auction_bid_record", data_list=deduplicated_list, ignore=True)
         return len(items)
     else:
         log.error(f"{inspect.currentframe().f_code.co_name} error: {response_json.get('message', '')}")

qiandao_spider/qd_all_sg_spider/qd_luckybag_huoying_spider.py

@@ -144,7 +144,7 @@ def parse_luckybag_data(log, items, sql_pool, sql_luckybag_list):
 
     if info_list:
         try:
-            sql_pool.insert_many(table="qiandao_luckybag_list_record", data_list=info_list)
+            sql_pool.insert_many(table="qiandao_luckybag_list_record", data_list=info_list, ignore=True)
         except pymysql.err.IntegrityError as e:
             if "Duplicate entry" in str(e):
                 logger.warning("存在重复的 luckybag_id，跳过插入")

qiandao_spider/qd_all_sg_spider/qd_luckybag_labubu_spider.py

@@ -144,7 +144,7 @@ def parse_luckybag_data(log, items, sql_pool, sql_luckybag_list):
 
     if info_list:
         try:
-            sql_pool.insert_many(table="qiandao_luckybag_list_record", data_list=info_list)
+            sql_pool.insert_many(table="qiandao_luckybag_list_record", data_list=info_list, ignore=True)
         except pymysql.err.IntegrityError as e:
             if "Duplicate entry" in str(e):
                 logger.warning("存在重复的 luckybag_id，跳过插入")

qiandao_spider/qd_all_sg_spider/qd_luckybag_mini_spider.py

@@ -144,7 +144,7 @@ def parse_luckybag_data(log, items, sql_pool, sql_luckybag_list):
 
     if info_list:
         try:
-            sql_pool.insert_many(table="qiandao_luckybag_list_record", data_list=info_list)
+            sql_pool.insert_many(table="qiandao_luckybag_list_record", data_list=info_list, ignore=True)
         except pymysql.err.IntegrityError as e:
             if "Duplicate entry" in str(e):
                 logger.warning("存在重复的 luckybag_id，跳过插入")

qiandao_spider/qd_all_sg_spider/qd_luckybag_mlp_spider.py

@@ -144,7 +144,7 @@ def parse_luckybag_data(log, items, sql_pool, sql_luckybag_list):
 
     if info_list:
         try:
-            sql_pool.insert_many(table="qiandao_luckybag_list_record", data_list=info_list)
+            sql_pool.insert_many(table="qiandao_luckybag_list_record", data_list=info_list, ignore=True)
         except pymysql.err.IntegrityError as e:
             if "Duplicate entry" in str(e):
                 logger.warning("存在重复的 luckybag_id，跳过插入")

qiandao_spider/qd_all_sg_spider/qd_sg_model_spider.py

@@ -197,7 +197,7 @@ def parse_product_data(log, items, tag_id, sql_pool, sql_p_list):
 
     if info_list:
         try:
-            sql_pool.insert_many(table="qiandao_sg_category_record", data_list=info_list)
+            sql_pool.insert_many(table="qiandao_sg_category_record", data_list=info_list, ignore=True)
         except  Exception as e:
             log.error(f" {inspect.currentframe().f_code.co_name} 插入失败:", e)
 
@@ -339,7 +339,7 @@ def parse_sold_data(log, items, sql_pool, spu_id, max_trading_time):
 
         if info_list:
             try:
-                sql_pool.insert_many(table="qiandao_sg_sold_record", data_list=info_list)
+                sql_pool.insert_many(table="qiandao_sg_sold_record", data_list=info_list, ignore=True)
             except Exception as e:
                 log.error(f" {inspect.currentframe().f_code.co_name} 插入失败:", e)
         return has_new_data  # 返回是否有新数据

qiandao_spider/qd_all_sg_spider/qd_sg_switch_spider.py

@@ -189,7 +189,7 @@ def parse_product_data(log, items, tag_id, sql_pool, sql_p_list):
 
     if info_list:
         try:
-            sql_pool.insert_many(table="qiandao_sg_category_record", data_list=info_list)
+            sql_pool.insert_many(table="qiandao_sg_category_record", data_list=info_list, ignore=True)
         except  Exception as e:
             log.error(f" {inspect.currentframe().f_code.co_name} 插入失败:", e)
 
@@ -331,7 +331,7 @@ def parse_sold_data(log, items, sql_pool, spu_id, max_trading_time):
 
         if info_list:
             try:
-                sql_pool.insert_many(table="qiandao_sg_sold_record", data_list=info_list)
+                sql_pool.insert_many(table="qiandao_sg_sold_record", data_list=info_list, ignore=True)
             except Exception as e:
                 log.error(f" {inspect.currentframe().f_code.co_name} 插入失败:", e)
         return has_new_data  # 返回是否有新数据

qiandao_spider/qd_all_sg_spider/qd_sg_tcg_spider.py

@@ -201,7 +201,7 @@ def parse_product_data(log, items, tag_id, sql_pool, sql_p_list):
 
     if info_list:
         try:
-            sql_pool.insert_many(table="qiandao_sg_category_record", data_list=info_list)
+            sql_pool.insert_many(table="qiandao_sg_category_record", data_list=info_list, ignore=True)
         except  Exception as e:
             log.error(f" {inspect.currentframe().f_code.co_name} 插入失败:", e)
 
@@ -343,7 +343,7 @@ def parse_sold_data(log, items, sql_pool, spu_id, max_trading_time):
 
         if info_list:
             try:
-                sql_pool.insert_many(table="qiandao_sg_sold_record", data_list=info_list)
+                sql_pool.insert_many(table="qiandao_sg_sold_record", data_list=info_list, ignore=True)
             except Exception as e:
                 log.error(f" {inspect.currentframe().f_code.co_name} 插入失败:", e)
         return has_new_data  # 返回是否有新数据