|
@@ -1,8 +1,9 @@
|
|
|
# -*- coding: utf-8 -*-
|
|
# -*- coding: utf-8 -*-
|
|
|
# Author : Charley
|
|
# Author : Charley
|
|
|
-# Python : 3.10.8
|
|
|
|
|
|
|
+# Python : 3.12.10
|
|
|
# Date : 2025/3/25 14:14
|
|
# Date : 2025/3/25 14:14
|
|
|
import re
|
|
import re
|
|
|
|
|
+import time
|
|
|
import pymysql
|
|
import pymysql
|
|
|
import YamlLoader
|
|
import YamlLoader
|
|
|
from loguru import logger
|
|
from loguru import logger
|
|
@@ -50,27 +51,128 @@ class MySQLConnectionPool:
|
|
|
write_timeout=30 # 写入超时时间(秒)
|
|
write_timeout=30 # 写入超时时间(秒)
|
|
|
)
|
|
)
|
|
|
|
|
|
|
|
|
|
+ # def _execute(self, query, args=None, commit=False):
|
|
|
|
|
+ # """
|
|
|
|
|
+ # 执行SQL
|
|
|
|
|
+ # :param query: SQL语句
|
|
|
|
|
+ # :param args: SQL参数
|
|
|
|
|
+ # :param commit: 是否提交事务
|
|
|
|
|
+ # :return: 查询结果
|
|
|
|
|
+ # """
|
|
|
|
|
+ # try:
|
|
|
|
|
+ # with self.pool.connection() as conn:
|
|
|
|
|
+ # with conn.cursor() as cursor:
|
|
|
|
|
+ # cursor.execute(query, args)
|
|
|
|
|
+ # if commit:
|
|
|
|
|
+ # conn.commit()
|
|
|
|
|
+ # self.log.debug(f"sql _execute, Query: {query}, Rows: {cursor.rowcount}")
|
|
|
|
|
+ # return cursor
|
|
|
|
|
+ # except Exception as e:
|
|
|
|
|
+ # if commit and conn:
|
|
|
|
|
+ # conn.rollback()
|
|
|
|
|
+ # self.log.exception(f"Error executing query: {e}, Query: {query}, Args: {args}")
|
|
|
|
|
+ # raise e
|
|
|
|
|
+
|
|
|
def _execute(self, query, args=None, commit=False):
|
|
def _execute(self, query, args=None, commit=False):
|
|
|
"""
|
|
"""
|
|
|
- 执行SQL
|
|
|
|
|
- :param query: SQL语句
|
|
|
|
|
- :param args: SQL参数
|
|
|
|
|
- :param commit: 是否提交事务
|
|
|
|
|
- :return: 查询结果
|
|
|
|
|
|
|
+ 执行SQL(带断连重试,指数退避,覆盖分钟级网络波动)。
|
|
|
|
|
+
|
|
|
|
|
+ Args:
|
|
|
|
|
+ query (str): SQL语句。
|
|
|
|
|
+ args: SQL参数,可以是元组、列表或 None。
|
|
|
|
|
+ commit (bool, optional): 是否提交事务。Defaults to False。
|
|
|
|
|
+
|
|
|
|
|
+ Returns:
|
|
|
|
|
+ cursor: pymysql cursor,调用方可读取 rowcount / lastrowid / fetchone 等。
|
|
|
|
|
+
|
|
|
|
|
+ Raises:
|
|
|
|
|
+ pymysql.err.IntegrityError: 唯一键冲突等完整性错误。
|
|
|
|
|
+ pymysql.err.OperationalError: 重试耗尽后仍无法连接数据库。
|
|
|
"""
|
|
"""
|
|
|
- try:
|
|
|
|
|
- with self.pool.connection() as conn:
|
|
|
|
|
- with conn.cursor() as cursor:
|
|
|
|
|
- cursor.execute(query, args)
|
|
|
|
|
- if commit:
|
|
|
|
|
- conn.commit()
|
|
|
|
|
- self.log.debug(f"sql _execute, Query: {query}, Rows: {cursor.rowcount}")
|
|
|
|
|
- return cursor
|
|
|
|
|
- except Exception as e:
|
|
|
|
|
- if commit and conn:
|
|
|
|
|
- conn.rollback()
|
|
|
|
|
- self.log.exception(f"Error executing query: {e}, Query: {query}, Args: {args}")
|
|
|
|
|
- raise e
|
|
|
|
|
|
|
+ for attempt in range(15): # 指数退避:5s→10s→20s…最多 300s,覆盖约 10 分钟波动
|
|
|
|
|
+ try:
|
|
|
|
|
+ with self.pool.connection() as conn:
|
|
|
|
|
+ with conn.cursor() as cursor:
|
|
|
|
|
+ cursor.execute(query, args)
|
|
|
|
|
+ if commit:
|
|
|
|
|
+ conn.commit()
|
|
|
|
|
+ self.log.debug(f"sql _execute, Query: {query}, Rows: {cursor.rowcount}")
|
|
|
|
|
+ return cursor
|
|
|
|
|
+ except (pymysql.err.InterfaceError, pymysql.err.OperationalError) as e:
|
|
|
|
|
+ errno = e.args[0] if e.args else None
|
|
|
|
|
+ if errno in (2006, 2013): # MySQL server gone / lost connection during query
|
|
|
|
|
+ delay = min(5 * (2 ** attempt), 300)
|
|
|
|
|
+ self.log.warning(
|
|
|
|
|
+ f"数据库连接断开 (errno={errno}),{delay}s 后重试 "
|
|
|
|
|
+ f"(attempt {attempt + 1}/15): {e}"
|
|
|
|
|
+ )
|
|
|
|
|
+ time.sleep(delay)
|
|
|
|
|
+ continue
|
|
|
|
|
+ self.log.error(f"Error executing query: {e}, Query: {query}")
|
|
|
|
|
+ raise
|
|
|
|
|
+ except pymysql.err.IntegrityError:
|
|
|
|
|
+ raise
|
|
|
|
|
+ except Exception as e:
|
|
|
|
|
+ self.log.exception(f"Error executing query: {e}, Query: {query}, Args: {args}")
|
|
|
|
|
+ raise
|
|
|
|
|
+ raise pymysql.err.OperationalError(2013, f"重试 15 次后仍无法连接数据库,Query: {query[:100]}")
|
|
|
|
|
+
|
|
|
|
|
+ def _execute_many(self, query, args_list, commit=True):
|
|
|
|
|
+ """
|
|
|
|
|
+ executemany 的断连重试包装,指数退避,覆盖分钟级网络波动。
|
|
|
|
|
+
|
|
|
|
|
+ Args:
|
|
|
|
|
+ query (str): SQL语句。
|
|
|
|
|
+ args_list (list): 参数列表。
|
|
|
|
|
+ commit (bool, optional): 是否提交事务。Defaults to True。
|
|
|
|
|
+
|
|
|
|
|
+ Returns:
|
|
|
|
|
+ int: 影响行数(cursor.rowcount)。
|
|
|
|
|
+
|
|
|
|
|
+ Raises:
|
|
|
|
|
+ pymysql.err.IntegrityError: 唯一键冲突等完整性错误。
|
|
|
|
|
+ pymysql.err.OperationalError: 重试耗尽后仍无法连接数据库。
|
|
|
|
|
+ """
|
|
|
|
|
+ for attempt in range(15):
|
|
|
|
|
+ try:
|
|
|
|
|
+ with self.pool.connection() as conn:
|
|
|
|
|
+ with conn.cursor() as cursor:
|
|
|
|
|
+ cursor.executemany(query, args_list)
|
|
|
|
|
+ if commit:
|
|
|
|
|
+ conn.commit()
|
|
|
|
|
+ return cursor.rowcount
|
|
|
|
|
+ except (pymysql.err.InterfaceError, pymysql.err.OperationalError) as e:
|
|
|
|
|
+ errno = e.args[0] if e.args else None
|
|
|
|
|
+ if errno in (2006, 2013):
|
|
|
|
|
+ delay = min(5 * (2 ** attempt), 300)
|
|
|
|
|
+ self.log.warning(
|
|
|
|
|
+ f"数据库连接断开 (errno={errno}),{delay}s 后重试 "
|
|
|
|
|
+ f"(attempt {attempt + 1}/15): {e}"
|
|
|
|
|
+ )
|
|
|
|
|
+ time.sleep(delay)
|
|
|
|
|
+ continue
|
|
|
|
|
+ if commit:
|
|
|
|
|
+ try:
|
|
|
|
|
+ conn.rollback()
|
|
|
|
|
+ except Exception:
|
|
|
|
|
+ pass
|
|
|
|
|
+ raise
|
|
|
|
|
+ except pymysql.err.IntegrityError:
|
|
|
|
|
+ if commit:
|
|
|
|
|
+ try:
|
|
|
|
|
+ conn.rollback()
|
|
|
|
|
+ except Exception:
|
|
|
|
|
+ pass
|
|
|
|
|
+ raise
|
|
|
|
|
+ except Exception as e:
|
|
|
|
|
+ if commit:
|
|
|
|
|
+ try:
|
|
|
|
|
+ conn.rollback()
|
|
|
|
|
+ except Exception:
|
|
|
|
|
+ pass
|
|
|
|
|
+ self.log.exception(f"Error executing many: {e}, Query: {query}")
|
|
|
|
|
+ raise
|
|
|
|
|
+ raise pymysql.err.OperationalError(2013, f"重试 15 次后仍无法连接数据库,Query: {query[:100]}")
|
|
|
|
|
|
|
|
def select_one(self, query, args=None):
|
|
def select_one(self, query, args=None):
|
|
|
"""
|
|
"""
|
|
@@ -180,13 +282,14 @@ class MySQLConnectionPool:
|
|
|
return cursor.lastrowid
|
|
return cursor.lastrowid
|
|
|
except pymysql.err.IntegrityError as e:
|
|
except pymysql.err.IntegrityError as e:
|
|
|
if "Duplicate entry" in str(e):
|
|
if "Duplicate entry" in str(e):
|
|
|
- self.log.warning(f"插入失败:重复条目,已跳过。错误详情: {e}")
|
|
|
|
|
|
|
+ # 重复条目用 warning 简短输出,不打印堆栈
|
|
|
|
|
+ self.log.warning(f"插入跳过-重复条目 Table: {table}, {e.args[1] if len(e.args) > 1 else e}")
|
|
|
return -1 # 返回 -1 表示重复条目被跳过
|
|
return -1 # 返回 -1 表示重复条目被跳过
|
|
|
else:
|
|
else:
|
|
|
- self.log.exception(f"数据库完整性错误: {e}")
|
|
|
|
|
|
|
+ self.log.error(f"数据库完整性错误 Table: {table}, Error: {e}")
|
|
|
raise
|
|
raise
|
|
|
except Exception as e:
|
|
except Exception as e:
|
|
|
- self.log.exception(f"未知错误: {e}")
|
|
|
|
|
|
|
+ self.log.error(f"insert_one_or_dict 失败 Table: {table}, Error: {e}")
|
|
|
raise
|
|
raise
|
|
|
|
|
|
|
|
def insert_many(self, table=None, data_list=None, query=None, args_list=None, batch_size=1000, commit=True,
|
|
def insert_many(self, table=None, data_list=None, query=None, args_list=None, batch_size=1000, commit=True,
|
|
@@ -220,37 +323,26 @@ class MySQLConnectionPool:
|
|
|
for i in range(0, len(args_list), batch_size):
|
|
for i in range(0, len(args_list), batch_size):
|
|
|
batch = args_list[i:i + batch_size]
|
|
batch = args_list[i:i + batch_size]
|
|
|
try:
|
|
try:
|
|
|
- with self.pool.connection() as conn:
|
|
|
|
|
- with conn.cursor() as cursor:
|
|
|
|
|
- cursor.executemany(query, batch)
|
|
|
|
|
- if commit:
|
|
|
|
|
- conn.commit()
|
|
|
|
|
- total += cursor.rowcount
|
|
|
|
|
|
|
+ total += self._execute_many(query, batch, commit=commit)
|
|
|
except pymysql.err.IntegrityError as e:
|
|
except pymysql.err.IntegrityError as e:
|
|
|
# 处理唯一索引冲突
|
|
# 处理唯一索引冲突
|
|
|
if "Duplicate entry" in str(e):
|
|
if "Duplicate entry" in str(e):
|
|
|
if ignore:
|
|
if ignore:
|
|
|
- # 如果使用了 INSERT IGNORE,理论上不会进这里,但以防万一
|
|
|
|
|
self.log.warning(f"批量插入遇到重复条目(ignore模式): {e}")
|
|
self.log.warning(f"批量插入遇到重复条目(ignore模式): {e}")
|
|
|
else:
|
|
else:
|
|
|
# 没有使用 IGNORE,降级为逐条插入
|
|
# 没有使用 IGNORE,降级为逐条插入
|
|
|
self.log.warning(f"批量插入遇到重复条目,开始逐条插入。错误: {e}")
|
|
self.log.warning(f"批量插入遇到重复条目,开始逐条插入。错误: {e}")
|
|
|
- if commit:
|
|
|
|
|
- conn.rollback()
|
|
|
|
|
-
|
|
|
|
|
rowcount = 0
|
|
rowcount = 0
|
|
|
for j, args in enumerate(batch):
|
|
for j, args in enumerate(batch):
|
|
|
try:
|
|
try:
|
|
|
if data_list:
|
|
if data_list:
|
|
|
- # 字典模式
|
|
|
|
|
self.insert_one_or_dict(
|
|
self.insert_one_or_dict(
|
|
|
table=table,
|
|
table=table,
|
|
|
data=dict(zip(data_list[0].keys(), args)),
|
|
data=dict(zip(data_list[0].keys(), args)),
|
|
|
commit=commit,
|
|
commit=commit,
|
|
|
- ignore=False # 单条插入时手动捕获重复
|
|
|
|
|
|
|
+ ignore=False
|
|
|
)
|
|
)
|
|
|
else:
|
|
else:
|
|
|
- # 原始SQL模式
|
|
|
|
|
self.insert_one(query, args)
|
|
self.insert_one(query, args)
|
|
|
rowcount += 1
|
|
rowcount += 1
|
|
|
except pymysql.err.IntegrityError as e2:
|
|
except pymysql.err.IntegrityError as e2:
|
|
@@ -263,16 +355,10 @@ class MySQLConnectionPool:
|
|
|
total += rowcount
|
|
total += rowcount
|
|
|
self.log.info(f"批次逐条插入完成: 成功{rowcount}/{len(batch)}条")
|
|
self.log.info(f"批次逐条插入完成: 成功{rowcount}/{len(batch)}条")
|
|
|
else:
|
|
else:
|
|
|
- # 其他完整性错误
|
|
|
|
|
self.log.exception(f"数据库完整性错误: {e}")
|
|
self.log.exception(f"数据库完整性错误: {e}")
|
|
|
- if commit:
|
|
|
|
|
- conn.rollback()
|
|
|
|
|
raise e
|
|
raise e
|
|
|
except Exception as e:
|
|
except Exception as e:
|
|
|
- # 其他数据库错误
|
|
|
|
|
self.log.exception(f"批量插入失败: {e}")
|
|
self.log.exception(f"批量插入失败: {e}")
|
|
|
- if commit:
|
|
|
|
|
- conn.rollback()
|
|
|
|
|
raise e
|
|
raise e
|
|
|
if table:
|
|
if table:
|
|
|
self.log.info(f"sql insert_many, Table: {table}, Total Rows: {total}")
|
|
self.log.info(f"sql insert_many, Table: {table}, Total Rows: {total}")
|
|
@@ -308,17 +394,10 @@ class MySQLConnectionPool:
|
|
|
for i in range(0, len(args_list), batch_size):
|
|
for i in range(0, len(args_list), batch_size):
|
|
|
batch = args_list[i:i + batch_size]
|
|
batch = args_list[i:i + batch_size]
|
|
|
try:
|
|
try:
|
|
|
- with self.pool.connection() as conn:
|
|
|
|
|
- with conn.cursor() as cursor:
|
|
|
|
|
- cursor.executemany(query, batch)
|
|
|
|
|
- if commit:
|
|
|
|
|
- conn.commit()
|
|
|
|
|
- total += cursor.rowcount
|
|
|
|
|
|
|
+ total += self._execute_many(query, batch, commit=commit)
|
|
|
except pymysql.err.IntegrityError as e:
|
|
except pymysql.err.IntegrityError as e:
|
|
|
if "Duplicate entry" in str(e) and not ignore:
|
|
if "Duplicate entry" in str(e) and not ignore:
|
|
|
self.log.warning(f"批量插入遇到重复,降级为逐条插入: {e}")
|
|
self.log.warning(f"批量插入遇到重复,降级为逐条插入: {e}")
|
|
|
- if commit:
|
|
|
|
|
- conn.rollback()
|
|
|
|
|
rowcount = 0
|
|
rowcount = 0
|
|
|
for args in batch:
|
|
for args in batch:
|
|
|
try:
|
|
try:
|
|
@@ -334,13 +413,9 @@ class MySQLConnectionPool:
|
|
|
total += rowcount
|
|
total += rowcount
|
|
|
else:
|
|
else:
|
|
|
self.log.exception(f"数据库完整性错误: {e}")
|
|
self.log.exception(f"数据库完整性错误: {e}")
|
|
|
- if commit:
|
|
|
|
|
- conn.rollback()
|
|
|
|
|
raise e
|
|
raise e
|
|
|
except Exception as e:
|
|
except Exception as e:
|
|
|
self.log.exception(f"批量插入失败: {e}")
|
|
self.log.exception(f"批量插入失败: {e}")
|
|
|
- if commit:
|
|
|
|
|
- conn.rollback()
|
|
|
|
|
raise e
|
|
raise e
|
|
|
self.log.info(f"sql insert_many_two, Table: {table}, Total Rows: {total}")
|
|
self.log.info(f"sql insert_many_two, Table: {table}, Total Rows: {total}")
|
|
|
return total
|
|
return total
|
|
@@ -356,11 +431,8 @@ class MySQLConnectionPool:
|
|
|
for i in range(0, len(args_list), batch_size):
|
|
for i in range(0, len(args_list), batch_size):
|
|
|
batch = args_list[i:i + batch_size]
|
|
batch = args_list[i:i + batch_size]
|
|
|
try:
|
|
try:
|
|
|
- with self.pool.connection() as conn:
|
|
|
|
|
- with conn.cursor() as cursor:
|
|
|
|
|
- cursor.executemany(query, batch)
|
|
|
|
|
- conn.commit()
|
|
|
|
|
- self.log.debug(f"insert_too_many -> Total Rows: {len(batch)}")
|
|
|
|
|
|
|
+ self._execute_many(query, batch, commit=True)
|
|
|
|
|
+ self.log.debug(f"insert_too_many -> Total Rows: {len(batch)}")
|
|
|
except Exception as e:
|
|
except Exception as e:
|
|
|
self.log.error(f"insert_too_many error. Trying single insert. Error: {e}")
|
|
self.log.error(f"insert_too_many error. Trying single insert. Error: {e}")
|
|
|
# 当前批次降级为单条插入
|
|
# 当前批次降级为单条插入
|
|
@@ -382,17 +454,11 @@ class MySQLConnectionPool:
|
|
|
:param query: 更新语句
|
|
:param query: 更新语句
|
|
|
:param args_list: 更新参数列表
|
|
:param args_list: 更新参数列表
|
|
|
"""
|
|
"""
|
|
|
- conn = None
|
|
|
|
|
- cursor = None
|
|
|
|
|
try:
|
|
try:
|
|
|
- conn = self.pool.connection()
|
|
|
|
|
- cursor = conn.cursor()
|
|
|
|
|
- cursor.executemany(query, args_list)
|
|
|
|
|
- conn.commit()
|
|
|
|
|
|
|
+ self._execute_many(query, args_list, commit=True)
|
|
|
self.log.debug(f"sql update_all, SQL: {query}, Rows: {len(args_list)}")
|
|
self.log.debug(f"sql update_all, SQL: {query}, Rows: {len(args_list)}")
|
|
|
self.log.info('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>data update_all 更新中>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
|
|
self.log.info('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>data update_all 更新中>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
|
|
|
except Exception as e:
|
|
except Exception as e:
|
|
|
- conn.rollback()
|
|
|
|
|
self.log.error(f"Error executing query: {e}")
|
|
self.log.error(f"Error executing query: {e}")
|
|
|
# 如果批量更新失败,则逐条更新
|
|
# 如果批量更新失败,则逐条更新
|
|
|
rowcount = 0
|
|
rowcount = 0
|
|
@@ -400,11 +466,6 @@ class MySQLConnectionPool:
|
|
|
self.update_one(query, args)
|
|
self.update_one(query, args)
|
|
|
rowcount += 1
|
|
rowcount += 1
|
|
|
self.log.debug(f'Batch update failed. Updated {rowcount} rows individually.')
|
|
self.log.debug(f'Batch update failed. Updated {rowcount} rows individually.')
|
|
|
- finally:
|
|
|
|
|
- if cursor:
|
|
|
|
|
- cursor.close()
|
|
|
|
|
- if conn:
|
|
|
|
|
- conn.close()
|
|
|
|
|
|
|
|
|
|
def update_one_or_dict(self, table=None, data=None, condition=None, query=None, args=None, commit=True):
|
|
def update_one_or_dict(self, table=None, data=None, condition=None, query=None, args=None, commit=True):
|
|
|
"""
|
|
"""
|
|
@@ -526,13 +587,9 @@ class MySQLConnectionPool:
|
|
|
|
|
|
|
|
# 执行批量更新
|
|
# 执行批量更新
|
|
|
try:
|
|
try:
|
|
|
- with self.pool.connection() as conn:
|
|
|
|
|
- with conn.cursor() as cursor:
|
|
|
|
|
- cursor.executemany(base_query, batch_args)
|
|
|
|
|
- if commit:
|
|
|
|
|
- conn.commit()
|
|
|
|
|
- total += cursor.rowcount
|
|
|
|
|
- self.log.debug(f"Batch update succeeded. Rows: {cursor.rowcount}")
|
|
|
|
|
|
|
+ rowcount = self._execute_many(base_query, batch_args, commit=commit)
|
|
|
|
|
+ total += rowcount
|
|
|
|
|
+ self.log.debug(f"Batch update succeeded. Rows: {rowcount}")
|
|
|
except Exception as e:
|
|
except Exception as e:
|
|
|
if commit:
|
|
if commit:
|
|
|
conn.rollback()
|
|
conn.rollback()
|
|
@@ -555,16 +612,10 @@ class MySQLConnectionPool:
|
|
|
for i in range(0, len(args_list), batch_size):
|
|
for i in range(0, len(args_list), batch_size):
|
|
|
batch_args = args_list[i:i + batch_size]
|
|
batch_args = args_list[i:i + batch_size]
|
|
|
try:
|
|
try:
|
|
|
- with self.pool.connection() as conn:
|
|
|
|
|
- with conn.cursor() as cursor:
|
|
|
|
|
- cursor.executemany(query, batch_args)
|
|
|
|
|
- if commit:
|
|
|
|
|
- conn.commit()
|
|
|
|
|
- total += cursor.rowcount
|
|
|
|
|
- self.log.debug(f"Batch update succeeded. Rows: {cursor.rowcount}")
|
|
|
|
|
|
|
+ rowcount = self._execute_many(query, batch_args, commit=commit)
|
|
|
|
|
+ total += rowcount
|
|
|
|
|
+ self.log.debug(f"Batch update succeeded. Rows: {rowcount}")
|
|
|
except Exception as e:
|
|
except Exception as e:
|
|
|
- if commit:
|
|
|
|
|
- conn.rollback()
|
|
|
|
|
self.log.error(f"Batch update failed: {e}")
|
|
self.log.error(f"Batch update failed: {e}")
|
|
|
# 降级为单条更新
|
|
# 降级为单条更新
|
|
|
for args in batch_args:
|
|
for args in batch_args:
|