Metrix/CapacityReport
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor Dockerfile for Python 3.13, enhance health check, and add server info retrieval. Implement LOAD DATA INFILE support in DatabaseManager for improved CSV import performance. Introduce script editor for SQL management with save and format functionalities. Update UI for database management and script editing, including status indicators and improved layout.

Browse Source
This commit is contained in:
nixevol
2026-01-14 14:47:25 +08:00
parent f389c0722d
commit 866fc1cb06
127 changed files with 56470 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/database.py
+119 -4
View File
@@ -97,6 +97,55 @@ class DatabaseManager:
except Exception as e:
return False, str(e)
def check_load_data_support(self) -> Tuple[bool, str]:
"""
检测数据库是否支持 LOAD DATA LOCAL INFILE
Returns:
(是否支持, 详细信息)
"""
try:
with self.get_connection() as conn:
with conn.cursor() as cursor:
# 检查服务器端 local_infile 变量
cursor.execute("SHOW VARIABLES LIKE 'local_infile'")
result = cursor.fetchone()
if result:
value = result.get('Value', '').upper()
if value == 'ON':
return True, "服务器已启用 local_infile"
else:
return False, f"服务器 local_infile={value}，需要设置为 ON"
else:
return False, "无法获取 local_infile 变量"
except Exception as e:
return False, f"检测失败: {str(e)}"
def get_server_info(self) -> Dict[str, Any]:
"""获取数据库服务器信息"""
try:
with self.get_connection() as conn:
with conn.cursor() as cursor:
# 获取版本
cursor.execute("SELECT VERSION() as version")
version = cursor.fetchone().get('version', 'Unknown')
# 检查 LOAD DATA 支持
load_data_supported, load_data_msg = self.check_load_data_support()
return {
"version": version,
"load_data_infile": load_data_supported,
"load_data_message": load_data_msg
}
except Exception as e:
return {
"version": "Unknown",
"load_data_infile": False,
"load_data_message": str(e)
}
def get_tables(self) -> List[str]:
"""获取所有表名"""
with self.get_connection() as conn:
@@ -211,10 +260,13 @@ class DatabaseManager:
return False, str(e)
def bulk_insert(self, table_name: str, columns: List[str], data: List[Tuple],
batch_size: int = 5000) -> int:
batch_size: int = 5000, conn=None) -> int:
"""
高性能批量插入
使用 executemany + 批量提交，比 to_sql 快 5-10 倍
Args:
conn: 可选，复用已有连接
"""
if not data:
return 0
@@ -224,8 +276,9 @@ class DatabaseManager:
column_names = ', '.join([f'`{col}`' for col in columns])
sql = f"INSERT INTO `{table_name}` ({column_names}) VALUES ({placeholders})"
with self.get_fast_connection() as conn:
with conn.cursor() as cursor:
def do_insert(connection):
nonlocal total_inserted
with connection.cursor() as cursor:
# 优化插入性能的设置
cursor.execute("SET autocommit=0")
cursor.execute("SET unique_checks=0")
@@ -238,13 +291,75 @@ class DatabaseManager:
total_inserted += len(batch)
# 提交并恢复设置
conn.commit()
connection.commit()
cursor.execute("SET unique_checks=1")
cursor.execute("SET foreign_key_checks=1")
cursor.execute("SET autocommit=1")
if conn:
do_insert(conn)
else:
with self.get_fast_connection() as connection:
do_insert(connection)
return total_inserted
def load_data_infile(self, table_name: str, columns: List[str],
temp_file: str, conn=None) -> int:
"""
使用 LOAD DATA LOCAL INFILE 高速导入 CSV 文件
比 executemany 快 10-50 倍
Args:
table_name: 目标表名
columns: 列名列表
temp_file: 临时 CSV 文件路径
conn: 可选，复用已有连接
Returns:
导入的行数
"""
column_names = ', '.join([f'`{col}`' for col in columns])
# 使用正斜杠路径（MySQL 兼容）
file_path = temp_file.replace('\\', '/')
sql = f"""
LOAD DATA LOCAL INFILE '{file_path}'
INTO TABLE `{table_name}`
CHARACTER SET utf8mb4
FIELDS TERMINATED BY ','
OPTIONALLY ENCLOSED BY '"'
LINES TERMINATED BY '\\n'
IGNORE 1 LINES
({column_names})
"""
def do_load(connection):
with connection.cursor() as cursor:
# 优化导入性能的设置
cursor.execute("SET autocommit=0")
cursor.execute("SET unique_checks=0")
cursor.execute("SET foreign_key_checks=0")
# 执行 LOAD DATA
cursor.execute(sql)
row_count = cursor.rowcount
# 提交并恢复设置
connection.commit()
cursor.execute("SET unique_checks=1")
cursor.execute("SET foreign_key_checks=1")
cursor.execute("SET autocommit=1")
return row_count
if conn:
return do_load(conn)
else:
with self.get_fast_connection() as connection:
return do_load(connection)
# 字段类型到 MySQL 类型的映射
TYPE_MAPPING = {
'string': 'VARCHAR(255)',
app/main.py
+142 -9
View File
@@ -60,6 +60,54 @@ global_task_lock: Dict[str, Any] = {
}
# ==================== 健康检查 ====================
@app.get("/health")
async def health_check():
"""
健康检查接口（用于 Docker/K8s 健康检查）
返回:
- status: 服务状态 (healthy/unhealthy)
- timestamp: 当前时间戳
- version: 应用版本
- checks: 各组件检查结果
"""
checks = {
"app": {"status": "ok"},
"database": {"status": "unknown"},
}
# 检查数据库连接
try:
db_manager = DatabaseManager(config.mysql_config)
server_info = db_manager.get_server_info()
if server_info:
checks["database"] = {
"status": "ok",
"version": server_info.get("version", "unknown"),
"load_data_infile": server_info.get("load_data_support", False)
}
else:
checks["database"] = {"status": "error", "message": "无法获取数据库信息"}
except Exception as e:
checks["database"] = {"status": "error", "message": str(e)}
# 综合判断健康状态
is_healthy = all(
c.get("status") == "ok"
for c in checks.values()
)
return {
"status": "healthy" if is_healthy else "unhealthy",
"timestamp": datetime.now().isoformat(),
"version": "2.0.0",
"uptime_pid": os.getpid(),
"checks": checks
}
# ==================== 页面路由 ====================
@app.get("/", response_class=HTMLResponse)
@@ -240,7 +288,6 @@ async def test_task_api():
@app.get("/api/task/status")
@app.post("/api/task/status")
async def get_global_task_status():
"""获取全局任务状态（是否有任务在上传或处理中）"""
# 自动清理：如果任务已完成但还锁定，则自动解锁
@@ -324,7 +371,6 @@ async def unlock_task(task_id: str = Body(None, embed=True)):
@app.get("/api/process/active")
@app.post("/api/process/active")
async def get_active_task():
"""获取当前正在进行的任务（全局状态）- 兼容旧接口"""
return await get_global_task_status()
@@ -472,16 +518,28 @@ async def get_history_detail(record_id: str = Body(..., embed=True)):
# ==================== 数据库管理 API ====================
@app.get("/api/database/test")
@app.post("/api/database/test")
async def test_database():
"""测试数据库连接（支持 GET 和 POST，无需传参）"""
"""测试数据库连接"""
db = DatabaseManager(config)
success, message = db.test_connection()
db.dispose()
return {"success": success, "message": message}
@app.get("/api/database/info")
async def get_database_info():
"""获取数据库服务器信息（包括是否支持 LOAD DATA INFILE）"""
db = DatabaseManager(config)
try:
info = db.get_server_info()
return {"success": True, **info}
except Exception as e:
return {"success": False, "error": str(e)}
finally:
db.dispose()
@app.get("/api/database/tables")
@app.post("/api/database/tables")
async def get_tables():
@@ -682,14 +740,23 @@ async def update_extract_fields(fields: List[Dict[str, Any]] = Body(...)):
# ==================== 清理 API ====================
@app.get("/api/cache/size")
@app.post("/api/cache/size")
async def get_cache_size():
"""获取 cache 目录占用大小（无需传参）"""
"""获取 cache 目录占用大小"""
try:
total_size = 0
file_count = 0
dir_count = 0
# 如果 cache 目录不存在，直接返回 0
if not CACHE_DIR.exists():
return {
"success": True,
"size_bytes": 0,
"size_formatted": "0 B",
"file_count": 0,
"dir_count": 0
}
def get_dir_size(path: Path):
"""递归计算目录大小"""
nonlocal total_size, file_count, dir_count
@@ -705,12 +772,18 @@ async def get_cache_size():
pass
# 计算 cache 目录大小（排除 history.json）
for item in CACHE_DIR.iterdir():
if item.name != "history.json":
get_dir_size(item)
try:
for item in CACHE_DIR.iterdir():
if item.name != "history.json":
get_dir_size(item)
except (PermissionError, OSError) as e:
# 如果无法访问目录，返回 0 而不是失败
pass
# 格式化大小
def format_size(size_bytes):
if size_bytes == 0:
return "0 B"
for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
if size_bytes < 1024.0:
return f"{size_bytes:.2f} {unit}"
@@ -898,6 +971,66 @@ async def get_service_status():
}
# ==================== SQL 脚本编辑 API ====================
@app.get("/api/script/content")
async def get_script_content():
"""获取 SQL 脚本内容"""
from app.config import SQL_SCRIPT
try:
if SQL_SCRIPT.exists():
content = SQL_SCRIPT.read_text(encoding='utf-8')
# 获取文件修改时间
mtime = SQL_SCRIPT.stat().st_mtime
from datetime import datetime
modified = datetime.fromtimestamp(mtime).strftime('%Y-%m-%d %H:%M:%S')
return {
"success": True,
"content": content,
"modified": modified,
"path": str(SQL_SCRIPT)
}
else:
return {
"success": True,
"content": "# SQL 脚本文件不存在，请在此编写脚本\n",
"modified": None,
"path": str(SQL_SCRIPT)
}
except Exception as e:
return {"success": False, "error": str(e)}
@app.post("/api/script/save")
async def save_script_content(content: str = Body(..., embed=True)):
"""保存 SQL 脚本内容"""
from app.config import SQL_SCRIPT
try:
# 备份原文件
if SQL_SCRIPT.exists():
backup_path = SQL_SCRIPT.with_suffix('.sql.bak')
import shutil
shutil.copy(SQL_SCRIPT, backup_path)
# 保存新内容
SQL_SCRIPT.write_text(content, encoding='utf-8')
# 获取新的修改时间
mtime = SQL_SCRIPT.stat().st_mtime
from datetime import datetime
modified = datetime.fromtimestamp(mtime).strftime('%Y-%m-%d %H:%M:%S')
return {
"success": True,
"message": "脚本保存成功",
"modified": modified
}
except Exception as e:
return {"success": False, "error": str(e)}
# ==================== 启动入口 ====================
if __name__ == "__main__":
app/processor.py
+135 -25
View File
@@ -84,6 +84,10 @@ class DataProcessor:
# 预编译字段映射，避免重复查找
self._field_map, self._type_map = self._build_field_map()
# LOAD DATA INFILE 支持状态（在首次使用时检测）
self._load_data_supported: Optional[bool] = None
self._load_data_checked = False
def _build_field_map(self) -> Tuple[Dict[str, str], Dict[str, str]]:
"""
@@ -138,6 +142,9 @@ class DataProcessor:
self.results["error"] = str(e)
finally:
# 清理临时目录
self._cleanup_temp_dir()
# 释放数据库连接
self.db.dispose()
return self.results
@@ -264,11 +271,20 @@ class DataProcessor:
return 'gbk'
return 'utf-8'
def _process_csv_file_fast(self, csv_file: Path, table_name: str) -> int:
def _process_csv_file_fast(self, csv_file: Path, table_name: str,
conn=None, table_created: bool = False) -> Tuple[int, bool]:
"""
高性能处理单个 CSV 文件
使用批量插入代替 to_sql，性能提升 5-10 倍
根据配置的字段类型进行数据转换
使用 LOAD DATA LOCAL INFILE，比 executemany 快 10-50 倍
Args:
csv_file: CSV 文件路径
table_name: 目标表名
conn: 数据库连接（复用）
table_created: 表是否已创建
Returns:
(导入行数, 表是否已创建)
"""
encoding = self._detect_encoding(csv_file)
rel_path = csv_file.relative_to(self.work_dir)
@@ -294,7 +310,7 @@ class DataProcessor:
if len(col_mapping) <= 3:
if 'kpis' in str(csv_file).lower():
self.logger.warning(f"跳过非数据文件: {rel_path}")
return 0
return 0, table_created
raise ValueError(f"字段匹配不足: {rel_path}")
# 选择需要的列并重命名
@@ -340,16 +356,105 @@ class DataProcessor:
if mask.any():
df_result.loc[mask, col] = df_result.loc[mask, col].str[:255]
# 确保表存在（传递类型信息）
self.db.create_table_from_columns(table_name, target_cols, column_types)
# 确保表存在（只在第一次创建）
if not table_created:
self.db.create_table_from_columns(table_name, target_cols, column_types)
table_created = True
# 转换为元组列表，用于批量插入
data_tuples = [tuple(row) for row in df_result.values]
# 使用 LOAD DATA INFILE 导入
inserted = self._load_data_infile(df_result, table_name, target_cols, conn)
return inserted, table_created
def _get_temp_dir(self) -> Path:
"""获取临时目录（使用工作目录下的 .temp 子目录）"""
temp_dir = self.work_dir / '.temp'
temp_dir.mkdir(exist_ok=True)
return temp_dir
def _cleanup_temp_dir(self):
"""清理临时目录"""
temp_dir = self.work_dir / '.temp'
if temp_dir.exists():
try:
import shutil
shutil.rmtree(temp_dir)
except Exception:
pass
def _check_load_data_support(self) -> bool:
"""检测是否支持 LOAD DATA INFILE（只检测一次）"""
if self._load_data_checked:
return self._load_data_supported or False
self._load_data_checked = True
supported, message = self.db.check_load_data_support()
self._load_data_supported = supported
if supported:
self.logger.info(f"LOAD DATA INFILE: 已启用 ({message})")
else:
self.logger.warning(f"LOAD DATA INFILE: 不可用 ({message})，将使用批量插入模式")
return supported
def _load_data_infile(self, df: pd.DataFrame, table_name: str,
columns: List[str], conn=None) -> int:
"""
使用 LOAD DATA LOCAL INFILE 导入数据
如果失败则自动回退到 bulk_insert 方式
临时文件放在工作目录的 .temp 子目录中
"""
import tempfile
# 检测是否支持 LOAD DATA INFILE
if not self._check_load_data_support():
# 不支持，直接使用 bulk_insert
return self._bulk_insert_fallback(df, table_name, columns, conn)
# 获取临时目录
temp_dir = self._get_temp_dir()
temp_file = None
try:
# 写入临时 CSV 文件
with tempfile.NamedTemporaryFile(
mode='w',
suffix='.csv',
delete=False,
encoding='utf-8',
newline='',
dir=str(temp_dir) # 使用指定的临时目录
) as f:
temp_file = f.name
# 写入 CSV（带表头，用于 IGNORE 1 LINES）
df.to_csv(f, index=False, header=True, na_rep='\\N')
# 使用 LOAD DATA LOCAL INFILE 导入
inserted = self.db.load_data_infile(table_name, columns, temp_file, conn)
return inserted
except Exception as e:
# LOAD DATA 失败，标记为不支持并回退
self.logger.warning(f"LOAD DATA INFILE 执行失败: {e}，回退到批量插入模式")
self._load_data_supported = False
return self._bulk_insert_fallback(df, table_name, columns, conn)
finally:
# 清理临时文件
if temp_file and os.path.exists(temp_file):
try:
os.remove(temp_file)
except Exception:
pass
def _bulk_insert_fallback(self, df: pd.DataFrame, table_name: str,
columns: List[str], conn=None) -> int:
"""批量插入回退方案"""
# 转换为元组列表
data_tuples = [tuple(row) for row in df.values]
# 使用批量插入
inserted = self.db.bulk_insert(table_name, target_cols, data_tuples, self.BATCH_SIZE)
return inserted
return self.db.bulk_insert(table_name, columns, data_tuples, self.BATCH_SIZE, conn)
# 支持的日期时间格式列表
DATETIME_FORMATS = [
@@ -510,7 +615,7 @@ class DataProcessor:
return data_dirs
def _process_csv_files(self):
"""处理所有 CSV 文件（高性能版）"""
"""处理所有 CSV 文件（高性能版 - 使用 LOAD DATA INFILE + 连接复用）"""
self.logger.info("正在处理 CSV 文件并上传到数据库...")
# 查找数据目录
@@ -520,7 +625,7 @@ class DataProcessor:
self.logger.warning("未找到任何数据目录")
return
# 按目录分组处理
# 按目录分组处理，使用连接复用
for table_name, subdir in data_dirs.items():
self.logger.info(f"处理目录: {subdir.relative_to(self.work_dir)} -> 表: {table_name}")
@@ -533,20 +638,25 @@ class DataProcessor:
total_rows = 0
start_time = time.time()
table_created = False
for i, csv_file in enumerate(csv_files, 1):
try:
rows = self._process_csv_file_fast(csv_file, table_name)
total_rows += rows
# 每处理 10 个文件报告一次进度
if i % 10 == 0:
elapsed = round(time.time() - start_time, 1)
self.logger.info(f"进度: {i}/{len(csv_files)} 文件, 已导入 {total_rows} 行, 耗时 {elapsed}s")
# 使用连接复用：一个表的所有 CSV 文件共用一个连接
with self.db.get_fast_connection() as conn:
for i, csv_file in enumerate(csv_files, 1):
try:
rows, table_created = self._process_csv_file_fast(
csv_file, table_name, conn, table_created
)
total_rows += rows
except Exception as e:
rel_path = csv_file.relative_to(self.work_dir)
self.logger.error(f"CSV 处理失败 {rel_path}: {e}")
# 每处理 10 个文件报告一次进度
if i % 10 == 0:
elapsed = round(time.time() - start_time, 1)
self.logger.info(f"进度: {i}/{len(csv_files)} 文件, 已导入 {total_rows} 行, 耗时 {elapsed}s")
except Exception as e:
rel_path = csv_file.relative_to(self.work_dir)
self.logger.error(f"CSV 处理失败 {rel_path}: {e}")
elapsed = round(time.time() - start_time, 2)
speed = round(total_rows / elapsed) if elapsed > 0 else 0