diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..5495c0c
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,46 @@
+# Git
+.git
+.gitignore
+
+# Python
+__pycache__
+*.py[cod]
+*$py.class
+*.so
+.Python
+.env
+.venv
+venv/
+ENV/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# Cache and logs
+cache/
+logs/
+*.log
+
+# Docker
+docker-compose*.yml
+Dockerfile*
+.dockerignore
+
+# MySQL data
+mysql/
+
+# Temporary files
+*.tmp
+*.temp
+.temp/
+
+# OS files
+.DS_Store
+Thumbs.db
+
+# Documentation
+*.md
+!README.md
diff --git a/.gitignore b/.gitignore
index 18b5d6d..ede13d1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,7 +10,6 @@ dist/
downloads/
eggs/
.eggs/
-lib/
lib64/
parts/
sdist/
diff --git a/DEPLOY.md b/DEPLOY.md
new file mode 100644
index 0000000..db06dca
--- /dev/null
+++ b/DEPLOY.md
@@ -0,0 +1,178 @@
+# CapacityReport Docker 部署指南
+
+## 环境要求
+
+- Docker Engine 20.10+
+- Docker Compose 2.0+
+
+## 快速启动
+
+```bash
+# 启动所有服务
+docker-compose up -d
+
+# 查看日志
+docker-compose logs -f
+
+# 停止服务
+docker-compose down
+```
+
+## 内网离线部署
+
+### 步骤1：准备镜像（在有外网的机器上）
+
+```bash
+# 拉取所需镜像
+docker pull python:3.11-slim
+docker pull mysql:8.0
+
+# 导出为文件
+docker save -o capacity-images.tar python:3.11-slim mysql:8.0
+
+# 将 capacity-images.tar 传输到目标机器
+```
+
+### 步骤2：加载镜像（在目标机器上）
+
+```bash
+# 加载镜像
+docker load -i capacity-images.tar
+```
+
+### 步骤3：构建并启动
+
+```bash
+# 构建应用镜像
+docker-compose build
+
+# 启动服务
+docker-compose up -d
+```
+
+## 配置说明
+
+### 数据库连接
+
+Docker 环境下，需要修改 `Configure.json` 中的数据库地址：
+
+```json
+{
+ "MySQL_DBInfo": {
+ "host": "mysql",
+ "port": 3306,
+ "user": "root",
+ "passwd": "gmcc123",
+ "dbname": "CapacityReport"
+ }
+}
+```
+
+**注意**：
+- Docker Compose 中 host 应设置为 `mysql`（服务名），而不是 `localhost` 或 `127.0.0.1`
+- 数据库 `CapacityReport` 会在容器首次启动时自动创建（使用 utf8mb4 字符集）
+- 如果数据目录已存在（从持久化卷恢复），数据库不会被重复创建，这是安全的
+
+### 端口映射
+
+| 服务 | 端口 | 说明 |
+|-----|------|-----|
+| app | 9081 | Web 应用 |
+| mysql | 3306 | MySQL 数据库 |
+
+## LOAD DATA INFILE 支持
+
+MySQL 容器已配置支持 `LOAD DATA LOCAL INFILE`：
+
+1. 服务端配置：`mysql/conf.d/custom.cnf` 中 `local_infile = ON`
+2. 启动参数：`--local-infile=ON`
+3. Python 客户端：PyMySQL 连接时 `local_infile=True`
+
+验证是否启用：
+
+```bash
+docker exec -it capacity-mysql mysql -uroot -pgmcc123 -e "SHOW VARIABLES LIKE 'local_infile';"
+```
+
+预期输出：
+```
++---------------+-------+
+| Variable_name | Value |
++---------------+-------+
+| local_infile | ON |
++---------------+-------+
+```
+
+## 数据持久化
+
+| 路径 | 说明 |
+|-----|-----|
+| ./cache | 处理缓存和历史记录 |
+| ./logs | 应用日志 |
+| mysql-data | MySQL 数据（Docker 卷） |
+
+## 常用命令
+
+```bash
+# 查看运行状态
+docker-compose ps
+
+# 查看应用日志
+docker-compose logs -f app
+
+# 查看 MySQL 日志
+docker-compose logs -f mysql
+
+# 重启应用
+docker-compose restart app
+
+# 进入应用容器
+docker exec -it capacity-report bash
+
+# 进入 MySQL
+docker exec -it capacity-mysql mysql -uroot -pgmcc123
+
+# 备份数据库
+docker exec capacity-mysql mysqldump -uroot -pgmcc123 CapacityReport > backup.sql
+
+# 恢复数据库
+docker exec -i capacity-mysql mysql -uroot -pgmcc123 CapacityReport < backup.sql
+
+# 清空数据重新开始
+docker-compose down -v
+docker-compose up -d
+```
+
+## 故障排查
+
+### 应用无法连接 MySQL
+
+1. 检查 MySQL 是否就绪：
+ ```bash
+ docker-compose logs mysql
+ ```
+
+2. 验证 MySQL 健康状态：
+ ```bash
+ docker exec capacity-mysql mysqladmin ping -uroot -pgmcc123
+ ```
+
+3. 检查 Configure.json 中 host 设置是否为 `mysql`
+
+### LOAD DATA INFILE 不工作
+
+1. 检查 MySQL 变量：
+ ```bash
+ docker exec -it capacity-mysql mysql -uroot -pgmcc123 -e "SHOW VARIABLES LIKE 'local_infile';"
+ ```
+
+2. 检查配置文件：
+ ```bash
+ docker exec -it capacity-mysql cat /etc/mysql/conf.d/custom.cnf
+ ```
+
+### cache 目录权限问题
+
+```bash
+chmod -R 777 cache/
+```
diff --git a/Dockerfile b/Dockerfile
index db9bae7..5f6a4e1 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,24 +1,41 @@
-FROM python:3.11
+# CapacityReport Dockerfile
+# 容量报表处理系统 - 用于内网离线部署
+
+FROM python:3.13-slim
+
+LABEL maintainer="CapacityReport"
+LABEL description="容量报表处理系统"
WORKDIR /app
-# 复制依赖文件
+# 安装系统依赖
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ gcc \
+ libmariadb-dev \
+ curl \
+ && rm -rf /var/lib/apt/lists/*
+
+# 先复制依赖文件（利用缓存）
COPY requirements.txt ./
-# 安装依赖
+# 安装 Python 依赖
RUN pip install --no-cache-dir -r requirements.txt
-# 创建日志目录
-RUN mkdir -p /var/log/supervisor /var/run
+# 创建目录
+RUN mkdir -p /var/log/supervisor /var/run /app/cache /app/static/lib
# 复制应用代码
COPY . .
# 设置启动脚本权限
-RUN chmod +x /app/start.sh
+RUN chmod +x /app/start.sh 2>/dev/null || true
# 暴露端口
EXPOSE 9081
-# 使用启动脚本
-CMD ["/app/start.sh"]
+# 健康检查
+HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \
+ CMD curl -f http://localhost:9081/health || exit 1
+
+# 启动命令
+CMD ["supervisord", "-c", "/app/supervisord.conf"]
diff --git a/app/database.py b/app/database.py
index c969dfa..cec2ac2 100644
--- a/app/database.py
+++ b/app/database.py
@@ -97,6 +97,55 @@ class DatabaseManager:
except Exception as e:
return False, str(e)
+ def check_load_data_support(self) -> Tuple[bool, str]:
+ """
+ 检测数据库是否支持 LOAD DATA LOCAL INFILE
+
+ Returns:
+ (是否支持, 详细信息)
+ """
+ try:
+ with self.get_connection() as conn:
+ with conn.cursor() as cursor:
+ # 检查服务器端 local_infile 变量
+ cursor.execute("SHOW VARIABLES LIKE 'local_infile'")
+ result = cursor.fetchone()
+
+ if result:
+ value = result.get('Value', '').upper()
+ if value == 'ON':
+ return True, "服务器已启用 local_infile"
+ else:
+ return False, f"服务器 local_infile={value}，需要设置为 ON"
+ else:
+ return False, "无法获取 local_infile 变量"
+ except Exception as e:
+ return False, f"检测失败: {str(e)}"
+
+ def get_server_info(self) -> Dict[str, Any]:
+ """获取数据库服务器信息"""
+ try:
+ with self.get_connection() as conn:
+ with conn.cursor() as cursor:
+ # 获取版本
+ cursor.execute("SELECT VERSION() as version")
+ version = cursor.fetchone().get('version', 'Unknown')
+
+ # 检查 LOAD DATA 支持
+ load_data_supported, load_data_msg = self.check_load_data_support()
+
+ return {
+ "version": version,
+ "load_data_infile": load_data_supported,
+ "load_data_message": load_data_msg
+ }
+ except Exception as e:
+ return {
+ "version": "Unknown",
+ "load_data_infile": False,
+ "load_data_message": str(e)
+ }
+
def get_tables(self) -> List[str]:
"""获取所有表名"""
with self.get_connection() as conn:
@@ -211,10 +260,13 @@ class DatabaseManager:
return False, str(e)
def bulk_insert(self, table_name: str, columns: List[str], data: List[Tuple],
- batch_size: int = 5000) -> int:
+ batch_size: int = 5000, conn=None) -> int:
"""
高性能批量插入
使用 executemany + 批量提交，比 to_sql 快 5-10 倍
+
+ Args:
+ conn: 可选，复用已有连接
"""
if not data:
return 0
@@ -224,8 +276,9 @@ class DatabaseManager:
column_names = ', '.join([f'`{col}`' for col in columns])
sql = f"INSERT INTO `{table_name}` ({column_names}) VALUES ({placeholders})"
- with self.get_fast_connection() as conn:
- with conn.cursor() as cursor:
+ def do_insert(connection):
+ nonlocal total_inserted
+ with connection.cursor() as cursor:
# 优化插入性能的设置
cursor.execute("SET autocommit=0")
cursor.execute("SET unique_checks=0")
@@ -238,13 +291,75 @@ class DatabaseManager:
total_inserted += len(batch)
# 提交并恢复设置
- conn.commit()
+ connection.commit()
cursor.execute("SET unique_checks=1")
cursor.execute("SET foreign_key_checks=1")
cursor.execute("SET autocommit=1")
+ if conn:
+ do_insert(conn)
+ else:
+ with self.get_fast_connection() as connection:
+ do_insert(connection)
+
return total_inserted
+ def load_data_infile(self, table_name: str, columns: List[str],
+ temp_file: str, conn=None) -> int:
+ """
+ 使用 LOAD DATA LOCAL INFILE 高速导入 CSV 文件
+ 比 executemany 快 10-50 倍
+
+ Args:
+ table_name: 目标表名
+ columns: 列名列表
+ temp_file: 临时 CSV 文件路径
+ conn: 可选，复用已有连接
+
+ Returns:
+ 导入的行数
+ """
+ column_names = ', '.join([f'`{col}`' for col in columns])
+
+ # 使用正斜杠路径（MySQL 兼容）
+ file_path = temp_file.replace('\\', '/')
+
+ sql = f"""
+ LOAD DATA LOCAL INFILE '{file_path}'
+ INTO TABLE `{table_name}`
+ CHARACTER SET utf8mb4
+ FIELDS TERMINATED BY ','
+ OPTIONALLY ENCLOSED BY '"'
+ LINES TERMINATED BY '\\n'
+ IGNORE 1 LINES
+ ({column_names})
+ """
+
+ def do_load(connection):
+ with connection.cursor() as cursor:
+ # 优化导入性能的设置
+ cursor.execute("SET autocommit=0")
+ cursor.execute("SET unique_checks=0")
+ cursor.execute("SET foreign_key_checks=0")
+
+ # 执行 LOAD DATA
+ cursor.execute(sql)
+ row_count = cursor.rowcount
+
+ # 提交并恢复设置
+ connection.commit()
+ cursor.execute("SET unique_checks=1")
+ cursor.execute("SET foreign_key_checks=1")
+ cursor.execute("SET autocommit=1")
+
+ return row_count
+
+ if conn:
+ return do_load(conn)
+ else:
+ with self.get_fast_connection() as connection:
+ return do_load(connection)
+
# 字段类型到 MySQL 类型的映射
TYPE_MAPPING = {
'string': 'VARCHAR(255)',
diff --git a/app/main.py b/app/main.py
index 918a9f6..4772268 100644
--- a/app/main.py
+++ b/app/main.py
@@ -60,6 +60,54 @@ global_task_lock: Dict[str, Any] = {
}
+# ==================== 健康检查 ====================
+
+@app.get("/health")
+async def health_check():
+ """
+ 健康检查接口（用于 Docker/K8s 健康检查）
+
+ 返回:
+ - status: 服务状态 (healthy/unhealthy)
+ - timestamp: 当前时间戳
+ - version: 应用版本
+ - checks: 各组件检查结果
+ """
+ checks = {
+ "app": {"status": "ok"},
+ "database": {"status": "unknown"},
+ }
+
+ # 检查数据库连接
+ try:
+ db_manager = DatabaseManager(config.mysql_config)
+ server_info = db_manager.get_server_info()
+ if server_info:
+ checks["database"] = {
+ "status": "ok",
+ "version": server_info.get("version", "unknown"),
+ "load_data_infile": server_info.get("load_data_support", False)
+ }
+ else:
+ checks["database"] = {"status": "error", "message": "无法获取数据库信息"}
+ except Exception as e:
+ checks["database"] = {"status": "error", "message": str(e)}
+
+ # 综合判断健康状态
+ is_healthy = all(
+ c.get("status") == "ok"
+ for c in checks.values()
+ )
+
+ return {
+ "status": "healthy" if is_healthy else "unhealthy",
+ "timestamp": datetime.now().isoformat(),
+ "version": "2.0.0",
+ "uptime_pid": os.getpid(),
+ "checks": checks
+ }
+
+
# ==================== 页面路由 ====================
@app.get("/", response_class=HTMLResponse)
@@ -240,7 +288,6 @@ async def test_task_api():
@app.get("/api/task/status")
-@app.post("/api/task/status")
async def get_global_task_status():
"""获取全局任务状态（是否有任务在上传或处理中）"""
# 自动清理：如果任务已完成但还锁定，则自动解锁
@@ -324,7 +371,6 @@ async def unlock_task(task_id: str = Body(None, embed=True)):
@app.get("/api/process/active")
-@app.post("/api/process/active")
async def get_active_task():
"""获取当前正在进行的任务（全局状态）- 兼容旧接口"""
return await get_global_task_status()
@@ -472,16 +518,28 @@ async def get_history_detail(record_id: str = Body(..., embed=True)):
# ==================== 数据库管理 API ====================
-@app.get("/api/database/test")
@app.post("/api/database/test")
async def test_database():
- """测试数据库连接（支持 GET 和 POST，无需传参）"""
+ """测试数据库连接"""
db = DatabaseManager(config)
success, message = db.test_connection()
db.dispose()
return {"success": success, "message": message}
+@app.get("/api/database/info")
+async def get_database_info():
+ """获取数据库服务器信息（包括是否支持 LOAD DATA INFILE）"""
+ db = DatabaseManager(config)
+ try:
+ info = db.get_server_info()
+ return {"success": True, **info}
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+ finally:
+ db.dispose()
+
+
@app.get("/api/database/tables")
@app.post("/api/database/tables")
async def get_tables():
@@ -682,14 +740,23 @@ async def update_extract_fields(fields: List[Dict[str, Any]] = Body(...)):
# ==================== 清理 API ====================
@app.get("/api/cache/size")
-@app.post("/api/cache/size")
async def get_cache_size():
- """获取 cache 目录占用大小（无需传参）"""
+ """获取 cache 目录占用大小"""
try:
total_size = 0
file_count = 0
dir_count = 0
+ # 如果 cache 目录不存在，直接返回 0
+ if not CACHE_DIR.exists():
+ return {
+ "success": True,
+ "size_bytes": 0,
+ "size_formatted": "0 B",
+ "file_count": 0,
+ "dir_count": 0
+ }
+
def get_dir_size(path: Path):
"""递归计算目录大小"""
nonlocal total_size, file_count, dir_count
@@ -705,12 +772,18 @@ async def get_cache_size():
pass
# 计算 cache 目录大小（排除 history.json）
- for item in CACHE_DIR.iterdir():
- if item.name != "history.json":
- get_dir_size(item)
+ try:
+ for item in CACHE_DIR.iterdir():
+ if item.name != "history.json":
+ get_dir_size(item)
+ except (PermissionError, OSError) as e:
+ # 如果无法访问目录，返回 0 而不是失败
+ pass
# 格式化大小
def format_size(size_bytes):
+ if size_bytes == 0:
+ return "0 B"
for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
if size_bytes < 1024.0:
return f"{size_bytes:.2f} {unit}"
@@ -898,6 +971,66 @@ async def get_service_status():
}
+# ==================== SQL 脚本编辑 API ====================
+
+@app.get("/api/script/content")
+async def get_script_content():
+ """获取 SQL 脚本内容"""
+ from app.config import SQL_SCRIPT
+
+ try:
+ if SQL_SCRIPT.exists():
+ content = SQL_SCRIPT.read_text(encoding='utf-8')
+ # 获取文件修改时间
+ mtime = SQL_SCRIPT.stat().st_mtime
+ from datetime import datetime
+ modified = datetime.fromtimestamp(mtime).strftime('%Y-%m-%d %H:%M:%S')
+ return {
+ "success": True,
+ "content": content,
+ "modified": modified,
+ "path": str(SQL_SCRIPT)
+ }
+ else:
+ return {
+ "success": True,
+ "content": "# SQL 脚本文件不存在，请在此编写脚本\n",
+ "modified": None,
+ "path": str(SQL_SCRIPT)
+ }
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
+
+@app.post("/api/script/save")
+async def save_script_content(content: str = Body(..., embed=True)):
+ """保存 SQL 脚本内容"""
+ from app.config import SQL_SCRIPT
+
+ try:
+ # 备份原文件
+ if SQL_SCRIPT.exists():
+ backup_path = SQL_SCRIPT.with_suffix('.sql.bak')
+ import shutil
+ shutil.copy(SQL_SCRIPT, backup_path)
+
+ # 保存新内容
+ SQL_SCRIPT.write_text(content, encoding='utf-8')
+
+ # 获取新的修改时间
+ mtime = SQL_SCRIPT.stat().st_mtime
+ from datetime import datetime
+ modified = datetime.fromtimestamp(mtime).strftime('%Y-%m-%d %H:%M:%S')
+
+ return {
+ "success": True,
+ "message": "脚本保存成功",
+ "modified": modified
+ }
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
+
# ==================== 启动入口 ====================
if __name__ == "__main__":
diff --git a/app/processor.py b/app/processor.py
index 9f8b784..f03b59b 100644
--- a/app/processor.py
+++ b/app/processor.py
@@ -84,6 +84,10 @@ class DataProcessor:
# 预编译字段映射，避免重复查找
self._field_map, self._type_map = self._build_field_map()
+
+ # LOAD DATA INFILE 支持状态（在首次使用时检测）
+ self._load_data_supported: Optional[bool] = None
+ self._load_data_checked = False
def _build_field_map(self) -> Tuple[Dict[str, str], Dict[str, str]]:
"""
@@ -138,6 +142,9 @@ class DataProcessor:
self.results["error"] = str(e)
finally:
+ # 清理临时目录
+ self._cleanup_temp_dir()
+ # 释放数据库连接
self.db.dispose()
return self.results
@@ -264,11 +271,20 @@ class DataProcessor:
return 'gbk'
return 'utf-8'
- def _process_csv_file_fast(self, csv_file: Path, table_name: str) -> int:
+ def _process_csv_file_fast(self, csv_file: Path, table_name: str,
+ conn=None, table_created: bool = False) -> Tuple[int, bool]:
"""
高性能处理单个 CSV 文件
- 使用批量插入代替 to_sql，性能提升 5-10 倍
- 根据配置的字段类型进行数据转换
+ 使用 LOAD DATA LOCAL INFILE，比 executemany 快 10-50 倍
+
+ Args:
+ csv_file: CSV 文件路径
+ table_name: 目标表名
+ conn: 数据库连接（复用）
+ table_created: 表是否已创建
+
+ Returns:
+ (导入行数, 表是否已创建)
"""
encoding = self._detect_encoding(csv_file)
rel_path = csv_file.relative_to(self.work_dir)
@@ -294,7 +310,7 @@ class DataProcessor:
if len(col_mapping) <= 3:
if 'kpis' in str(csv_file).lower():
self.logger.warning(f"跳过非数据文件: {rel_path}")
- return 0
+ return 0, table_created
raise ValueError(f"字段匹配不足: {rel_path}")
# 选择需要的列并重命名
@@ -340,16 +356,105 @@ class DataProcessor:
if mask.any():
df_result.loc[mask, col] = df_result.loc[mask, col].str[:255]
- # 确保表存在（传递类型信息）
- self.db.create_table_from_columns(table_name, target_cols, column_types)
+ # 确保表存在（只在第一次创建）
+ if not table_created:
+ self.db.create_table_from_columns(table_name, target_cols, column_types)
+ table_created = True
- # 转换为元组列表，用于批量插入
- data_tuples = [tuple(row) for row in df_result.values]
+ # 使用 LOAD DATA INFILE 导入
+ inserted = self._load_data_infile(df_result, table_name, target_cols, conn)
+ return inserted, table_created
+
+ def _get_temp_dir(self) -> Path:
+ """获取临时目录（使用工作目录下的 .temp 子目录）"""
+ temp_dir = self.work_dir / '.temp'
+ temp_dir.mkdir(exist_ok=True)
+ return temp_dir
+
+ def _cleanup_temp_dir(self):
+ """清理临时目录"""
+ temp_dir = self.work_dir / '.temp'
+ if temp_dir.exists():
+ try:
+ import shutil
+ shutil.rmtree(temp_dir)
+ except Exception:
+ pass
+
+ def _check_load_data_support(self) -> bool:
+ """检测是否支持 LOAD DATA INFILE（只检测一次）"""
+ if self._load_data_checked:
+ return self._load_data_supported or False
+
+ self._load_data_checked = True
+ supported, message = self.db.check_load_data_support()
+ self._load_data_supported = supported
+
+ if supported:
+ self.logger.info(f"LOAD DATA INFILE: 已启用 ({message})")
+ else:
+ self.logger.warning(f"LOAD DATA INFILE: 不可用 ({message})，将使用批量插入模式")
+
+ return supported
+
+ def _load_data_infile(self, df: pd.DataFrame, table_name: str,
+ columns: List[str], conn=None) -> int:
+ """
+ 使用 LOAD DATA LOCAL INFILE 导入数据
+ 如果失败则自动回退到 bulk_insert 方式
+ 临时文件放在工作目录的 .temp 子目录中
+ """
+ import tempfile
+
+ # 检测是否支持 LOAD DATA INFILE
+ if not self._check_load_data_support():
+ # 不支持，直接使用 bulk_insert
+ return self._bulk_insert_fallback(df, table_name, columns, conn)
+
+ # 获取临时目录
+ temp_dir = self._get_temp_dir()
+ temp_file = None
+
+ try:
+ # 写入临时 CSV 文件
+ with tempfile.NamedTemporaryFile(
+ mode='w',
+ suffix='.csv',
+ delete=False,
+ encoding='utf-8',
+ newline='',
+ dir=str(temp_dir) # 使用指定的临时目录
+ ) as f:
+ temp_file = f.name
+ # 写入 CSV（带表头，用于 IGNORE 1 LINES）
+ df.to_csv(f, index=False, header=True, na_rep='\\N')
+
+ # 使用 LOAD DATA LOCAL INFILE 导入
+ inserted = self.db.load_data_infile(table_name, columns, temp_file, conn)
+ return inserted
+
+ except Exception as e:
+ # LOAD DATA 失败，标记为不支持并回退
+ self.logger.warning(f"LOAD DATA INFILE 执行失败: {e}，回退到批量插入模式")
+ self._load_data_supported = False
+ return self._bulk_insert_fallback(df, table_name, columns, conn)
+
+ finally:
+ # 清理临时文件
+ if temp_file and os.path.exists(temp_file):
+ try:
+ os.remove(temp_file)
+ except Exception:
+ pass
+
+ def _bulk_insert_fallback(self, df: pd.DataFrame, table_name: str,
+ columns: List[str], conn=None) -> int:
+ """批量插入回退方案"""
+ # 转换为元组列表
+ data_tuples = [tuple(row) for row in df.values]
# 使用批量插入
- inserted = self.db.bulk_insert(table_name, target_cols, data_tuples, self.BATCH_SIZE)
-
- return inserted
+ return self.db.bulk_insert(table_name, columns, data_tuples, self.BATCH_SIZE, conn)
# 支持的日期时间格式列表
DATETIME_FORMATS = [
@@ -510,7 +615,7 @@ class DataProcessor:
return data_dirs
def _process_csv_files(self):
- """处理所有 CSV 文件（高性能版）"""
+ """处理所有 CSV 文件（高性能版 - 使用 LOAD DATA INFILE + 连接复用）"""
self.logger.info("正在处理 CSV 文件并上传到数据库...")
# 查找数据目录
@@ -520,7 +625,7 @@ class DataProcessor:
self.logger.warning("未找到任何数据目录")
return
- # 按目录分组处理
+ # 按目录分组处理，使用连接复用
for table_name, subdir in data_dirs.items():
self.logger.info(f"处理目录: {subdir.relative_to(self.work_dir)} -> 表: {table_name}")
@@ -533,20 +638,25 @@ class DataProcessor:
total_rows = 0
start_time = time.time()
+ table_created = False
- for i, csv_file in enumerate(csv_files, 1):
- try:
- rows = self._process_csv_file_fast(csv_file, table_name)
- total_rows += rows
-
- # 每处理 10 个文件报告一次进度
- if i % 10 == 0:
- elapsed = round(time.time() - start_time, 1)
- self.logger.info(f"进度: {i}/{len(csv_files)} 文件, 已导入 {total_rows} 行, 耗时 {elapsed}s")
+ # 使用连接复用：一个表的所有 CSV 文件共用一个连接
+ with self.db.get_fast_connection() as conn:
+ for i, csv_file in enumerate(csv_files, 1):
+ try:
+ rows, table_created = self._process_csv_file_fast(
+ csv_file, table_name, conn, table_created
+ )
+ total_rows += rows
- except Exception as e:
- rel_path = csv_file.relative_to(self.work_dir)
- self.logger.error(f"CSV 处理失败 {rel_path}: {e}")
+ # 每处理 10 个文件报告一次进度
+ if i % 10 == 0:
+ elapsed = round(time.time() - start_time, 1)
+ self.logger.info(f"进度: {i}/{len(csv_files)} 文件, 已导入 {total_rows} 行, 耗时 {elapsed}s")
+
+ except Exception as e:
+ rel_path = csv_file.relative_to(self.work_dir)
+ self.logger.error(f"CSV 处理失败 {rel_path}: {e}")
elapsed = round(time.time() - start_time, 2)
speed = round(total_rows / elapsed) if elapsed > 0 else 0
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 0000000..199d87f
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,97 @@
+# CapacityReport Docker 编排文件
+# 用于内网离线部署
+#
+# 使用方法:
+# docker-compose up -d # 启动所有服务
+# docker-compose down # 停止所有服务
+# docker-compose logs -f app # 查看应用日志
+# docker-compose logs -f mysql # 查看 MySQL 日志
+#
+# 内网部署准备（在有外网的机器上）:
+# docker pull python:3.11-slim
+# docker pull mysql:8.0 # 或指定版本: mysql:8.0.44
+# docker save -o capacity-images.tar python:3.11-slim mysql:8.0
+# # 将 capacity-images.tar 传到目标机器后执行:
+# docker load -i capacity-images.tar
+
+version: '3.8'
+
+services:
+ # 容量报表应用
+ app:
+ build:
+ context: .
+ dockerfile: Dockerfile
+ container_name: capacity-report
+ restart: unless-stopped
+ ports:
+ - "9081:9081"
+ volumes:
+ # 持久化数据
+ - ./cache:/app/cache
+ - ./Configure.json:/app/Configure.json
+ - ./ReportScript.sql:/app/ReportScript.sql
+ # 日志目录
+ - ./logs:/var/log/supervisor
+ environment:
+ - TZ=Asia/Shanghai
+ - PYTHONUNBUFFERED=1
+ depends_on:
+ mysql:
+ condition: service_healthy
+ networks:
+ - capacity-network
+
+ # MySQL 数据库
+ # 版本说明：
+ # - mysql:8.0 # 自动使用最新的 8.0.x 版本（推荐，当前最新为 8.0.44）
+ # - mysql:8.0.44 # 固定版本，适合生产环境（2025-11-04 发布）
+ # - mysql:8.0.43 # 其他稳定版本
+ # 注意：MySQL 9.5 目前没有官方 Docker 镜像，建议使用 8.0 系列
+ # LOAD DATA INFILE 支持：MySQL 8.0 完全支持，已通过配置启用
+ mysql:
+ image: mysql:8.0
+ container_name: capacity-mysql
+ restart: unless-stopped
+ ports:
+ - "3306:3306"
+ volumes:
+ # 数据持久化
+ - mysql-data:/var/lib/mysql
+ # 自定义配置（启用 LOAD DATA INFILE）
+ - ./mysql/conf.d:/etc/mysql/conf.d:ro
+ # 初始化脚本（仅在首次启动时执行）
+ - ./mysql/init:/docker-entrypoint-initdb.d:ro
+ environment:
+ - TZ=Asia/Shanghai
+ - MYSQL_ROOT_PASSWORD=gmcc123
+ # MYSQL_DATABASE: 仅在首次初始化时创建数据库
+ # 如果数据目录已存在，此变量会被忽略（安全）
+ # 初始化脚本 mysql/init/01-init-db.sql 会确保数据库以 utf8mb4 字符集创建
+ - MYSQL_DATABASE=CapacityReport
+ command:
+ - --character-set-server=utf8mb4
+ - --collation-server=utf8mb4_unicode_ci
+ - --local-infile=ON
+ - --secure-file-priv=
+ - --max_allowed_packet=256M
+ - --innodb_buffer_pool_size=512M
+ - --innodb_log_file_size=128M
+ - --innodb_flush_log_at_trx_commit=2
+ - --sync_binlog=0
+ networks:
+ - capacity-network
+ healthcheck:
+ test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "root", "-pgmcc123"]
+ interval: 10s
+ timeout: 5s
+ retries: 5
+ start_period: 30s
+
+networks:
+ capacity-network:
+ driver: bridge
+
+volumes:
+ mysql-data:
+ driver: local
diff --git a/mysql/conf.d/custom.cnf b/mysql/conf.d/custom.cnf
new file mode 100644
index 0000000..80ea686
--- /dev/null
+++ b/mysql/conf.d/custom.cnf
@@ -0,0 +1,35 @@
+# MySQL 自定义配置 - CapacityReport
+# 启用 LOAD DATA LOCAL INFILE 支持
+
+[mysqld]
+# 字符集
+character-set-server = utf8mb4
+collation-server = utf8mb4_unicode_ci
+
+# 启用 LOAD DATA LOCAL INFILE
+local_infile = ON
+secure_file_priv = ""
+
+# 性能优化
+max_allowed_packet = 256M
+innodb_buffer_pool_size = 512M
+innodb_log_file_size = 128M
+innodb_flush_log_at_trx_commit = 2
+sync_binlog = 0
+
+# 连接设置
+max_connections = 200
+wait_timeout = 28800
+interactive_timeout = 28800
+
+# 日志（可选，生产环境建议注释掉）
+# general_log = 1
+# general_log_file = /var/lib/mysql/general.log
+
+[mysql]
+# 客户端设置
+local_infile = ON
+default-character-set = utf8mb4
+
+[client]
+default-character-set = utf8mb4
diff --git a/mysql/init/01-init-db.sql b/mysql/init/01-init-db.sql
new file mode 100644
index 0000000..afef3b5
--- /dev/null
+++ b/mysql/init/01-init-db.sql
@@ -0,0 +1,13 @@
+-- MySQL 初始化脚本 - 确保数据库以正确的字符集创建
+-- 此脚本仅在首次初始化时执行（数据目录为空时）
+
+-- 如果数据库不存在，则创建（使用 utf8mb4 字符集）
+CREATE DATABASE IF NOT EXISTS CapacityReport
+ CHARACTER SET utf8mb4
+ COLLATE utf8mb4_unicode_ci;
+
+-- 切换到数据库
+USE CapacityReport;
+
+-- 输出确认信息
+SELECT 'CapacityReport database initialized with utf8mb4 charset' AS status;
diff --git a/start.sh b/start.sh
new file mode 100644
index 0000000..0d30307
--- /dev/null
+++ b/start.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+# CapacityReport 启动脚本
+
+echo "Starting CapacityReport..."
+
+# 创建必要的目录
+mkdir -p /var/log/supervisor /var/run /app/cache
+
+# 启动 supervisor
+exec supervisord -c /app/supervisord.conf
diff --git a/static/css/fonts.css b/static/css/fonts.css
new file mode 100644
index 0000000..62be0c9
--- /dev/null
+++ b/static/css/fonts.css
@@ -0,0 +1,43 @@
+/*
+ * 本地字体配置
+ *
+ * 如果需要使用 Noto Sans SC 字体：
+ * 1. 下载字体文件到 static/fonts/ 目录
+ * 2. 取消下方 @font-face 的注释
+ *
+ * 默认使用系统字体，无需额外下载
+ */
+
+/*
+@font-face {
+ font-family: 'Noto Sans SC';
+ font-style: normal;
+ font-weight: 400;
+ font-display: swap;
+ src: url('/static/fonts/NotoSansSC-Regular.woff2') format('woff2');
+}
+
+@font-face {
+ font-family: 'Noto Sans SC';
+ font-style: normal;
+ font-weight: 500;
+ font-display: swap;
+ src: url('/static/fonts/NotoSansSC-Medium.woff2') format('woff2');
+}
+
+@font-face {
+ font-family: 'Noto Sans SC';
+ font-style: normal;
+ font-weight: 600;
+ font-display: swap;
+ src: url('/static/fonts/NotoSansSC-SemiBold.woff2') format('woff2');
+}
+
+@font-face {
+ font-family: 'Noto Sans SC';
+ font-style: normal;
+ font-weight: 700;
+ font-display: swap;
+ src: url('/static/fonts/NotoSansSC-Bold.woff2') format('woff2');
+}
+*/
diff --git a/static/css/style.css b/static/css/style.css
index 9951991..94db699 100644
--- a/static/css/style.css
+++ b/static/css/style.css
@@ -1036,6 +1036,77 @@ body {
.status-dot.connected { background: var(--td-success-color); }
.status-dot.error { background: var(--td-error-color); }
+/* 数据库服务器信息 */
+.db-server-info {
+ padding: 10px 12px;
+ background: var(--td-bg-color-secondarycontainer);
+ border-radius: var(--td-radius-default);
+ border: 1px solid var(--td-border-color-light);
+ margin-top: 8px;
+ font-size: 12px;
+ flex-shrink: 0;
+}
+
+.server-info-item {
+ display: flex;
+ justify-content: space-between;
+ align-items: center;
+ padding: 4px 0;
+}
+
+.server-info-item:not(:last-child) {
+ border-bottom: 1px dashed var(--td-border-color-light);
+}
+
+.server-info-item .info-label {
+ color: var(--td-text-color-secondary);
+}
+
+.server-info-item .info-value {
+ color: var(--td-text-color-primary);
+ font-weight: 500;
+ display: flex;
+ align-items: center;
+ justify-content: flex-end;
+ min-width: 80px;
+}
+
+.support-badge,
+.version-badge {
+ display: inline-block;
+ padding: 2px 8px;
+ border-radius: 10px;
+ font-size: 11px;
+ font-weight: 500;
+ white-space: nowrap;
+}
+
+.support-badge.supported {
+ background: var(--td-success-color-light);
+ color: var(--td-success-color);
+}
+
+.support-badge.unsupported {
+ background: var(--td-warning-color-light);
+ color: var(--td-warning-color);
+}
+
+.support-badge.unknown,
+.version-badge.unknown {
+ background: var(--td-bg-color-container-hover);
+ color: var(--td-text-color-placeholder);
+}
+
+.version-badge.connected {
+ background: rgba(0, 200, 83, 0.15);
+ color: #00c853;
+}
+
+.version-badge.error {
+ background: var(--td-error-color-light);
+ color: var(--td-error-color);
+}
+
.table-list {
flex: 1;
background: var(--td-bg-color-container);
@@ -1212,9 +1283,21 @@ body {
}
.settings-card .card-header {
+ display: flex;
+ flex-direction: column;
+ gap: 8px;
+ align-items: flex-start;
+}
+
+.settings-card .card-header-top {
display: flex;
justify-content: space-between;
align-items: center;
+ width: 100%;
+}
+
+.field-mapping-card .card-header .form-hint {
+ margin: 0;
}
.config-update-time,
@@ -1409,6 +1492,7 @@ body {
.field-mapping-item {
position: relative;
padding: 12px 14px;
+ padding-top: 32px;
background: var(--td-bg-color-container);
border: 1px solid var(--td-border-color);
border-radius: var(--td-radius-default);
@@ -1438,6 +1522,13 @@ body {
box-shadow: 0 1px 2px rgba(0, 0, 0, 0.1);
}
+.field-mapping-item > .remove-mapping {
+ position: absolute;
+ top: 4px;
+ right: 4px;
+ z-index: 1;
+}
+
.field-mapping-header {
display: flex;
align-items: flex-end;
@@ -1503,11 +1594,19 @@ body {
flex-direction: column;
}
-.extract-list > label {
+.extract-list-header {
+ display: flex;
+ align-items: center;
+ justify-content: space-between;
+ margin-bottom: 8px;
+ gap: 12px;
+}
+
+.extract-list-header > label {
font-size: 12px;
color: var(--td-text-color-primary);
font-weight: 500;
- margin-bottom: 8px;
+ flex-shrink: 0;
}
/* 提取来源列表 - 树状显示 */
@@ -1584,11 +1683,14 @@ body {
.add-extract-row {
display: flex;
gap: 8px;
+ align-items: center;
+ flex: 1;
justify-content: flex-end;
}
.add-extract-row .extract-input {
- flex: 0 0 75%;
+ flex: 1;
+ min-width: 250px;
height: 28px;
font-size: 13px;
}
@@ -2019,3 +2121,96 @@ body {
grid-template-columns: 1fr;
}
}
+
+
+/* ==================== 脚本编辑器样式 ==================== */
+
+/* 页面头部脚本路径 */
+.script-path {
+ font-size: 12px;
+ color: var(--td-text-color-secondary);
+ margin-left: 12px;
+ max-width: 400px;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+}
+
+/* 修改时间显示 */
+.script-modified {
+ font-size: 12px;
+ color: var(--td-text-color-secondary);
+ margin-right: 12px;
+}
+
+/* 保存按钮修改状态 */
+#saveScript.modified {
+ background: var(--td-warning-color);
+ border-color: var(--td-warning-color);
+ color: white;
+}
+
+#saveScript.modified:hover {
+ background: #d96a1f;
+ border-color: #d96a1f;
+}
+
+/* 脚本页面主体 */
+.script-page-body {
+ display: flex;
+ flex-direction: column;
+ height: calc(100% - 64px);
+ padding: 0 !important;
+ overflow: hidden;
+}
+
+/* 编辑器容器 */
+.script-editor-container {
+ flex: 1;
+ overflow: hidden;
+ border: 1px solid var(--td-border-color);
+ border-radius: var(--td-radius-default);
+ margin: 16px;
+ margin-bottom: 0;
+ background: var(--td-bg-color-container);
+}
+
+.script-editor {
+ width: 100%;
+ height: 100%;
+}
+
+/* 状态栏 */
+.script-status-bar {
+ display: flex;
+ justify-content: space-between;
+ align-items: center;
+ padding: 6px 16px;
+ margin: 8px 16px 16px;
+ background: var(--td-bg-color-secondarycontainer);
+ border-radius: var(--td-radius-default);
+ font-size: 12px;
+ color: var(--td-text-color-secondary);
+}
+
+#editorCursor {
+ font-family: 'JetBrains Mono', 'Fira Code', 'Consolas', monospace;
+}
+
+#editorStatus {
+ font-weight: 500;
+}
+
+/* Monaco 编辑器主题适配 */
+[data-theme="dark"] .script-editor-container {
+ border-color: var(--td-border-color);
+}
+
+/* 格式化按钮样式 */
+#formatScript {
+ border-color: var(--td-border-color);
+}
+
+#formatScript:hover {
+ background: var(--td-bg-color-container-hover);
+}
diff --git a/static/index.html b/static/index.html
index a81941b..63d23e3 100644
--- a/static/index.html
+++ b/static/index.html
@@ -5,10 +5,11 @@
容量报表处理程序
-
-
-
+
+
+
+