Metrix/CapacityReport
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 增加处理历史保留配置

Browse Source
This commit is contained in:
nixevol
2026-05-19 12:01:26 +08:00
parent f1075695a1
commit a3cc9f347c
9 changed files with 208 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/routers/config.py
+20 -6
View File
@@ -1,12 +1,13 @@
import json
from datetime import datetime
from typing import Any
from urllib.parse import quote
from fastapi import APIRouter, Body, HTTPException, UploadFile, File
from fastapi.responses import FileResponse
from fastapi.responses import Response
from app import state
from app.config import CONFIG_FILE, RemoteDataConfig
from app.config import HistoryRetentionConfig, RemoteDataConfig
router = APIRouter(tags=["config"])
@@ -46,6 +47,13 @@ async def update_remote_config(config: dict[str, Any] = Body(...)):
return {"success": True, "message": "远程数据配置已更新", "update": state.config.update}
@router.post("/api/config/history-retention")
async def update_history_retention(config: dict[str, Any] = Body(...)):
state.config.history_retention = HistoryRetentionConfig.from_dict(config)
state.config.save()
return {"success": True, "message": "处理历史保留配置已更新", "update": state.config.update}
@router.post("/api/config/sheet-filter")
async def update_sheet_filter(filters: list[str] = Body(...)):
state.config.sheet_filter = filters
@@ -62,12 +70,14 @@ async def update_extract_fields(fields: list[dict[str, Any]] = Body(...)):
@router.get("/api/config/download")
async def download_config():
if not CONFIG_FILE.exists():
raise HTTPException(status_code=404, detail="配置文件不存在")
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
filename = f"Configure_{timestamp}.json"
return FileResponse(path=str(CONFIG_FILE), filename=filename, media_type="application/json")
content = json.dumps(state.config.to_file_dict(), ensure_ascii=False, indent=2)
return Response(
content=content,
media_type="application/json",
headers={"Content-Disposition": f"attachment; filename*=UTF-8''{quote(filename)}"},
)
@router.post("/api/config/upload")
@@ -107,3 +117,7 @@ def _apply_config_data(data: dict[str, Any]) -> None:
remote_data = data.get("RemoteData")
if isinstance(remote_data, dict):
state.config.remote_data = RemoteDataConfig.from_dict(remote_data)
history_retention = data.get("HistoryRetention")
if isinstance(history_retention, dict):
state.config.history_retention = HistoryRetentionConfig.from_dict(history_retention)
app/api/routers/remote.py
+4
View File
@@ -124,6 +124,10 @@ def _run_remote_processing(
"status": "failed",
}
finally:
try:
state.apply_history_retention()
except Exception as exc:
print(f"自动清理处理历史失败: {exc}")
state.reset_task_lock()
app/api/routers/tasks.py
+4 -1
View File
@@ -163,5 +163,8 @@ def _run_processing(task_id: str, work_dir: Path, logger: ProcessLogger) -> None
"status": "failed",
}
finally:
try:
state.apply_history_retention()
except Exception as exc:
print(f"自动清理处理历史失败: {exc}")
state.reset_task_lock()
app/config.py
+49 -8
View File
@@ -2,10 +2,10 @@
配置管理模块
"""
import json
from pathlib import Path
from datetime import datetime
from typing import Any, Dict, List, Optional
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Dict, List
BASE_DIR = Path(__file__).resolve().parent.parent
@@ -98,11 +98,44 @@ class RemoteDataConfig:
).normalized()
@dataclass
class HistoryRetentionConfig:
enabled: bool = False
keep_count: int = 20
def normalized(self) -> "HistoryRetentionConfig":
try:
keep_count = int(self.keep_count)
except (TypeError, ValueError):
keep_count = 20
return HistoryRetentionConfig(
enabled=bool(self.enabled),
keep_count=max(keep_count, 0),
)
def to_dict(self) -> Dict[str, Any]:
normalized = self.normalized()
return {
"enabled": normalized.enabled,
"keep_count": normalized.keep_count,
}
@classmethod
def from_dict(cls, data: Dict[str, Any] | None) -> "HistoryRetentionConfig":
data = data or {}
return cls(
enabled=bool(data.get("enabled", False)),
keep_count=data.get("keep_count", 20),
).normalized()
@dataclass
class AppConfig:
update: str = ""
mysql: MySQLConfig = field(default_factory=MySQLConfig)
remote_data: RemoteDataConfig = field(default_factory=RemoteDataConfig)
history_retention: HistoryRetentionConfig = field(default_factory=HistoryRetentionConfig)
sheet_filter: List[str] = field(default_factory=list)
extract_fields: List[Dict[str, Any]] = field(default_factory=list)
@@ -124,20 +157,28 @@ class AppConfig:
dbname=mysql_data.get("dbname", "CapacityReport")
)
remote_config = RemoteDataConfig.from_dict(data.get("RemoteData"))
history_retention = HistoryRetentionConfig.from_dict(data.get("HistoryRetention"))
return cls(
update=data.get("Update", ""),
mysql=mysql_config,
remote_data=remote_config,
history_retention=history_retention,
sheet_filter=data.get("SheetFilter", []),
extract_fields=data.get("ExtractField", [])
)
def save(self):
"""保存配置到 Configure.json，并自动更新 Update 时间"""
self.update = datetime.now().strftime("%Y/%m/%d %H:%M:%S")
data = {
data = self.to_file_dict()
with open(CONFIG_FILE, 'w', encoding='utf-8') as f:
json.dump(data, f, ensure_ascii=False, indent=2)
def to_file_dict(self) -> Dict[str, Any]:
"""转换为配置文件结构（包含敏感字段，用于保存和下载）"""
return {
"Update": self.update,
"MySQL_DBInfo": {
"host": self.mysql.host,
@@ -147,12 +188,10 @@ class AppConfig:
"dbname": self.mysql.dbname
},
"RemoteData": self.remote_data.normalized().to_dict(include_password=True),
"HistoryRetention": self.history_retention.normalized().to_dict(),
"SheetFilter": self.sheet_filter,
"ExtractField": self.extract_fields
}
with open(CONFIG_FILE, 'w', encoding='utf-8') as f:
json.dump(data, f, ensure_ascii=False, indent=2)
def to_dict(self) -> Dict[str, Any]:
"""转换为字典（用于返回给前端，隐藏密码）"""
@@ -165,6 +204,7 @@ class AppConfig:
"dbname": self.mysql.dbname
},
"remote_data": self.remote_data.normalized().to_dict(),
"history_retention": self.history_retention.normalized().to_dict(),
"sheet_filter": self.sheet_filter,
"extract_fields": self.extract_fields
}
@@ -181,6 +221,7 @@ class AppConfig:
"dbname": self.mysql.dbname
},
"remote_data": self.remote_data.normalized().to_dict(include_password=True),
"history_retention": self.history_retention.normalized().to_dict(),
"sheet_filter": self.sheet_filter,
"extract_fields": self.extract_fields
}
app/history.py
+47 -19
View File
@@ -55,6 +55,23 @@ class HistoryManager:
def _save(self, records: List[Dict[str, Any]]):
"""保存历史记录"""
HISTORY_FILE.write_text(json.dumps(records, ensure_ascii=False, indent=2), encoding='utf-8')
def _delete_work_dir(self, work_dir: str | None) -> None:
if not work_dir:
return
try:
work_path = Path(work_dir).resolve()
cache_path = CACHE_DIR.resolve()
if work_path.exists() and work_path.is_dir():
try:
work_path.relative_to(cache_path)
shutil.rmtree(work_path)
except ValueError:
print(f"警告: 尝试删除cache目录外的文件: {work_dir}")
except Exception as e:
print(f"删除文件目录失败: {work_dir}, 错误: {e}")
def create(self, work_dir: Path, file_count: int, record_id: Optional[str] = None) -> HistoryRecord:
"""创建新的历史记录"""
@@ -162,28 +179,39 @@ class HistoryManager:
if len(new_records) < len(records):
self._save(new_records)
# 删除对应的文件目录（安全检查：确保路径在cache目录内）
if work_dir:
try:
work_path = Path(work_dir).resolve()
cache_path = CACHE_DIR.resolve()
# 安全检查：确保要删除的目录在cache目录内
if work_path.exists() and work_path.is_dir():
# 检查路径是否在cache目录内
try:
work_path.relative_to(cache_path)
# 路径安全，可以删除
shutil.rmtree(work_path)
except ValueError:
# 路径不在cache目录内，跳过删除（安全保护）
print(f"警告: 尝试删除cache目录外的文件: {work_dir}")
except Exception as e:
# 记录错误但不影响删除历史记录的操作
print(f"删除文件目录失败: {work_dir}, 错误: {e}")
self._delete_work_dir(work_dir)
return True
return False
def prune_finished(self, keep_count: int) -> int:
"""按保留数量清理已结束的处理历史，处理中和等待中的记录不会被删除"""
keep_count = max(int(keep_count), 0)
records = self._load()
kept_finished = 0
pruned: List[Dict[str, Any]] = []
remaining: List[Dict[str, Any]] = []
for rec in records:
status = rec.get("status")
if status not in {"completed", "failed"}:
remaining.append(rec)
continue
if kept_finished < keep_count:
remaining.append(rec)
kept_finished += 1
else:
pruned.append(rec)
if not pruned:
return 0
self._save(remaining)
for rec in pruned:
self._delete_work_dir(rec.get("work_dir"))
return len(pruned)
def clear(self) -> int:
"""清空所有历史记录，同时清空cache目录（保留history.json）"""
app/state.py
+6
View File
@@ -23,3 +23,9 @@ def reset_task_lock() -> None:
global_task_lock["stage"] = None
global_task_lock["started_at"] = None
def apply_history_retention() -> int:
retention = config.history_retention.normalized()
if not retention.enabled:
return 0
return history_manager.prune_finished(retention.keep_count)