Metrix/CapacityReport
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 增加处理历史保留配置

Browse Source
This commit is contained in:
nixevol
2026-05-19 12:01:26 +08:00
parent f1075695a1
commit a3cc9f347c
9 changed files with 208 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/routers/config.py
+20 -6
View File
@@ -1,12 +1,13 @@
import json
from datetime import datetime
from typing import Any
from urllib.parse import quote
from fastapi import APIRouter, Body, HTTPException, UploadFile, File
from fastapi.responses import FileResponse
from fastapi.responses import Response
from app import state
from app.config import CONFIG_FILE, RemoteDataConfig
from app.config import HistoryRetentionConfig, RemoteDataConfig
router = APIRouter(tags=["config"])
@@ -46,6 +47,13 @@ async def update_remote_config(config: dict[str, Any] = Body(...)):
return {"success": True, "message": "远程数据配置已更新", "update": state.config.update}
@router.post("/api/config/history-retention")
async def update_history_retention(config: dict[str, Any] = Body(...)):
state.config.history_retention = HistoryRetentionConfig.from_dict(config)
state.config.save()
return {"success": True, "message": "处理历史保留配置已更新", "update": state.config.update}
@router.post("/api/config/sheet-filter")
async def update_sheet_filter(filters: list[str] = Body(...)):
state.config.sheet_filter = filters
@@ -62,12 +70,14 @@ async def update_extract_fields(fields: list[dict[str, Any]] = Body(...)):
@router.get("/api/config/download")
async def download_config():
if not CONFIG_FILE.exists():
raise HTTPException(status_code=404, detail="配置文件不存在")
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
filename = f"Configure_{timestamp}.json"
return FileResponse(path=str(CONFIG_FILE), filename=filename, media_type="application/json")
content = json.dumps(state.config.to_file_dict(), ensure_ascii=False, indent=2)
return Response(
content=content,
media_type="application/json",
headers={"Content-Disposition": f"attachment; filename*=UTF-8''{quote(filename)}"},
)
@router.post("/api/config/upload")
@@ -107,3 +117,7 @@ def _apply_config_data(data: dict[str, Any]) -> None:
remote_data = data.get("RemoteData")
if isinstance(remote_data, dict):
state.config.remote_data = RemoteDataConfig.from_dict(remote_data)
history_retention = data.get("HistoryRetention")
if isinstance(history_retention, dict):
state.config.history_retention = HistoryRetentionConfig.from_dict(history_retention)
app/api/routers/remote.py
+4
View File
@@ -124,6 +124,10 @@ def _run_remote_processing(
"status": "failed",
}
finally:
try:
state.apply_history_retention()
except Exception as exc:
print(f"自动清理处理历史失败: {exc}")
state.reset_task_lock()
app/api/routers/tasks.py
+4 -1
View File
@@ -163,5 +163,8 @@ def _run_processing(task_id: str, work_dir: Path, logger: ProcessLogger) -> None
"status": "failed",
}
finally:
try:
state.apply_history_retention()
except Exception as exc:
print(f"自动清理处理历史失败: {exc}")
state.reset_task_lock()
app/config.py
+47 -6
View File
@@ -2,10 +2,10 @@
配置管理模块
"""
import json
from pathlib import Path
from datetime import datetime
from typing import Any, Dict, List, Optional
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Dict, List
BASE_DIR = Path(__file__).resolve().parent.parent
@@ -98,11 +98,44 @@ class RemoteDataConfig:
).normalized()
@dataclass
class HistoryRetentionConfig:
enabled: bool = False
keep_count: int = 20
def normalized(self) -> "HistoryRetentionConfig":
try:
keep_count = int(self.keep_count)
except (TypeError, ValueError):
keep_count = 20
return HistoryRetentionConfig(
enabled=bool(self.enabled),
keep_count=max(keep_count, 0),
)
def to_dict(self) -> Dict[str, Any]:
normalized = self.normalized()
return {
"enabled": normalized.enabled,
"keep_count": normalized.keep_count,
}
@classmethod
def from_dict(cls, data: Dict[str, Any] | None) -> "HistoryRetentionConfig":
data = data or {}
return cls(
enabled=bool(data.get("enabled", False)),
keep_count=data.get("keep_count", 20),
).normalized()
@dataclass
class AppConfig:
update: str = ""
mysql: MySQLConfig = field(default_factory=MySQLConfig)
remote_data: RemoteDataConfig = field(default_factory=RemoteDataConfig)
history_retention: HistoryRetentionConfig = field(default_factory=HistoryRetentionConfig)
sheet_filter: List[str] = field(default_factory=list)
extract_fields: List[Dict[str, Any]] = field(default_factory=list)
@@ -124,11 +157,13 @@ class AppConfig:
dbname=mysql_data.get("dbname", "CapacityReport")
)
remote_config = RemoteDataConfig.from_dict(data.get("RemoteData"))
history_retention = HistoryRetentionConfig.from_dict(data.get("HistoryRetention"))
return cls(
update=data.get("Update", ""),
mysql=mysql_config,
remote_data=remote_config,
history_retention=history_retention,
sheet_filter=data.get("SheetFilter", []),
extract_fields=data.get("ExtractField", [])
)
@@ -136,8 +171,14 @@ class AppConfig:
def save(self):
"""保存配置到 Configure.json，并自动更新 Update 时间"""
self.update = datetime.now().strftime("%Y/%m/%d %H:%M:%S")
data = self.to_file_dict()
data = {
with open(CONFIG_FILE, 'w', encoding='utf-8') as f:
json.dump(data, f, ensure_ascii=False, indent=2)
def to_file_dict(self) -> Dict[str, Any]:
"""转换为配置文件结构（包含敏感字段，用于保存和下载）"""
return {
"Update": self.update,
"MySQL_DBInfo": {
"host": self.mysql.host,
@@ -147,13 +188,11 @@ class AppConfig:
"dbname": self.mysql.dbname
},
"RemoteData": self.remote_data.normalized().to_dict(include_password=True),
"HistoryRetention": self.history_retention.normalized().to_dict(),
"SheetFilter": self.sheet_filter,
"ExtractField": self.extract_fields
}
with open(CONFIG_FILE, 'w', encoding='utf-8') as f:
json.dump(data, f, ensure_ascii=False, indent=2)
def to_dict(self) -> Dict[str, Any]:
"""转换为字典（用于返回给前端，隐藏密码）"""
return {
@@ -165,6 +204,7 @@ class AppConfig:
"dbname": self.mysql.dbname
},
"remote_data": self.remote_data.normalized().to_dict(),
"history_retention": self.history_retention.normalized().to_dict(),
"sheet_filter": self.sheet_filter,
"extract_fields": self.extract_fields
}
@@ -181,6 +221,7 @@ class AppConfig:
"dbname": self.mysql.dbname
},
"remote_data": self.remote_data.normalized().to_dict(include_password=True),
"history_retention": self.history_retention.normalized().to_dict(),
"sheet_filter": self.sheet_filter,
"extract_fields": self.extract_fields
}
app/history.py
+47 -19
View File
@@ -56,6 +56,23 @@ class HistoryManager:
"""保存历史记录"""
HISTORY_FILE.write_text(json.dumps(records, ensure_ascii=False, indent=2), encoding='utf-8')
def _delete_work_dir(self, work_dir: str | None) -> None:
if not work_dir:
return
try:
work_path = Path(work_dir).resolve()
cache_path = CACHE_DIR.resolve()
if work_path.exists() and work_path.is_dir():
try:
work_path.relative_to(cache_path)
shutil.rmtree(work_path)
except ValueError:
print(f"警告: 尝试删除cache目录外的文件: {work_dir}")
except Exception as e:
print(f"删除文件目录失败: {work_dir}, 错误: {e}")
def create(self, work_dir: Path, file_count: int, record_id: Optional[str] = None) -> HistoryRecord:
"""创建新的历史记录"""
# 如果提供了 record_id，使用它；否则生成新的
@@ -162,29 +179,40 @@ class HistoryManager:
if len(new_records) < len(records):
self._save(new_records)
# 删除对应的文件目录（安全检查：确保路径在cache目录内）
if work_dir:
try:
work_path = Path(work_dir).resolve()
cache_path = CACHE_DIR.resolve()
# 安全检查：确保要删除的目录在cache目录内
if work_path.exists() and work_path.is_dir():
# 检查路径是否在cache目录内
try:
work_path.relative_to(cache_path)
# 路径安全，可以删除
shutil.rmtree(work_path)
except ValueError:
# 路径不在cache目录内，跳过删除（安全保护）
print(f"警告: 尝试删除cache目录外的文件: {work_dir}")
except Exception as e:
# 记录错误但不影响删除历史记录的操作
print(f"删除文件目录失败: {work_dir}, 错误: {e}")
self._delete_work_dir(work_dir)
return True
return False
def prune_finished(self, keep_count: int) -> int:
"""按保留数量清理已结束的处理历史，处理中和等待中的记录不会被删除"""
keep_count = max(int(keep_count), 0)
records = self._load()
kept_finished = 0
pruned: List[Dict[str, Any]] = []
remaining: List[Dict[str, Any]] = []
for rec in records:
status = rec.get("status")
if status not in {"completed", "failed"}:
remaining.append(rec)
continue
if kept_finished < keep_count:
remaining.append(rec)
kept_finished += 1
else:
pruned.append(rec)
if not pruned:
return 0
self._save(remaining)
for rec in pruned:
self._delete_work_dir(rec.get("work_dir"))
return len(pruned)
def clear(self) -> int:
"""清空所有历史记录，同时清空cache目录（保留history.json）"""
records = self._load()
app/state.py
+6
View File
@@ -23,3 +23,9 @@ def reset_task_lock() -> None:
global_task_lock["stage"] = None
global_task_lock["started_at"] = None
def apply_history_retention() -> int:
retention = config.history_retention.normalized()
if not retention.enabled:
return 0
return history_manager.prune_finished(retention.keep_count)
docs/project_context.md
+9
View File
@@ -1,5 +1,14 @@
# 项目上下文记录
## 2026-05-19：新增处理历史保留配置并完善配置导入导出
- `app/config.py` 新增 `HistoryRetention` 配置块，包含 `enabled` 和 `keep_count`；`keep_count=0` 表示不保留已结束处理历史，关闭开关时不自动删除历史。
- `app/history.py` 新增按保留数量清理已结束历史的能力，只清理 `completed/failed` 记录及其 `cache/<task_id>` 工作目录，不删除 `pending/processing` 记录。
- `app/api/routers/tasks.py` 和 `app/api/routers/remote.py` 在本地上传处理、远程下载并处理任务结束后自动应用历史保留规则。
- `app/api/routers/config.py` 新增 `/api/config/history-retention` 保存接口；配置下载改为从当前内存配置生成完整 JSON，确保导出的配置始终包含 `RemoteData` 和 `HistoryRetention`；配置上传也会恢复这两个配置块。
- `frontend/src/components/SettingsPanel.vue` 在连接配置页新增“处理历史保留”卡片，可设置自动清理开关和保留最近次数。
- 已执行 `.venv\Scripts\python.exe -m compileall app`、临时目录历史清理验证、配置导入导出验证和 `npm run build`；已用本机 Chrome DevTools 验证设置页新增卡片可见。
## 2026-05-19：调整上传页远程下载入口
- `frontend/src/components/FileWorkflow.vue` 将“远程下载并处理”按钮移动到拖拽上传框内部，删除独立的“远程自动化”卡片，上传页首屏只保留一个主要操作区域。
frontend/src/components/SettingsPanel.vue
+63 -1
View File
@@ -124,6 +124,38 @@
</n-space>
</template>
</n-card>
<n-card title="处理历史保留" size="small" class="work-card">
<n-form label-placement="top">
<n-grid :cols="12" :x-gap="12">
<n-gi :span="6">
<n-form-item label="自动清理处理历史">
<n-switch v-model:value="historyRetentionForm.enabled" />
</n-form-item>
</n-gi>
<n-gi :span="6">
<n-form-item label="保留最近次数">
<n-input-number
v-model:value="historyRetentionForm.keep_count"
class="full-width"
:min="0"
:precision="0"
:disabled="!historyRetentionForm.enabled"
/>
</n-form-item>
</n-gi>
</n-grid>
<p class="form-hint">关闭后不自动删除处理历史；启用后会在任务结束时自动清理，设置为 0 表示不保留历史，否则只保留最近指定次数的处理历史。</p>
</n-form>
<template #footer>
<n-space justify="end">
<n-button type="primary" :loading="savingHistoryRetention" @click="saveHistoryRetention">
保存配置
</n-button>
</n-space>
</template>
</n-card>
</div>
</n-tab-pane>
@@ -282,7 +314,7 @@ import { useMessage, type SelectOption } from 'naive-ui';
import { CloseOutline, CloudDownloadOutline, CloudUploadOutline } from '@vicons/ionicons5';
import { apiGet, apiPost, downloadGet, upload } from '../api/client';
import type { ApiMessage, AppConfig, RemoteDataConfig } from '../types';
import type { ApiMessage, AppConfig, HistoryRetentionConfig, RemoteDataConfig } from '../types';
import { resetPageHeader, setPageHeader } from '../composables/pageHeader';
interface ExtractFieldConfig {
@@ -300,6 +332,7 @@ const testingDb = ref(false);
const testingRemote = ref(false);
const savingMysql = ref(false);
const savingRemote = ref(false);
const savingHistoryRetention = ref(false);
const savingSheetFilter = ref(false);
const savingExtractFields = ref(false);
const changingPassword = ref(false);
@@ -330,6 +363,11 @@ const remoteForm = reactive<RemoteDataConfig>({
auto_delete_source: false
});
const historyRetentionForm = reactive<HistoryRetentionConfig>({
enabled: false,
keep_count: 20
});
const passwordForm = reactive({
current_password: '',
new_password: '',
@@ -384,6 +422,7 @@ async function loadConfig() {
mysqlForm.passwd = config.mysql.passwd || '';
mysqlForm.dbname = config.mysql.dbname;
Object.assign(remoteForm, normalizeRemoteConfig(config.remote_data));
Object.assign(historyRetentionForm, normalizeHistoryRetentionConfig(config.history_retention));
sheetFilters.value = [...config.sheet_filter];
extractFields.value = normalizeExtractFields(config.extract_fields);
resetExtractInputs();
@@ -488,6 +527,22 @@ async function testDatabase() {
}
}
async function saveHistoryRetention() {
savingHistoryRetention.value = true;
try {
const result = await apiPost<ApiMessage>('/api/config/history-retention', {
enabled: historyRetentionForm.enabled,
keep_count: Math.max(Number(historyRetentionForm.keep_count) || 0, 0)
});
configUpdate.value = result.update || configUpdate.value;
message.success(result.message || '处理历史保留配置已保存');
} catch (error) {
message.error(error instanceof Error ? error.message : '保存处理历史保留配置失败');
} finally {
savingHistoryRetention.value = false;
}
}
function addSheetFilter() {
const value = newSheetFilter.value.trim();
if (!value) return;
@@ -643,6 +698,13 @@ function normalizeRemoteConfig(config: RemoteDataConfig | undefined): RemoteData
};
}
function normalizeHistoryRetentionConfig(config: HistoryRetentionConfig | undefined): HistoryRetentionConfig {
return {
enabled: Boolean(config?.enabled),
keep_count: Math.max(Number(config?.keep_count ?? 20) || 0, 0)
};
}
function normalizeExtractFields(fields: Array<Record<string, unknown>>): ExtractFieldConfig[] {
return fields.map(field => {
const extract = Array.isArray(field.Extract) ? uniqueStrings(field.Extract) : [];
frontend/src/types.ts
+6
View File
@@ -58,6 +58,7 @@ export interface AppConfig {
dbname: string;
};
remote_data: RemoteDataConfig;
history_retention: HistoryRetentionConfig;
sheet_filter: string[];
extract_fields: Array<Record<string, unknown>>;
}
@@ -75,6 +76,11 @@ export interface RemoteDataConfig {
auto_delete_source: boolean;
}
export interface HistoryRetentionConfig {
enabled: boolean;
keep_count: number;
}
export interface TableInfo {
name: string;
columns: Array<{