Metrix/CapacityReport
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 拆分设置页并支持远程源文件清理

Browse Source
This commit is contained in:
nixevol
2026-05-19 09:22:42 +08:00
parent f2bef1f2f7
commit 596fec5192
8 changed files with 413 additions and 249 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1 -1
View File
@@ -89,7 +89,7 @@ Vite 会把 `/api` 和 `/health` 代理到 `http://localhost:9081`。
`Configure.json` 主要包含：
- `MySQL_DBInfo`：MySQL 连接信息
- `RemoteData`：FTP/SFTP 远程数据源配置，用于递归下载目录后自动处理
- `RemoteData`：FTP/SFTP 远程数据源配置，用于递归下载目录后自动处理，可选择处理成功后删除远程源文件
- `SheetFilter`：Excel Sheet 过滤规则
- `ExtractField`：字段抽取映射配置
app/api/routers/remote.py
+7
View File
@@ -98,6 +98,13 @@ def _run_remote_processing(
processor = DataProcessor(state.config, work_dir, logger)
result = processor.process()
status = "completed" if result.get("success") else "failed"
if status == "completed" and remote_config.auto_delete_source:
try:
deleted_count = downloader.delete_source_files(download_result.remote_files)
logger.success(f"远程源文件清理完成，共删除 {deleted_count} 个文件，目录已保留")
except Exception as exc:
logger.warning(f"远程源文件清理失败，数据处理结果已保留: {exc}")
state.history_manager.update(
task_id,
status=status,
app/config.py
+4
View File
@@ -34,6 +34,7 @@ class RemoteDataConfig:
remote_dir: str = "/"
passive: bool = True
timeout: int = 30
auto_delete_source: bool = False
def normalized(self) -> "RemoteDataConfig":
protocol = self.protocol.lower().strip()
@@ -51,6 +52,7 @@ class RemoteDataConfig:
remote_dir=(self.remote_dir or "/").strip() or "/",
passive=bool(self.passive),
timeout=max(int(self.timeout or 30), 1),
auto_delete_source=bool(self.auto_delete_source),
)
def to_dict(self, include_password: bool = False) -> Dict[str, Any]:
@@ -63,6 +65,7 @@ class RemoteDataConfig:
"remote_dir": self.remote_dir,
"passive": self.passive,
"timeout": self.timeout,
"auto_delete_source": self.auto_delete_source,
}
if include_password:
data["passwd"] = self.passwd
@@ -91,6 +94,7 @@ class RemoteDataConfig:
remote_dir=str(data.get("remote_dir", "/")),
passive=bool(data.get("passive", True)),
timeout=timeout,
auto_delete_source=bool(data.get("auto_delete_source", False)),
).normalized()
app/services/remote_download.py
+88 -2
View File
@@ -1,10 +1,10 @@
from __future__ import annotations
import stat
from dataclasses import dataclass
from dataclasses import dataclass, field
from ftplib import FTP
from pathlib import Path
from typing import Callable
from typing import Callable, Iterable
from app.config import RemoteDataConfig
@@ -16,6 +16,7 @@ LogFn = Callable[[str], None]
class RemoteDownloadResult:
file_count: int = 0
total_bytes: int = 0
remote_files: list[str] = field(default_factory=list)
class RemoteDownloadError(RuntimeError):
@@ -48,6 +49,18 @@ class RemoteDataDownloader:
return self._download_ftp(destination)
return self._download_sftp(destination)
def delete_source_files(self, remote_files: Iterable[str] | None = None) -> int:
self._validate_config()
source_files = list(remote_files or [])
if source_files:
if self.config.protocol == "ftp":
return self._delete_ftp_file_paths(source_files)
return self._delete_sftp_file_paths(source_files)
if self.config.protocol == "ftp":
return self._delete_ftp_source_files()
return self._delete_sftp_source_files()
def _validate_config(self) -> None:
if self.config.protocol not in {"ftp", "sftp"}:
raise RemoteDownloadError("远程协议只支持 FTP 或 SFTP")
@@ -149,6 +162,38 @@ class RemoteDataDownloader:
ftp.retrbinary(f"RETR {remote_path}", file.write)
result.file_count += 1
result.total_bytes += expected_size or local_path.stat().st_size
result.remote_files.append(remote_path)
def _delete_ftp_source_files(self) -> int:
with self._ftp_client() as ftp:
self._log(f"开始清理 FTP 源文件: {self.config.remote_dir}")
return self._delete_ftp_files(ftp, self.config.remote_dir)
def _delete_ftp_files(self, ftp: FTP, remote_dir: str) -> int:
deleted_count = 0
for name, entry_type, _ in self._list_ftp_entries(ftp, remote_dir):
if name in {".", ".."}:
continue
remote_path = self._join_remote_path(remote_dir, name)
if entry_type == "dir" or (entry_type == "unknown" and self._ftp_is_dir(ftp, remote_path)):
deleted_count += self._delete_ftp_files(ftp, remote_path)
continue
self._log(f"删除远程文件: {remote_path}")
ftp.delete(remote_path)
deleted_count += 1
return deleted_count
def _delete_ftp_file_paths(self, remote_files: list[str]) -> int:
deleted_count = 0
with self._ftp_client() as ftp:
self._log(f"开始清理 FTP 源文件，共 {len(remote_files)} 个")
for remote_path in remote_files:
self._log(f"删除远程文件: {remote_path}")
ftp.delete(remote_path)
deleted_count += 1
return deleted_count
def _sftp_ssh_client(self):
try:
@@ -206,6 +251,47 @@ class RemoteDataDownloader:
sftp.get(remote_path, str(local_path))
result.file_count += 1
result.total_bytes += int(getattr(attrs, "st_size", 0) or local_path.stat().st_size)
result.remote_files.append(remote_path)
def _delete_sftp_source_files(self) -> int:
ssh = self._sftp_ssh_client()
try:
with ssh.open_sftp() as sftp:
self._log(f"开始清理 SFTP 源文件: {self.config.remote_dir}")
return self._delete_sftp_files(sftp, self.config.remote_dir)
finally:
ssh.close()
def _delete_sftp_files(self, sftp, remote_path: str) -> int:
attrs = sftp.stat(remote_path)
if stat.S_ISDIR(attrs.st_mode):
deleted_count = 0
for item in sftp.listdir_attr(remote_path):
if item.filename in {".", ".."}:
continue
deleted_count += self._delete_sftp_files(
sftp,
self._join_remote_path(remote_path, item.filename),
)
return deleted_count
self._log(f"删除远程文件: {remote_path}")
sftp.remove(remote_path)
return 1
def _delete_sftp_file_paths(self, remote_files: list[str]) -> int:
deleted_count = 0
ssh = self._sftp_ssh_client()
try:
with ssh.open_sftp() as sftp:
self._log(f"开始清理 SFTP 源文件，共 {len(remote_files)} 个")
for remote_path in remote_files:
self._log(f"删除远程文件: {remote_path}")
sftp.remove(remote_path)
deleted_count += 1
finally:
ssh.close()
return deleted_count
@staticmethod
def _join_remote_path(parent: str, child: str) -> str:
docs/project_context.md
+9 -1
View File
@@ -1,8 +1,16 @@
# 项目上下文记录
## 2026-05-19：拆分系统设置页并支持远程源文件自动清理
- `frontend/src/components/SettingsPanel.vue` 的系统设置改为 Naive UI Tabs：连接配置页放数据库配置和远程数据源，规则映射页放 Sheet 过滤规则和字段映射配置，修改密码独立一页，避免设置内容堆在一个长页面。
- `RemoteData` 配置新增 `auto_delete_source` 开关；前端在远程数据源配置中显示“处理成功后删除源文件”，默认关闭。
- `app/services/remote_download.py` 新增远程源文件清理能力，下载阶段会记录本次实际下载的远程文件路径；自动清理只删除这些文件，不删除目录，也不会删除处理期间新进入远程目录的文件。
- `app/api/routers/remote.py` 在远程下载并处理成功后才会执行源文件清理；清理失败只写入警告日志，不改变已完成的数据处理结果。
- 已执行 `.venv\Scripts\python.exe -m compileall app` 和 `npm run build`，均通过；未启动浏览器或 headless Chrome。
## 2026-05-18：新增 FTP/SFTP 远程自动化处理
- `Configure.json` 新增 `RemoteData` 配置，包含启用状态、协议、主机、端口、用户名、密码、远程目录、FTP 被动模式和超时时间；`app/config.py` 会兼容旧配置并在保存时写回该配置块。
- `Configure.json` 新增 `RemoteData` 配置，包含启用状态、协议、主机、端口、用户名、密码、远程目录、FTP 被动模式、超时时间和源文件自动清理开关；`app/config.py` 会兼容旧配置并在保存时写回该配置块。
- 新增 `app/services/remote_download.py`，FTP 使用标准库 `ftplib`，SFTP 使用 `paramiko`，会递归下载远程目录下的全部文件和文件夹到本地任务缓存目录。
- 新增 `app/api/routers/remote.py`，`POST /api/remote/test` 用于测试远程连接，`POST /api/remote/start` 会创建历史任务、下载远程数据并复用 `DataProcessor` 完成现有处理流程。
- `frontend/src/components/SettingsPanel.vue` 新增“远程数据源”配置卡片，支持 FTP/SFTP 切换、保存和测试连接；`frontend/src/components/FileWorkflow.vue` 新增“远程下载并处理”入口。
frontend/src/components/SettingsPanel.vue
+57 -29
View File
@@ -2,12 +2,18 @@
<div class="settings-workspace">
<input ref="configInput" class="hidden-input" type="file" accept=".json" @change="uploadConfigFile" />
<div class="settings-column">
<n-card title="数据库配置" size="small" class="work-card">
<template #header-extra>
<n-card size="small" class="work-card settings-tabs-card">
<template #header>
<div class="settings-card-header">
<span>系统设置</span>
<span class="muted-line">更新时间：{{ configUpdate || '-' }}</span>
</div>
</template>
<n-tabs type="line" animated class="settings-tabs">
<n-tab-pane name="connection" tab="连接配置">
<div class="settings-section-grid">
<n-card title="数据库配置" size="small" class="work-card">
<n-form label-placement="top">
<n-grid :cols="12" :x-gap="12">
<n-gi :span="8">
@@ -48,9 +54,18 @@
<n-card title="远程数据源" size="small" class="work-card">
<n-form label-placement="top">
<n-grid :cols="12" :x-gap="12">
<n-gi :span="6">
<n-form-item label="启用远程自动化">
<n-switch v-model:value="remoteForm.enabled" />
</n-form-item>
</n-gi>
<n-gi :span="6">
<n-form-item label="处理成功后删除源文件">
<n-switch v-model:value="remoteForm.auto_delete_source" />
</n-form-item>
</n-gi>
</n-grid>
<n-grid :cols="12" :x-gap="12">
<n-gi :span="4">
<n-form-item label="协议">
@@ -99,7 +114,7 @@
<n-form-item label="远程目录">
<n-input v-model:value="remoteForm.remote_dir" placeholder="/CapacityReportData" />
</n-form-item>
<p class="form-hint">自动化执行会递归下载该目录下的全部文件和文件夹到本地缓存，再按现有处理流程入库和执行脚本。</p>
<p class="form-hint">自动化执行会递归下载该目录下的全部文件和文件夹到本地缓存，再按现有处理流程入库和执行脚本；自动删除源文件只会在处理成功后删除远程文件，保留目录结构。</p>
</n-form>
<template #footer>
@@ -109,7 +124,11 @@
</n-space>
</template>
</n-card>
</div>
</n-tab-pane>
<n-tab-pane name="rules" tab="规则映射">
<div class="settings-rules-grid">
<n-card title="Sheet 过滤规则" size="small" class="work-card">
<n-space vertical>
<p class="form-hint">匹配这些关键词的 Sheet 将被跳过处理</p>
@@ -141,28 +160,6 @@
</template>
</n-card>
<n-card title="修改登录密码" size="small" class="work-card">
<n-form label-placement="top">
<n-form-item label="当前密码">
<n-input v-model:value="passwordForm.current_password" type="password" show-password-on="click" />
</n-form-item>
<n-form-item label="新密码">
<n-input v-model:value="passwordForm.new_password" type="password" show-password-on="click" />
</n-form-item>
<n-form-item label="确认新密码">
<n-input v-model:value="passwordForm.confirm_password" type="password" show-password-on="click" />
</n-form-item>
</n-form>
<template #footer>
<n-space justify="end">
<n-button type="primary" :loading="changingPassword" @click="changePassword">修改密码</n-button>
</n-space>
</template>
</n-card>
</div>
<n-card size="small" class="work-card field-mapping-card">
<template #header>
<div class="field-card-header">
@@ -249,6 +246,34 @@
</template>
</n-card>
</div>
</n-tab-pane>
<n-tab-pane name="security" tab="修改密码">
<div class="settings-security-panel">
<n-card title="修改登录密码" size="small" class="work-card">
<n-form label-placement="top">
<n-form-item label="当前密码">
<n-input v-model:value="passwordForm.current_password" type="password" show-password-on="click" />
</n-form-item>
<n-form-item label="新密码">
<n-input v-model:value="passwordForm.new_password" type="password" show-password-on="click" />
</n-form-item>
<n-form-item label="确认新密码">
<n-input v-model:value="passwordForm.confirm_password" type="password" show-password-on="click" />
</n-form-item>
</n-form>
<template #footer>
<n-space justify="end">
<n-button type="primary" :loading="changingPassword" @click="changePassword">修改密码</n-button>
</n-space>
</template>
</n-card>
</div>
</n-tab-pane>
</n-tabs>
</n-card>
</div>
</template>
<script setup lang="ts">
@@ -301,7 +326,8 @@ const remoteForm = reactive<RemoteDataConfig>({
passwd: '',
remote_dir: '/',
passive: true,
timeout: 30
timeout: 30,
auto_delete_source: false
});
const passwordForm = reactive({
@@ -408,7 +434,8 @@ function getRemotePayload(): RemoteDataConfig {
passwd: remoteForm.passwd || '',
remote_dir: remoteForm.remote_dir.trim() || '/',
passive: remoteForm.passive,
timeout: remoteForm.timeout || 30
timeout: remoteForm.timeout || 30,
auto_delete_source: remoteForm.auto_delete_source
};
}
@@ -611,7 +638,8 @@ function normalizeRemoteConfig(config: RemoteDataConfig | undefined): RemoteData
passwd: config?.passwd || '',
remote_dir: config?.remote_dir || '/',
passive: config?.passive ?? true,
timeout: config?.timeout || 30
timeout: config?.timeout || 30,
auto_delete_source: Boolean(config?.auto_delete_source)
};
}
frontend/src/styles.css
+38 -8
View File
@@ -1175,21 +1175,49 @@ select {
/* ==================== Settings ==================== */
.settings-workspace {
display: grid;
grid-template-columns: 1fr 1.5fr;
align-items: start;
gap: 20px;
min-height: 100%;
padding: 24px 32px 40px;
}
.settings-column {
display: flex;
.settings-tabs-card {
min-width: 0;
flex-direction: column;
}
.settings-card-header {
display: flex;
align-items: center;
justify-content: space-between;
min-width: 0;
gap: 16px;
}
.settings-tabs-card > .n-card__content {
padding-top: 8px;
}
.settings-tabs .n-tab-pane {
padding-top: 16px;
}
.settings-section-grid,
.settings-rules-grid {
display: grid;
align-items: start;
gap: 20px;
}
.settings-section-grid {
grid-template-columns: repeat(2, minmax(0, 1fr));
}
.settings-rules-grid {
grid-template-columns: minmax(280px, 0.72fr) minmax(0, 1.4fr);
}
.settings-security-panel {
max-width: 560px;
}
.card-actions {
display: flex;
align-items: center;
@@ -1505,7 +1533,8 @@ select {
/* ==================== Responsive ==================== */
@media (max-width: 1200px) {
.settings-workspace {
.settings-section-grid,
.settings-rules-grid {
grid-template-columns: 1fr;
}
@@ -1557,6 +1586,7 @@ select {
}
.card-actions,
.settings-card-header,
.field-card-title-row,
.extract-list-header {
align-items: stretch;
frontend/src/types.ts
+1
View File
@@ -72,6 +72,7 @@ export interface RemoteDataConfig {
remote_dir: string;
passive: boolean;
timeout: number;
auto_delete_source: boolean;
}
export interface TableInfo {