diff --git a/README.md b/README.md index 40bb437..da3c204 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ Vite 会把 `/api` 和 `/health` 代理到 `http://localhost:9081`。 `Configure.json` 主要包含： - `MySQL_DBInfo`：MySQL 连接信息 -- `RemoteData`：FTP/SFTP 远程数据源配置，用于递归下载目录后自动处理 +- `RemoteData`：FTP/SFTP 远程数据源配置，用于递归下载目录后自动处理，可选择处理成功后删除远程源文件 - `SheetFilter`：Excel Sheet 过滤规则 - `ExtractField`：字段抽取映射配置 diff --git a/app/api/routers/remote.py b/app/api/routers/remote.py index dc6aeac..73c787c 100644 --- a/app/api/routers/remote.py +++ b/app/api/routers/remote.py @@ -98,6 +98,13 @@ def _run_remote_processing( processor = DataProcessor(state.config, work_dir, logger) result = processor.process() status = "completed" if result.get("success") else "failed" + if status == "completed" and remote_config.auto_delete_source: + try: + deleted_count = downloader.delete_source_files(download_result.remote_files) + logger.success(f"远程源文件清理完成，共删除 {deleted_count} 个文件，目录已保留") + except Exception as exc: + logger.warning(f"远程源文件清理失败，数据处理结果已保留: {exc}") + state.history_manager.update( task_id, status=status, diff --git a/app/config.py b/app/config.py index 2fc2401..7c3d914 100644 --- a/app/config.py +++ b/app/config.py @@ -34,6 +34,7 @@ class RemoteDataConfig: remote_dir: str = "/" passive: bool = True timeout: int = 30 + auto_delete_source: bool = False def normalized(self) -> "RemoteDataConfig": protocol = self.protocol.lower().strip() @@ -51,6 +52,7 @@ class RemoteDataConfig: remote_dir=(self.remote_dir or "/").strip() or "/", passive=bool(self.passive), timeout=max(int(self.timeout or 30), 1), + auto_delete_source=bool(self.auto_delete_source), ) def to_dict(self, include_password: bool = False) -> Dict[str, Any]: @@ -63,6 +65,7 @@ class RemoteDataConfig: "remote_dir": self.remote_dir, "passive": self.passive, "timeout": self.timeout, + "auto_delete_source": self.auto_delete_source, } if include_password: data["passwd"] = self.passwd @@ -91,6 +94,7 @@ class RemoteDataConfig: remote_dir=str(data.get("remote_dir", "/")), passive=bool(data.get("passive", True)), timeout=timeout, + auto_delete_source=bool(data.get("auto_delete_source", False)), ).normalized() diff --git a/app/services/remote_download.py b/app/services/remote_download.py index 59cc2df..a489b95 100644 --- a/app/services/remote_download.py +++ b/app/services/remote_download.py @@ -1,10 +1,10 @@ from __future__ import annotations import stat -from dataclasses import dataclass +from dataclasses import dataclass, field from ftplib import FTP from pathlib import Path -from typing import Callable +from typing import Callable, Iterable from app.config import RemoteDataConfig @@ -16,6 +16,7 @@ LogFn = Callable[[str], None] class RemoteDownloadResult: file_count: int = 0 total_bytes: int = 0 + remote_files: list[str] = field(default_factory=list) class RemoteDownloadError(RuntimeError): @@ -48,6 +49,18 @@ class RemoteDataDownloader: return self._download_ftp(destination) return self._download_sftp(destination) + def delete_source_files(self, remote_files: Iterable[str] | None = None) -> int: + self._validate_config() + source_files = list(remote_files or []) + if source_files: + if self.config.protocol == "ftp": + return self._delete_ftp_file_paths(source_files) + return self._delete_sftp_file_paths(source_files) + + if self.config.protocol == "ftp": + return self._delete_ftp_source_files() + return self._delete_sftp_source_files() + def _validate_config(self) -> None: if self.config.protocol not in {"ftp", "sftp"}: raise RemoteDownloadError("远程协议只支持 FTP 或 SFTP") @@ -149,6 +162,38 @@ class RemoteDataDownloader: ftp.retrbinary(f"RETR {remote_path}", file.write) result.file_count += 1 result.total_bytes += expected_size or local_path.stat().st_size + result.remote_files.append(remote_path) + + def _delete_ftp_source_files(self) -> int: + with self._ftp_client() as ftp: + self._log(f"开始清理 FTP 源文件: {self.config.remote_dir}") + return self._delete_ftp_files(ftp, self.config.remote_dir) + + def _delete_ftp_files(self, ftp: FTP, remote_dir: str) -> int: + deleted_count = 0 + for name, entry_type, _ in self._list_ftp_entries(ftp, remote_dir): + if name in {".", ".."}: + continue + + remote_path = self._join_remote_path(remote_dir, name) + if entry_type == "dir" or (entry_type == "unknown" and self._ftp_is_dir(ftp, remote_path)): + deleted_count += self._delete_ftp_files(ftp, remote_path) + continue + + self._log(f"删除远程文件: {remote_path}") + ftp.delete(remote_path) + deleted_count += 1 + return deleted_count + + def _delete_ftp_file_paths(self, remote_files: list[str]) -> int: + deleted_count = 0 + with self._ftp_client() as ftp: + self._log(f"开始清理 FTP 源文件，共 {len(remote_files)} 个") + for remote_path in remote_files: + self._log(f"删除远程文件: {remote_path}") + ftp.delete(remote_path) + deleted_count += 1 + return deleted_count def _sftp_ssh_client(self): try: @@ -206,6 +251,47 @@ class RemoteDataDownloader: sftp.get(remote_path, str(local_path)) result.file_count += 1 result.total_bytes += int(getattr(attrs, "st_size", 0) or local_path.stat().st_size) + result.remote_files.append(remote_path) + + def _delete_sftp_source_files(self) -> int: + ssh = self._sftp_ssh_client() + try: + with ssh.open_sftp() as sftp: + self._log(f"开始清理 SFTP 源文件: {self.config.remote_dir}") + return self._delete_sftp_files(sftp, self.config.remote_dir) + finally: + ssh.close() + + def _delete_sftp_files(self, sftp, remote_path: str) -> int: + attrs = sftp.stat(remote_path) + if stat.S_ISDIR(attrs.st_mode): + deleted_count = 0 + for item in sftp.listdir_attr(remote_path): + if item.filename in {".", ".."}: + continue + deleted_count += self._delete_sftp_files( + sftp, + self._join_remote_path(remote_path, item.filename), + ) + return deleted_count + + self._log(f"删除远程文件: {remote_path}") + sftp.remove(remote_path) + return 1 + + def _delete_sftp_file_paths(self, remote_files: list[str]) -> int: + deleted_count = 0 + ssh = self._sftp_ssh_client() + try: + with ssh.open_sftp() as sftp: + self._log(f"开始清理 SFTP 源文件，共 {len(remote_files)} 个") + for remote_path in remote_files: + self._log(f"删除远程文件: {remote_path}") + sftp.remove(remote_path) + deleted_count += 1 + finally: + ssh.close() + return deleted_count @staticmethod def _join_remote_path(parent: str, child: str) -> str: diff --git a/docs/project_context.md b/docs/project_context.md index 8aff0f7..fa97a92 100644 --- a/docs/project_context.md +++ b/docs/project_context.md @@ -1,8 +1,16 @@ # 项目上下文记录 +## 2026-05-19：拆分系统设置页并支持远程源文件自动清理 + +- `frontend/src/components/SettingsPanel.vue` 的系统设置改为 Naive UI Tabs：连接配置页放数据库配置和远程数据源，规则映射页放 Sheet 过滤规则和字段映射配置，修改密码独立一页，避免设置内容堆在一个长页面。 +- `RemoteData` 配置新增 `auto_delete_source` 开关；前端在远程数据源配置中显示“处理成功后删除源文件”，默认关闭。 +- `app/services/remote_download.py` 新增远程源文件清理能力，下载阶段会记录本次实际下载的远程文件路径；自动清理只删除这些文件，不删除目录，也不会删除处理期间新进入远程目录的文件。 +- `app/api/routers/remote.py` 在远程下载并处理成功后才会执行源文件清理；清理失败只写入警告日志，不改变已完成的数据处理结果。 +- 已执行 `.venv\Scripts\python.exe -m compileall app` 和 `npm run build`，均通过；未启动浏览器或 headless Chrome。 + ## 2026-05-18：新增 FTP/SFTP 远程自动化处理 -- `Configure.json` 新增 `RemoteData` 配置，包含启用状态、协议、主机、端口、用户名、密码、远程目录、FTP 被动模式和超时时间；`app/config.py` 会兼容旧配置并在保存时写回该配置块。 +- `Configure.json` 新增 `RemoteData` 配置，包含启用状态、协议、主机、端口、用户名、密码、远程目录、FTP 被动模式、超时时间和源文件自动清理开关；`app/config.py` 会兼容旧配置并在保存时写回该配置块。 - 新增 `app/services/remote_download.py`，FTP 使用标准库 `ftplib`，SFTP 使用 `paramiko`，会递归下载远程目录下的全部文件和文件夹到本地任务缓存目录。 - 新增 `app/api/routers/remote.py`，`POST /api/remote/test` 用于测试远程连接，`POST /api/remote/start` 会创建历史任务、下载远程数据并复用 `DataProcessor` 完成现有处理流程。 - `frontend/src/components/SettingsPanel.vue` 新增“远程数据源”配置卡片，支持 FTP/SFTP 切换、保存和测试连接；`frontend/src/components/FileWorkflow.vue` 新增“远程下载并处理”入口。 diff --git a/frontend/src/components/SettingsPanel.vue b/frontend/src/components/SettingsPanel.vue index cada288..014408b 100644 --- a/frontend/src/components/SettingsPanel.vue +++ b/frontend/src/components/SettingsPanel.vue @@ -2,251 +2,276 @@
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

自动化执行会递归下载该目录下的全部文件和文件夹到本地缓存，再按现有处理流程入库和执行脚本。

- - - - - - - -

匹配这些关键词的 Sheet 将被跳过处理

-
- - {{ filter }} - - -
- - - 添加 - - - - - - - - - - - - - - - - - - - - - - -
- - + - - + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + -
-
{{ index + 1 }}
- - - + + -
- - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

自动化执行会递归下载该目录下的全部文件和文件夹到本地缓存，再按现有处理流程入库和执行脚本；自动删除源文件只会在处理成功后删除远程文件，保留目录结构。

+ + + +
+ -
-
- 提取来源（{{ field.Extract.length }} 个） - - - 添加 - -
+ +
+ + +

匹配这些关键词的 Sheet 将被跳过处理

+
+ + {{ filter }} + + +
+ + + 添加 + + -
-
- {{ source }} - 删除 -
-
-
暂无提取来源
+ + + + + + + + + +
+
{{ index + 1 }}
+ + + + +
+ + + + + + +
+ +
+
+ 提取来源（{{ field.Extract.length }} 个） + + + 添加 + +
+ +
+
+ {{ source }} + 删除 +
+
+
暂无提取来源
+
+
+ + + +
-
- + - + +
+ + + + + + + + + + + + + + + +
+ +
@@ -301,7 +326,8 @@ const remoteForm = reactive({ passwd: '', remote_dir: '/', passive: true, - timeout: 30 + timeout: 30, + auto_delete_source: false }); const passwordForm = reactive({ @@ -408,7 +434,8 @@ function getRemotePayload(): RemoteDataConfig { passwd: remoteForm.passwd || '', remote_dir: remoteForm.remote_dir.trim() || '/', passive: remoteForm.passive, - timeout: remoteForm.timeout || 30 + timeout: remoteForm.timeout || 30, + auto_delete_source: remoteForm.auto_delete_source }; } @@ -611,7 +638,8 @@ function normalizeRemoteConfig(config: RemoteDataConfig | undefined): RemoteData passwd: config?.passwd || '', remote_dir: config?.remote_dir || '/', passive: config?.passive ?? true, - timeout: config?.timeout || 30 + timeout: config?.timeout || 30, + auto_delete_source: Boolean(config?.auto_delete_source) }; } diff --git a/frontend/src/styles.css b/frontend/src/styles.css index 1eb7f53..df6613a 100644 --- a/frontend/src/styles.css +++ b/frontend/src/styles.css @@ -1175,21 +1175,49 @@ select { /* ==================== Settings ==================== */ .settings-workspace { - display: grid; - grid-template-columns: 1fr 1.5fr; - align-items: start; - gap: 20px; min-height: 100%; padding: 24px 32px 40px; } -.settings-column { - display: flex; +.settings-tabs-card { min-width: 0; - flex-direction: column; +} + +.settings-card-header { + display: flex; + align-items: center; + justify-content: space-between; + min-width: 0; + gap: 16px; +} + +.settings-tabs-card > .n-card__content { + padding-top: 8px; +} + +.settings-tabs .n-tab-pane { + padding-top: 16px; +} + +.settings-section-grid, +.settings-rules-grid { + display: grid; + align-items: start; gap: 20px; } +.settings-section-grid { + grid-template-columns: repeat(2, minmax(0, 1fr)); +} + +.settings-rules-grid { + grid-template-columns: minmax(280px, 0.72fr) minmax(0, 1.4fr); +} + +.settings-security-panel { + max-width: 560px; +} + .card-actions { display: flex; align-items: center; @@ -1505,7 +1533,8 @@ select { /* ==================== Responsive ==================== */ @media (max-width: 1200px) { - .settings-workspace { + .settings-section-grid, + .settings-rules-grid { grid-template-columns: 1fr; } @@ -1557,6 +1586,7 @@ select { } .card-actions, + .settings-card-header, .field-card-title-row, .extract-list-header { align-items: stretch; diff --git a/frontend/src/types.ts b/frontend/src/types.ts index b408901..1e01b30 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -72,6 +72,7 @@ export interface RemoteDataConfig { remote_dir: string; passive: boolean; timeout: number; + auto_delete_source: boolean; } export interface TableInfo {