feat: 新增远程数据自动化处理

This commit is contained in:
2026-05-18 18:21:20 +08:00
parent 3750160891
commit f2bef1f2f7
12 changed files with 698 additions and 5 deletions
+3
View File
@@ -89,6 +89,7 @@ Vite 会把 `/api` 和 `/health` 代理到 `http://localhost:9081`。
`Configure.json` 主要包含: `Configure.json` 主要包含:
- `MySQL_DBInfo`:MySQL 连接信息 - `MySQL_DBInfo`:MySQL 连接信息
- `RemoteData`:FTP/SFTP 远程数据源配置,用于递归下载目录后自动处理
- `SheetFilter`:Excel Sheet 过滤规则 - `SheetFilter`:Excel Sheet 过滤规则
- `ExtractField`:字段抽取映射配置 - `ExtractField`:字段抽取映射配置
@@ -114,6 +115,8 @@ Docker 镜像构建采用多阶段流程:先在 Node 阶段执行前端构建
- `POST /api/login`:登录 - `POST /api/login`:登录
- `POST /api/change-password`:修改密码 - `POST /api/change-password`:修改密码
- `POST /api/upload`:上传文件 - `POST /api/upload`:上传文件
- `POST /api/remote/test`:测试 FTP/SFTP 远程数据源
- `POST /api/remote/start`:从远程目录下载数据并自动启动处理
- `POST /api/process/start`:启动处理 - `POST /api/process/start`:启动处理
- `POST /api/process/status`:查询处理状态 - `POST /api/process/status`:查询处理状态
- `GET /api/history`:历史记录 - `GET /api/history`:历史记录
+11 -1
View File
@@ -6,7 +6,7 @@ from fastapi import APIRouter, Body, HTTPException, UploadFile, File
from fastapi.responses import FileResponse from fastapi.responses import FileResponse
from app import state from app import state
from app.config import CONFIG_FILE from app.config import CONFIG_FILE, RemoteDataConfig
router = APIRouter(tags=["config"]) router = APIRouter(tags=["config"])
@@ -39,6 +39,13 @@ async def update_mysql_config(
return {"success": True, "message": "数据库配置已更新", "update": state.config.update} return {"success": True, "message": "数据库配置已更新", "update": state.config.update}
@router.post("/api/config/remote")
async def update_remote_config(config: dict[str, Any] = Body(...)):
state.config.remote_data = RemoteDataConfig.from_dict(config)
state.config.save()
return {"success": True, "message": "远程数据配置已更新", "update": state.config.update}
@router.post("/api/config/sheet-filter") @router.post("/api/config/sheet-filter")
async def update_sheet_filter(filters: list[str] = Body(...)): async def update_sheet_filter(filters: list[str] = Body(...)):
state.config.sheet_filter = filters state.config.sheet_filter = filters
@@ -97,3 +104,6 @@ def _apply_config_data(data: dict[str, Any]) -> None:
if "ExtractField" in data: if "ExtractField" in data:
state.config.extract_fields = data["ExtractField"] if isinstance(data["ExtractField"], list) else [] state.config.extract_fields = data["ExtractField"] if isinstance(data["ExtractField"], list) else []
remote_data = data.get("RemoteData")
if isinstance(remote_data, dict):
state.config.remote_data = RemoteDataConfig.from_dict(remote_data)
+129
View File
@@ -0,0 +1,129 @@
from datetime import datetime
from pathlib import Path
from threading import Thread
from typing import Any
from fastapi import APIRouter, Body, HTTPException
from app import state
from app.config import CACHE_DIR, RemoteDataConfig
from app.processor import DataProcessor, ProcessLogger
from app.services.remote_download import RemoteDataDownloader
router = APIRouter(tags=["remote"])
@router.post("/api/remote/test")
async def test_remote_connection(config: dict[str, Any] | None = Body(None)):
remote_config = RemoteDataConfig.from_dict(config) if config else state.config.remote_data
try:
RemoteDataDownloader(remote_config).test_connection()
return {"success": True, "message": "远程服务器连接成功"}
except Exception as exc:
return {"success": False, "message": f"远程服务器连接失败: {exc}"}
@router.post("/api/remote/start")
async def start_remote_processing():
if state.global_task_lock["locked"]:
raise HTTPException(status_code=409, detail="已有任务在运行,请等待当前任务完成")
remote_config = state.config.remote_data.normalized()
if not remote_config.enabled:
raise HTTPException(status_code=400, detail="请先启用远程数据配置")
task_id = datetime.now().strftime("%Y%m%d_%H%M%S")
work_dir = CACHE_DIR / task_id
work_dir.mkdir(parents=True, exist_ok=True)
logs: list[str] = []
def log_callback(message: str) -> None:
logs.append(message)
state.processing_tasks[task_id] = {"logs": logs.copy(), "status": "processing"}
logger = ProcessLogger(log_file=work_dir / "log.txt", callback=log_callback)
state.history_manager.create(work_dir, 0, record_id=task_id)
state.history_manager.update(task_id, status="processing")
state.processing_tasks[task_id] = {"logs": [], "status": "processing"}
state.global_task_lock.update(
{
"locked": True,
"task_id": task_id,
"stage": "downloading",
"started_at": datetime.now().isoformat(),
}
)
thread = Thread(
target=_run_remote_processing,
args=(task_id, work_dir, remote_config, logger),
daemon=True,
)
thread.start()
return {
"success": True,
"message": "远程下载处理任务已启动",
"task_id": task_id,
"stage": "downloading",
}
def _run_remote_processing(
task_id: str,
work_dir: Path,
remote_config: RemoteDataConfig,
logger: ProcessLogger,
) -> None:
try:
logger.info(
f"开始远程下载,协议: {remote_config.protocol.upper()},"
f"服务器: {remote_config.host}:{remote_config.port},目录: {remote_config.remote_dir}"
)
downloader = RemoteDataDownloader(remote_config, logger.info)
download_result = downloader.download_to(work_dir)
logger.success(
f"远程下载完成,共 {download_result.file_count} 个文件,"
f"{_format_bytes(download_result.total_bytes)}"
)
if download_result.file_count == 0:
raise RuntimeError("远程目录中未下载到任何文件")
state.history_manager.update(task_id, file_count=download_result.file_count)
state.global_task_lock["stage"] = "processing"
processor = DataProcessor(state.config, work_dir, logger)
result = processor.process()
status = "completed" if result.get("success") else "failed"
state.history_manager.update(
task_id,
status=status,
elapsed_time=result.get("elapsed_time", 0),
error=result.get("error"),
result_tables=["4G_结果表", "5G_结果表"],
)
state.processing_tasks[task_id] = {
"logs": state.history_manager.get_logs(task_id),
"status": status,
}
except Exception as exc:
logger.error(f"远程自动化任务失败: {exc}")
state.history_manager.update(task_id, status="failed", error=str(exc))
state.processing_tasks[task_id] = {
"logs": state.history_manager.get_logs(task_id),
"status": "failed",
}
finally:
state.reset_task_lock()
def _format_bytes(size: int) -> str:
value = float(size)
for unit in ("B", "KB", "MB", "GB"):
if value < 1024:
return f"{value:.1f} {unit}"
value /= 1024
return f"{value:.1f} TB"
+77
View File
@@ -23,10 +23,82 @@ class MySQLConfig:
dbname: str = "CapacityReport" dbname: str = "CapacityReport"
@dataclass
class RemoteDataConfig:
enabled: bool = False
protocol: str = "sftp"
host: str = ""
port: int = 22
user: str = ""
passwd: str = ""
remote_dir: str = "/"
passive: bool = True
timeout: int = 30
def normalized(self) -> "RemoteDataConfig":
protocol = self.protocol.lower().strip()
if protocol not in {"ftp", "sftp"}:
protocol = "sftp"
port = self.port or (22 if protocol == "sftp" else 21)
return RemoteDataConfig(
enabled=bool(self.enabled),
protocol=protocol,
host=self.host.strip(),
port=port,
user=self.user.strip(),
passwd=self.passwd,
remote_dir=(self.remote_dir or "/").strip() or "/",
passive=bool(self.passive),
timeout=max(int(self.timeout or 30), 1),
)
def to_dict(self, include_password: bool = False) -> Dict[str, Any]:
data = {
"enabled": self.enabled,
"protocol": self.protocol,
"host": self.host,
"port": self.port,
"user": self.user,
"remote_dir": self.remote_dir,
"passive": self.passive,
"timeout": self.timeout,
}
if include_password:
data["passwd"] = self.passwd
return data
@classmethod
def from_dict(cls, data: Dict[str, Any] | None) -> "RemoteDataConfig":
data = data or {}
protocol = str(data.get("protocol", "sftp")).lower()
default_port = 22 if protocol == "sftp" else 21
try:
port = int(data.get("port") or default_port)
except (TypeError, ValueError):
port = default_port
try:
timeout = int(data.get("timeout") or 30)
except (TypeError, ValueError):
timeout = 30
return cls(
enabled=bool(data.get("enabled", False)),
protocol=protocol,
host=str(data.get("host", "")),
port=port,
user=str(data.get("user", "")),
passwd=str(data.get("passwd", "")),
remote_dir=str(data.get("remote_dir", "/")),
passive=bool(data.get("passive", True)),
timeout=timeout,
).normalized()
@dataclass @dataclass
class AppConfig: class AppConfig:
update: str = "" update: str = ""
mysql: MySQLConfig = field(default_factory=MySQLConfig) mysql: MySQLConfig = field(default_factory=MySQLConfig)
remote_data: RemoteDataConfig = field(default_factory=RemoteDataConfig)
sheet_filter: List[str] = field(default_factory=list) sheet_filter: List[str] = field(default_factory=list)
extract_fields: List[Dict[str, Any]] = field(default_factory=list) extract_fields: List[Dict[str, Any]] = field(default_factory=list)
@@ -47,10 +119,12 @@ class AppConfig:
passwd=mysql_data.get("passwd", ""), passwd=mysql_data.get("passwd", ""),
dbname=mysql_data.get("dbname", "CapacityReport") dbname=mysql_data.get("dbname", "CapacityReport")
) )
remote_config = RemoteDataConfig.from_dict(data.get("RemoteData"))
return cls( return cls(
update=data.get("Update", ""), update=data.get("Update", ""),
mysql=mysql_config, mysql=mysql_config,
remote_data=remote_config,
sheet_filter=data.get("SheetFilter", []), sheet_filter=data.get("SheetFilter", []),
extract_fields=data.get("ExtractField", []) extract_fields=data.get("ExtractField", [])
) )
@@ -68,6 +142,7 @@ class AppConfig:
"passwd": self.mysql.passwd, "passwd": self.mysql.passwd,
"dbname": self.mysql.dbname "dbname": self.mysql.dbname
}, },
"RemoteData": self.remote_data.normalized().to_dict(include_password=True),
"SheetFilter": self.sheet_filter, "SheetFilter": self.sheet_filter,
"ExtractField": self.extract_fields "ExtractField": self.extract_fields
} }
@@ -85,6 +160,7 @@ class AppConfig:
"user": self.mysql.user, "user": self.mysql.user,
"dbname": self.mysql.dbname "dbname": self.mysql.dbname
}, },
"remote_data": self.remote_data.normalized().to_dict(),
"sheet_filter": self.sheet_filter, "sheet_filter": self.sheet_filter,
"extract_fields": self.extract_fields "extract_fields": self.extract_fields
} }
@@ -100,6 +176,7 @@ class AppConfig:
"passwd": self.mysql.passwd, "passwd": self.mysql.passwd,
"dbname": self.mysql.dbname "dbname": self.mysql.dbname
}, },
"remote_data": self.remote_data.normalized().to_dict(include_password=True),
"sheet_filter": self.sheet_filter, "sheet_filter": self.sheet_filter,
"extract_fields": self.extract_fields "extract_fields": self.extract_fields
} }
+2 -1
View File
@@ -8,7 +8,7 @@ from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import FileResponse, JSONResponse from fastapi.responses import FileResponse, JSONResponse
from app import state from app import state
from app.api.routers import auth, cache, config, database, health, history, script, service, tasks, upload from app.api.routers import auth, cache, config, database, health, history, remote, script, service, tasks, upload
from app.auth import verify_jwt_token from app.auth import verify_jwt_token
from app.config import BASE_DIR from app.config import BASE_DIR
@@ -65,6 +65,7 @@ def register_routes(app: FastAPI) -> None:
auth.router, auth.router,
health.router, health.router,
upload.router, upload.router,
remote.router,
tasks.router, tasks.router,
history.router, history.router,
database.router, database.router,
+217
View File
@@ -0,0 +1,217 @@
from __future__ import annotations
import stat
from dataclasses import dataclass
from ftplib import FTP
from pathlib import Path
from typing import Callable
from app.config import RemoteDataConfig
LogFn = Callable[[str], None]
@dataclass
class RemoteDownloadResult:
file_count: int = 0
total_bytes: int = 0
class RemoteDownloadError(RuntimeError):
pass
class RemoteDataDownloader:
def __init__(self, config: RemoteDataConfig, logger: LogFn | None = None):
self.config = config.normalized()
self.logger = logger
def test_connection(self) -> None:
self._validate_config()
if self.config.protocol == "ftp":
with self._ftp_client() as ftp:
ftp.cwd(self.config.remote_dir)
return
ssh = self._sftp_ssh_client()
try:
with ssh.open_sftp() as sftp:
sftp.stat(self.config.remote_dir)
finally:
ssh.close()
def download_to(self, destination: Path) -> RemoteDownloadResult:
self._validate_config()
destination.mkdir(parents=True, exist_ok=True)
if self.config.protocol == "ftp":
return self._download_ftp(destination)
return self._download_sftp(destination)
def _validate_config(self) -> None:
if self.config.protocol not in {"ftp", "sftp"}:
raise RemoteDownloadError("远程协议只支持 FTP 或 SFTP")
if not self.config.host:
raise RemoteDownloadError("请填写远程服务器地址")
if not self.config.user:
raise RemoteDownloadError("请填写远程服务器用户名")
if not self.config.remote_dir:
raise RemoteDownloadError("请填写远程数据目录")
def _log(self, message: str) -> None:
if self.logger:
self.logger(message)
def _ftp_client(self) -> FTP:
ftp = FTP()
ftp.connect(self.config.host, self.config.port, timeout=self.config.timeout)
ftp.login(self.config.user, self.config.passwd)
ftp.set_pasv(self.config.passive)
return ftp
def _download_ftp(self, destination: Path) -> RemoteDownloadResult:
result = RemoteDownloadResult()
with self._ftp_client() as ftp:
self._log(f"已连接 FTP: {self.config.host}:{self.config.port}")
self._download_ftp_dir(ftp, self.config.remote_dir, destination, result)
return result
def _download_ftp_dir(
self,
ftp: FTP,
remote_dir: str,
local_dir: Path,
result: RemoteDownloadResult,
) -> None:
local_dir.mkdir(parents=True, exist_ok=True)
entries = self._list_ftp_entries(ftp, remote_dir)
for name, entry_type, size in entries:
if name in {".", ".."}:
continue
remote_path = self._join_remote_path(remote_dir, name)
local_path = local_dir / name
if entry_type == "dir":
self._download_ftp_dir(ftp, remote_path, local_path, result)
continue
if entry_type == "unknown" and self._ftp_is_dir(ftp, remote_path):
self._download_ftp_dir(ftp, remote_path, local_path, result)
continue
self._download_ftp_file(ftp, remote_path, local_path, result, size)
def _list_ftp_entries(self, ftp: FTP, remote_dir: str) -> list[tuple[str, str, int]]:
try:
return [
(
name,
facts.get("type", "unknown"),
int(facts.get("size", "0") or 0),
)
for name, facts in ftp.mlsd(remote_dir)
]
except Exception:
names = ftp.nlst(remote_dir)
entries = []
for name in names:
clean_name = Path(name.replace("\\", "/")).name
if clean_name:
entries.append((clean_name, "unknown", 0))
return entries
def _ftp_is_dir(self, ftp: FTP, remote_path: str) -> bool:
current = ftp.pwd()
try:
ftp.cwd(remote_path)
return True
except Exception:
return False
finally:
try:
ftp.cwd(current)
except Exception:
pass
def _download_ftp_file(
self,
ftp: FTP,
remote_path: str,
local_path: Path,
result: RemoteDownloadResult,
expected_size: int = 0,
) -> None:
local_path.parent.mkdir(parents=True, exist_ok=True)
self._log(f"下载: {remote_path}")
with local_path.open("wb") as file:
ftp.retrbinary(f"RETR {remote_path}", file.write)
result.file_count += 1
result.total_bytes += expected_size or local_path.stat().st_size
def _sftp_ssh_client(self):
try:
import paramiko
except ImportError as exc:
raise RemoteDownloadError("SFTP 功能需要安装 paramiko,请执行 uv pip install -r requirements.txt") from exc
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(
hostname=self.config.host,
port=self.config.port,
username=self.config.user,
password=self.config.passwd,
timeout=self.config.timeout,
banner_timeout=self.config.timeout,
auth_timeout=self.config.timeout,
)
return ssh
def _download_sftp(self, destination: Path) -> RemoteDownloadResult:
result = RemoteDownloadResult()
ssh = self._sftp_ssh_client()
try:
with ssh.open_sftp() as sftp:
self._log(f"已连接 SFTP: {self.config.host}:{self.config.port}")
self._download_sftp_path(sftp, self.config.remote_dir, destination, result)
finally:
ssh.close()
return result
def _download_sftp_path(
self,
sftp,
remote_path: str,
local_path: Path,
result: RemoteDownloadResult,
) -> None:
attrs = sftp.stat(remote_path)
if stat.S_ISDIR(attrs.st_mode):
local_path.mkdir(parents=True, exist_ok=True)
for item in sftp.listdir_attr(remote_path):
if item.filename in {".", ".."}:
continue
self._download_sftp_path(
sftp,
self._join_remote_path(remote_path, item.filename),
local_path / item.filename,
result,
)
return
local_path.parent.mkdir(parents=True, exist_ok=True)
self._log(f"下载: {remote_path}")
sftp.get(remote_path, str(local_path))
result.file_count += 1
result.total_bytes += int(getattr(attrs, "st_size", 0) or local_path.stat().st_size)
@staticmethod
def _join_remote_path(parent: str, child: str) -> str:
parent = parent.replace("\\", "/").rstrip("/")
if not parent:
return child
if parent == "/":
return f"/{child}"
return f"{parent}/{child}"
+9
View File
@@ -1,5 +1,14 @@
# 项目上下文记录 # 项目上下文记录
## 2026-05-18:新增 FTP/SFTP 远程自动化处理
- `Configure.json` 新增 `RemoteData` 配置,包含启用状态、协议、主机、端口、用户名、密码、远程目录、FTP 被动模式和超时时间;`app/config.py` 会兼容旧配置并在保存时写回该配置块。
- 新增 `app/services/remote_download.py`,FTP 使用标准库 `ftplib`,SFTP 使用 `paramiko`,会递归下载远程目录下的全部文件和文件夹到本地任务缓存目录。
- 新增 `app/api/routers/remote.py`,`POST /api/remote/test` 用于测试远程连接,`POST /api/remote/start` 会创建历史任务、下载远程数据并复用 `DataProcessor` 完成现有处理流程。
- `frontend/src/components/SettingsPanel.vue` 新增“远程数据源”配置卡片,支持 FTP/SFTP 切换、保存和测试连接;`frontend/src/components/FileWorkflow.vue` 新增“远程下载并处理”入口。
- 远程任务启动后会把全局任务阶段设置为 `downloading`,前端处理进度页会显示“远程下载中...”,下载完成后再切换为既有数据处理流程。
- 新增依赖 `paramiko`,当前 `.venv` 已执行 `uv pip install -r requirements.txt`;已执行 `.venv\Scripts\python.exe -m compileall app` 和 `npm run build`,均通过;未启动浏览器或 headless Chrome。
## 2026-05-18:完善服务重启交互和运行时兼容 ## 2026-05-18:完善服务重启交互和运行时兼容
- `frontend/src/AppShell.vue` 的重启按钮点击后会先弹出确认框,确认后显示全屏“正在重启服务”遮罩和旋转加载动画,避免用户重复操作。 - `frontend/src/AppShell.vue` 的重启按钮点击后会先弹出确认框,确认后显示全屏“正在重启服务”遮罩和旋转加载动画,避免用户重复操作。
+42 -2
View File
@@ -40,6 +40,17 @@
/> />
</section> </section>
<section v-if="!taskInProgress" class="remote-run-card">
<div class="remote-run-copy">
<h4>远程自动化</h4>
<p>从已配置的 FTP/SFTP 目录递归下载数据,然后自动开始处理。</p>
</div>
<n-button type="primary" secondary :loading="remoteStarting" :disabled="working" @click="startRemoteProcessing">
<template #icon><n-icon><CloudDownloadOutline /></n-icon></template>
远程下载并处理
</n-button>
</section>
<div v-if="!taskInProgress && files.length > 0" class="file-list"> <div v-if="!taskInProgress && files.length > 0" class="file-list">
<div class="file-list-header"> <div class="file-list-header">
<h4>已选择文件</h4> <h4>已选择文件</h4>
@@ -133,6 +144,7 @@
import { computed, onBeforeUnmount, onMounted, ref } from 'vue'; import { computed, onBeforeUnmount, onMounted, ref } from 'vue';
import { useMessage } from 'naive-ui'; import { useMessage } from 'naive-ui';
import { import {
CloudDownloadOutline,
CloudUploadOutline, CloudUploadOutline,
DocumentTextOutline, DocumentTextOutline,
FolderOpenOutline, FolderOpenOutline,
@@ -160,7 +172,9 @@ interface DroppedFile {
interface UploadResponse { interface UploadResponse {
success: boolean; success: boolean;
task_id: string; task_id: string;
file_count: number; file_count?: number;
message?: string;
stage?: string;
} }
const validExtensions = new Set(['.zip', '.xlsx', '.xls', '.csv']); const validExtensions = new Set(['.zip', '.xlsx', '.xls', '.csv']);
@@ -170,6 +184,7 @@ const folderInput = ref<HTMLInputElement | null>(null);
const files = ref<PickedFile[]>([]); const files = ref<PickedFile[]>([]);
const uploadProgress = ref(0); const uploadProgress = ref(0);
const working = ref(false); const working = ref(false);
const remoteStarting = ref(false);
const isDragging = ref(false); const isDragging = ref(false);
const taskStatus = ref<TaskStatus | null>(null); const taskStatus = ref<TaskStatus | null>(null);
const activeTask = ref<ActiveTask | null>(null); const activeTask = ref<ActiveTask | null>(null);
@@ -194,6 +209,7 @@ const statusClass = computed(() => {
const processStatusText = computed(() => { const processStatusText = computed(() => {
if (taskStatus.value?.status === 'completed') return '处理完成'; if (taskStatus.value?.status === 'completed') return '处理完成';
if (taskStatus.value?.status === 'failed') return '处理失败'; if (taskStatus.value?.status === 'failed') return '处理失败';
if (activeTask.value?.stage === 'downloading') return '远程下载中...';
if (activeTask.value?.stage === 'uploading') return '上传中...'; if (activeTask.value?.stage === 'uploading') return '上传中...';
return '处理中...'; return '处理中...';
}); });
@@ -396,7 +412,7 @@ async function uploadAndStart() {
item.status = 'uploaded'; item.status = 'uploaded';
item.progress = 100; item.progress = 100;
}); });
message.success(`上传完成:${result.file_count} 个文件`); message.success(`上传完成:${result.file_count ?? files.value.length} 个文件`);
await apiPost('/api/process/start', { task_id: result.task_id }); await apiPost('/api/process/start', { task_id: result.task_id });
taskStatus.value = { task_id: result.task_id, status: 'processing', logs: ['任务已提交,等待处理日志...'] }; taskStatus.value = { task_id: result.task_id, status: 'processing', logs: ['任务已提交,等待处理日志...'] };
startPolling(result.task_id); startPolling(result.task_id);
@@ -412,6 +428,30 @@ async function uploadAndStart() {
} }
} }
async function startRemoteProcessing() {
if (working.value || remoteStarting.value) return;
remoteStarting.value = true;
try {
const result = await apiPost<UploadResponse>('/api/remote/start');
message.success(result.message || '远程自动化任务已启动');
files.value = [];
uploadProgress.value = 0;
activeTask.value = {
has_active: true,
task_id: result.task_id,
stage: result.stage || 'downloading',
started_at: new Date().toISOString()
};
taskStatus.value = { task_id: result.task_id, status: 'processing', logs: ['远程下载任务已提交,等待处理日志...'] };
startPolling(result.task_id);
} catch (error) {
message.error(error instanceof Error ? error.message : '启动远程自动化任务失败');
} finally {
remoteStarting.value = false;
}
}
function updateUploadingFiles(progress: number) { function updateUploadingFiles(progress: number) {
files.value.forEach(item => { files.value.forEach(item => {
if (item.status === 'uploading') { if (item.status === 'uploading') {
+159 -1
View File
@@ -46,6 +46,70 @@
</template> </template>
</n-card> </n-card>
<n-card title="远程数据源" size="small" class="work-card">
<n-form label-placement="top">
<n-form-item label="启用远程自动化">
<n-switch v-model:value="remoteForm.enabled" />
</n-form-item>
<n-grid :cols="12" :x-gap="12">
<n-gi :span="4">
<n-form-item label="协议">
<n-select
v-model:value="remoteForm.protocol"
:options="remoteProtocolOptions"
@update:value="updateRemoteProtocol"
/>
</n-form-item>
</n-gi>
<n-gi :span="8">
<n-form-item label="服务器地址">
<n-input v-model:value="remoteForm.host" placeholder="192.168.1.10" />
</n-form-item>
</n-gi>
</n-grid>
<n-grid :cols="12" :x-gap="12">
<n-gi :span="4">
<n-form-item label="端口">
<n-input-number v-model:value="remoteForm.port" class="full-width" :min="1" :max="65535" />
</n-form-item>
</n-gi>
<n-gi :span="4">
<n-form-item label="超时秒数">
<n-input-number v-model:value="remoteForm.timeout" class="full-width" :min="1" :max="600" />
</n-form-item>
</n-gi>
<n-gi :span="4">
<n-form-item label="FTP 被动模式">
<n-switch v-model:value="remoteForm.passive" :disabled="remoteForm.protocol !== 'ftp'" />
</n-form-item>
</n-gi>
</n-grid>
<n-grid :cols="12" :x-gap="12">
<n-gi :span="6">
<n-form-item label="用户名">
<n-input v-model:value="remoteForm.user" placeholder="remote user" />
</n-form-item>
</n-gi>
<n-gi :span="6">
<n-form-item label="密码">
<n-input v-model:value="remoteForm.passwd" type="password" show-password-on="click" />
</n-form-item>
</n-gi>
</n-grid>
<n-form-item label="远程目录">
<n-input v-model:value="remoteForm.remote_dir" placeholder="/CapacityReportData" />
</n-form-item>
<p class="form-hint">自动化执行会递归下载该目录下的全部文件和文件夹到本地缓存,再按现有处理流程入库和执行脚本。</p>
</n-form>
<template #footer>
<n-space justify="end">
<n-button type="primary" :loading="savingRemote" @click="saveRemote">保存配置</n-button>
<n-button :loading="testingRemote" @click="testRemote">测试连接</n-button>
</n-space>
</template>
</n-card>
<n-card title="Sheet 过滤规则" size="small" class="work-card"> <n-card title="Sheet 过滤规则" size="small" class="work-card">
<n-space vertical> <n-space vertical>
<p class="form-hint">匹配这些关键词的 Sheet 将被跳过处理</p> <p class="form-hint">匹配这些关键词的 Sheet 将被跳过处理</p>
@@ -193,7 +257,7 @@ import { useMessage, type SelectOption } from 'naive-ui';
import { CloseOutline, CloudDownloadOutline, CloudUploadOutline } from '@vicons/ionicons5'; import { CloseOutline, CloudDownloadOutline, CloudUploadOutline } from '@vicons/ionicons5';
import { apiGet, apiPost, downloadGet, upload } from '../api/client'; import { apiGet, apiPost, downloadGet, upload } from '../api/client';
import type { ApiMessage, AppConfig } from '../types'; import type { ApiMessage, AppConfig, RemoteDataConfig } from '../types';
import { resetPageHeader, setPageHeader } from '../composables/pageHeader'; import { resetPageHeader, setPageHeader } from '../composables/pageHeader';
interface ExtractFieldConfig { interface ExtractFieldConfig {
@@ -208,7 +272,9 @@ const configInput = ref<HTMLInputElement | null>(null);
const configUpdate = ref(''); const configUpdate = ref('');
const loading = ref(false); const loading = ref(false);
const testingDb = ref(false); const testingDb = ref(false);
const testingRemote = ref(false);
const savingMysql = ref(false); const savingMysql = ref(false);
const savingRemote = ref(false);
const savingSheetFilter = ref(false); const savingSheetFilter = ref(false);
const savingExtractFields = ref(false); const savingExtractFields = ref(false);
const changingPassword = ref(false); const changingPassword = ref(false);
@@ -226,6 +292,18 @@ const mysqlForm = reactive({
dbname: '' dbname: ''
}); });
const remoteForm = reactive<RemoteDataConfig>({
enabled: false,
protocol: 'sftp',
host: '',
port: 22,
user: '',
passwd: '',
remote_dir: '/',
passive: true,
timeout: 30
});
const passwordForm = reactive({ const passwordForm = reactive({
current_password: '', current_password: '',
new_password: '', new_password: '',
@@ -238,6 +316,10 @@ const fieldTypeOptions: SelectOption[] = [
{ label: '小数', value: 'float' }, { label: '小数', value: 'float' },
{ label: '日期时间', value: 'datetime' } { label: '日期时间', value: 'datetime' }
]; ];
const remoteProtocolOptions: SelectOption[] = [
{ label: 'SFTP', value: 'sftp' },
{ label: 'FTP', value: 'ftp' }
];
const visibleFieldMappings = computed(() => { const visibleFieldMappings = computed(() => {
const keyword = fieldSearch.value.trim().toLowerCase(); const keyword = fieldSearch.value.trim().toLowerCase();
@@ -275,6 +357,7 @@ async function loadConfig() {
mysqlForm.user = config.mysql.user; mysqlForm.user = config.mysql.user;
mysqlForm.passwd = config.mysql.passwd || ''; mysqlForm.passwd = config.mysql.passwd || '';
mysqlForm.dbname = config.mysql.dbname; mysqlForm.dbname = config.mysql.dbname;
Object.assign(remoteForm, normalizeRemoteConfig(config.remote_data));
sheetFilters.value = [...config.sheet_filter]; sheetFilters.value = [...config.sheet_filter];
extractFields.value = normalizeExtractFields(config.extract_fields); extractFields.value = normalizeExtractFields(config.extract_fields);
resetExtractInputs(); resetExtractInputs();
@@ -285,6 +368,15 @@ async function loadConfig() {
} }
} }
function updateRemoteProtocol(value: string) {
if (value === 'sftp' && (!remoteForm.port || remoteForm.port === 21)) {
remoteForm.port = 22;
}
if (value === 'ftp' && (!remoteForm.port || remoteForm.port === 22)) {
remoteForm.port = 21;
}
}
async function saveMysql() { async function saveMysql() {
if (!mysqlForm.host || !mysqlForm.user || !mysqlForm.dbname) { if (!mysqlForm.host || !mysqlForm.user || !mysqlForm.dbname) {
message.warning('请填写完整数据库配置'); message.warning('请填写完整数据库配置');
@@ -306,6 +398,57 @@ async function saveMysql() {
} }
} }
function getRemotePayload(): RemoteDataConfig {
return {
...remoteForm,
protocol: remoteForm.protocol,
host: remoteForm.host.trim(),
port: remoteForm.port || (remoteForm.protocol === 'sftp' ? 22 : 21),
user: remoteForm.user.trim(),
passwd: remoteForm.passwd || '',
remote_dir: remoteForm.remote_dir.trim() || '/',
passive: remoteForm.passive,
timeout: remoteForm.timeout || 30
};
}
function validateRemoteForm(): boolean {
if (!remoteForm.host || !remoteForm.user || !remoteForm.remote_dir) {
message.warning('请填写完整远程数据源配置');
return false;
}
return true;
}
async function saveRemote() {
if (!validateRemoteForm()) return;
savingRemote.value = true;
try {
const result = await apiPost<ApiMessage>('/api/config/remote', getRemotePayload());
configUpdate.value = result.update || configUpdate.value;
message.success(result.message || '远程数据源配置已保存');
} catch (error) {
message.error(error instanceof Error ? error.message : '保存远程数据源配置失败');
} finally {
savingRemote.value = false;
}
}
async function testRemote() {
if (!validateRemoteForm()) return;
testingRemote.value = true;
try {
const result = await apiPost<ApiMessage>('/api/remote/test', getRemotePayload());
message[result.success ? 'success' : 'error'](result.message || '远程连接测试完成');
} catch (error) {
message.error(error instanceof Error ? error.message : '远程连接测试失败');
} finally {
testingRemote.value = false;
}
}
async function testDatabase() { async function testDatabase() {
testingDb.value = true; testingDb.value = true;
try { try {
@@ -457,6 +600,21 @@ async function uploadConfigFile(event: Event) {
} }
} }
function normalizeRemoteConfig(config: RemoteDataConfig | undefined): RemoteDataConfig {
const protocol = config?.protocol === 'ftp' ? 'ftp' : 'sftp';
return {
enabled: Boolean(config?.enabled),
protocol,
host: config?.host || '',
port: config?.port || (protocol === 'sftp' ? 22 : 21),
user: config?.user || '',
passwd: config?.passwd || '',
remote_dir: config?.remote_dir || '/',
passive: config?.passive ?? true,
timeout: config?.timeout || 30
};
}
function normalizeExtractFields(fields: Array<Record<string, unknown>>): ExtractFieldConfig[] { function normalizeExtractFields(fields: Array<Record<string, unknown>>): ExtractFieldConfig[] {
return fields.map(field => { return fields.map(field => {
const extract = Array.isArray(field.Extract) ? uniqueStrings(field.Extract) : []; const extract = Array.isArray(field.Extract) ? uniqueStrings(field.Extract) : [];
+35
View File
@@ -593,6 +593,36 @@ select {
font-size: 13px !important; font-size: 13px !important;
} }
.remote-run-card {
display: flex;
align-items: center;
justify-content: space-between;
gap: 16px;
margin-top: 16px;
padding: 16px 20px;
background: var(--td-bg-color-container);
border: 1px solid var(--td-border-color-light);
border-radius: var(--td-radius-default);
}
.remote-run-copy {
min-width: 0;
}
.remote-run-copy h4 {
margin: 0 0 4px;
color: var(--td-text-color-primary);
font-size: 15px;
font-weight: 600;
}
.remote-run-copy p {
margin: 0;
color: var(--td-text-color-secondary);
font-size: 13px;
line-height: 20px;
}
.file-list { .file-list {
margin-top: 20px; margin-top: 20px;
} }
@@ -1521,6 +1551,11 @@ select {
justify-content: flex-end; justify-content: flex-end;
} }
.remote-run-card {
align-items: stretch;
flex-direction: column;
}
.card-actions, .card-actions,
.field-card-title-row, .field-card-title-row,
.extract-list-header { .extract-list-header {
+13
View File
@@ -57,10 +57,23 @@ export interface AppConfig {
passwd?: string; passwd?: string;
dbname: string; dbname: string;
}; };
remote_data: RemoteDataConfig;
sheet_filter: string[]; sheet_filter: string[];
extract_fields: Array<Record<string, unknown>>; extract_fields: Array<Record<string, unknown>>;
} }
export interface RemoteDataConfig {
enabled: boolean;
protocol: 'ftp' | 'sftp';
host: string;
port: number;
user: string;
passwd?: string;
remote_dir: string;
passive: boolean;
timeout: number;
}
export interface TableInfo { export interface TableInfo {
name: string; name: string;
columns: Array<{ columns: Array<{
+1
View File
@@ -12,6 +12,7 @@ cryptography
pandas pandas
openpyxl openpyxl
chardet chardet
paramiko
# Process manager # Process manager
supervisor supervisor