feat: 增加远程自动调度处理

app/services/auto_scheduler.py

@@ -0,0 +1,408 @@
+from __future__ import annotations
+
+import json
+import threading
+from collections import defaultdict
+from dataclasses import dataclass
+from datetime import date, datetime, timedelta
+from typing import Any
+
+from fastapi import HTTPException
+
+from app import state
+from app.config import CACHE_DIR, AutoSchedulerConfig, RemoteDataConfig
+from app.services.remote_download import RemoteDataDownloader, RemoteFileInfo
+from app.utils.file_dates import extract_file_date, required_week_days
+
+
+READY_DIR = CACHE_DIR / "auto_scheduler"
+READY_FLAG = READY_DIR / "ready.flag"
+DISABLED_CHECK_SECONDS = 60
+STARTUP_CHECK_SECONDS = 5
+FAILURE_RESULTS = {"scan_failed", "trigger_failed", "source_cleanup_failed", "failed", "invalid_flag"}
+
+
+@dataclass(frozen=True)
+class DirectoryReadyStatus:
+    directory: str
+    ready: bool
+    found_days: list[date]
+    missing_days: list[date]
+    file_count: int
+    error: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "ready": self.ready,
+            "found_days": [item.isoformat() for item in self.found_days],
+            "missing_days": [item.isoformat() for item in self.missing_days],
+            "found_count": len(self.found_days),
+            "required_count": len(self.found_days) + len(self.missing_days),
+            "file_count": self.file_count,
+            "error": self.error,
+        }
+
+
+class AutoScheduler:
+    def __init__(self) -> None:
+        self._stop_event = threading.Event()
+        self._check_lock = threading.Lock()
+        self._status_lock = threading.Lock()
+        self._thread: threading.Thread | None = None
+        self._running = False
+        self._last_check_at: datetime | None = None
+        self._next_check_at: datetime | None = None
+        self._last_result = "not_started"
+        self._last_message = "自动调度器尚未检查"
+        self._directory_status: dict[str, dict[str, Any]] = {}
+        self._task_running = False
+        self._task_id: str | None = None
+        self._failure_count = 0
+
+    def start(self) -> None:
+        if self._thread and self._thread.is_alive():
+            return
+        self._running = True
+        self._stop_event.clear()
+        self._set_next_check(datetime.now() + timedelta(seconds=STARTUP_CHECK_SECONDS))
+        self._thread = threading.Thread(target=self._run_loop, name="auto-scheduler", daemon=True)
+        self._thread.start()
+
+    def stop(self) -> None:
+        self._running = False
+        self._stop_event.set()
+        if self._thread and self._thread.is_alive():
+            self._thread.join(timeout=5)
+
+    def get_status(self) -> dict[str, Any]:
+        config = state.current_config().remote_data.normalized()
+        scheduler = config.auto_scheduler.normalized()
+        target_days = required_week_days(scheduler.week_offset)
+        ready_flag = self._read_ready_flag()
+
+        with self._status_lock:
+            return {
+                "enabled": scheduler.enabled,
+                "running": self._running,
+                "check_interval_hours": scheduler.check_interval_hours,
+                "expected_directories": scheduler.expected_directories,
+                "week_offset": scheduler.week_offset,
+                "auto_delete_source": config.auto_delete_source,
+                "next_check_at": self._format_dt(self._next_check_at),
+                "last_check_at": self._format_dt(self._last_check_at),
+                "last_result": self._last_result,
+                "last_message": self._last_message,
+                "failure_count": self._failure_count,
+                "task_running": self._task_running,
+                "task_id": self._task_id,
+                "ready_flag": ready_flag,
+                "target_week": {
+                    "start": target_days[0].isoformat(),
+                    "end": target_days[-1].isoformat(),
+                    "days": [item.isoformat() for item in target_days],
+                },
+                "directory_status": self._directory_status,
+            }
+
+    def check_and_run(self, manual: bool = False) -> dict[str, Any]:
+        if not self._check_lock.acquire(blocking=False):
+            return self._finish_check("busy", "自动调度器正在检查中", manual)
+
+        try:
+            return self._check_and_run_locked(manual)
+        finally:
+            self._check_lock.release()
+
+    def _run_loop(self) -> None:
+        while not self._stop_event.wait(self._seconds_until_next_check()):
+            self.check_and_run(manual=False)
+
+    def _check_and_run_locked(self, manual: bool) -> dict[str, Any]:
+        now = datetime.now()
+        remote_config = state.current_config().remote_data.normalized()
+        scheduler = remote_config.auto_scheduler.normalized()
+        self._last_check_at = now
+
+        if not scheduler.enabled:
+            self._set_next_check(now + timedelta(seconds=DISABLED_CHECK_SECONDS))
+            return self._finish_check("disabled", "自动调度未启用", manual)
+
+        self._set_next_check(now + timedelta(hours=scheduler.check_interval_hours))
+
+        if not remote_config.enabled:
+            return self._finish_check("remote_disabled", "远程数据源未启用", manual)
+
+        if state.global_task_lock["locked"]:
+            return self._finish_check("task_running", "已有任务在运行，本轮自动调度跳过", manual)
+
+        ready_flag = self._read_ready_flag()
+        if ready_flag["exists"]:
+            if ready_flag.get("invalid"):
+                self._clear_ready_flag()
+                return self._finish_check("invalid_flag", "就绪标识格式错误，已清除，本轮不触发处理", manual)
+            target_dates = self._target_dates_from_flag(ready_flag, scheduler)
+            return self._trigger_processing(target_dates, ready_flag, manual)
+
+        target_days = required_week_days(scheduler.week_offset)
+        directory_status = self._check_remote_ready(remote_config, scheduler, target_days)
+        ready = bool(directory_status) and all(item.ready for item in directory_status.values())
+        self._set_directory_status(directory_status)
+
+        error_count = sum(1 for item in directory_status.values() if item.error)
+        if error_count:
+            return self._finish_check(
+                "scan_failed",
+                f"远程目录扫描失败，{error_count}/{len(directory_status)} 个目录无法访问或扫描失败",
+                manual,
+            )
+
+        if ready:
+            self._mark_ready(target_days, directory_status)
+            return self._finish_check(
+                "marked_ready",
+                "远程数据已满足目标周 7 天，已写入就绪标识，下次检查将自动处理",
+                manual,
+            )
+
+        ready_count = sum(1 for item in directory_status.values() if item.ready)
+        return self._finish_check(
+            "waiting",
+            f"远程数据未就绪，{ready_count}/{len(directory_status)} 个目录满足目标周 7 天",
+            manual,
+        )
+
+    def _check_remote_ready(
+        self,
+        remote_config: RemoteDataConfig,
+        scheduler: AutoSchedulerConfig,
+        target_days: list[date],
+    ) -> dict[str, DirectoryReadyStatus]:
+        downloader = RemoteDataDownloader(remote_config)
+        expected_directories = scheduler.expected_directories
+        if expected_directories:
+            return {
+                directory: self._directory_ready_status(
+                    directory,
+                    self._safe_list_remote_zip_files(downloader, directory),
+                    target_days,
+                )
+                for directory in expected_directories
+            }
+
+        files = self._safe_list_remote_zip_files(downloader, None)
+        if files is None:
+            return {
+                ".": DirectoryReadyStatus(
+                    directory=".",
+                    ready=False,
+                    found_days=[],
+                    missing_days=target_days,
+                    file_count=0,
+                    error="远程目录扫描失败",
+                )
+            }
+
+        grouped: dict[str, list[RemoteFileInfo]] = defaultdict(list)
+        for remote_file in files:
+            grouped[remote_file.parent or "."].append(remote_file)
+
+        if not grouped:
+            return {
+                ".": DirectoryReadyStatus(
+                    directory=".",
+                    ready=False,
+                    found_days=[],
+                    missing_days=target_days,
+                    file_count=0,
+                    error="远程目录未找到 ZIP 文件",
+                )
+            }
+
+        return {
+            directory: self._directory_ready_status(directory, directory_files, target_days)
+            for directory, directory_files in sorted(grouped.items(), key=lambda item: item[0])
+        }
+
+    def _safe_list_remote_zip_files(
+        self,
+        downloader: RemoteDataDownloader,
+        directory: str | None,
+    ) -> list[RemoteFileInfo] | None:
+        try:
+            return downloader.list_remote_zip_files(directory)
+        except Exception:
+            return None
+
+    def _directory_ready_status(
+        self,
+        directory: str,
+        files: list[RemoteFileInfo] | None,
+        target_days: list[date],
+    ) -> DirectoryReadyStatus:
+        if files is None:
+            return DirectoryReadyStatus(
+                directory=directory,
+                ready=False,
+                found_days=[],
+                missing_days=target_days,
+                file_count=0,
+                error="远程目录不存在或无法访问",
+            )
+
+        required = set(target_days)
+        found = {
+            file_date
+            for remote_file in files
+            if (file_date := extract_file_date(remote_file.name)) in required
+        }
+        missing = [item for item in target_days if item not in found]
+        return DirectoryReadyStatus(
+            directory=directory,
+            ready=not missing,
+            found_days=sorted(found),
+            missing_days=missing,
+            file_count=len(files),
+        )
+
+    def _trigger_processing(
+        self,
+        target_dates: list[date],
+        ready_flag: dict[str, Any],
+        manual: bool,
+    ) -> dict[str, Any]:
+        from app.api.routers.remote import start_remote_processing_job
+
+        try:
+            result = start_remote_processing_job(
+                source="scheduler",
+                on_finish=self._on_processing_finish,
+                target_dates=target_dates,
+            )
+        except HTTPException as exc:
+            return self._finish_check("trigger_failed", str(exc.detail), manual)
+        except Exception as exc:
+            return self._finish_check("trigger_failed", f"自动调度触发失败: {exc}", manual)
+
+        task_id = result.get("task_id")
+        with self._status_lock:
+            self._task_running = True
+            self._task_id = str(task_id) if task_id else None
+            self._last_result = "triggered"
+            self._last_message = "已根据就绪标识触发远程下载并处理"
+        return {
+            "success": True,
+            "result": "triggered",
+            "message": "已根据就绪标识触发远程下载并处理",
+            "task_id": task_id,
+            "ready_flag": ready_flag,
+            "status": self.get_status(),
+        }
+
+    def _mark_ready(
+        self,
+        target_days: list[date],
+        directory_status: dict[str, DirectoryReadyStatus],
+    ) -> None:
+        READY_DIR.mkdir(parents=True, exist_ok=True)
+        payload = {
+            "ready_at": datetime.now().isoformat(timespec="seconds"),
+            "week_start": target_days[0].isoformat(),
+            "week_end": target_days[-1].isoformat(),
+            "target_dates": [item.isoformat() for item in target_days],
+            "directories": {
+                name: item.to_dict()
+                for name, item in directory_status.items()
+            },
+        }
+        READY_FLAG.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+
+    def _clear_ready_flag(self) -> None:
+        try:
+            READY_FLAG.unlink(missing_ok=True)
+        except OSError:
+            pass
+
+    def _read_ready_flag(self) -> dict[str, Any]:
+        if not READY_FLAG.exists():
+            return {"exists": False}
+        try:
+            data = json.loads(READY_FLAG.read_text(encoding="utf-8"))
+            if isinstance(data, dict):
+                return {"exists": True, **data}
+        except Exception as exc:
+            return {"exists": True, "invalid": True, "error": str(exc)}
+        return {"exists": True, "invalid": True, "error": "ready.flag 格式错误"}
+
+    def _target_dates_from_flag(
+        self,
+        ready_flag: dict[str, Any],
+        scheduler: AutoSchedulerConfig,
+    ) -> list[date]:
+        raw_dates = ready_flag.get("target_dates")
+        if isinstance(raw_dates, list):
+            dates: list[date] = []
+            for raw_date in raw_dates:
+                try:
+                    dates.append(date.fromisoformat(str(raw_date)))
+                except ValueError:
+                    continue
+            if dates:
+                return sorted(dates)
+        return required_week_days(scheduler.week_offset)
+
+    def _on_processing_finish(self, task_id: str, status: str) -> None:
+        if status == "completed":
+            self._clear_ready_flag()
+            message = "自动调度任务处理成功，已清除就绪标识"
+            result = "completed"
+        else:
+            message = "自动调度任务未完成，就绪标识已保留，下次检查会重试"
+            result = status or "failed"
+
+        with self._status_lock:
+            self._task_running = False
+            self._task_id = task_id
+            self._last_result = result
+            self._last_message = message
+            self._failure_count = 0 if result == "completed" else self._failure_count + 1
+
+    def _finish_check(self, result: str, message: str, manual: bool) -> dict[str, Any]:
+        with self._status_lock:
+            self._last_result = result
+            self._last_message = message
+            if result in FAILURE_RESULTS:
+                self._failure_count += 1
+            elif result not in {"busy", "task_running"}:
+                self._failure_count = 0
+            if result not in {"triggered", "task_running"}:
+                self._task_running = False
+
+        return {
+            "success": result not in FAILURE_RESULTS,
+            "manual": manual,
+            "result": result,
+            "message": message,
+            "status": self.get_status(),
+        }
+
+    def _set_directory_status(self, directory_status: dict[str, DirectoryReadyStatus]) -> None:
+        with self._status_lock:
+            self._directory_status = {
+                name: item.to_dict()
+                for name, item in directory_status.items()
+            }
+
+    def _set_next_check(self, value: datetime) -> None:
+        with self._status_lock:
+            self._next_check_at = value
+
+    def _seconds_until_next_check(self) -> float:
+        with self._status_lock:
+            next_check_at = self._next_check_at
+        if not next_check_at:
+            return STARTUP_CHECK_SECONDS
+        return max((next_check_at - datetime.now()).total_seconds(), 0)
+
+    @staticmethod
+    def _format_dt(value: datetime | None) -> str | None:
+        return value.isoformat(timespec="seconds") if value else None

app/services/remote_download.py

@@ -1,12 +1,15 @@
 from __future__ import annotations
 
 import stat
+from collections import defaultdict
 from dataclasses import dataclass, field
+from datetime import date
 from ftplib import FTP
 from pathlib import Path
 from typing import Callable, Iterable
 
 from app.config import RemoteDataConfig
+from app.utils.file_dates import extract_file_date, select_recent_items_by_directory
 
 
 LogFn = Callable[[str], None]
@@ -19,6 +22,15 @@ class RemoteDownloadResult:
     remote_files: list[str] = field(default_factory=list)
 
 
+@dataclass(frozen=True)
+class RemoteFileInfo:
+    path: str
+    relative_path: str
+    parent: str
+    name: str
+    size: int = 0
+
+
 class RemoteDownloadError(RuntimeError):
     pass
 
@@ -42,13 +54,71 @@ class RemoteDataDownloader:
         finally:
             ssh.close()
 
-    def download_to(self, destination: Path) -> RemoteDownloadResult:
+    def download_to(self, destination: Path, target_dates: Iterable[date] | None = None) -> RemoteDownloadResult:
         self._validate_config()
         destination.mkdir(parents=True, exist_ok=True)
+        zip_files = self.list_remote_zip_files()
+        date_filter = set(target_dates or [])
+        if date_filter:
+            selected_files = self._select_files_by_dates(zip_files, date_filter)
+            return self._download_selected_files(destination, selected_files)
+
+        if zip_files:
+            selected_files, summaries = select_recent_items_by_directory(
+                zip_files,
+                parent_key=lambda item: item.parent,
+                name_key=lambda item: item.name,
+            )
+            for summary in summaries:
+                if summary.max_date and summary.start_date and summary.skipped_count:
+                    self._log(
+                        f"远程目录 {summary.directory or '.'}: 仅下载 "
+                        f"{summary.start_date.isoformat()} 至 {summary.max_date.isoformat()} "
+                        f"的 {summary.selected_count}/{summary.total_count} 个 ZIP 文件，"
+                        f"跳过 {summary.skipped_count} 个旧文件"
+                    )
+            return self._download_selected_files(destination, selected_files)
+
         if self.config.protocol == "ftp":
             return self._download_ftp(destination)
         return self._download_sftp(destination)
 
+    def _select_files_by_dates(self, zip_files: list[RemoteFileInfo], target_dates: set[date]) -> list[RemoteFileInfo]:
+        selected_files: list[RemoteFileInfo] = []
+        grouped: dict[str, list[RemoteFileInfo]] = defaultdict(list)
+        for remote_file in zip_files:
+            grouped[remote_file.parent].append(remote_file)
+
+        for parent, files in sorted(grouped.items(), key=lambda item: item[0]):
+            selected = [
+                remote_file
+                for remote_file in files
+                if extract_file_date(remote_file.name) in target_dates
+            ]
+            selected_files.extend(selected)
+            skipped_count = len(files) - len(selected)
+            if skipped_count:
+                first_day = min(target_dates).isoformat()
+                last_day = max(target_dates).isoformat()
+                self._log(
+                    f"远程目录 {parent or '.'}: 仅下载调度目标日期 "
+                    f"{first_day} 至 {last_day} 的 {len(selected)}/{len(files)} 个 ZIP 文件，"
+                    f"跳过 {skipped_count} 个非目标日期文件"
+                )
+
+        return selected_files
+
+    def list_remote_zip_files(self, directory: str | None = None) -> list[RemoteFileInfo]:
+        self._validate_config()
+        remote_dir = (
+            self._join_remote_path(self.config.remote_dir, directory.strip("/"))
+            if directory
+            else self.config.remote_dir
+        )
+        if self.config.protocol == "ftp":
+            return self._list_ftp_zip_files(remote_dir)
+        return self._list_sftp_zip_files(remote_dir)
+
     def delete_source_files(self, remote_files: Iterable[str] | None = None) -> int:
         self._validate_config()
         source_files = list(remote_files or [])
@@ -164,6 +234,59 @@ class RemoteDataDownloader:
         result.total_bytes += expected_size or local_path.stat().st_size
         result.remote_files.append(remote_path)
 
+    def _download_selected_files(self, destination: Path, remote_files: list[RemoteFileInfo]) -> RemoteDownloadResult:
+        result = RemoteDownloadResult()
+        if not remote_files:
+            return result
+
+        if self.config.protocol == "ftp":
+            with self._ftp_client() as ftp:
+                self._log(f"已连接 FTP: {self.config.host}:{self.config.port}")
+                for remote_file in remote_files:
+                    self._download_ftp_file(
+                        ftp,
+                        remote_file.path,
+                        destination / remote_file.relative_path,
+                        result,
+                        remote_file.size,
+                    )
+            return result
+
+        ssh = self._sftp_ssh_client()
+        try:
+            with ssh.open_sftp() as sftp:
+                self._log(f"已连接 SFTP: {self.config.host}:{self.config.port}")
+                for remote_file in remote_files:
+                    self._download_sftp_file(
+                        sftp,
+                        remote_file.path,
+                        destination / remote_file.relative_path,
+                        result,
+                        remote_file.size,
+                    )
+        finally:
+            ssh.close()
+        return result
+
+    def _list_ftp_zip_files(self, remote_dir: str) -> list[RemoteFileInfo]:
+        with self._ftp_client() as ftp:
+            return self._collect_ftp_zip_files(ftp, remote_dir)
+
+    def _collect_ftp_zip_files(self, ftp: FTP, remote_dir: str) -> list[RemoteFileInfo]:
+        files: list[RemoteFileInfo] = []
+        for name, entry_type, size in self._list_ftp_entries(ftp, remote_dir):
+            if name in {".", ".."}:
+                continue
+
+            remote_path = self._join_remote_path(remote_dir, name)
+            if entry_type == "dir" or (entry_type == "unknown" and self._ftp_is_dir(ftp, remote_path)):
+                files.extend(self._collect_ftp_zip_files(ftp, remote_path))
+                continue
+
+            if name.lower().endswith(".zip"):
+                files.append(self._remote_file_info(remote_path, size))
+        return files
+
     def _delete_ftp_source_files(self) -> int:
         with self._ftp_client() as ftp:
             self._log(f"开始清理 FTP 源文件: {self.config.remote_dir}")
@@ -246,13 +369,50 @@ class RemoteDataDownloader:
                 )
             return
 
+        self._download_sftp_file(sftp, remote_path, local_path, result, int(getattr(attrs, "st_size", 0) or 0))
+
+    def _download_sftp_file(
+        self,
+        sftp,
+        remote_path: str,
+        local_path: Path,
+        result: RemoteDownloadResult,
+        expected_size: int = 0,
+    ) -> None:
         local_path.parent.mkdir(parents=True, exist_ok=True)
         self._log(f"下载: {remote_path}")
         sftp.get(remote_path, str(local_path))
         result.file_count += 1
-        result.total_bytes += int(getattr(attrs, "st_size", 0) or local_path.stat().st_size)
+        result.total_bytes += expected_size or local_path.stat().st_size
         result.remote_files.append(remote_path)
 
+    def _list_sftp_zip_files(self, remote_dir: str) -> list[RemoteFileInfo]:
+        ssh = self._sftp_ssh_client()
+        try:
+            with ssh.open_sftp() as sftp:
+                return self._collect_sftp_zip_files(sftp, remote_dir)
+        finally:
+            ssh.close()
+
+    def _collect_sftp_zip_files(self, sftp, remote_path: str) -> list[RemoteFileInfo]:
+        attrs = sftp.stat(remote_path)
+        if not stat.S_ISDIR(attrs.st_mode):
+            name = Path(remote_path.replace("\\", "/")).name
+            if name.lower().endswith(".zip"):
+                return [self._remote_file_info(remote_path, int(getattr(attrs, "st_size", 0) or 0))]
+            return []
+
+        files: list[RemoteFileInfo] = []
+        for item in sftp.listdir_attr(remote_path):
+            if item.filename in {".", ".."}:
+                continue
+            child_path = self._join_remote_path(remote_path, item.filename)
+            if stat.S_ISDIR(item.st_mode):
+                files.extend(self._collect_sftp_zip_files(sftp, child_path))
+            elif item.filename.lower().endswith(".zip"):
+                files.append(self._remote_file_info(child_path, int(getattr(item, "st_size", 0) or 0)))
+        return files
+
     def _delete_sftp_source_files(self) -> int:
         ssh = self._sftp_ssh_client()
         try:
@@ -301,3 +461,22 @@ class RemoteDataDownloader:
         if parent == "/":
             return f"/{child}"
         return f"{parent}/{child}"
+
+    def _remote_file_info(self, remote_path: str, size: int = 0) -> RemoteFileInfo:
+        normalized_root = self.config.remote_dir.replace("\\", "/").rstrip("/")
+        normalized_path = remote_path.replace("\\", "/")
+        if normalized_root and normalized_root != "/" and normalized_path.startswith(f"{normalized_root}/"):
+            relative_path = normalized_path[len(normalized_root) + 1 :]
+        else:
+            relative_path = normalized_path.lstrip("/")
+        relative = Path(relative_path)
+        parent = str(relative.parent).replace("\\", "/")
+        if parent == ".":
+            parent = ""
+        return RemoteFileInfo(
+            path=remote_path,
+            relative_path=relative_path,
+            parent=parent,
+            name=relative.name,
+            size=size,
+        )