feat: 实现 CellData 预处理与单独刷新

app/api/routers/cell_data.py

@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+import time
+from datetime import datetime
+from pathlib import Path
+from threading import Thread
+
+from fastapi import APIRouter, Body, HTTPException
+
+from app import state
+from app.api.routers.task_runtime import set_task_stage
+from app.config import AppConfig, CACHE_DIR
+from app.processor import ProcessLogger
+from app.services.cell_data import refresh_cell_data
+
+router = APIRouter(tags=["cell-data"])
+
+
+@router.post("/api/cell-data/process/start")
+async def start_cell_data_processing():
+    if state.global_task_lock["locked"]:
+        raise HTTPException(status_code=409, detail="已有任务在运行，请等待当前任务完成")
+
+    task_id = "cell_data_" + datetime.now().strftime("%Y%m%d_%H%M%S")
+    work_dir = CACHE_DIR / task_id
+    work_dir.mkdir(parents=True, exist_ok=True)
+    logs: list[str] = []
+    current_stage = "locating"
+
+    def log_callback(message: str) -> None:
+        logs.append(message)
+        set_task_stage(task_id, current_stage, logs)
+
+    def stage_callback(stage: str) -> None:
+        nonlocal current_stage
+        current_stage = stage
+        set_task_stage(task_id, current_stage, logs)
+
+    logger = ProcessLogger(
+        log_file=work_dir / "log.txt",
+        callback=log_callback,
+        stage_callback=stage_callback,
+    )
+    app_config = state.current_config()
+    state.processing_tasks[task_id] = {"logs": [], "status": "processing", "stage": current_stage}
+    state.global_task_lock.update(
+        {
+            "locked": True,
+            "task_id": task_id,
+            "stage": current_stage,
+            "started_at": datetime.now().isoformat(),
+        }
+    )
+
+    thread = Thread(target=_run_cell_data_processing, args=(task_id, work_dir, logger, app_config), daemon=True)
+    thread.start()
+    return {"success": True, "message": "CellData 处理已启动", "task_id": task_id, "stage": current_stage}
+
+
+@router.post("/api/cell-data/process/status")
+async def get_cell_data_processing_status(task_id: str = Body(..., embed=True)):
+    if task_id in state.processing_tasks:
+        task = state.processing_tasks[task_id]
+        return {
+            "task_id": task_id,
+            "status": task.get("status", "processing"),
+            "stage": task.get("stage", "processing"),
+            "logs": task.get("logs", []),
+            "error": task.get("error"),
+            "result": task.get("result"),
+            "elapsed_time": task.get("elapsed_time"),
+        }
+    raise HTTPException(status_code=404, detail="任务不存在")
+
+
+def _run_cell_data_processing(task_id: str, work_dir: Path, logger: ProcessLogger, app_config: AppConfig) -> None:
+    started = time.time()
+    try:
+        result = refresh_cell_data(app_config, work_dir, logger)
+        elapsed = round(time.time() - started, 2)
+        state.processing_tasks[task_id] = {
+            "logs": logger.get_logs(),
+            "status": "completed",
+            "stage": "completed",
+            "elapsed_time": elapsed,
+            "result": {
+                "selected_files": result.selected_files,
+                "parsed_rows": result.parsed_rows,
+                "imported_rows": result.imported_rows,
+                "skipped_rows": result.skipped_rows,
+            },
+        }
+    except Exception as exc:
+        logger.error(str(exc))
+        state.processing_tasks[task_id] = {
+            "logs": logger.get_logs(),
+            "status": "failed",
+            "stage": "failed",
+            "error": str(exc),
+            "elapsed_time": round(time.time() - started, 2),
+        }
+    finally:
+        state.reset_task_lock()

app/api/routers/config.py

@@ -1,4 +1,5 @@
 import json
+import re
 from datetime import datetime
 from typing import Any
 from urllib.parse import quote
@@ -11,6 +12,7 @@ from app import state
 from app.config import (
     CellDataConfig,
     DataMappingsConfig,
+    DEFAULT_CELL_DATA_MAPPING,
     HistoryRetentionConfig,
     MetrixConfig,
     MySQLConfig,
@@ -103,6 +105,11 @@ async def update_cell_data_remote_config(config: dict[str, Any] = Body(...)):
     state.config.cell_data = CellDataConfig(
         remote_data=RemoteDataConfig.from_dict(config),
         mysql=current.mysql,
+        scan_paths=current.scan_paths,
+        year_dir_regex=current.year_dir_regex,
+        file_name_regex=current.file_name_regex,
+        file_time_regex=current.file_time_regex,
+        mapping=current.mapping,
     ).normalized()
     state.config.save()
     return {"success": True, "message": "CellData 远程数据源配置已更新", "update": state.config.update}
@@ -115,6 +122,11 @@ async def update_cell_data_mysql_config(config: dict[str, Any] = Body(...)):
     state.config.cell_data = CellDataConfig(
         remote_data=current.remote_data,
         mysql=MySQLConfig.from_dict(config, default_dbname="celldata"),
+        scan_paths=current.scan_paths,
+        year_dir_regex=current.year_dir_regex,
+        file_name_regex=current.file_name_regex,
+        file_time_regex=current.file_time_regex,
+        mapping=current.mapping,
     ).normalized()
     state.config.save()
     return {"success": True, "message": "CellData 数据库配置已更新", "update": state.config.update}
@@ -136,6 +148,36 @@ async def test_cell_data_mysql_connection(config: dict[str, Any] | None = Body(N
     return _test_mysql_config(mysql_config)
 
 
+@router.post("/api/config/cell-data/settings")
+async def update_cell_data_settings(config: dict[str, Any] = Body(...)):
+    validation = _validate_cell_data_settings(config)
+    if not validation["success"]:
+        raise HTTPException(status_code=400, detail=validation["message"])
+    state.reload_config()
+    current = state.config.cell_data.normalized()
+    state.config.cell_data = CellDataConfig(
+        remote_data=current.remote_data,
+        mysql=current.mysql,
+        scan_paths=config.get("scan_paths", current.scan_paths),
+        year_dir_regex=str(config.get("year_dir_regex", current.year_dir_regex)),
+        file_name_regex=str(config.get("file_name_regex", current.file_name_regex)),
+        file_time_regex=str(config.get("file_time_regex", current.file_time_regex)),
+        mapping=config.get("mapping", current.mapping),
+    ).normalized()
+    state.config.save()
+    return {"success": True, "message": "CellData 规则已更新", "update": state.config.update}
+
+
+@router.post("/api/config/cell-data/settings/validate")
+async def validate_cell_data_settings(config: dict[str, Any] = Body(...)):
+    return _validate_cell_data_settings(config)
+
+
+@router.get("/api/config/cell-data/mapping/default")
+async def get_default_cell_data_mapping():
+    return DEFAULT_CELL_DATA_MAPPING
+
+
 @router.post("/api/config/history-retention")
 async def update_history_retention(config: dict[str, Any] = Body(...)):
     state.reload_config()
@@ -254,3 +296,45 @@ def _test_mysql_config(config: MySQLConfig) -> dict[str, Any]:
     except Exception as exc:
         return {"success": False, "message": str(exc)}
 
+
+def _validate_cell_data_settings(config: dict[str, Any]) -> dict[str, Any]:
+    scan_paths = config.get("scan_paths", [])
+    if not isinstance(scan_paths, list) or not any(str(path).strip() for path in scan_paths):
+        return {"success": False, "message": "请至少配置一个扫描路径"}
+
+    for key in ("year_dir_regex", "file_name_regex", "file_time_regex"):
+        try:
+            re.compile(str(config.get(key, "")))
+        except re.error as exc:
+            return {"success": False, "message": f"{key} 正则无效: {exc}"}
+
+    mapping = config.get("mapping")
+    if not isinstance(mapping, dict):
+        return {"success": False, "message": "映射规则必须是 JSON 对象"}
+    if not str(mapping.get("target_table", "")).strip():
+        return {"success": False, "message": "映射规则缺少 target_table"}
+    key_config = mapping.get("key")
+    if not isinstance(key_config, dict) or not key_config.get("field") or not key_config.get("expr"):
+        return {"success": False, "message": "映射规则缺少 key.field 或 key.expr"}
+    sources = mapping.get("sources")
+    if not isinstance(sources, list) or not sources:
+        return {"success": False, "message": "映射规则至少需要一个 sources 项"}
+
+    for index, source in enumerate(sources, 1):
+        if not isinstance(source, dict):
+            return {"success": False, "message": f"sources 第 {index} 项必须是对象"}
+        if not source.get("band") or not source.get("file_prefix"):
+            return {"success": False, "message": f"sources 第 {index} 项缺少 band 或 file_prefix"}
+        fields = source.get("fields")
+        if not isinstance(fields, dict) or not fields:
+            return {"success": False, "message": f"sources 第 {index} 项缺少 fields"}
+        for target, rule in fields.items():
+            if not str(target).strip():
+                return {"success": False, "message": f"sources 第 {index} 项存在空目标字段"}
+            if isinstance(rule, str) and rule.strip():
+                continue
+            if isinstance(rule, dict) and "value" in rule:
+                continue
+            return {"success": False, "message": f"{target} 的映射规则无效"}
+    return {"success": True, "message": "映射规则有效"}
+

app/api/routers/remote.py

@@ -14,6 +14,7 @@ from app.api.routers.task_runtime import (
 )
 from app.config import AppConfig, CACHE_DIR, RemoteDataConfig
 from app.processor import DataProcessor, ProcessLogger
+from app.services.cell_data import refresh_cell_data
 from app.services.license import LicenseError, check_processing_allowed
 from app.services.platform import PlatformStorageDownloader, make_source_downloader
 from app.services.pipeline import RESULT_TABLES, run_import_and_report
@@ -150,6 +151,8 @@ def _run_remote_processing(
 
         state.history_manager.update(task_id, file_count=download_result.file_count)
 
+        refresh_cell_data(app_config, work_dir, logger)
+
         if app_config.warehouse_type == "metrix":
             started = time.time()
             run_import_and_report(work_dir, app_config, logger)

app/api/routers/tasks.py

@@ -13,6 +13,7 @@ from app.api.routers.task_runtime import (
 )
 from app.config import AppConfig
 from app.processor import DataProcessor, ProcessLogger
+from app.services.cell_data import refresh_cell_data
 from app.services.license import LicenseError, check_processing_allowed
 
 
@@ -168,6 +169,7 @@ def _task_finished(task_id: str) -> bool:
 
 def _run_processing(task_id: str, work_dir: Path, logger: ProcessLogger, app_config: AppConfig) -> None:
     try:
+        refresh_cell_data(app_config, work_dir, logger)
         if app_config.warehouse_type == "metrix":
             import time