feat: 新增 CellData 数据源和数据库配置
This commit is contained in:
+182
-48
@@ -1,5 +1,7 @@
|
||||
{
|
||||
"Update": "2026/05/25 17:57:25",
|
||||
"Update": "2026/06/24 23:16:46",
|
||||
"SourceType": "sftp",
|
||||
"WarehouseType": "mysql",
|
||||
"MySQL_DBInfo": {
|
||||
"host": "127.0.0.1",
|
||||
"port": 3306,
|
||||
@@ -7,6 +9,183 @@
|
||||
"passwd": "123456",
|
||||
"dbname": "CapacityReport"
|
||||
},
|
||||
"Metrix": {
|
||||
"base_url": "http://host.docker.internal:8000",
|
||||
"storage_id": "",
|
||||
"database_conn_id": "",
|
||||
"target_database": "",
|
||||
"recent_days": 7,
|
||||
"token": ""
|
||||
},
|
||||
"DataMappings": {
|
||||
"directories": [
|
||||
{
|
||||
"path": "4G",
|
||||
"table": "4G_UD",
|
||||
"ready_rule": "daily"
|
||||
},
|
||||
{
|
||||
"path": "5G",
|
||||
"table": "5G_UD",
|
||||
"ready_rule": "daily"
|
||||
},
|
||||
{
|
||||
"path": "RJ/2.6G/2.6RJGD",
|
||||
"table": "2_6GRJGD",
|
||||
"ready_rule": "auto"
|
||||
},
|
||||
{
|
||||
"path": "RJ/2.6G/2.6RJYD",
|
||||
"table": "2_6GRJYD",
|
||||
"ready_rule": "auto"
|
||||
},
|
||||
{
|
||||
"path": "RJ/700M/700RJGD",
|
||||
"table": "700MRJGD",
|
||||
"ready_rule": "auto"
|
||||
},
|
||||
{
|
||||
"path": "RJ/700M/700RJYD",
|
||||
"table": "700MRJYD",
|
||||
"ready_rule": "auto"
|
||||
}
|
||||
],
|
||||
"table_field_mappings": {
|
||||
"2_6GRJGD": [
|
||||
{
|
||||
"Source": "开始时间",
|
||||
"Target": "开始时间",
|
||||
"Type": "datetime"
|
||||
},
|
||||
{
|
||||
"Source": "结束时间",
|
||||
"Target": "结束时间",
|
||||
"Type": "datetime"
|
||||
},
|
||||
{
|
||||
"Source": "gNBId",
|
||||
"Target": "gNBId",
|
||||
"Type": "string"
|
||||
},
|
||||
{
|
||||
"Source": "cellId",
|
||||
"Target": "cellId",
|
||||
"Type": "string"
|
||||
},
|
||||
{
|
||||
"Source": "gNBplmn",
|
||||
"Target": "gNBplmn",
|
||||
"Type": "string"
|
||||
},
|
||||
{
|
||||
"Source": "5G上下行总流量(上行PDCP PDU数据量+下行PDCP成功发送数据量)-GD(GB)",
|
||||
"Target": "上下行总流量_GB",
|
||||
"Type": "float"
|
||||
}
|
||||
],
|
||||
"2_6GRJYD": [
|
||||
{
|
||||
"Source": "开始时间",
|
||||
"Target": "开始时间",
|
||||
"Type": "datetime"
|
||||
},
|
||||
{
|
||||
"Source": "结束时间",
|
||||
"Target": "结束时间",
|
||||
"Type": "datetime"
|
||||
},
|
||||
{
|
||||
"Source": "gNBId",
|
||||
"Target": "gNBId",
|
||||
"Type": "string"
|
||||
},
|
||||
{
|
||||
"Source": "cellId",
|
||||
"Target": "cellId",
|
||||
"Type": "string"
|
||||
},
|
||||
{
|
||||
"Source": "gNBplmn",
|
||||
"Target": "gNBplmn",
|
||||
"Type": "string"
|
||||
},
|
||||
{
|
||||
"Source": "5G上下行总流量(上行PDCP PDU数据量+下行PDCP成功发送数据量)-YD(GB)",
|
||||
"Target": "上下行总流量_GB",
|
||||
"Type": "float"
|
||||
}
|
||||
],
|
||||
"700MRJGD": [
|
||||
{
|
||||
"Source": "开始时间",
|
||||
"Target": "开始时间",
|
||||
"Type": "datetime"
|
||||
},
|
||||
{
|
||||
"Source": "结束时间",
|
||||
"Target": "结束时间",
|
||||
"Type": "datetime"
|
||||
},
|
||||
{
|
||||
"Source": "gNBId",
|
||||
"Target": "gNBId",
|
||||
"Type": "string"
|
||||
},
|
||||
{
|
||||
"Source": "cellId",
|
||||
"Target": "cellId",
|
||||
"Type": "string"
|
||||
},
|
||||
{
|
||||
"Source": "gNBplmn",
|
||||
"Target": "gNBplmn",
|
||||
"Type": "string"
|
||||
},
|
||||
{
|
||||
"Source": "5G上下行总流量(上行PDCP PDU数据量+下行PDCP成功发送数据量)-GD(GB)",
|
||||
"Target": "上下行总流量_GB",
|
||||
"Type": "float"
|
||||
}
|
||||
],
|
||||
"700MRJYD": [
|
||||
{
|
||||
"Source": "开始时间",
|
||||
"Target": "开始时间",
|
||||
"Type": "datetime"
|
||||
},
|
||||
{
|
||||
"Source": "结束时间",
|
||||
"Target": "结束时间",
|
||||
"Type": "datetime"
|
||||
},
|
||||
{
|
||||
"Source": "gNBId",
|
||||
"Target": "gNBId",
|
||||
"Type": "string"
|
||||
},
|
||||
{
|
||||
"Source": "cellId",
|
||||
"Target": "cellId",
|
||||
"Type": "string"
|
||||
},
|
||||
{
|
||||
"Source": "gNBplmn",
|
||||
"Target": "gNBplmn",
|
||||
"Type": "string"
|
||||
},
|
||||
{
|
||||
"Source": "5G上下行总流量(上行PDCP PDU数据量+下行PDCP成功发送数据量)-YD(GB)",
|
||||
"Target": "上下行总流量_GB",
|
||||
"Type": "float"
|
||||
},
|
||||
{
|
||||
"Source": "5G上下行总流量(上行PDCP PDU数据量+下行PDCP成功发送数据量)(GB)-YD(GB)",
|
||||
"Target": "上下行总流量_GB",
|
||||
"Type": "float"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"RemoteData": {
|
||||
"enabled": true,
|
||||
"protocol": "sftp",
|
||||
@@ -18,7 +197,7 @@
|
||||
"timeout": 30,
|
||||
"auto_delete_source": true,
|
||||
"auto_scheduler": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"check_interval_hours": 1,
|
||||
"expected_directories": [],
|
||||
"week_offset": 0
|
||||
@@ -29,51 +208,6 @@
|
||||
"enabled": true,
|
||||
"keep_count": 1
|
||||
},
|
||||
"DataMappings": {
|
||||
"directories": [
|
||||
{"path": "4G", "table": "4G_UD", "ready_rule": "daily"},
|
||||
{"path": "5G", "table": "5G_UD", "ready_rule": "daily"},
|
||||
{"path": "RJ/2.6G/2.6RJGD", "table": "2_6GRJGD", "ready_rule": "auto"},
|
||||
{"path": "RJ/2.6G/2.6RJYD", "table": "2_6GRJYD", "ready_rule": "auto"},
|
||||
{"path": "RJ/700M/700RJGD", "table": "700MRJGD", "ready_rule": "auto"},
|
||||
{"path": "RJ/700M/700RJYD", "table": "700MRJYD", "ready_rule": "auto"}
|
||||
],
|
||||
"table_field_mappings": {
|
||||
"2_6GRJGD": [
|
||||
{"Source": "开始时间", "Target": "开始时间", "Type": "datetime"},
|
||||
{"Source": "结束时间", "Target": "结束时间", "Type": "datetime"},
|
||||
{"Source": "gNBId", "Target": "gNBId", "Type": "string"},
|
||||
{"Source": "cellId", "Target": "cellId", "Type": "string"},
|
||||
{"Source": "gNBplmn", "Target": "gNBplmn", "Type": "string"},
|
||||
{"Source": "5G上下行总流量(上行PDCP PDU数据量+下行PDCP成功发送数据量)-GD(GB)", "Target": "上下行总流量_GB", "Type": "float"}
|
||||
],
|
||||
"2_6GRJYD": [
|
||||
{"Source": "开始时间", "Target": "开始时间", "Type": "datetime"},
|
||||
{"Source": "结束时间", "Target": "结束时间", "Type": "datetime"},
|
||||
{"Source": "gNBId", "Target": "gNBId", "Type": "string"},
|
||||
{"Source": "cellId", "Target": "cellId", "Type": "string"},
|
||||
{"Source": "gNBplmn", "Target": "gNBplmn", "Type": "string"},
|
||||
{"Source": "5G上下行总流量(上行PDCP PDU数据量+下行PDCP成功发送数据量)-YD(GB)", "Target": "上下行总流量_GB", "Type": "float"}
|
||||
],
|
||||
"700MRJGD": [
|
||||
{"Source": "开始时间", "Target": "开始时间", "Type": "datetime"},
|
||||
{"Source": "结束时间", "Target": "结束时间", "Type": "datetime"},
|
||||
{"Source": "gNBId", "Target": "gNBId", "Type": "string"},
|
||||
{"Source": "cellId", "Target": "cellId", "Type": "string"},
|
||||
{"Source": "gNBplmn", "Target": "gNBplmn", "Type": "string"},
|
||||
{"Source": "5G上下行总流量(上行PDCP PDU数据量+下行PDCP成功发送数据量)-GD(GB)", "Target": "上下行总流量_GB", "Type": "float"}
|
||||
],
|
||||
"700MRJYD": [
|
||||
{"Source": "开始时间", "Target": "开始时间", "Type": "datetime"},
|
||||
{"Source": "结束时间", "Target": "结束时间", "Type": "datetime"},
|
||||
{"Source": "gNBId", "Target": "gNBId", "Type": "string"},
|
||||
{"Source": "cellId", "Target": "cellId", "Type": "string"},
|
||||
{"Source": "gNBplmn", "Target": "gNBplmn", "Type": "string"},
|
||||
{"Source": "5G上下行总流量(上行PDCP PDU数据量+下行PDCP成功发送数据量)-YD(GB)", "Target": "上下行总流量_GB", "Type": "float"},
|
||||
{"Source": "5G上下行总流量(上行PDCP PDU数据量+下行PDCP成功发送数据量)(GB)-YD(GB)", "Target": "上下行总流量_GB", "Type": "float"}
|
||||
]
|
||||
}
|
||||
},
|
||||
"SheetFilter": [
|
||||
"指标(计数器)",
|
||||
"Template"
|
||||
@@ -334,4 +468,4 @@
|
||||
"Type": "float"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -3,18 +3,22 @@ from datetime import datetime
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
import pymysql
|
||||
from fastapi import APIRouter, Body, HTTPException, UploadFile, File
|
||||
from fastapi.responses import Response
|
||||
|
||||
from app import state
|
||||
from app.config import (
|
||||
CellDataConfig,
|
||||
DataMappingsConfig,
|
||||
HistoryRetentionConfig,
|
||||
MetrixConfig,
|
||||
MySQLConfig,
|
||||
RemoteDataConfig,
|
||||
SOURCE_TYPES,
|
||||
WAREHOUSE_TYPES,
|
||||
)
|
||||
from app.services.remote_download import RemoteDataDownloader
|
||||
|
||||
|
||||
router = APIRouter(tags=["config"])
|
||||
@@ -92,6 +96,46 @@ async def update_data_mappings_config(config: dict[str, Any] = Body(...)):
|
||||
return {"success": True, "message": "数据目录映射已更新", "update": state.config.update}
|
||||
|
||||
|
||||
@router.post("/api/config/cell-data/remote")
|
||||
async def update_cell_data_remote_config(config: dict[str, Any] = Body(...)):
|
||||
state.reload_config()
|
||||
current = state.config.cell_data.normalized()
|
||||
state.config.cell_data = CellDataConfig(
|
||||
remote_data=RemoteDataConfig.from_dict(config),
|
||||
mysql=current.mysql,
|
||||
).normalized()
|
||||
state.config.save()
|
||||
return {"success": True, "message": "CellData 远程数据源配置已更新", "update": state.config.update}
|
||||
|
||||
|
||||
@router.post("/api/config/cell-data/mysql")
|
||||
async def update_cell_data_mysql_config(config: dict[str, Any] = Body(...)):
|
||||
state.reload_config()
|
||||
current = state.config.cell_data.normalized()
|
||||
state.config.cell_data = CellDataConfig(
|
||||
remote_data=current.remote_data,
|
||||
mysql=MySQLConfig.from_dict(config, default_dbname="celldata"),
|
||||
).normalized()
|
||||
state.config.save()
|
||||
return {"success": True, "message": "CellData 数据库配置已更新", "update": state.config.update}
|
||||
|
||||
|
||||
@router.post("/api/config/cell-data/remote/test")
|
||||
async def test_cell_data_remote_connection(config: dict[str, Any] | None = Body(None)):
|
||||
try:
|
||||
remote_config = RemoteDataConfig.from_dict(config) if config else state.current_config().cell_data.remote_data
|
||||
RemoteDataDownloader(remote_config).test_connection()
|
||||
return {"success": True, "message": "CellData 远程服务器连接成功"}
|
||||
except Exception as exc:
|
||||
return {"success": False, "message": f"连接失败: {exc}"}
|
||||
|
||||
|
||||
@router.post("/api/config/cell-data/mysql/test")
|
||||
async def test_cell_data_mysql_connection(config: dict[str, Any] | None = Body(None)):
|
||||
mysql_config = MySQLConfig.from_dict(config, default_dbname="celldata") if config else state.current_config().cell_data.mysql
|
||||
return _test_mysql_config(mysql_config)
|
||||
|
||||
|
||||
@router.post("/api/config/history-retention")
|
||||
async def update_history_retention(config: dict[str, Any] = Body(...)):
|
||||
state.reload_config()
|
||||
@@ -167,9 +211,7 @@ def _apply_config_data(data: dict[str, Any]) -> None:
|
||||
|
||||
mysql_data = data.get("MySQL_DBInfo")
|
||||
if isinstance(mysql_data, dict):
|
||||
for key in ("host", "port", "user", "passwd", "dbname"):
|
||||
if key in mysql_data:
|
||||
setattr(state.config.mysql, key, mysql_data[key])
|
||||
state.config.mysql = MySQLConfig.from_dict(mysql_data)
|
||||
|
||||
if "SheetFilter" in data:
|
||||
state.config.sheet_filter = data["SheetFilter"] if isinstance(data["SheetFilter"], list) else []
|
||||
@@ -181,7 +223,34 @@ def _apply_config_data(data: dict[str, Any]) -> None:
|
||||
if isinstance(remote_data, dict):
|
||||
state.config.remote_data = RemoteDataConfig.from_dict(remote_data)
|
||||
|
||||
cell_data = data.get("CellData")
|
||||
if isinstance(cell_data, dict):
|
||||
state.config.cell_data = CellDataConfig.from_dict(cell_data)
|
||||
|
||||
history_retention = data.get("HistoryRetention")
|
||||
if isinstance(history_retention, dict):
|
||||
state.config.history_retention = HistoryRetentionConfig.from_dict(history_retention)
|
||||
|
||||
|
||||
def _test_mysql_config(config: MySQLConfig) -> dict[str, Any]:
|
||||
normalized = config.normalized()
|
||||
try:
|
||||
conn = pymysql.connect(
|
||||
host=normalized.host,
|
||||
port=normalized.port,
|
||||
user=normalized.user,
|
||||
password=normalized.passwd,
|
||||
database=normalized.dbname,
|
||||
charset="utf8mb4",
|
||||
cursorclass=pymysql.cursors.DictCursor,
|
||||
connect_timeout=10,
|
||||
)
|
||||
try:
|
||||
with conn.cursor() as cursor:
|
||||
cursor.execute("SELECT 1")
|
||||
finally:
|
||||
conn.close()
|
||||
return {"success": True, "message": "连接成功"}
|
||||
except Exception as exc:
|
||||
return {"success": False, "message": str(exc)}
|
||||
|
||||
|
||||
+83
-28
@@ -33,6 +33,42 @@ class MySQLConfig:
|
||||
passwd: str = ""
|
||||
dbname: str = "CapacityReport"
|
||||
|
||||
def normalized(self) -> "MySQLConfig":
|
||||
try:
|
||||
port = int(self.port or 3306)
|
||||
except (TypeError, ValueError):
|
||||
port = 3306
|
||||
return MySQLConfig(
|
||||
host=str(self.host or "").strip() or "localhost",
|
||||
port=port,
|
||||
user=str(self.user or "").strip(),
|
||||
passwd=self.passwd or "",
|
||||
dbname=str(self.dbname or "").strip(),
|
||||
)
|
||||
|
||||
def to_dict(self, include_password: bool = False) -> Dict[str, Any]:
|
||||
normalized = self.normalized()
|
||||
data = {
|
||||
"host": normalized.host,
|
||||
"port": normalized.port,
|
||||
"user": normalized.user,
|
||||
"dbname": normalized.dbname,
|
||||
}
|
||||
if include_password:
|
||||
data["passwd"] = normalized.passwd
|
||||
return data
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any] | None, default_dbname: str = "CapacityReport") -> "MySQLConfig":
|
||||
data = data or {}
|
||||
return cls(
|
||||
host=str(data.get("host", "localhost")),
|
||||
port=data.get("port", 3306),
|
||||
user=str(data.get("user", "root")),
|
||||
passwd=str(data.get("passwd", "")),
|
||||
dbname=str(data.get("dbname", default_dbname)),
|
||||
).normalized()
|
||||
|
||||
|
||||
# Source backends: direct FTP/SFTP, or Metrix storage platform.
|
||||
SOURCE_TYPES = ("ftp", "sftp", "metrix")
|
||||
@@ -292,6 +328,43 @@ class RemoteDataConfig:
|
||||
).normalized()
|
||||
|
||||
|
||||
@dataclass
|
||||
class CellDataConfig:
|
||||
"""CellData side data source and database connection used by later processing steps."""
|
||||
remote_data: RemoteDataConfig = field(
|
||||
default_factory=lambda: RemoteDataConfig(
|
||||
enabled=False,
|
||||
protocol="sftp",
|
||||
remote_dir="/CellData",
|
||||
)
|
||||
)
|
||||
mysql: MySQLConfig = field(default_factory=lambda: MySQLConfig(dbname="celldata"))
|
||||
|
||||
def normalized(self) -> "CellDataConfig":
|
||||
return CellDataConfig(
|
||||
remote_data=self.remote_data.normalized(),
|
||||
mysql=self.mysql.normalized(),
|
||||
)
|
||||
|
||||
def to_dict(self, include_password: bool = False) -> Dict[str, Any]:
|
||||
normalized = self.normalized()
|
||||
return {
|
||||
"remote_data": normalized.remote_data.to_dict(include_password=include_password),
|
||||
"mysql": normalized.mysql.to_dict(include_password=include_password),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any] | None) -> "CellDataConfig":
|
||||
data = data or {}
|
||||
default = cls()
|
||||
remote_data = data.get("RemoteData") or data.get("remote_data")
|
||||
mysql_data = data.get("MySQL_DBInfo") or data.get("mysql")
|
||||
return cls(
|
||||
remote_data=RemoteDataConfig.from_dict(remote_data) if isinstance(remote_data, dict) else default.remote_data,
|
||||
mysql=MySQLConfig.from_dict(mysql_data, default_dbname="celldata") if isinstance(mysql_data, dict) else default.mysql,
|
||||
).normalized()
|
||||
|
||||
|
||||
@dataclass
|
||||
class HistoryRetentionConfig:
|
||||
enabled: bool = False
|
||||
@@ -346,6 +419,7 @@ class AppConfig:
|
||||
metrix: MetrixConfig = field(default_factory=MetrixConfig)
|
||||
data_mappings: DataMappingsConfig = field(default_factory=DataMappingsConfig)
|
||||
remote_data: RemoteDataConfig = field(default_factory=RemoteDataConfig)
|
||||
cell_data: CellDataConfig = field(default_factory=CellDataConfig)
|
||||
history_retention: HistoryRetentionConfig = field(default_factory=HistoryRetentionConfig)
|
||||
sheet_filter: List[str] = field(default_factory=list)
|
||||
extract_fields: List[Dict[str, Any]] = field(default_factory=list)
|
||||
@@ -359,17 +433,11 @@ class AppConfig:
|
||||
with open(CONFIG_FILE, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
|
||||
mysql_data = data.get("MySQL_DBInfo", {})
|
||||
mysql_config = MySQLConfig(
|
||||
host=mysql_data.get("host", "localhost"),
|
||||
port=mysql_data.get("port", 3306),
|
||||
user=mysql_data.get("user", "root"),
|
||||
passwd=mysql_data.get("passwd", ""),
|
||||
dbname=mysql_data.get("dbname", "CapacityReport")
|
||||
)
|
||||
mysql_config = MySQLConfig.from_dict(data.get("MySQL_DBInfo"))
|
||||
remote_config = RemoteDataConfig.from_dict(data.get("RemoteData"))
|
||||
metrix_config = MetrixConfig.from_dict(data.get("Metrix"))
|
||||
data_mappings = DataMappingsConfig.from_dict(data.get("DataMappings"))
|
||||
cell_data = CellDataConfig.from_dict(data.get("CellData"))
|
||||
history_retention = HistoryRetentionConfig.from_dict(data.get("HistoryRetention"))
|
||||
|
||||
return cls(
|
||||
@@ -380,6 +448,7 @@ class AppConfig:
|
||||
metrix=metrix_config,
|
||||
data_mappings=data_mappings,
|
||||
remote_data=remote_config,
|
||||
cell_data=cell_data,
|
||||
history_retention=history_retention,
|
||||
sheet_filter=data.get("SheetFilter", []),
|
||||
extract_fields=data.get("ExtractField", [])
|
||||
@@ -399,16 +468,11 @@ class AppConfig:
|
||||
"Update": self.update,
|
||||
"SourceType": self.source_type,
|
||||
"WarehouseType": self.warehouse_type,
|
||||
"MySQL_DBInfo": {
|
||||
"host": self.mysql.host,
|
||||
"port": self.mysql.port,
|
||||
"user": self.mysql.user,
|
||||
"passwd": self.mysql.passwd,
|
||||
"dbname": self.mysql.dbname
|
||||
},
|
||||
"MySQL_DBInfo": self.mysql.normalized().to_dict(include_password=True),
|
||||
"Metrix": self.metrix.normalized().to_dict(include_token=True),
|
||||
"DataMappings": self.data_mappings.normalized().to_dict(),
|
||||
"RemoteData": self.remote_data.normalized().to_dict(include_password=True),
|
||||
"CellData": self.cell_data.normalized().to_dict(include_password=True),
|
||||
"HistoryRetention": self.history_retention.normalized().to_dict(),
|
||||
"SheetFilter": self.sheet_filter,
|
||||
"ExtractField": self.extract_fields
|
||||
@@ -420,15 +484,11 @@ class AppConfig:
|
||||
"update": self.update,
|
||||
"source_type": self.source_type,
|
||||
"warehouse_type": self.warehouse_type,
|
||||
"mysql": {
|
||||
"host": self.mysql.host,
|
||||
"port": self.mysql.port,
|
||||
"user": self.mysql.user,
|
||||
"dbname": self.mysql.dbname
|
||||
},
|
||||
"mysql": self.mysql.normalized().to_dict(),
|
||||
"metrix": self.metrix.normalized().to_dict(),
|
||||
"data_mappings": self.data_mappings.normalized().to_dict(),
|
||||
"remote_data": self.remote_data.normalized().to_dict(),
|
||||
"cell_data": self.cell_data.normalized().to_dict(),
|
||||
"history_retention": self.history_retention.normalized().to_dict(),
|
||||
"sheet_filter": self.sheet_filter,
|
||||
"extract_fields": self.extract_fields
|
||||
@@ -440,16 +500,11 @@ class AppConfig:
|
||||
"update": self.update,
|
||||
"source_type": self.source_type,
|
||||
"warehouse_type": self.warehouse_type,
|
||||
"mysql": {
|
||||
"host": self.mysql.host,
|
||||
"port": self.mysql.port,
|
||||
"user": self.mysql.user,
|
||||
"passwd": self.mysql.passwd,
|
||||
"dbname": self.mysql.dbname
|
||||
},
|
||||
"mysql": self.mysql.normalized().to_dict(include_password=True),
|
||||
"metrix": self.metrix.normalized().to_dict(include_token=True),
|
||||
"data_mappings": self.data_mappings.normalized().to_dict(),
|
||||
"remote_data": self.remote_data.normalized().to_dict(include_password=True),
|
||||
"cell_data": self.cell_data.normalized().to_dict(include_password=True),
|
||||
"history_retention": self.history_retention.normalized().to_dict(),
|
||||
"sheet_filter": self.sheet_filter,
|
||||
"extract_fields": self.extract_fields
|
||||
|
||||
@@ -703,3 +703,11 @@
|
||||
- 自动调度:原 RJ 专用检查改为通用自动粒度检查。`ready_rule=auto` 的目录会从普通每日扫描中排除,并单独按最新 ZIP 判断日/周粒度;如果配置里只有自动粒度目录,只要这些目录就绪也可触发调度。
|
||||
- 前端:设置页「规则映射」左侧独立滚动配置栏中只保留一个「数据目录映射」卡片,每行可编辑目录、暂存表与就绪规则,并保存到 `/api/config/data-mappings`。后续新增目录类映射继续加同一张表,不再新增配置卡片。
|
||||
- 验证:`python -m compileall -q app` 通过;`frontend` `npm run build` 通过(仅既有大 chunk 提示);构建产物与 Python 缓存已清理。
|
||||
|
||||
## 2026-06-24:新增 CellData 远程源与数据库配置入口
|
||||
|
||||
- 背景:后续需要引入 CellData 自动化处理,处理完成报表 SQL 后还会从 CellData 数据库表匹配数据并写入结果表。本轮先只落配置与界面,不接入实际提取/处理/入库流水线。
|
||||
- 配置:新增顶层 `CellData` 配置块,包含 `RemoteData`(FTP/SFTP 连接,默认远程目录 `/CellData`,不参与现有自动调度)与 `MySQL_DBInfo`(默认库名 `celldata`)。CellData 数据库可与主仓库 MySQL 相同,也可指向独立数据库。
|
||||
- 后端接口:新增 `/api/config/cell-data/remote`、`/api/config/cell-data/mysql` 保存接口,以及 `/api/config/cell-data/remote/test`、`/api/config/cell-data/mysql/test` 测试接口;测试逻辑分别复用 `RemoteDataDownloader` 和独立 PyMySQL `SELECT 1`。
|
||||
- 前端:系统设置「数据库」页新增「CellData 数据库配置」卡片;「远程数据源」页新增「CellData 数据源」卡片;「数据源 / 仓库」页说明 CellData 为独立辅助数据源,不影响主数据源/仓库选择。
|
||||
- 验证:`python -m compileall -q app` 通过;`frontend` `npm run build` 通过(仅既有大 chunk 提示);构建产物与 Python 缓存已清理。
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
<n-radio-button value="metrix">Metrix 数据库平台</n-radio-button>
|
||||
</n-radio-group>
|
||||
</n-form-item>
|
||||
<p class="form-hint">两侧可独立选择:源用 FTP/SFTP 或 Metrix 存储平台;仓库用本地 MySQL 或 Metrix 数据库平台。选 Metrix 时在下方填写连接;选 FTP/SFTP 或 MySQL 时在对应标签页填写。源目录在「远程数据源」标签页的「远程目录」。</p>
|
||||
<p class="form-hint">两侧可独立选择:源用 FTP/SFTP 或 Metrix 存储平台;仓库用本地 MySQL 或 Metrix 数据库平台。CellData 是后续处理使用的辅助数据源,独立配置,不影响这里的主数据源/仓库选择。</p>
|
||||
</n-form>
|
||||
<template #footer>
|
||||
<n-space justify="end">
|
||||
@@ -118,6 +118,46 @@
|
||||
</template>
|
||||
</n-card>
|
||||
|
||||
<n-card title="CellData 数据库配置" size="small" class="work-card">
|
||||
<n-form label-placement="top">
|
||||
<n-grid :cols="12" :x-gap="12">
|
||||
<n-gi :span="8">
|
||||
<n-form-item label="主机地址">
|
||||
<n-input v-model:value="cellDataMysqlForm.host" placeholder="localhost" />
|
||||
</n-form-item>
|
||||
</n-gi>
|
||||
<n-gi :span="4">
|
||||
<n-form-item label="端口">
|
||||
<n-input-number v-model:value="cellDataMysqlForm.port" class="full-width" :min="1" :max="65535" />
|
||||
</n-form-item>
|
||||
</n-gi>
|
||||
</n-grid>
|
||||
<n-form-item label="数据库名">
|
||||
<n-input v-model:value="cellDataMysqlForm.dbname" placeholder="celldata" />
|
||||
</n-form-item>
|
||||
<n-grid :cols="12" :x-gap="12">
|
||||
<n-gi :span="6">
|
||||
<n-form-item label="用户名">
|
||||
<n-input v-model:value="cellDataMysqlForm.user" placeholder="root" />
|
||||
</n-form-item>
|
||||
</n-gi>
|
||||
<n-gi :span="6">
|
||||
<n-form-item label="密码">
|
||||
<n-input v-model:value="cellDataMysqlForm.passwd" type="password" show-password-on="click" />
|
||||
</n-form-item>
|
||||
</n-gi>
|
||||
</n-grid>
|
||||
<p class="form-hint">后续处理会从该库复制 CellData 所需表到临时处理区,产出结果后再清理。通常可与主 MySQL 相同,也可以指向独立 CellData 数据库。</p>
|
||||
</n-form>
|
||||
|
||||
<template #footer>
|
||||
<n-space justify="end">
|
||||
<n-button type="primary" :loading="savingCellDataMysql" @click="saveCellDataMysql">保存配置</n-button>
|
||||
<n-button :loading="testingCellDataDb" @click="testCellDataDatabase">测试连接</n-button>
|
||||
</n-space>
|
||||
</template>
|
||||
</n-card>
|
||||
|
||||
<n-card title="处理历史保留" size="small" class="work-card work-card-narrow">
|
||||
<n-form label-placement="top">
|
||||
<n-grid :cols="12" :x-gap="12">
|
||||
@@ -214,6 +254,58 @@
|
||||
</n-space>
|
||||
</template>
|
||||
</n-card>
|
||||
|
||||
<n-card title="CellData 数据源" size="small" class="work-card settings-remote-card">
|
||||
<n-form label-placement="top">
|
||||
<div class="remote-connection-row">
|
||||
<n-form-item label="协议">
|
||||
<n-select
|
||||
v-model:value="cellDataRemoteForm.protocol"
|
||||
:options="remoteProtocolOptions"
|
||||
@update:value="updateCellDataRemoteProtocol"
|
||||
/>
|
||||
</n-form-item>
|
||||
<n-form-item label="服务器地址">
|
||||
<n-input v-model:value="cellDataRemoteForm.host" placeholder="192.168.1.10" />
|
||||
</n-form-item>
|
||||
<n-form-item label="端口">
|
||||
<n-input-number v-model:value="cellDataRemoteForm.port" class="full-width" :min="1" :max="65535" />
|
||||
</n-form-item>
|
||||
<n-form-item label="超时秒数">
|
||||
<n-input-number v-model:value="cellDataRemoteForm.timeout" class="full-width" :min="1" :max="600" />
|
||||
</n-form-item>
|
||||
<n-form-item label="FTP 被动模式" class="remote-passive-field">
|
||||
<n-switch v-model:value="cellDataRemoteForm.passive" :disabled="cellDataRemoteForm.protocol !== 'ftp'" />
|
||||
</n-form-item>
|
||||
</div>
|
||||
<n-grid :cols="12" :x-gap="12">
|
||||
<n-gi :span="6">
|
||||
<n-form-item label="用户名">
|
||||
<n-input v-model:value="cellDataRemoteForm.user" placeholder="remote user" />
|
||||
</n-form-item>
|
||||
</n-gi>
|
||||
<n-gi :span="6">
|
||||
<n-form-item label="密码">
|
||||
<n-input v-model:value="cellDataRemoteForm.passwd" type="password" show-password-on="click" />
|
||||
</n-form-item>
|
||||
</n-gi>
|
||||
</n-grid>
|
||||
<n-form-item label="远程目录">
|
||||
<n-input v-model:value="cellDataRemoteForm.remote_dir" placeholder="/CellData" />
|
||||
</n-form-item>
|
||||
<n-form-item label="启用 CellData 数据源">
|
||||
<n-switch v-model:value="cellDataRemoteForm.enabled" />
|
||||
</n-form-item>
|
||||
<p class="form-hint">用于后续 CellData 存储、提取、处理和入库流程;当前只保存连接信息并支持连接测试,不参与现有容量报表处理链路。</p>
|
||||
</n-form>
|
||||
|
||||
<template #footer>
|
||||
<n-space justify="end">
|
||||
<n-button type="primary" :loading="savingCellDataRemote" @click="saveCellDataRemote">保存配置</n-button>
|
||||
<n-button :loading="testingCellDataRemote" @click="testCellDataRemote">测试连接</n-button>
|
||||
</n-space>
|
||||
</template>
|
||||
</n-card>
|
||||
</div>
|
||||
</n-tab-pane>
|
||||
|
||||
@@ -531,6 +623,7 @@ import { apiGet, apiPost, downloadGet, upload } from '../api/client';
|
||||
import type {
|
||||
ApiMessage,
|
||||
AppConfig,
|
||||
CellDataConfig,
|
||||
DataDirectoryMapping,
|
||||
DataMappingsConfig,
|
||||
HistoryRetentionConfig,
|
||||
@@ -557,10 +650,14 @@ const configUpdateText = computed(() => `更新时间:${configUpdate.value ||
|
||||
const loading = ref(false);
|
||||
const testingDb = ref(false);
|
||||
const testingRemote = ref(false);
|
||||
const testingCellDataDb = ref(false);
|
||||
const testingCellDataRemote = ref(false);
|
||||
const loadingSchedulerStatus = ref(false);
|
||||
const triggeringScheduler = ref(false);
|
||||
const savingMysql = ref(false);
|
||||
const savingRemote = ref(false);
|
||||
const savingCellDataMysql = ref(false);
|
||||
const savingCellDataRemote = ref(false);
|
||||
const savingBackend = ref(false);
|
||||
const savingMetrix = ref(false);
|
||||
const sourceType = ref<'ftp' | 'sftp' | 'metrix'>('sftp');
|
||||
@@ -605,6 +702,25 @@ const remoteForm = reactive<RemoteDataConfig>({
|
||||
}
|
||||
});
|
||||
|
||||
const cellDataRemoteForm = reactive<RemoteDataConfig>({
|
||||
enabled: false,
|
||||
protocol: 'sftp',
|
||||
host: '',
|
||||
port: 22,
|
||||
user: '',
|
||||
passwd: '',
|
||||
remote_dir: '/CellData',
|
||||
passive: true,
|
||||
timeout: 30,
|
||||
auto_delete_source: false,
|
||||
auto_scheduler: {
|
||||
enabled: false,
|
||||
check_interval_hours: 1,
|
||||
expected_directories: [],
|
||||
week_offset: 0
|
||||
}
|
||||
});
|
||||
|
||||
const metrixForm = reactive<MetrixConfig>({
|
||||
base_url: 'http://host.docker.internal:8000',
|
||||
token: '',
|
||||
@@ -628,6 +744,14 @@ const newDirectoryMapping = reactive<DataDirectoryMapping>({
|
||||
ready_rule: 'daily'
|
||||
});
|
||||
|
||||
const cellDataMysqlForm = reactive({
|
||||
host: '',
|
||||
port: 3306 as number | null,
|
||||
user: '',
|
||||
passwd: '',
|
||||
dbname: 'celldata'
|
||||
});
|
||||
|
||||
const historyRetentionForm = reactive<HistoryRetentionConfig>({
|
||||
enabled: false,
|
||||
keep_count: 20
|
||||
@@ -761,6 +885,8 @@ async function loadConfig() {
|
||||
mysqlForm.passwd = config.mysql.passwd || '';
|
||||
mysqlForm.dbname = config.mysql.dbname;
|
||||
Object.assign(remoteForm, normalizeRemoteConfig(config.remote_data));
|
||||
Object.assign(cellDataRemoteForm, normalizeRemoteConfig(config.cell_data?.remote_data, '/CellData'));
|
||||
Object.assign(cellDataMysqlForm, normalizeCellDataMysqlConfig(config.cell_data?.mysql));
|
||||
Object.assign(historyRetentionForm, normalizeHistoryRetentionConfig(config.history_retention));
|
||||
sheetFilters.value = [...config.sheet_filter];
|
||||
extractFields.value = normalizeExtractFields(config.extract_fields);
|
||||
@@ -782,6 +908,15 @@ function updateRemoteProtocol(value: string) {
|
||||
}
|
||||
}
|
||||
|
||||
function updateCellDataRemoteProtocol(value: string) {
|
||||
if (value === 'sftp' && (!cellDataRemoteForm.port || cellDataRemoteForm.port === 21)) {
|
||||
cellDataRemoteForm.port = 22;
|
||||
}
|
||||
if (value === 'ftp' && (!cellDataRemoteForm.port || cellDataRemoteForm.port === 22)) {
|
||||
cellDataRemoteForm.port = 21;
|
||||
}
|
||||
}
|
||||
|
||||
async function saveBackend() {
|
||||
savingBackend.value = true;
|
||||
try {
|
||||
@@ -839,6 +974,27 @@ async function saveMysql() {
|
||||
}
|
||||
}
|
||||
|
||||
async function saveCellDataMysql() {
|
||||
if (!cellDataMysqlForm.host || !cellDataMysqlForm.user || !cellDataMysqlForm.dbname) {
|
||||
message.warning('请填写完整 CellData 数据库配置');
|
||||
return;
|
||||
}
|
||||
|
||||
savingCellDataMysql.value = true;
|
||||
try {
|
||||
const result = await apiPost<ApiMessage>('/api/config/cell-data/mysql', {
|
||||
...cellDataMysqlForm,
|
||||
port: cellDataMysqlForm.port || 3306
|
||||
});
|
||||
configUpdate.value = result.update || configUpdate.value;
|
||||
message.success(result.message || 'CellData 数据库配置已保存');
|
||||
} catch (error) {
|
||||
message.error(error instanceof Error ? error.message : '保存 CellData 数据库配置失败');
|
||||
} finally {
|
||||
savingCellDataMysql.value = false;
|
||||
}
|
||||
}
|
||||
|
||||
function getRemotePayload(): RemoteDataConfig {
|
||||
enforceRemoteSchedulerRules();
|
||||
const autoScheduler = normalizeAutoSchedulerConfig(remoteForm.auto_scheduler);
|
||||
@@ -857,6 +1013,27 @@ function getRemotePayload(): RemoteDataConfig {
|
||||
};
|
||||
}
|
||||
|
||||
function getCellDataRemotePayload(): RemoteDataConfig {
|
||||
return {
|
||||
...cellDataRemoteForm,
|
||||
protocol: cellDataRemoteForm.protocol,
|
||||
host: cellDataRemoteForm.host.trim(),
|
||||
port: cellDataRemoteForm.port || (cellDataRemoteForm.protocol === 'sftp' ? 22 : 21),
|
||||
user: cellDataRemoteForm.user.trim(),
|
||||
passwd: cellDataRemoteForm.passwd || '',
|
||||
remote_dir: cellDataRemoteForm.remote_dir.trim() || '/CellData',
|
||||
passive: cellDataRemoteForm.passive,
|
||||
timeout: cellDataRemoteForm.timeout || 30,
|
||||
auto_delete_source: false,
|
||||
auto_scheduler: {
|
||||
enabled: false,
|
||||
check_interval_hours: 1,
|
||||
expected_directories: [],
|
||||
week_offset: 0
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function validateRemoteForm(): boolean {
|
||||
if (!remoteForm.host || !remoteForm.user || !remoteForm.remote_dir) {
|
||||
message.warning('请填写完整远程数据源配置');
|
||||
@@ -865,6 +1042,14 @@ function validateRemoteForm(): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
function validateCellDataRemoteForm(): boolean {
|
||||
if (!cellDataRemoteForm.host || !cellDataRemoteForm.user || !cellDataRemoteForm.remote_dir) {
|
||||
message.warning('请填写完整 CellData 远程数据源配置');
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
async function saveRemote() {
|
||||
if (!validateRemoteForm()) return;
|
||||
|
||||
@@ -881,6 +1066,21 @@ async function saveRemote() {
|
||||
}
|
||||
}
|
||||
|
||||
async function saveCellDataRemote() {
|
||||
if (!validateCellDataRemoteForm()) return;
|
||||
|
||||
savingCellDataRemote.value = true;
|
||||
try {
|
||||
const result = await apiPost<ApiMessage>('/api/config/cell-data/remote', getCellDataRemotePayload());
|
||||
configUpdate.value = result.update || configUpdate.value;
|
||||
message.success(result.message || 'CellData 远程数据源配置已保存');
|
||||
} catch (error) {
|
||||
message.error(error instanceof Error ? error.message : '保存 CellData 远程数据源配置失败');
|
||||
} finally {
|
||||
savingCellDataRemote.value = false;
|
||||
}
|
||||
}
|
||||
|
||||
function addDirectoryMapping() {
|
||||
const path = normalizeDirectoryText(newDirectoryMapping.path);
|
||||
const table = newDirectoryMapping.table.trim();
|
||||
@@ -985,6 +1185,20 @@ async function testRemote() {
|
||||
}
|
||||
}
|
||||
|
||||
async function testCellDataRemote() {
|
||||
if (!validateCellDataRemoteForm()) return;
|
||||
|
||||
testingCellDataRemote.value = true;
|
||||
try {
|
||||
const result = await apiPost<ApiMessage>('/api/config/cell-data/remote/test', getCellDataRemotePayload());
|
||||
message[result.success ? 'success' : 'error'](result.message || 'CellData 远程连接测试完成');
|
||||
} catch (error) {
|
||||
message.error(error instanceof Error ? error.message : 'CellData 远程连接测试失败');
|
||||
} finally {
|
||||
testingCellDataRemote.value = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function testDatabase() {
|
||||
testingDb.value = true;
|
||||
try {
|
||||
@@ -997,6 +1211,21 @@ async function testDatabase() {
|
||||
}
|
||||
}
|
||||
|
||||
async function testCellDataDatabase() {
|
||||
testingCellDataDb.value = true;
|
||||
try {
|
||||
const result = await apiPost<ApiMessage>('/api/config/cell-data/mysql/test', {
|
||||
...cellDataMysqlForm,
|
||||
port: cellDataMysqlForm.port || 3306
|
||||
});
|
||||
message[result.success ? 'success' : 'error'](result.message || 'CellData 数据库连接测试完成');
|
||||
} catch (error) {
|
||||
message.error(error instanceof Error ? error.message : 'CellData 数据库连接测试失败');
|
||||
} finally {
|
||||
testingCellDataDb.value = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function saveHistoryRetention() {
|
||||
savingHistoryRetention.value = true;
|
||||
try {
|
||||
@@ -1156,7 +1385,7 @@ async function uploadConfigFile(event: Event) {
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeRemoteConfig(config: RemoteDataConfig | undefined): RemoteDataConfig {
|
||||
function normalizeRemoteConfig(config: RemoteDataConfig | undefined, defaultRemoteDir = '/'): RemoteDataConfig {
|
||||
const protocol = config?.protocol === 'ftp' ? 'ftp' : 'sftp';
|
||||
const autoScheduler = normalizeAutoSchedulerConfig(config?.auto_scheduler);
|
||||
return {
|
||||
@@ -1166,7 +1395,7 @@ function normalizeRemoteConfig(config: RemoteDataConfig | undefined): RemoteData
|
||||
port: config?.port || (protocol === 'sftp' ? 22 : 21),
|
||||
user: config?.user || '',
|
||||
passwd: config?.passwd || '',
|
||||
remote_dir: config?.remote_dir || '/',
|
||||
remote_dir: config?.remote_dir || defaultRemoteDir,
|
||||
passive: config?.passive ?? true,
|
||||
timeout: config?.timeout || 30,
|
||||
auto_delete_source: Boolean(config?.auto_delete_source) || autoScheduler.enabled,
|
||||
@@ -1174,6 +1403,16 @@ function normalizeRemoteConfig(config: RemoteDataConfig | undefined): RemoteData
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeCellDataMysqlConfig(config: CellDataConfig['mysql'] | undefined) {
|
||||
return {
|
||||
host: config?.host || 'localhost',
|
||||
port: config?.port || 3306,
|
||||
user: config?.user || 'root',
|
||||
passwd: config?.passwd || '',
|
||||
dbname: config?.dbname || 'celldata'
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeDataMappingsConfig(config: DataMappingsConfig | undefined): DataMappingsConfig {
|
||||
const directories: DataDirectoryMapping[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
@@ -100,6 +100,7 @@ export interface AppConfig {
|
||||
metrix: MetrixConfig;
|
||||
data_mappings: DataMappingsConfig;
|
||||
remote_data: RemoteDataConfig;
|
||||
cell_data: CellDataConfig;
|
||||
history_retention: HistoryRetentionConfig;
|
||||
sheet_filter: string[];
|
||||
extract_fields: Array<Record<string, unknown>>;
|
||||
@@ -139,6 +140,17 @@ export interface RemoteDataConfig {
|
||||
auto_scheduler: RemoteAutoSchedulerConfig;
|
||||
}
|
||||
|
||||
export interface CellDataConfig {
|
||||
remote_data: RemoteDataConfig;
|
||||
mysql: {
|
||||
host: string;
|
||||
port: number;
|
||||
user: string;
|
||||
passwd?: string;
|
||||
dbname: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface RemoteAutoSchedulerConfig {
|
||||
enabled: boolean;
|
||||
check_interval_hours: number;
|
||||
|
||||
Reference in New Issue
Block a user