feat: 统一数据目录映射配置
This commit is contained in:
+8
-7
@@ -29,13 +29,14 @@
|
|||||||
"enabled": true,
|
"enabled": true,
|
||||||
"keep_count": 1
|
"keep_count": 1
|
||||||
},
|
},
|
||||||
"RJData": {
|
"DataMappings": {
|
||||||
"enabled": true,
|
"directories": [
|
||||||
"weekly_directories": [
|
{"path": "4G", "table": "4G_UD", "ready_rule": "daily"},
|
||||||
"RJ/2.6G/2.6RJGD",
|
{"path": "5G", "table": "5G_UD", "ready_rule": "daily"},
|
||||||
"RJ/2.6G/2.6RJYD",
|
{"path": "RJ/2.6G/2.6RJGD", "table": "2_6GRJGD", "ready_rule": "auto"},
|
||||||
"RJ/700M/700RJGD",
|
{"path": "RJ/2.6G/2.6RJYD", "table": "2_6GRJYD", "ready_rule": "auto"},
|
||||||
"RJ/700M/700RJYD"
|
{"path": "RJ/700M/700RJGD", "table": "700MRJGD", "ready_rule": "auto"},
|
||||||
|
{"path": "RJ/700M/700RJYD", "table": "700MRJYD", "ready_rule": "auto"}
|
||||||
],
|
],
|
||||||
"table_field_mappings": {
|
"table_field_mappings": {
|
||||||
"2_6GRJGD": [
|
"2_6GRJGD": [
|
||||||
|
|||||||
@@ -8,9 +8,9 @@ from fastapi.responses import Response
|
|||||||
|
|
||||||
from app import state
|
from app import state
|
||||||
from app.config import (
|
from app.config import (
|
||||||
|
DataMappingsConfig,
|
||||||
HistoryRetentionConfig,
|
HistoryRetentionConfig,
|
||||||
MetrixConfig,
|
MetrixConfig,
|
||||||
RJDataConfig,
|
|
||||||
RemoteDataConfig,
|
RemoteDataConfig,
|
||||||
SOURCE_TYPES,
|
SOURCE_TYPES,
|
||||||
WAREHOUSE_TYPES,
|
WAREHOUSE_TYPES,
|
||||||
@@ -80,6 +80,18 @@ async def update_metrix_config(config: dict[str, Any] = Body(...)):
|
|||||||
return {"success": True, "message": "Metrix 连接配置已更新", "update": state.config.update}
|
return {"success": True, "message": "Metrix 连接配置已更新", "update": state.config.update}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/api/config/data-mappings")
|
||||||
|
async def update_data_mappings_config(config: dict[str, Any] = Body(...)):
|
||||||
|
state.reload_config()
|
||||||
|
current = state.config.data_mappings.normalized()
|
||||||
|
payload = dict(config)
|
||||||
|
if "table_field_mappings" not in payload:
|
||||||
|
payload["table_field_mappings"] = current.table_field_mappings
|
||||||
|
state.config.data_mappings = DataMappingsConfig.from_dict(payload)
|
||||||
|
state.config.save()
|
||||||
|
return {"success": True, "message": "数据目录映射已更新", "update": state.config.update}
|
||||||
|
|
||||||
|
|
||||||
@router.post("/api/config/history-retention")
|
@router.post("/api/config/history-retention")
|
||||||
async def update_history_retention(config: dict[str, Any] = Body(...)):
|
async def update_history_retention(config: dict[str, Any] = Body(...)):
|
||||||
state.reload_config()
|
state.reload_config()
|
||||||
@@ -149,6 +161,10 @@ def _apply_config_data(data: dict[str, Any]) -> None:
|
|||||||
if isinstance(metrix_data, dict):
|
if isinstance(metrix_data, dict):
|
||||||
state.config.metrix = MetrixConfig.from_dict(metrix_data)
|
state.config.metrix = MetrixConfig.from_dict(metrix_data)
|
||||||
|
|
||||||
|
data_mappings = data.get("DataMappings")
|
||||||
|
if isinstance(data_mappings, dict):
|
||||||
|
state.config.data_mappings = DataMappingsConfig.from_dict(data_mappings)
|
||||||
|
|
||||||
mysql_data = data.get("MySQL_DBInfo")
|
mysql_data = data.get("MySQL_DBInfo")
|
||||||
if isinstance(mysql_data, dict):
|
if isinstance(mysql_data, dict):
|
||||||
for key in ("host", "port", "user", "passwd", "dbname"):
|
for key in ("host", "port", "user", "passwd", "dbname"):
|
||||||
@@ -169,6 +185,3 @@ def _apply_config_data(data: dict[str, Any]) -> None:
|
|||||||
if isinstance(history_retention, dict):
|
if isinstance(history_retention, dict):
|
||||||
state.config.history_retention = HistoryRetentionConfig.from_dict(history_retention)
|
state.config.history_retention = HistoryRetentionConfig.from_dict(history_retention)
|
||||||
|
|
||||||
rj_data = data.get("RJData")
|
|
||||||
if isinstance(rj_data, dict):
|
|
||||||
state.config.rj_data = RJDataConfig.from_dict(rj_data)
|
|
||||||
|
|||||||
+71
-68
@@ -46,7 +46,7 @@ class MetrixConfig:
|
|||||||
|
|
||||||
Metrix appears in the UI as two connection types ("存储平台"/"数据库平台") that share the
|
Metrix appears in the UI as two connection types ("存储平台"/"数据库平台") that share the
|
||||||
same base_url + token; storage_id is the file source, database_conn_id + target_database
|
same base_url + token; storage_id is the file source, database_conn_id + target_database
|
||||||
are the warehouse. data_dir_to_table maps data sub-dirs to staging tables (Metrix mode only).
|
are the warehouse.
|
||||||
"""
|
"""
|
||||||
base_url: str = "http://host.docker.internal:8000"
|
base_url: str = "http://host.docker.internal:8000"
|
||||||
token: str = ""
|
token: str = ""
|
||||||
@@ -54,18 +54,12 @@ class MetrixConfig:
|
|||||||
database_conn_id: str = ""
|
database_conn_id: str = ""
|
||||||
target_database: str = ""
|
target_database: str = ""
|
||||||
recent_days: int = 7
|
recent_days: int = 7
|
||||||
data_dir_to_table: Dict[str, str] = field(default_factory=lambda: {"4G": "4G_UD", "5G": "5G_UD"})
|
|
||||||
|
|
||||||
def normalized(self) -> "MetrixConfig":
|
def normalized(self) -> "MetrixConfig":
|
||||||
try:
|
try:
|
||||||
recent_days = max(int(self.recent_days), 1)
|
recent_days = max(int(self.recent_days), 1)
|
||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
recent_days = 7
|
recent_days = 7
|
||||||
mapping = {
|
|
||||||
str(k).strip(): str(v).strip()
|
|
||||||
for k, v in (self.data_dir_to_table or {}).items()
|
|
||||||
if str(k).strip() and str(v).strip()
|
|
||||||
}
|
|
||||||
return MetrixConfig(
|
return MetrixConfig(
|
||||||
base_url=str(self.base_url or "").strip(),
|
base_url=str(self.base_url or "").strip(),
|
||||||
token=str(self.token or "").strip(),
|
token=str(self.token or "").strip(),
|
||||||
@@ -73,7 +67,6 @@ class MetrixConfig:
|
|||||||
database_conn_id=str(self.database_conn_id or "").strip(),
|
database_conn_id=str(self.database_conn_id or "").strip(),
|
||||||
target_database=str(self.target_database or "").strip(),
|
target_database=str(self.target_database or "").strip(),
|
||||||
recent_days=recent_days,
|
recent_days=recent_days,
|
||||||
data_dir_to_table=mapping or {"4G": "4G_UD", "5G": "5G_UD"},
|
|
||||||
)
|
)
|
||||||
|
|
||||||
def to_dict(self, include_token: bool = False) -> Dict[str, Any]:
|
def to_dict(self, include_token: bool = False) -> Dict[str, Any]:
|
||||||
@@ -84,7 +77,6 @@ class MetrixConfig:
|
|||||||
"database_conn_id": n.database_conn_id,
|
"database_conn_id": n.database_conn_id,
|
||||||
"target_database": n.target_database,
|
"target_database": n.target_database,
|
||||||
"recent_days": n.recent_days,
|
"recent_days": n.recent_days,
|
||||||
"data_dir_to_table": n.data_dir_to_table,
|
|
||||||
}
|
}
|
||||||
if include_token:
|
if include_token:
|
||||||
data["token"] = n.token
|
data["token"] = n.token
|
||||||
@@ -93,7 +85,6 @@ class MetrixConfig:
|
|||||||
@classmethod
|
@classmethod
|
||||||
def from_dict(cls, data: Dict[str, Any] | None) -> "MetrixConfig":
|
def from_dict(cls, data: Dict[str, Any] | None) -> "MetrixConfig":
|
||||||
data = data or {}
|
data = data or {}
|
||||||
mapping = data.get("data_dir_to_table")
|
|
||||||
return cls(
|
return cls(
|
||||||
base_url=str(data.get("base_url", "http://host.docker.internal:8000")),
|
base_url=str(data.get("base_url", "http://host.docker.internal:8000")),
|
||||||
token=str(data.get("token", "")),
|
token=str(data.get("token", "")),
|
||||||
@@ -101,7 +92,70 @@ class MetrixConfig:
|
|||||||
database_conn_id=str(data.get("database_conn_id", "")),
|
database_conn_id=str(data.get("database_conn_id", "")),
|
||||||
target_database=str(data.get("target_database", "")),
|
target_database=str(data.get("target_database", "")),
|
||||||
recent_days=data.get("recent_days", 7),
|
recent_days=data.get("recent_days", 7),
|
||||||
data_dir_to_table=mapping if isinstance(mapping, dict) else {"4G": "4G_UD", "5G": "5G_UD"},
|
).normalized()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class DataMappingsConfig:
|
||||||
|
"""Source directories mapped to staging tables."""
|
||||||
|
directories: List[Dict[str, str]] = field(
|
||||||
|
default_factory=lambda: [
|
||||||
|
{"path": "4G", "table": "4G_UD", "ready_rule": "daily"},
|
||||||
|
{"path": "5G", "table": "5G_UD", "ready_rule": "daily"},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
table_field_mappings: Dict[str, List[Dict[str, Any]]] = field(default_factory=dict)
|
||||||
|
|
||||||
|
def normalized(self) -> "DataMappingsConfig":
|
||||||
|
directories = []
|
||||||
|
seen = set()
|
||||||
|
for item in self.directories or []:
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
continue
|
||||||
|
path = str(item.get("path", "")).replace("\\", "/").strip().strip("/")
|
||||||
|
table = str(item.get("table", "")).strip()
|
||||||
|
ready_rule = str(item.get("ready_rule", "daily")).strip().lower()
|
||||||
|
if ready_rule not in {"daily", "auto"}:
|
||||||
|
ready_rule = "daily"
|
||||||
|
if not path or not table:
|
||||||
|
continue
|
||||||
|
key = (path, table, ready_rule)
|
||||||
|
if key in seen:
|
||||||
|
continue
|
||||||
|
seen.add(key)
|
||||||
|
directories.append({"path": path, "table": table, "ready_rule": ready_rule})
|
||||||
|
|
||||||
|
if not directories:
|
||||||
|
directories = [
|
||||||
|
{"path": "4G", "table": "4G_UD", "ready_rule": "daily"},
|
||||||
|
{"path": "5G", "table": "5G_UD", "ready_rule": "daily"},
|
||||||
|
]
|
||||||
|
|
||||||
|
mappings = {}
|
||||||
|
for table_name, fields in (self.table_field_mappings or {}).items():
|
||||||
|
if isinstance(fields, list):
|
||||||
|
mappings[table_name] = [
|
||||||
|
{k: v for k, v in f.items() if k in ("Source", "Target", "Type")}
|
||||||
|
for f in fields
|
||||||
|
if isinstance(f, dict) and "Source" in f and "Target" in f
|
||||||
|
]
|
||||||
|
|
||||||
|
return DataMappingsConfig(directories=directories, table_field_mappings=mappings)
|
||||||
|
|
||||||
|
def to_dict(self) -> Dict[str, Any]:
|
||||||
|
normalized = self.normalized()
|
||||||
|
return {
|
||||||
|
"directories": normalized.directories,
|
||||||
|
"table_field_mappings": normalized.table_field_mappings,
|
||||||
|
}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_dict(cls, data: Dict[str, Any] | None) -> "DataMappingsConfig":
|
||||||
|
data = data or {}
|
||||||
|
directories = data.get("directories", [])
|
||||||
|
return cls(
|
||||||
|
directories=directories if isinstance(directories, list) else [],
|
||||||
|
table_field_mappings=data.get("table_field_mappings", {}) if isinstance(data.get("table_field_mappings"), dict) else {},
|
||||||
).normalized()
|
).normalized()
|
||||||
|
|
||||||
|
|
||||||
@@ -238,57 +292,6 @@ class RemoteDataConfig:
|
|||||||
).normalized()
|
).normalized()
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class RJDataConfig:
|
|
||||||
"""RJ数据配置 - 周数据处理"""
|
|
||||||
enabled: bool = False
|
|
||||||
weekly_directories: List[str] = field(default_factory=list)
|
|
||||||
# 字段映射: {表名: [{Source, Target, Type}, ...]}
|
|
||||||
table_field_mappings: Dict[str, List[Dict[str, Any]]] = field(default_factory=dict)
|
|
||||||
|
|
||||||
def normalized(self) -> "RJDataConfig":
|
|
||||||
directories = []
|
|
||||||
seen = set()
|
|
||||||
for d in self.weekly_directories or []:
|
|
||||||
normalized = str(d).replace("\\", "/").strip().strip("/")
|
|
||||||
if normalized and normalized not in seen:
|
|
||||||
directories.append(normalized)
|
|
||||||
seen.add(normalized)
|
|
||||||
|
|
||||||
# 标准化字段映射
|
|
||||||
mappings = {}
|
|
||||||
for table_name, fields in (self.table_field_mappings or {}).items():
|
|
||||||
if isinstance(fields, list):
|
|
||||||
mappings[table_name] = [
|
|
||||||
{k: v for k, v in f.items() if k in ("Source", "Target", "Type")}
|
|
||||||
for f in fields
|
|
||||||
if isinstance(f, dict) and "Source" in f and "Target" in f
|
|
||||||
]
|
|
||||||
|
|
||||||
return RJDataConfig(
|
|
||||||
enabled=bool(self.enabled),
|
|
||||||
weekly_directories=directories,
|
|
||||||
table_field_mappings=mappings,
|
|
||||||
)
|
|
||||||
|
|
||||||
def to_dict(self) -> Dict[str, Any]:
|
|
||||||
normalized = self.normalized()
|
|
||||||
return {
|
|
||||||
"enabled": normalized.enabled,
|
|
||||||
"weekly_directories": normalized.weekly_directories,
|
|
||||||
"table_field_mappings": normalized.table_field_mappings,
|
|
||||||
}
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_dict(cls, data: Dict[str, Any] | None) -> "RJDataConfig":
|
|
||||||
data = data or {}
|
|
||||||
return cls(
|
|
||||||
enabled=bool(data.get("enabled", False)),
|
|
||||||
weekly_directories=data.get("weekly_directories", []) if isinstance(data.get("weekly_directories"), list) else [],
|
|
||||||
table_field_mappings=data.get("table_field_mappings", {}) if isinstance(data.get("table_field_mappings"), dict) else {},
|
|
||||||
).normalized()
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class HistoryRetentionConfig:
|
class HistoryRetentionConfig:
|
||||||
enabled: bool = False
|
enabled: bool = False
|
||||||
@@ -341,9 +344,9 @@ class AppConfig:
|
|||||||
warehouse_type: str = "mysql"
|
warehouse_type: str = "mysql"
|
||||||
mysql: MySQLConfig = field(default_factory=MySQLConfig)
|
mysql: MySQLConfig = field(default_factory=MySQLConfig)
|
||||||
metrix: MetrixConfig = field(default_factory=MetrixConfig)
|
metrix: MetrixConfig = field(default_factory=MetrixConfig)
|
||||||
|
data_mappings: DataMappingsConfig = field(default_factory=DataMappingsConfig)
|
||||||
remote_data: RemoteDataConfig = field(default_factory=RemoteDataConfig)
|
remote_data: RemoteDataConfig = field(default_factory=RemoteDataConfig)
|
||||||
history_retention: HistoryRetentionConfig = field(default_factory=HistoryRetentionConfig)
|
history_retention: HistoryRetentionConfig = field(default_factory=HistoryRetentionConfig)
|
||||||
rj_data: RJDataConfig = field(default_factory=RJDataConfig)
|
|
||||||
sheet_filter: List[str] = field(default_factory=list)
|
sheet_filter: List[str] = field(default_factory=list)
|
||||||
extract_fields: List[Dict[str, Any]] = field(default_factory=list)
|
extract_fields: List[Dict[str, Any]] = field(default_factory=list)
|
||||||
|
|
||||||
@@ -366,8 +369,8 @@ class AppConfig:
|
|||||||
)
|
)
|
||||||
remote_config = RemoteDataConfig.from_dict(data.get("RemoteData"))
|
remote_config = RemoteDataConfig.from_dict(data.get("RemoteData"))
|
||||||
metrix_config = MetrixConfig.from_dict(data.get("Metrix"))
|
metrix_config = MetrixConfig.from_dict(data.get("Metrix"))
|
||||||
|
data_mappings = DataMappingsConfig.from_dict(data.get("DataMappings"))
|
||||||
history_retention = HistoryRetentionConfig.from_dict(data.get("HistoryRetention"))
|
history_retention = HistoryRetentionConfig.from_dict(data.get("HistoryRetention"))
|
||||||
rj_data = RJDataConfig.from_dict(data.get("RJData"))
|
|
||||||
|
|
||||||
return cls(
|
return cls(
|
||||||
update=data.get("Update", ""),
|
update=data.get("Update", ""),
|
||||||
@@ -375,9 +378,9 @@ class AppConfig:
|
|||||||
warehouse_type=_normalize_warehouse_type(data.get("WarehouseType")),
|
warehouse_type=_normalize_warehouse_type(data.get("WarehouseType")),
|
||||||
mysql=mysql_config,
|
mysql=mysql_config,
|
||||||
metrix=metrix_config,
|
metrix=metrix_config,
|
||||||
|
data_mappings=data_mappings,
|
||||||
remote_data=remote_config,
|
remote_data=remote_config,
|
||||||
history_retention=history_retention,
|
history_retention=history_retention,
|
||||||
rj_data=rj_data,
|
|
||||||
sheet_filter=data.get("SheetFilter", []),
|
sheet_filter=data.get("SheetFilter", []),
|
||||||
extract_fields=data.get("ExtractField", [])
|
extract_fields=data.get("ExtractField", [])
|
||||||
)
|
)
|
||||||
@@ -404,9 +407,9 @@ class AppConfig:
|
|||||||
"dbname": self.mysql.dbname
|
"dbname": self.mysql.dbname
|
||||||
},
|
},
|
||||||
"Metrix": self.metrix.normalized().to_dict(include_token=True),
|
"Metrix": self.metrix.normalized().to_dict(include_token=True),
|
||||||
|
"DataMappings": self.data_mappings.normalized().to_dict(),
|
||||||
"RemoteData": self.remote_data.normalized().to_dict(include_password=True),
|
"RemoteData": self.remote_data.normalized().to_dict(include_password=True),
|
||||||
"HistoryRetention": self.history_retention.normalized().to_dict(),
|
"HistoryRetention": self.history_retention.normalized().to_dict(),
|
||||||
"RJData": self.rj_data.normalized().to_dict(),
|
|
||||||
"SheetFilter": self.sheet_filter,
|
"SheetFilter": self.sheet_filter,
|
||||||
"ExtractField": self.extract_fields
|
"ExtractField": self.extract_fields
|
||||||
}
|
}
|
||||||
@@ -424,9 +427,9 @@ class AppConfig:
|
|||||||
"dbname": self.mysql.dbname
|
"dbname": self.mysql.dbname
|
||||||
},
|
},
|
||||||
"metrix": self.metrix.normalized().to_dict(),
|
"metrix": self.metrix.normalized().to_dict(),
|
||||||
|
"data_mappings": self.data_mappings.normalized().to_dict(),
|
||||||
"remote_data": self.remote_data.normalized().to_dict(),
|
"remote_data": self.remote_data.normalized().to_dict(),
|
||||||
"history_retention": self.history_retention.normalized().to_dict(),
|
"history_retention": self.history_retention.normalized().to_dict(),
|
||||||
"rj_data": self.rj_data.normalized().to_dict(),
|
|
||||||
"sheet_filter": self.sheet_filter,
|
"sheet_filter": self.sheet_filter,
|
||||||
"extract_fields": self.extract_fields
|
"extract_fields": self.extract_fields
|
||||||
}
|
}
|
||||||
@@ -445,9 +448,9 @@ class AppConfig:
|
|||||||
"dbname": self.mysql.dbname
|
"dbname": self.mysql.dbname
|
||||||
},
|
},
|
||||||
"metrix": self.metrix.normalized().to_dict(include_token=True),
|
"metrix": self.metrix.normalized().to_dict(include_token=True),
|
||||||
|
"data_mappings": self.data_mappings.normalized().to_dict(),
|
||||||
"remote_data": self.remote_data.normalized().to_dict(include_password=True),
|
"remote_data": self.remote_data.normalized().to_dict(include_password=True),
|
||||||
"history_retention": self.history_retention.normalized().to_dict(),
|
"history_retention": self.history_retention.normalized().to_dict(),
|
||||||
"rj_data": self.rj_data.normalized().to_dict(),
|
|
||||||
"sheet_filter": self.sheet_filter,
|
"sheet_filter": self.sheet_filter,
|
||||||
"extract_fields": self.extract_fields
|
"extract_fields": self.extract_fields
|
||||||
}
|
}
|
||||||
|
|||||||
+31
-88
@@ -123,21 +123,18 @@ class DataProcessor:
|
|||||||
self._field_map, self._type_map = self._build_field_map()
|
self._field_map, self._type_map = self._build_field_map()
|
||||||
self._apply_sql_script_type_hints()
|
self._apply_sql_script_type_hints()
|
||||||
|
|
||||||
# 预编译RJ字段映射
|
# 预编译按表字段映射
|
||||||
self._rj_field_maps: Dict[str, Tuple[Dict[str, str], Dict[str, str]]] = {}
|
self._table_field_maps: Dict[str, Tuple[Dict[str, str], Dict[str, str]]] = {}
|
||||||
self._build_rj_field_maps()
|
self._build_table_field_maps()
|
||||||
|
|
||||||
# LOAD DATA INFILE 支持状态(在首次使用时检测)
|
# LOAD DATA INFILE 支持状态(在首次使用时检测)
|
||||||
self._load_data_supported: Optional[bool] = None
|
self._load_data_supported: Optional[bool] = None
|
||||||
self._load_data_checked = False
|
self._load_data_checked = False
|
||||||
|
|
||||||
def _build_rj_field_maps(self) -> None:
|
def _build_table_field_maps(self) -> None:
|
||||||
"""预构建RJ表的字段映射"""
|
"""预构建按表覆盖的字段映射"""
|
||||||
rj_config = self.config.rj_data.normalized()
|
mappings = self.config.data_mappings.normalized().table_field_mappings
|
||||||
if not rj_config.enabled:
|
for table_name, fields in mappings.items():
|
||||||
return
|
|
||||||
|
|
||||||
for table_name, fields in rj_config.table_field_mappings.items():
|
|
||||||
field_map = {}
|
field_map = {}
|
||||||
type_map = {}
|
type_map = {}
|
||||||
for field_def in fields:
|
for field_def in fields:
|
||||||
@@ -147,7 +144,7 @@ class DataProcessor:
|
|||||||
if source and target:
|
if source and target:
|
||||||
field_map[source] = target
|
field_map[source] = target
|
||||||
type_map[target] = field_type
|
type_map[target] = field_type
|
||||||
self._rj_field_maps[table_name] = (field_map, type_map)
|
self._table_field_maps[table_name] = (field_map, type_map)
|
||||||
|
|
||||||
def _build_field_map(self) -> Tuple[Dict[str, str], Dict[str, str]]:
|
def _build_field_map(self) -> Tuple[Dict[str, str], Dict[str, str]]:
|
||||||
"""
|
"""
|
||||||
@@ -186,9 +183,8 @@ class DataProcessor:
|
|||||||
field_map: {源字段名: 目标字段名}
|
field_map: {源字段名: 目标字段名}
|
||||||
type_map: {目标字段名: 字段类型}
|
type_map: {目标字段名: 字段类型}
|
||||||
"""
|
"""
|
||||||
# 检查是否是RJ表(使用预构建的缓存)
|
if table_name in self._table_field_maps:
|
||||||
if table_name in self._rj_field_maps:
|
return self._table_field_maps[table_name]
|
||||||
return self._rj_field_maps[table_name]
|
|
||||||
|
|
||||||
# 使用全局映射
|
# 使用全局映射
|
||||||
return self._field_map, self._type_map
|
return self._field_map, self._type_map
|
||||||
@@ -520,7 +516,7 @@ class DataProcessor:
|
|||||||
keep_default_na=False # 不使用默认的 NA 值
|
keep_default_na=False # 不使用默认的 NA 值
|
||||||
)
|
)
|
||||||
|
|
||||||
# 获取字段映射(RJ表使用专用映射,其他表使用全局映射)
|
# 获取字段映射(按表配置优先,其他表使用全局映射)
|
||||||
field_map, type_map = self._get_field_map_for_table(table_name)
|
field_map, type_map = self._get_field_map_for_table(table_name)
|
||||||
|
|
||||||
# 快速字段匹配(使用预编译的映射表)
|
# 快速字段匹配(使用预编译的映射表)
|
||||||
@@ -829,83 +825,27 @@ class DataProcessor:
|
|||||||
.str.replace(' ', '', regex=False)
|
.str.replace(' ', '', regex=False)
|
||||||
)
|
)
|
||||||
|
|
||||||
# RJ目录名到表名的映射
|
def _find_data_directories(self) -> Dict[str, List[Path]]:
|
||||||
RJ_DIR_TO_TABLE = {
|
|
||||||
"2.6RJGD": "2_6GRJGD",
|
|
||||||
"2.6RJYD": "2_6GRJYD",
|
|
||||||
"700RJGD": "700MRJGD",
|
|
||||||
"700RJYD": "700MRJYD",
|
|
||||||
}
|
|
||||||
|
|
||||||
def _find_data_directories(self) -> Dict[str, Path]:
|
|
||||||
"""
|
"""
|
||||||
查找包含数据文件的目录,返回 {表名: 目录路径}
|
查找包含数据文件的目录,返回 {表名: [目录路径]}
|
||||||
支持 4G/5G 和 RJ 数据目录
|
|
||||||
"""
|
"""
|
||||||
data_dirs = {}
|
data_dirs: Dict[str, List[Path]] = {}
|
||||||
target_names = {'4G', '5G', '4g', '5g'}
|
|
||||||
|
|
||||||
self.logger.info(f"开始查找数据目录,工作目录: {self.work_dir}")
|
self.logger.info(f"开始查找数据目录,工作目录: {self.work_dir}")
|
||||||
|
|
||||||
# 递归查找所有名为 4G 或 5G 的目录
|
for item in self.config.data_mappings.normalized().directories:
|
||||||
found_dirs = []
|
subdir = self.work_dir / item["path"]
|
||||||
for subdir in self.work_dir.rglob('*'):
|
if subdir.exists() and subdir.is_dir():
|
||||||
if subdir.is_dir() and subdir.name in target_names:
|
self._add_data_directory(data_dirs, item["table"], subdir)
|
||||||
found_dirs.append(subdir)
|
self.logger.info(f"发现数据目录: {subdir.relative_to(self.work_dir)} -> 表: {item['table']}")
|
||||||
|
|
||||||
self.logger.info(f"找到 {len(found_dirs)} 个候选目录")
|
|
||||||
|
|
||||||
for subdir in found_dirs:
|
|
||||||
table_name = f"{subdir.name.upper()}_UD"
|
|
||||||
if table_name not in data_dirs:
|
|
||||||
data_dirs[table_name] = subdir
|
|
||||||
self.logger.info(f"发现数据目录: {subdir.relative_to(self.work_dir)} -> 表: {table_name}")
|
|
||||||
|
|
||||||
# 查找 RJ 数据目录
|
|
||||||
rj_config = self.config.rj_data.normalized()
|
|
||||||
if rj_config.enabled:
|
|
||||||
self._find_rj_data_directories(data_dirs)
|
|
||||||
|
|
||||||
if not data_dirs:
|
|
||||||
self.logger.warning("未找到 4G/5G 目录,使用直接子目录")
|
|
||||||
for subdir in self.work_dir.iterdir():
|
|
||||||
if subdir.is_dir():
|
|
||||||
table_name = f"{subdir.name}_UD"
|
|
||||||
data_dirs[table_name] = subdir
|
|
||||||
self.logger.info(f"使用直接子目录: {subdir.relative_to(self.work_dir)} -> 表: {table_name}")
|
|
||||||
|
|
||||||
return data_dirs
|
return data_dirs
|
||||||
|
|
||||||
def _find_rj_data_directories(self, data_dirs: Dict[str, Path]) -> None:
|
@staticmethod
|
||||||
"""查找RJ数据目录"""
|
def _add_data_directory(data_dirs: Dict[str, List[Path]], table_name: str, directory: Path) -> None:
|
||||||
self.logger.info("查找RJ数据目录...")
|
existing = data_dirs.setdefault(table_name, [])
|
||||||
|
resolved = directory.resolve()
|
||||||
# RJ目录结构: RJ/2.6G/2.6RJGD, RJ/2.6G/2.6RJYD 等
|
if all(path.resolve() != resolved for path in existing):
|
||||||
# 只搜索工作目录下的特定路径,避免全量递归
|
existing.append(directory)
|
||||||
rj_config = self.config.rj_data.normalized()
|
|
||||||
for weekly_dir in rj_config.weekly_directories:
|
|
||||||
# 构建本地路径: work_dir / RJ/2.6G/2.6RJGD
|
|
||||||
rj_path = self.work_dir / weekly_dir
|
|
||||||
if rj_path.exists() and rj_path.is_dir():
|
|
||||||
dir_name = rj_path.name
|
|
||||||
if dir_name in self.RJ_DIR_TO_TABLE:
|
|
||||||
table_name = self.RJ_DIR_TO_TABLE[dir_name]
|
|
||||||
if table_name not in data_dirs:
|
|
||||||
data_dirs[table_name] = rj_path
|
|
||||||
self.logger.info(f"发现RJ数据目录: {rj_path.relative_to(self.work_dir)} -> 表: {table_name}")
|
|
||||||
|
|
||||||
# 兜底: 如果配置的路径不存在,尝试从工作目录中查找
|
|
||||||
if not any(k in data_dirs for k in self.RJ_DIR_TO_TABLE.values()):
|
|
||||||
self.logger.info("配置的RJ路径不存在,尝试从工作目录中查找...")
|
|
||||||
for rj_dir in self.work_dir.rglob('*'):
|
|
||||||
if not rj_dir.is_dir():
|
|
||||||
continue
|
|
||||||
dir_name = rj_dir.name
|
|
||||||
if dir_name in self.RJ_DIR_TO_TABLE:
|
|
||||||
table_name = self.RJ_DIR_TO_TABLE[dir_name]
|
|
||||||
if table_name not in data_dirs:
|
|
||||||
data_dirs[table_name] = rj_dir
|
|
||||||
self.logger.info(f"发现RJ数据目录: {rj_dir.relative_to(self.work_dir)} -> 表: {table_name}")
|
|
||||||
|
|
||||||
def _process_csv_files(self):
|
def _process_csv_files(self):
|
||||||
"""处理所有 CSV 文件(使用 LOAD DATA INFILE + 连接复用)"""
|
"""处理所有 CSV 文件(使用 LOAD DATA INFILE + 连接复用)"""
|
||||||
@@ -919,14 +859,17 @@ class DataProcessor:
|
|||||||
return
|
return
|
||||||
|
|
||||||
# 按目录分组处理,使用连接复用
|
# 按目录分组处理,使用连接复用
|
||||||
for table_name, subdir in data_dirs.items():
|
for table_name, subdirs in data_dirs.items():
|
||||||
self.logger.info(f"处理目录: {subdir.relative_to(self.work_dir)} -> 表: {table_name}")
|
directory_names = ", ".join(str(subdir.relative_to(self.work_dir)) for subdir in subdirs)
|
||||||
|
self.logger.info(f"处理目录: {directory_names} -> 表: {table_name}")
|
||||||
|
|
||||||
# 删除旧表
|
# 删除旧表
|
||||||
self.db.drop_table(table_name)
|
self.db.drop_table(table_name)
|
||||||
|
|
||||||
# 处理该目录下的所有 CSV
|
# 处理该目录下的所有 CSV
|
||||||
csv_files = self._filter_recent_files(list(self._scan_files(subdir, ['.csv'])), "CSV", root=subdir)
|
csv_files: list[Path] = []
|
||||||
|
for subdir in subdirs:
|
||||||
|
csv_files.extend(self._filter_recent_files(list(self._scan_files(subdir, ['.csv'])), "CSV", root=subdir))
|
||||||
self.logger.info(f"找到 {len(csv_files)} 个 CSV 文件")
|
self.logger.info(f"找到 {len(csv_files)} 个 CSV 文件")
|
||||||
|
|
||||||
total_rows = 0
|
total_rows = 0
|
||||||
|
|||||||
@@ -24,8 +24,8 @@ FAILURE_RESULTS = {"scan_failed", "trigger_failed", "source_cleanup_failed", "fa
|
|||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class RJDirectoryReadyStatus:
|
class AutoDirectoryReadyStatus:
|
||||||
"""RJ 数据目录就绪状态,按最新文件自动识别日粒度或周粒度。"""
|
"""自动粒度目录就绪状态,按最新文件自动识别日粒度或周粒度。"""
|
||||||
directory: str
|
directory: str
|
||||||
ready: bool
|
ready: bool
|
||||||
granularity: str | None = None
|
granularity: str | None = None
|
||||||
@@ -115,7 +115,12 @@ class AutoScheduler:
|
|||||||
app_config = state.current_config()
|
app_config = state.current_config()
|
||||||
config = app_config.remote_data.normalized()
|
config = app_config.remote_data.normalized()
|
||||||
scheduler = config.auto_scheduler.normalized()
|
scheduler = config.auto_scheduler.normalized()
|
||||||
rj_config = app_config.rj_data.normalized()
|
data_mappings = app_config.data_mappings.normalized()
|
||||||
|
auto_directories = [
|
||||||
|
item["path"]
|
||||||
|
for item in data_mappings.directories
|
||||||
|
if item.get("ready_rule") == "auto"
|
||||||
|
]
|
||||||
target_days = required_week_days(scheduler.week_offset)
|
target_days = required_week_days(scheduler.week_offset)
|
||||||
ready_flag = self._read_ready_flag()
|
ready_flag = self._read_ready_flag()
|
||||||
|
|
||||||
@@ -127,9 +132,7 @@ class AutoScheduler:
|
|||||||
"expected_directories": scheduler.expected_directories,
|
"expected_directories": scheduler.expected_directories,
|
||||||
"week_offset": scheduler.week_offset,
|
"week_offset": scheduler.week_offset,
|
||||||
"auto_delete_source": config.auto_delete_source,
|
"auto_delete_source": config.auto_delete_source,
|
||||||
"rj_data_enabled": rj_config.enabled,
|
"auto_ready_directories": auto_directories,
|
||||||
"rj_directories": rj_config.weekly_directories,
|
|
||||||
"rj_weekly_directories": rj_config.weekly_directories,
|
|
||||||
"next_check_at": self._format_dt(self._next_check_at),
|
"next_check_at": self._format_dt(self._next_check_at),
|
||||||
"last_check_at": self._format_dt(self._last_check_at),
|
"last_check_at": self._format_dt(self._last_check_at),
|
||||||
"last_result": self._last_result,
|
"last_result": self._last_result,
|
||||||
@@ -188,15 +191,19 @@ class AutoScheduler:
|
|||||||
|
|
||||||
target_days = required_week_days(scheduler.week_offset)
|
target_days = required_week_days(scheduler.week_offset)
|
||||||
downloader = make_source_downloader(app_config)
|
downloader = make_source_downloader(app_config)
|
||||||
rj_config = app_config.rj_data.normalized()
|
data_mappings = app_config.data_mappings.normalized()
|
||||||
rj_directories = set(rj_config.weekly_directories) if rj_config.enabled else set()
|
auto_directories = {
|
||||||
|
item["path"]
|
||||||
|
for item in data_mappings.directories
|
||||||
|
if item.get("ready_rule") == "auto"
|
||||||
|
}
|
||||||
|
|
||||||
# 检查现有7天目录(4G/5G数据),RJ 目录单独按粒度判断,避免被普通7天规则误拦。
|
# 自动粒度目录单独判断,避免被普通 7 天规则误拦。
|
||||||
directory_status = self._check_remote_ready(
|
directory_status = self._check_remote_ready(
|
||||||
downloader,
|
downloader,
|
||||||
scheduler,
|
scheduler,
|
||||||
target_days,
|
target_days,
|
||||||
excluded_directories=rj_directories,
|
excluded_directories=auto_directories,
|
||||||
)
|
)
|
||||||
|
|
||||||
error_count = sum(1 for item in directory_status.values() if item.error)
|
error_count = sum(1 for item in directory_status.values() if item.error)
|
||||||
@@ -210,37 +217,37 @@ class AutoScheduler:
|
|||||||
|
|
||||||
active_status = [item for item in directory_status.values() if not item.skipped]
|
active_status = [item for item in directory_status.values() if not item.skipped]
|
||||||
skipped_count = len(directory_status) - len(active_status)
|
skipped_count = len(directory_status) - len(active_status)
|
||||||
daily_ready = bool(active_status) and all(item.ready for item in active_status)
|
|
||||||
|
|
||||||
# 检查 RJ 数据目录,按最新文件自动判断日粒度或周粒度。
|
# 检查自动粒度目录,按最新文件自动判断日粒度或周粒度。
|
||||||
rj_status = self._check_rj_ready(downloader, target_days)
|
auto_status = self._check_auto_ready(downloader, target_days, sorted(auto_directories))
|
||||||
rj_error_count = sum(1 for item in rj_status.values() if item.error)
|
auto_error_count = sum(1 for item in auto_status.values() if item.error)
|
||||||
self._set_combined_directory_status(directory_status, rj_status)
|
self._set_combined_directory_status(directory_status, auto_status)
|
||||||
if rj_error_count:
|
if auto_error_count:
|
||||||
return self._finish_check(
|
return self._finish_check(
|
||||||
"scan_failed",
|
"scan_failed",
|
||||||
f"RJ 远程目录扫描失败,{rj_error_count}/{len(rj_status)} 个目录无法访问或扫描失败",
|
f"自动粒度远程目录扫描失败,{auto_error_count}/{len(auto_status)} 个目录无法访问或扫描失败",
|
||||||
manual,
|
manual,
|
||||||
)
|
)
|
||||||
|
|
||||||
active_rj_status = [item for item in rj_status.values() if not item.skipped]
|
active_auto_status = [item for item in auto_status.values() if not item.skipped]
|
||||||
rj_ready = all(item.ready for item in active_rj_status)
|
daily_ready = all(item.ready for item in active_status) if active_status else not directory_status and bool(active_auto_status)
|
||||||
rj_status_text = ""
|
auto_ready = all(item.ready for item in active_auto_status)
|
||||||
|
auto_status_text = ""
|
||||||
|
|
||||||
if active_rj_status:
|
if active_auto_status:
|
||||||
rj_ready_count = sum(1 for item in active_rj_status if item.ready)
|
auto_ready_count = sum(1 for item in active_auto_status if item.ready)
|
||||||
rj_total = len(active_rj_status)
|
auto_total = len(active_auto_status)
|
||||||
if not rj_ready:
|
if not auto_ready:
|
||||||
rj_status_text = f",RJ 数据 {rj_ready_count}/{rj_total} 个有效目录就绪"
|
auto_status_text = f",自动粒度数据 {auto_ready_count}/{auto_total} 个有效目录就绪"
|
||||||
|
|
||||||
# 两个条件都满足才触发
|
# 两个条件都满足才触发
|
||||||
if daily_ready and rj_ready:
|
if daily_ready and auto_ready:
|
||||||
self._mark_ready(target_days, directory_status, rj_status)
|
self._mark_ready(target_days, directory_status, auto_status)
|
||||||
skipped_text = f",已跳过 {skipped_count} 个停推目录" if skipped_count else ""
|
skipped_text = f",已跳过 {skipped_count} 个停推目录" if skipped_count else ""
|
||||||
rj_text = ",RJ 数据已就绪" if active_rj_status else ""
|
auto_text = ",自动粒度数据已就绪" if active_auto_status else ""
|
||||||
return self._finish_check(
|
return self._finish_check(
|
||||||
"marked_ready",
|
"marked_ready",
|
||||||
f"远程数据已满足目标周 7 天{skipped_text}{rj_text},已写入就绪标识,下次检查将自动处理",
|
f"远程数据已满足目标周 7 天{skipped_text}{auto_text},已写入就绪标识,下次检查将自动处理",
|
||||||
manual,
|
manual,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -254,7 +261,7 @@ class AutoScheduler:
|
|||||||
if not active_status:
|
if not active_status:
|
||||||
return self._finish_check(
|
return self._finish_check(
|
||||||
"waiting",
|
"waiting",
|
||||||
f"远程普通日数据未发现有效目录,无法触发处理{rj_status_text}",
|
f"远程普通日数据未发现有效目录,无法触发处理{auto_status_text}",
|
||||||
manual,
|
manual,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -262,7 +269,7 @@ class AutoScheduler:
|
|||||||
skipped_text = f",跳过 {skipped_count} 个停推目录" if skipped_count else ""
|
skipped_text = f",跳过 {skipped_count} 个停推目录" if skipped_count else ""
|
||||||
return self._finish_check(
|
return self._finish_check(
|
||||||
"waiting",
|
"waiting",
|
||||||
f"远程数据未就绪,{ready_count}/{len(active_status)} 个有效目录满足目标周 7 天{skipped_text}{rj_status_text}",
|
f"远程数据未就绪,{ready_count}/{len(active_status)} 个有效目录满足目标周 7 天{skipped_text}{auto_status_text}",
|
||||||
manual,
|
manual,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -380,40 +387,40 @@ class AutoScheduler:
|
|||||||
file_count=len(files),
|
file_count=len(files),
|
||||||
)
|
)
|
||||||
|
|
||||||
def _check_rj_ready(
|
def _check_auto_ready(
|
||||||
self,
|
self,
|
||||||
downloader: RemoteDataDownloader,
|
downloader: RemoteDataDownloader,
|
||||||
target_days: list[date],
|
target_days: list[date],
|
||||||
) -> dict[str, RJDirectoryReadyStatus]:
|
directories: list[str],
|
||||||
"""检查 RJ 数据目录是否就绪,自动识别目录最新文件是日粒度还是周粒度。"""
|
) -> dict[str, AutoDirectoryReadyStatus]:
|
||||||
rj_config = state.current_config().rj_data.normalized()
|
"""检查自动粒度目录是否就绪,自动识别目录最新文件是日粒度还是周粒度。"""
|
||||||
if not rj_config.enabled:
|
if not directories:
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
result: dict[str, RJDirectoryReadyStatus] = {}
|
result: dict[str, AutoDirectoryReadyStatus] = {}
|
||||||
|
|
||||||
for directory in rj_config.weekly_directories:
|
for directory in directories:
|
||||||
result[directory] = self._check_single_rj_directory(downloader, directory, target_days)
|
result[directory] = self._check_single_auto_directory(downloader, directory, target_days)
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
def _check_single_rj_directory(
|
def _check_single_auto_directory(
|
||||||
self,
|
self,
|
||||||
downloader: RemoteDataDownloader,
|
downloader: RemoteDataDownloader,
|
||||||
directory: str,
|
directory: str,
|
||||||
target_days: list[date],
|
target_days: list[date],
|
||||||
) -> RJDirectoryReadyStatus:
|
) -> AutoDirectoryReadyStatus:
|
||||||
"""检查单个 RJ 目录是否包含目标周数据。"""
|
"""检查单个自动粒度目录是否包含目标周数据。"""
|
||||||
files = self._safe_list_remote_zip_files(downloader, directory)
|
files = self._safe_list_remote_zip_files(downloader, directory)
|
||||||
if files is None:
|
if files is None:
|
||||||
return RJDirectoryReadyStatus(
|
return AutoDirectoryReadyStatus(
|
||||||
directory=directory,
|
directory=directory,
|
||||||
ready=False,
|
ready=False,
|
||||||
error="远程目录不存在或无法访问",
|
error="远程目录不存在或无法访问",
|
||||||
)
|
)
|
||||||
|
|
||||||
if not files:
|
if not files:
|
||||||
return RJDirectoryReadyStatus(
|
return AutoDirectoryReadyStatus(
|
||||||
directory=directory,
|
directory=directory,
|
||||||
ready=True,
|
ready=True,
|
||||||
file_count=0,
|
file_count=0,
|
||||||
@@ -427,7 +434,7 @@ class AutoScheduler:
|
|||||||
if (date_range := parse_file_date_range(remote_file.name))
|
if (date_range := parse_file_date_range(remote_file.name))
|
||||||
]
|
]
|
||||||
if not parsed_files:
|
if not parsed_files:
|
||||||
return RJDirectoryReadyStatus(
|
return AutoDirectoryReadyStatus(
|
||||||
directory=directory,
|
directory=directory,
|
||||||
ready=False,
|
ready=False,
|
||||||
found_days=[],
|
found_days=[],
|
||||||
@@ -452,7 +459,7 @@ class AutoScheduler:
|
|||||||
missing = [item for item in target_days if item not in found]
|
missing = [item for item in target_days if item not in found]
|
||||||
|
|
||||||
if granularity == "daily":
|
if granularity == "daily":
|
||||||
return RJDirectoryReadyStatus(
|
return AutoDirectoryReadyStatus(
|
||||||
directory=directory,
|
directory=directory,
|
||||||
ready=not missing,
|
ready=not missing,
|
||||||
granularity=granularity,
|
granularity=granularity,
|
||||||
@@ -465,7 +472,7 @@ class AutoScheduler:
|
|||||||
for remote_file in files:
|
for remote_file in files:
|
||||||
date_range = parse_file_date_range(remote_file.name)
|
date_range = parse_file_date_range(remote_file.name)
|
||||||
if date_range and date_range.covers_all(required):
|
if date_range and date_range.covers_all(required):
|
||||||
return RJDirectoryReadyStatus(
|
return AutoDirectoryReadyStatus(
|
||||||
directory=directory,
|
directory=directory,
|
||||||
ready=True,
|
ready=True,
|
||||||
granularity=granularity,
|
granularity=granularity,
|
||||||
@@ -475,7 +482,7 @@ class AutoScheduler:
|
|||||||
file_count=len(files),
|
file_count=len(files),
|
||||||
)
|
)
|
||||||
|
|
||||||
return RJDirectoryReadyStatus(
|
return AutoDirectoryReadyStatus(
|
||||||
directory=directory,
|
directory=directory,
|
||||||
ready=False,
|
ready=False,
|
||||||
granularity=granularity,
|
granularity=granularity,
|
||||||
@@ -523,7 +530,7 @@ class AutoScheduler:
|
|||||||
self,
|
self,
|
||||||
target_days: list[date],
|
target_days: list[date],
|
||||||
directory_status: dict[str, DirectoryReadyStatus],
|
directory_status: dict[str, DirectoryReadyStatus],
|
||||||
rj_status: dict[str, RJDirectoryReadyStatus] | None = None,
|
auto_status: dict[str, AutoDirectoryReadyStatus] | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
READY_DIR.mkdir(parents=True, exist_ok=True)
|
READY_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
payload = {
|
payload = {
|
||||||
@@ -536,14 +543,10 @@ class AutoScheduler:
|
|||||||
for name, item in directory_status.items()
|
for name, item in directory_status.items()
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
if rj_status:
|
if auto_status:
|
||||||
payload["rj_directories"] = {
|
payload["auto_directories"] = {
|
||||||
name: item.to_dict()
|
name: item.to_dict()
|
||||||
for name, item in rj_status.items()
|
for name, item in auto_status.items()
|
||||||
}
|
|
||||||
payload["rj_weekly_directories"] = {
|
|
||||||
name: item.to_dict()
|
|
||||||
for name, item in rj_status.items()
|
|
||||||
}
|
}
|
||||||
READY_FLAG.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
READY_FLAG.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||||
|
|
||||||
@@ -619,7 +622,7 @@ class AutoScheduler:
|
|||||||
def _set_combined_directory_status(
|
def _set_combined_directory_status(
|
||||||
self,
|
self,
|
||||||
directory_status: dict[str, DirectoryReadyStatus],
|
directory_status: dict[str, DirectoryReadyStatus],
|
||||||
rj_status: dict[str, RJDirectoryReadyStatus],
|
auto_status: dict[str, AutoDirectoryReadyStatus],
|
||||||
) -> None:
|
) -> None:
|
||||||
with self._status_lock:
|
with self._status_lock:
|
||||||
combined = {
|
combined = {
|
||||||
@@ -629,7 +632,7 @@ class AutoScheduler:
|
|||||||
combined.update(
|
combined.update(
|
||||||
{
|
{
|
||||||
name: item.to_dict()
|
name: item.to_dict()
|
||||||
for name, item in rj_status.items()
|
for name, item in auto_status.items()
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
self._directory_status = combined
|
self._directory_status = combined
|
||||||
|
|||||||
@@ -41,13 +41,9 @@ class CsvProcessor:
|
|||||||
self.log = log
|
self.log = log
|
||||||
self.recent_days = int(config.get("recent_days", 7))
|
self.recent_days = int(config.get("recent_days", 7))
|
||||||
self.sheet_filter = set(config.get("sheet_filter", []))
|
self.sheet_filter = set(config.get("sheet_filter", []))
|
||||||
self.data_dir_to_table = {k.upper(): v for k, v in (config.get("data_dir_to_table") or {}).items()}
|
self.directories = _build_directory_mappings(config.get("directories") or [])
|
||||||
self.field_map, self.type_map = _build_global_map(config.get("extract_fields", []))
|
self.field_map, self.type_map = _build_global_map(config.get("extract_fields", []))
|
||||||
rj = config.get("rj") or {}
|
self.table_maps = _build_table_maps(config.get("table_field_mappings") or {})
|
||||||
self.rj_enabled = bool(rj.get("enabled"))
|
|
||||||
self.rj_weekly_dirs = list(rj.get("weekly_directories") or [])
|
|
||||||
self.rj_dir_to_table = dict(rj.get("dir_to_table") or {})
|
|
||||||
self.rj_maps = _build_rj_maps(rj.get("table_field_mappings") or {})
|
|
||||||
self.out_dir = self.work_dir / ".out"
|
self.out_dir = self.work_dir / ".out"
|
||||||
|
|
||||||
def process(self) -> dict[str, Path]:
|
def process(self) -> dict[str, Path]:
|
||||||
@@ -99,8 +95,10 @@ class CsvProcessor:
|
|||||||
return {}
|
return {}
|
||||||
self.out_dir.mkdir(parents=True, exist_ok=True)
|
self.out_dir.mkdir(parents=True, exist_ok=True)
|
||||||
result: dict[str, Path] = {}
|
result: dict[str, Path] = {}
|
||||||
for table, directory in data_dirs.items():
|
for table, directories in data_dirs.items():
|
||||||
csv_files = self._filter_recent(list(self._scan(directory, (".csv",))), "CSV", root=directory)
|
csv_files: list[Path] = []
|
||||||
|
for directory in directories:
|
||||||
|
csv_files.extend(self._filter_recent(list(self._scan(directory, (".csv",))), "CSV", root=directory))
|
||||||
if not csv_files:
|
if not csv_files:
|
||||||
continue
|
continue
|
||||||
field_map, type_map = self._maps_for_table(table)
|
field_map, type_map = self._maps_for_table(table)
|
||||||
@@ -185,30 +183,17 @@ class CsvProcessor:
|
|||||||
return out[union]
|
return out[union]
|
||||||
|
|
||||||
# --- directory detection --------------------------------------------
|
# --- directory detection --------------------------------------------
|
||||||
def _find_data_dirs(self) -> dict[str, Path]:
|
def _find_data_dirs(self) -> dict[str, list[Path]]:
|
||||||
data_dirs: dict[str, Path] = {}
|
data_dirs: dict[str, list[Path]] = {}
|
||||||
target_names = set(self.data_dir_to_table.keys())
|
for item in self.directories:
|
||||||
for sub in self.work_dir.rglob("*"):
|
directory = self.work_dir / item["path"]
|
||||||
if sub.is_dir() and sub.name.upper() in target_names:
|
if directory.exists() and directory.is_dir():
|
||||||
table = self.data_dir_to_table[sub.name.upper()]
|
_add_data_dir(data_dirs, item["table"], directory)
|
||||||
data_dirs.setdefault(table, sub)
|
|
||||||
if self.rj_enabled:
|
|
||||||
self._find_rj_dirs(data_dirs)
|
|
||||||
return data_dirs
|
return data_dirs
|
||||||
|
|
||||||
def _find_rj_dirs(self, data_dirs: dict[str, Path]) -> None:
|
|
||||||
for weekly in self.rj_weekly_dirs:
|
|
||||||
path = self.work_dir / weekly
|
|
||||||
if path.exists() and path.is_dir() and path.name in self.rj_dir_to_table:
|
|
||||||
data_dirs.setdefault(self.rj_dir_to_table[path.name], path)
|
|
||||||
if not any(table in data_dirs for table in self.rj_dir_to_table.values()):
|
|
||||||
for sub in self.work_dir.rglob("*"):
|
|
||||||
if sub.is_dir() and sub.name in self.rj_dir_to_table:
|
|
||||||
data_dirs.setdefault(self.rj_dir_to_table[sub.name], sub)
|
|
||||||
|
|
||||||
def _maps_for_table(self, table: str):
|
def _maps_for_table(self, table: str):
|
||||||
if table in self.rj_maps:
|
if table in self.table_maps:
|
||||||
return self.rj_maps[table]
|
return self.table_maps[table]
|
||||||
return self.field_map, self.type_map
|
return self.field_map, self.type_map
|
||||||
|
|
||||||
# --- helpers ---------------------------------------------------------
|
# --- helpers ---------------------------------------------------------
|
||||||
@@ -257,7 +242,32 @@ def _build_global_map(extract_fields: list[dict]) -> tuple[dict[str, str], dict[
|
|||||||
return field_map, type_map
|
return field_map, type_map
|
||||||
|
|
||||||
|
|
||||||
def _build_rj_maps(table_field_mappings: dict) -> dict[str, tuple[dict[str, str], dict[str, str]]]:
|
def _build_directory_mappings(items: list[dict]) -> list[dict[str, str]]:
|
||||||
|
mappings: list[dict[str, str]] = []
|
||||||
|
seen: set[tuple[str, str]] = set()
|
||||||
|
for item in items:
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
continue
|
||||||
|
path = str(item.get("path", "")).replace("\\", "/").strip().strip("/")
|
||||||
|
table = str(item.get("table", "")).strip()
|
||||||
|
if not path or not table:
|
||||||
|
continue
|
||||||
|
key = (path, table)
|
||||||
|
if key in seen:
|
||||||
|
continue
|
||||||
|
seen.add(key)
|
||||||
|
mappings.append({"path": path, "table": table})
|
||||||
|
return mappings
|
||||||
|
|
||||||
|
|
||||||
|
def _add_data_dir(data_dirs: dict[str, list[Path]], table: str, directory: Path) -> None:
|
||||||
|
existing = data_dirs.setdefault(table, [])
|
||||||
|
resolved = directory.resolve()
|
||||||
|
if all(path.resolve() != resolved for path in existing):
|
||||||
|
existing.append(directory)
|
||||||
|
|
||||||
|
|
||||||
|
def _build_table_maps(table_field_mappings: dict) -> dict[str, tuple[dict[str, str], dict[str, str]]]:
|
||||||
maps: dict[str, tuple[dict[str, str], dict[str, str]]] = {}
|
maps: dict[str, tuple[dict[str, str], dict[str, str]]] = {}
|
||||||
for table, fields in table_field_mappings.items():
|
for table, fields in table_field_mappings.items():
|
||||||
field_map: dict[str, str] = {}
|
field_map: dict[str, str] = {}
|
||||||
|
|||||||
@@ -11,12 +11,6 @@ from app.processor import ProcessLogger
|
|||||||
from app.services.csv_processor import CsvProcessor
|
from app.services.csv_processor import CsvProcessor
|
||||||
from app.services.platform import make_client
|
from app.services.platform import make_client
|
||||||
|
|
||||||
RJ_DIR_TO_TABLE = {
|
|
||||||
"2.6RJGD": "2_6GRJGD",
|
|
||||||
"2.6RJYD": "2_6GRJYD",
|
|
||||||
"700RJGD": "700MRJGD",
|
|
||||||
"700RJYD": "700MRJYD",
|
|
||||||
}
|
|
||||||
RESULT_TABLES = ["4G_结果表", "5G_结果表"]
|
RESULT_TABLES = ["4G_结果表", "5G_结果表"]
|
||||||
|
|
||||||
|
|
||||||
@@ -34,18 +28,13 @@ def validate_metrix(metrix: MetrixConfig) -> None:
|
|||||||
|
|
||||||
def build_processor_config(app_config: AppConfig) -> dict:
|
def build_processor_config(app_config: AppConfig) -> dict:
|
||||||
metrix = app_config.metrix.normalized()
|
metrix = app_config.metrix.normalized()
|
||||||
rj = app_config.rj_data.normalized()
|
mappings = app_config.data_mappings.normalized()
|
||||||
return {
|
return {
|
||||||
"recent_days": metrix.recent_days,
|
"recent_days": metrix.recent_days,
|
||||||
"sheet_filter": list(app_config.sheet_filter),
|
"sheet_filter": list(app_config.sheet_filter),
|
||||||
"data_dir_to_table": dict(metrix.data_dir_to_table),
|
"directories": list(mappings.directories),
|
||||||
"extract_fields": app_config.extract_fields,
|
"extract_fields": app_config.extract_fields,
|
||||||
"rj": {
|
"table_field_mappings": mappings.table_field_mappings,
|
||||||
"enabled": rj.enabled,
|
|
||||||
"weekly_directories": rj.weekly_directories,
|
|
||||||
"dir_to_table": RJ_DIR_TO_TABLE,
|
|
||||||
"table_field_mappings": rj.table_field_mappings,
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -694,3 +694,12 @@
|
|||||||
- 现状确认:`pipeline.py`/`warehouse.py` 所有 `run_script` 调用本就只传 `content`(报表 SQL 来自 `read_report_sql()` 读取本地 `ReportScript.sql`),从不传 `script_id`,行为已正确。
|
- 现状确认:`pipeline.py`/`warehouse.py` 所有 `run_script` 调用本就只传 `content`(报表 SQL 来自 `read_report_sql()` 读取本地 `ReportScript.sql`),从不传 `script_id`,行为已正确。
|
||||||
- 改动:`app/services/platform.py::run_script` 删除一直未被调用的 `script_id` 参数与对应 body 分支,方法只构造 `content`,从代码层面杜绝走平台库内脚本那条路;签名由 `(conn_id, script_id=None, content="", ...)` 改为 `(conn_id, content="", ...)`,现有调用全用 `content=` 关键字、`conn_id` 位置参,未受影响。
|
- 改动:`app/services/platform.py::run_script` 删除一直未被调用的 `script_id` 参数与对应 body 分支,方法只构造 `content`,从代码层面杜绝走平台库内脚本那条路;签名由 `(conn_id, script_id=None, content="", ...)` 改为 `(conn_id, content="", ...)`,现有调用全用 `content=` 关键字、`conn_id` 位置参,未受影响。
|
||||||
- 验证:`python -m compileall app` 通过;全仓 `app` 内除该行注释外无 `script_id` 引用。
|
- 验证:`python -m compileall app` 通过;全仓 `app` 内除该行注释外无 `script_id` 引用。
|
||||||
|
|
||||||
|
## 2026-06-24:数据目录映射统一为 DataMappings
|
||||||
|
|
||||||
|
- 背景:UD 与 RJ 在处理阶段本质都是「源目录 -> 暂存表」。此前 UD 使用 `UDData.directories`,RJ 使用 `RJData.weekly_directories` + 代码内置目录名到表名映射,概念重复且前端容易继续堆卡片。
|
||||||
|
- 配置:删除 `UDData` / `RJData` 两套配置,改为顶层 `DataMappings`。`DataMappings.directories` 每行结构为 `{path, table, ready_rule}`,`ready_rule=daily` 表示目标周每日 7 天检查,`ready_rule=auto` 表示按目录最新 ZIP 自动识别日粒度或周粒度;RJ 原字段映射并入 `DataMappings.table_field_mappings`,按目标表名覆盖全局字段映射。`MetrixConfig` 仍只保留平台连接信息。
|
||||||
|
- 处理链路:直连 MySQL 的 `DataProcessor` 与 Metrix 仓库模式的 `CsvProcessor` 都只读取 `DataMappings.directories`。一个表可对应多个目录:每个目录先按最近日期筛选 CSV,再合并导入同一张暂存表(Metrix 模式写 `.out/{table}.csv`,MySQL 模式逐文件导入同表)。表级字段映射存在时优先使用 `DataMappings.table_field_mappings[table]`,否则使用全局 `ExtractField`。
|
||||||
|
- 自动调度:原 RJ 专用检查改为通用自动粒度检查。`ready_rule=auto` 的目录会从普通每日扫描中排除,并单独按最新 ZIP 判断日/周粒度;如果配置里只有自动粒度目录,只要这些目录就绪也可触发调度。
|
||||||
|
- 前端:设置页「规则映射」左侧独立滚动配置栏中只保留一个「数据目录映射」卡片,每行可编辑目录、暂存表与就绪规则,并保存到 `/api/config/data-mappings`。后续新增目录类映射继续加同一张表,不再新增配置卡片。
|
||||||
|
- 验证:`python -m compileall -q app` 通过;`frontend` `npm run build` 通过(仅既有大 chunk 提示);构建产物与 Python 缓存已清理。
|
||||||
|
|||||||
@@ -337,36 +337,73 @@
|
|||||||
|
|
||||||
<n-tab-pane name="rules" tab="规则映射">
|
<n-tab-pane name="rules" tab="规则映射">
|
||||||
<div class="settings-rules-grid">
|
<div class="settings-rules-grid">
|
||||||
<n-card title="Sheet 过滤规则" size="small" class="work-card">
|
<div class="settings-rules-side">
|
||||||
<n-space vertical>
|
<n-card title="数据目录映射" size="small" class="work-card">
|
||||||
<p class="form-hint">匹配这些关键词的 Sheet 将被跳过处理</p>
|
<n-space vertical>
|
||||||
<div class="filter-tags">
|
<p class="form-hint">每行配置一个源目录到暂存表的映射。就绪规则选择“每日7天”时按目标周 7 天检查;选择“自动日/周”时按目录最新文件自动识别日粒度或周粒度。</p>
|
||||||
<n-tag
|
<div class="directory-mapping-list">
|
||||||
v-for="(filter, index) in sheetFilters"
|
<div
|
||||||
:key="`${filter}-${index}`"
|
v-for="(item, index) in dataMappingsForm.directories"
|
||||||
closable
|
:key="`${item.path}-${item.table}-${item.ready_rule}-${index}`"
|
||||||
@close="removeSheetFilter(index)"
|
class="directory-mapping-row"
|
||||||
>
|
>
|
||||||
{{ filter }}
|
<n-input v-model:value="item.path" size="small" placeholder="目录,例如 4G 或 RJ/700M/700RJYD" />
|
||||||
</n-tag>
|
<n-input v-model:value="item.table" size="small" placeholder="暂存表,例如 4G_UD" />
|
||||||
<n-empty v-if="sheetFilters.length === 0" size="small" description="暂无过滤规则" />
|
<n-select v-model:value="item.ready_rule" size="small" :options="readyRuleOptions" />
|
||||||
</div>
|
<n-button quaternary circle size="small" type="error" @click="removeDirectoryMapping(index)">
|
||||||
<n-input-group>
|
<template #icon>
|
||||||
<n-input
|
<n-icon><CloseOutline /></n-icon>
|
||||||
v-model:value="newSheetFilter"
|
</template>
|
||||||
placeholder="输入需要跳过的 Sheet 关键词"
|
</n-button>
|
||||||
@keydown.enter.prevent="addSheetFilter"
|
</div>
|
||||||
/>
|
<n-empty v-if="dataMappingsForm.directories.length === 0" size="small" description="暂无目录映射" />
|
||||||
<n-button @click="addSheetFilter">添加</n-button>
|
</div>
|
||||||
</n-input-group>
|
<n-input-group>
|
||||||
</n-space>
|
<n-input v-model:value="newDirectoryMapping.path" placeholder="目录路径" @keydown.enter.prevent="addDirectoryMapping" />
|
||||||
|
<n-input v-model:value="newDirectoryMapping.table" placeholder="暂存表名" @keydown.enter.prevent="addDirectoryMapping" />
|
||||||
<template #footer>
|
<n-select v-model:value="newDirectoryMapping.ready_rule" class="directory-ready-select" :options="readyRuleOptions" />
|
||||||
<n-space justify="end">
|
<n-button @click="addDirectoryMapping">添加</n-button>
|
||||||
<n-button type="primary" :loading="savingSheetFilter" @click="saveSheetFilter">保存规则</n-button>
|
</n-input-group>
|
||||||
</n-space>
|
</n-space>
|
||||||
</template>
|
|
||||||
</n-card>
|
<template #footer>
|
||||||
|
<n-space justify="end">
|
||||||
|
<n-button type="primary" :loading="savingDirectoryMappings" @click="saveDirectoryMappings">保存目录映射</n-button>
|
||||||
|
</n-space>
|
||||||
|
</template>
|
||||||
|
</n-card>
|
||||||
|
|
||||||
|
<n-card title="Sheet 过滤规则" size="small" class="work-card">
|
||||||
|
<n-space vertical>
|
||||||
|
<p class="form-hint">匹配这些关键词的 Sheet 将被跳过处理</p>
|
||||||
|
<div class="filter-tags">
|
||||||
|
<n-tag
|
||||||
|
v-for="(filter, index) in sheetFilters"
|
||||||
|
:key="`${filter}-${index}`"
|
||||||
|
closable
|
||||||
|
@close="removeSheetFilter(index)"
|
||||||
|
>
|
||||||
|
{{ filter }}
|
||||||
|
</n-tag>
|
||||||
|
<n-empty v-if="sheetFilters.length === 0" size="small" description="暂无过滤规则" />
|
||||||
|
</div>
|
||||||
|
<n-input-group>
|
||||||
|
<n-input
|
||||||
|
v-model:value="newSheetFilter"
|
||||||
|
placeholder="输入需要跳过的 Sheet 关键词"
|
||||||
|
@keydown.enter.prevent="addSheetFilter"
|
||||||
|
/>
|
||||||
|
<n-button @click="addSheetFilter">添加</n-button>
|
||||||
|
</n-input-group>
|
||||||
|
</n-space>
|
||||||
|
|
||||||
|
<template #footer>
|
||||||
|
<n-space justify="end">
|
||||||
|
<n-button type="primary" :loading="savingSheetFilter" @click="saveSheetFilter">保存规则</n-button>
|
||||||
|
</n-space>
|
||||||
|
</template>
|
||||||
|
</n-card>
|
||||||
|
</div>
|
||||||
|
|
||||||
<n-card size="small" class="work-card field-mapping-card">
|
<n-card size="small" class="work-card field-mapping-card">
|
||||||
<template #header>
|
<template #header>
|
||||||
@@ -494,6 +531,8 @@ import { apiGet, apiPost, downloadGet, upload } from '../api/client';
|
|||||||
import type {
|
import type {
|
||||||
ApiMessage,
|
ApiMessage,
|
||||||
AppConfig,
|
AppConfig,
|
||||||
|
DataDirectoryMapping,
|
||||||
|
DataMappingsConfig,
|
||||||
HistoryRetentionConfig,
|
HistoryRetentionConfig,
|
||||||
MetrixConfig,
|
MetrixConfig,
|
||||||
RemoteAutoSchedulerConfig,
|
RemoteAutoSchedulerConfig,
|
||||||
@@ -526,6 +565,7 @@ const savingBackend = ref(false);
|
|||||||
const savingMetrix = ref(false);
|
const savingMetrix = ref(false);
|
||||||
const sourceType = ref<'ftp' | 'sftp' | 'metrix'>('sftp');
|
const sourceType = ref<'ftp' | 'sftp' | 'metrix'>('sftp');
|
||||||
const warehouseType = ref<'mysql' | 'metrix'>('mysql');
|
const warehouseType = ref<'mysql' | 'metrix'>('mysql');
|
||||||
|
const savingDirectoryMappings = ref(false);
|
||||||
const savingHistoryRetention = ref(false);
|
const savingHistoryRetention = ref(false);
|
||||||
const savingSheetFilter = ref(false);
|
const savingSheetFilter = ref(false);
|
||||||
const savingExtractFields = ref(false);
|
const savingExtractFields = ref(false);
|
||||||
@@ -571,8 +611,21 @@ const metrixForm = reactive<MetrixConfig>({
|
|||||||
storage_id: '',
|
storage_id: '',
|
||||||
database_conn_id: '',
|
database_conn_id: '',
|
||||||
target_database: '',
|
target_database: '',
|
||||||
recent_days: 7,
|
recent_days: 7
|
||||||
data_dir_to_table: { '4G': '4G_UD', '5G': '5G_UD' }
|
});
|
||||||
|
|
||||||
|
const dataMappingsForm = reactive<DataMappingsConfig>({
|
||||||
|
directories: [
|
||||||
|
{ path: '4G', table: '4G_UD', ready_rule: 'daily' },
|
||||||
|
{ path: '5G', table: '5G_UD', ready_rule: 'daily' }
|
||||||
|
],
|
||||||
|
table_field_mappings: {}
|
||||||
|
});
|
||||||
|
|
||||||
|
const newDirectoryMapping = reactive<DataDirectoryMapping>({
|
||||||
|
path: '',
|
||||||
|
table: '',
|
||||||
|
ready_rule: 'daily'
|
||||||
});
|
});
|
||||||
|
|
||||||
const historyRetentionForm = reactive<HistoryRetentionConfig>({
|
const historyRetentionForm = reactive<HistoryRetentionConfig>({
|
||||||
@@ -600,6 +653,10 @@ const schedulerWeekOptions: SelectOption[] = [
|
|||||||
{ label: '上周', value: 0 },
|
{ label: '上周', value: 0 },
|
||||||
{ label: '上上周', value: -1 }
|
{ label: '上上周', value: -1 }
|
||||||
];
|
];
|
||||||
|
const readyRuleOptions: SelectOption[] = [
|
||||||
|
{ label: '每日7天', value: 'daily' },
|
||||||
|
{ label: '自动日/周', value: 'auto' }
|
||||||
|
];
|
||||||
|
|
||||||
const visibleFieldMappings = computed(() => {
|
const visibleFieldMappings = computed(() => {
|
||||||
const keyword = fieldSearch.value.trim().toLowerCase();
|
const keyword = fieldSearch.value.trim().toLowerCase();
|
||||||
@@ -696,8 +753,8 @@ async function loadConfig() {
|
|||||||
metrixForm.database_conn_id = config.metrix.database_conn_id || '';
|
metrixForm.database_conn_id = config.metrix.database_conn_id || '';
|
||||||
metrixForm.target_database = config.metrix.target_database || '';
|
metrixForm.target_database = config.metrix.target_database || '';
|
||||||
metrixForm.recent_days = Number(config.metrix.recent_days) || 7;
|
metrixForm.recent_days = Number(config.metrix.recent_days) || 7;
|
||||||
metrixForm.data_dir_to_table = config.metrix.data_dir_to_table || { '4G': '4G_UD', '5G': '5G_UD' };
|
|
||||||
}
|
}
|
||||||
|
Object.assign(dataMappingsForm, normalizeDataMappingsConfig(config.data_mappings));
|
||||||
mysqlForm.host = config.mysql.host;
|
mysqlForm.host = config.mysql.host;
|
||||||
mysqlForm.port = config.mysql.port;
|
mysqlForm.port = config.mysql.port;
|
||||||
mysqlForm.user = config.mysql.user;
|
mysqlForm.user = config.mysql.user;
|
||||||
@@ -824,6 +881,51 @@ async function saveRemote() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function addDirectoryMapping() {
|
||||||
|
const path = normalizeDirectoryText(newDirectoryMapping.path);
|
||||||
|
const table = newDirectoryMapping.table.trim();
|
||||||
|
const readyRule = newDirectoryMapping.ready_rule === 'auto' ? 'auto' : 'daily';
|
||||||
|
if (!path || !table) {
|
||||||
|
message.warning('请填写目录路径和暂存表名');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const exists = dataMappingsForm.directories.some(
|
||||||
|
item => normalizeDirectoryText(item.path) === path && item.table.trim() === table && item.ready_rule === readyRule
|
||||||
|
);
|
||||||
|
if (exists) {
|
||||||
|
message.warning('该目录映射已存在');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
dataMappingsForm.directories.push({ path, table, ready_rule: readyRule });
|
||||||
|
newDirectoryMapping.path = '';
|
||||||
|
newDirectoryMapping.table = '';
|
||||||
|
newDirectoryMapping.ready_rule = 'daily';
|
||||||
|
}
|
||||||
|
|
||||||
|
function removeDirectoryMapping(index: number) {
|
||||||
|
dataMappingsForm.directories.splice(index, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function saveDirectoryMappings() {
|
||||||
|
const config = normalizeDataMappingsConfig(dataMappingsForm);
|
||||||
|
if (config.directories.length === 0) {
|
||||||
|
message.warning('请至少保留一个数据目录映射');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
savingDirectoryMappings.value = true;
|
||||||
|
try {
|
||||||
|
const result = await apiPost<ApiMessage>('/api/config/data-mappings', config);
|
||||||
|
Object.assign(dataMappingsForm, config);
|
||||||
|
configUpdate.value = result.update || configUpdate.value;
|
||||||
|
message.success('数据目录映射已保存');
|
||||||
|
} catch (error) {
|
||||||
|
message.error(error instanceof Error ? error.message : '保存数据目录映射失败');
|
||||||
|
} finally {
|
||||||
|
savingDirectoryMappings.value = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function addSchedulerDirectory() {
|
function addSchedulerDirectory() {
|
||||||
const value = normalizeDirectoryText(newSchedulerDirectory.value);
|
const value = normalizeDirectoryText(newSchedulerDirectory.value);
|
||||||
if (!value) return;
|
if (!value) return;
|
||||||
@@ -1072,6 +1174,31 @@ function normalizeRemoteConfig(config: RemoteDataConfig | undefined): RemoteData
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function normalizeDataMappingsConfig(config: DataMappingsConfig | undefined): DataMappingsConfig {
|
||||||
|
const directories: DataDirectoryMapping[] = [];
|
||||||
|
const seen = new Set<string>();
|
||||||
|
for (const item of config?.directories || []) {
|
||||||
|
const path = normalizeDirectoryText(item.path || '');
|
||||||
|
const table = String(item.table || '').trim();
|
||||||
|
const readyRule = item.ready_rule === 'auto' ? 'auto' : 'daily';
|
||||||
|
if (!path || !table) continue;
|
||||||
|
const key = `${path}\n${table}\n${readyRule}`;
|
||||||
|
if (seen.has(key)) continue;
|
||||||
|
seen.add(key);
|
||||||
|
directories.push({ path, table, ready_rule: readyRule });
|
||||||
|
}
|
||||||
|
if (directories.length === 0) {
|
||||||
|
directories.push(
|
||||||
|
{ path: '4G', table: '4G_UD', ready_rule: 'daily' },
|
||||||
|
{ path: '5G', table: '5G_UD', ready_rule: 'daily' }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
directories,
|
||||||
|
table_field_mappings: config?.table_field_mappings || {}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function normalizeAutoSchedulerConfig(config: RemoteAutoSchedulerConfig | undefined): RemoteAutoSchedulerConfig {
|
function normalizeAutoSchedulerConfig(config: RemoteAutoSchedulerConfig | undefined): RemoteAutoSchedulerConfig {
|
||||||
return {
|
return {
|
||||||
enabled: Boolean(config?.enabled),
|
enabled: Boolean(config?.enabled),
|
||||||
|
|||||||
+34
-4
@@ -1483,14 +1483,13 @@ select {
|
|||||||
}
|
}
|
||||||
|
|
||||||
.settings-rules-grid {
|
.settings-rules-grid {
|
||||||
|
display: grid;
|
||||||
grid-template-columns: minmax(280px, 0.72fr) minmax(0, 1.4fr);
|
grid-template-columns: minmax(280px, 0.72fr) minmax(0, 1.4fr);
|
||||||
grid-template-rows: minmax(0, 1fr);
|
grid-template-rows: minmax(0, 1fr);
|
||||||
align-items: stretch;
|
align-items: stretch;
|
||||||
height: 100%;
|
height: 100%;
|
||||||
min-height: 0;
|
min-height: 0;
|
||||||
padding-right: 6px;
|
overflow: hidden;
|
||||||
overflow-x: hidden;
|
|
||||||
overflow-y: auto;
|
|
||||||
scrollbar-color: var(--td-scrollbar-color) transparent;
|
scrollbar-color: var(--td-scrollbar-color) transparent;
|
||||||
scrollbar-width: thin;
|
scrollbar-width: thin;
|
||||||
}
|
}
|
||||||
@@ -1509,10 +1508,22 @@ select {
|
|||||||
background: transparent;
|
background: transparent;
|
||||||
}
|
}
|
||||||
|
|
||||||
.settings-rules-grid > .work-card {
|
.settings-rules-grid > .work-card,
|
||||||
|
.settings-rules-side > .work-card {
|
||||||
min-width: 0;
|
min-width: 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.settings-rules-side {
|
||||||
|
display: flex;
|
||||||
|
min-width: 0;
|
||||||
|
min-height: 0;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 18px;
|
||||||
|
padding-right: 6px;
|
||||||
|
overflow-x: hidden;
|
||||||
|
overflow-y: auto;
|
||||||
|
}
|
||||||
|
|
||||||
.settings-security-panel {
|
.settings-security-panel {
|
||||||
max-width: 560px;
|
max-width: 560px;
|
||||||
}
|
}
|
||||||
@@ -1532,6 +1543,23 @@ select {
|
|||||||
min-height: 28px;
|
min-height: 28px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.directory-mapping-list {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 8px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.directory-mapping-row {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: minmax(0, 1fr) minmax(0, 1fr) 104px auto;
|
||||||
|
align-items: center;
|
||||||
|
gap: 8px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.directory-ready-select {
|
||||||
|
width: 112px;
|
||||||
|
}
|
||||||
|
|
||||||
.form-hint {
|
.form-hint {
|
||||||
margin: 0;
|
margin: 0;
|
||||||
color: var(--td-text-color-placeholder);
|
color: var(--td-text-color-placeholder);
|
||||||
@@ -1991,6 +2019,8 @@ select {
|
|||||||
}
|
}
|
||||||
|
|
||||||
.field-mapping-card {
|
.field-mapping-card {
|
||||||
|
grid-row: auto;
|
||||||
|
grid-column: auto;
|
||||||
height: min(760px, calc(100vh - 260px));
|
height: min(760px, calc(100vh - 260px));
|
||||||
min-height: 520px;
|
min-height: 520px;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -98,9 +98,9 @@ export interface AppConfig {
|
|||||||
dbname: string;
|
dbname: string;
|
||||||
};
|
};
|
||||||
metrix: MetrixConfig;
|
metrix: MetrixConfig;
|
||||||
|
data_mappings: DataMappingsConfig;
|
||||||
remote_data: RemoteDataConfig;
|
remote_data: RemoteDataConfig;
|
||||||
history_retention: HistoryRetentionConfig;
|
history_retention: HistoryRetentionConfig;
|
||||||
rj_data?: RJDataConfig;
|
|
||||||
sheet_filter: string[];
|
sheet_filter: string[];
|
||||||
extract_fields: Array<Record<string, unknown>>;
|
extract_fields: Array<Record<string, unknown>>;
|
||||||
}
|
}
|
||||||
@@ -112,12 +112,16 @@ export interface MetrixConfig {
|
|||||||
database_conn_id: string;
|
database_conn_id: string;
|
||||||
target_database: string;
|
target_database: string;
|
||||||
recent_days: number;
|
recent_days: number;
|
||||||
data_dir_to_table: Record<string, string>;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RJDataConfig {
|
export interface DataDirectoryMapping {
|
||||||
enabled: boolean;
|
path: string;
|
||||||
weekly_directories: string[];
|
table: string;
|
||||||
|
ready_rule: 'daily' | 'auto';
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DataMappingsConfig {
|
||||||
|
directories: DataDirectoryMapping[];
|
||||||
table_field_mappings: Record<string, Array<Record<string, unknown>>>;
|
table_field_mappings: Record<string, Array<Record<string, unknown>>>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user