Metrix/CapacityReport
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style: 优化 CellData 扫描路径配置布局

Browse Source
This commit is contained in:
nixevol
2026-06-25 05:07:47 +08:00
parent a7237cb016
commit d052558d58
7 changed files with 97 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/routers/config.py
+7 -1
View File
@@ -107,6 +107,8 @@ async def update_cell_data_remote_config(config: dict[str, Any] = Body(...)):
mysql=current.mysql,
scan_paths=current.scan_paths,
year_dir_regex=current.year_dir_regex,
month_dir_regex=current.month_dir_regex,
day_dir_regex=current.day_dir_regex,
file_name_regex=current.file_name_regex,
file_time_regex=current.file_time_regex,
mapping=current.mapping,
@@ -124,6 +126,8 @@ async def update_cell_data_mysql_config(config: dict[str, Any] = Body(...)):
mysql=MySQLConfig.from_dict(config, default_dbname="celldata"),
scan_paths=current.scan_paths,
year_dir_regex=current.year_dir_regex,
month_dir_regex=current.month_dir_regex,
day_dir_regex=current.day_dir_regex,
file_name_regex=current.file_name_regex,
file_time_regex=current.file_time_regex,
mapping=current.mapping,
@@ -160,6 +164,8 @@ async def update_cell_data_settings(config: dict[str, Any] = Body(...)):
mysql=current.mysql,
scan_paths=config.get("scan_paths", current.scan_paths),
year_dir_regex=str(config.get("year_dir_regex", current.year_dir_regex)),
month_dir_regex=str(config.get("month_dir_regex", current.month_dir_regex)),
day_dir_regex=str(config.get("day_dir_regex", current.day_dir_regex)),
file_name_regex=str(config.get("file_name_regex", current.file_name_regex)),
file_time_regex=str(config.get("file_time_regex", current.file_time_regex)),
mapping=config.get("mapping", current.mapping),
@@ -302,7 +308,7 @@ def _validate_cell_data_settings(config: dict[str, Any]) -> dict[str, Any]:
if not isinstance(scan_paths, list) or not any(str(path).strip() for path in scan_paths):
return {"success": False, "message": "请至少配置一个扫描路径"}
for key in ("year_dir_regex", "file_name_regex", "file_time_regex"):
for key in ("year_dir_regex", "month_dir_regex", "day_dir_regex", "file_name_regex", "file_time_regex"):
try:
re.compile(str(config.get(key, "")))
except re.error as exc:
app/config.py
+10
View File
@@ -80,6 +80,8 @@ DEFAULT_CELL_DATA_SCAN_PATHS = [
"/网优日常优化数据文档/日常性能报表/{maxyear}年/300表",
]
DEFAULT_CELL_DATA_YEAR_DIR_REGEX = r"(?P<year>\d{4})年"
DEFAULT_CELL_DATA_MONTH_DIR_REGEX = r"(?P<month>\d{1,2})月"
DEFAULT_CELL_DATA_DAY_DIR_REGEX = r"(?P<day>\d{1,2})日"
DEFAULT_CELL_DATA_FILE_NAME_REGEX = r"^Result_300_.*\.zip$"
DEFAULT_CELL_DATA_FILE_TIME_REGEX = r"(?P<timestamp>\d{14})(?=\.zip$)"
DEFAULT_CELL_DATA_MAPPING: Dict[str, Any] = {
@@ -435,6 +437,8 @@ class CellDataConfig:
mysql: MySQLConfig = field(default_factory=lambda: MySQLConfig(dbname="celldata"))
scan_paths: List[str] = field(default_factory=lambda: list(DEFAULT_CELL_DATA_SCAN_PATHS))
year_dir_regex: str = DEFAULT_CELL_DATA_YEAR_DIR_REGEX
month_dir_regex: str = DEFAULT_CELL_DATA_MONTH_DIR_REGEX
day_dir_regex: str = DEFAULT_CELL_DATA_DAY_DIR_REGEX
file_name_regex: str = DEFAULT_CELL_DATA_FILE_NAME_REGEX
file_time_regex: str = DEFAULT_CELL_DATA_FILE_TIME_REGEX
mapping: Dict[str, Any] = field(default_factory=lambda: deepcopy(DEFAULT_CELL_DATA_MAPPING))
@@ -456,6 +460,8 @@ class CellDataConfig:
mysql=self.mysql.normalized(),
scan_paths=scan_paths,
year_dir_regex=str(self.year_dir_regex or DEFAULT_CELL_DATA_YEAR_DIR_REGEX).strip() or DEFAULT_CELL_DATA_YEAR_DIR_REGEX,
month_dir_regex=str(self.month_dir_regex or DEFAULT_CELL_DATA_MONTH_DIR_REGEX).strip() or DEFAULT_CELL_DATA_MONTH_DIR_REGEX,
day_dir_regex=str(self.day_dir_regex or DEFAULT_CELL_DATA_DAY_DIR_REGEX).strip() or DEFAULT_CELL_DATA_DAY_DIR_REGEX,
file_name_regex=str(self.file_name_regex or DEFAULT_CELL_DATA_FILE_NAME_REGEX).strip() or DEFAULT_CELL_DATA_FILE_NAME_REGEX,
file_time_regex=str(self.file_time_regex or DEFAULT_CELL_DATA_FILE_TIME_REGEX).strip() or DEFAULT_CELL_DATA_FILE_TIME_REGEX,
mapping=mapping,
@@ -468,6 +474,8 @@ class CellDataConfig:
"mysql": normalized.mysql.to_dict(include_password=include_password),
"scan_paths": normalized.scan_paths,
"year_dir_regex": normalized.year_dir_regex,
"month_dir_regex": normalized.month_dir_regex,
"day_dir_regex": normalized.day_dir_regex,
"file_name_regex": normalized.file_name_regex,
"file_time_regex": normalized.file_time_regex,
"mapping": normalized.mapping,
@@ -484,6 +492,8 @@ class CellDataConfig:
mysql=MySQLConfig.from_dict(mysql_data, default_dbname="celldata") if isinstance(mysql_data, dict) else default.mysql,
scan_paths=data.get("scan_paths", default.scan_paths) if isinstance(data.get("scan_paths", default.scan_paths), list) else default.scan_paths,
year_dir_regex=str(data.get("year_dir_regex", default.year_dir_regex)),
month_dir_regex=str(data.get("month_dir_regex", default.month_dir_regex)),
day_dir_regex=str(data.get("day_dir_regex", default.day_dir_regex)),
file_name_regex=str(data.get("file_name_regex", default.file_name_regex)),
file_time_regex=str(data.get("file_time_regex", default.file_time_regex)),
mapping=data.get("mapping", default.mapping) if isinstance(data.get("mapping", default.mapping), dict) else default.mapping,
app/services/cell_data.py
+21 -18
View File
@@ -56,6 +56,8 @@ class CellDataProcessor:
self.logger = logger
self.downloader = RemoteDataDownloader(self.config.remote_data, self._log)
self.year_dir_re = re.compile(self.config.year_dir_regex)
self.month_dir_re = re.compile(self.config.month_dir_regex)
self.day_dir_re = re.compile(self.config.day_dir_regex)
self.file_name_re = re.compile(self.config.file_name_regex)
self.file_time_re = re.compile(self.config.file_time_regex)
@@ -135,26 +137,27 @@ class CellDataProcessor:
path = self._replace_date_placeholders(str(template).replace("\\", "/").strip())
if not path.startswith("/"):
path = self._join_remote_path(self.config.remote_data.remote_dir, path)
if "{maxyear}" not in path:
return path
parts = [part for part in path.split("/") if part]
maxyear_index = next((index for index, part in enumerate(parts) if "{maxyear}" in part), -1)
if maxyear_index < 0:
return path
parent = "/" + "/".join(parts[:maxyear_index]) if maxyear_index else "/"
year_segment = parts[maxyear_index]
years: list[int] = []
for entry in self._list_dir(parent):
if entry["type"] != "directory":
replacements = [
("{maxyear}", "year", self.year_dir_re, "年份"),
("{maxmonth}", "month", self.month_dir_re, "月份"),
("{maxday}", "day", self.day_dir_re, "日期"),
]
for token, group_name, pattern, label in replacements:
token_index = next((index for index, part in enumerate(parts) if token in part), -1)
if token_index < 0:
continue
match = self.year_dir_re.search(str(entry["name"]))
if match:
years.append(int(match.group("year")))
if not years:
raise RuntimeError(f"未找到年份目录: {parent}")
max_year = str(max(years))
parts[maxyear_index] = year_segment.replace("{maxyear}", max_year)
parent = "/" + "/".join(parts[:token_index]) if token_index else "/"
values: list[int] = []
for entry in self._list_dir(parent):
if entry["type"] != "directory":
continue
match = pattern.search(str(entry["name"]))
if match:
values.append(int(match.group(group_name)))
if not values:
raise RuntimeError(f"未找到{label}目录: {parent}")
parts[token_index] = parts[token_index].replace(token, str(max(values)))
return "/" + "/".join(parts)
def _replace_date_placeholders(self, path: str) -> str: