feat: 完善RJ数据自动调度

This commit is contained in:
2026-06-01 10:28:36 +08:00
parent 9711aa6270
commit e0056e609e
10 changed files with 328 additions and 113 deletions
+2
View File
@@ -69,11 +69,13 @@ api_tokens.json
*.csv *.csv
# IDE and OS # IDE and OS
.claude/
.vscode/ .vscode/
.idea/ .idea/
*.swp *.swp
*.swo *.swo
*~ *~
nul
.DS_Store .DS_Store
Thumbs.db Thumbs.db
desktop.ini desktop.ini
+20 -1
View File
@@ -33,7 +33,9 @@
"enabled": true, "enabled": true,
"weekly_directories": [ "weekly_directories": [
"RJ/2.6G/2.6RJGD", "RJ/2.6G/2.6RJGD",
"RJ/2.6G/2.6RJYD" "RJ/2.6G/2.6RJYD",
"RJ/700M/700RJGD",
"RJ/700M/700RJYD"
], ],
"table_field_mappings": { "table_field_mappings": {
"2_6GRJGD": [ "2_6GRJGD": [
@@ -51,6 +53,23 @@
{"Source": "cellId", "Target": "cellId", "Type": "string"}, {"Source": "cellId", "Target": "cellId", "Type": "string"},
{"Source": "gNBplmn", "Target": "gNBplmn", "Type": "string"}, {"Source": "gNBplmn", "Target": "gNBplmn", "Type": "string"},
{"Source": "5G上下行总流量(上行PDCP PDU数据量+下行PDCP成功发送数据量)-YD(GB)", "Target": "上下行总流量_GB", "Type": "float"} {"Source": "5G上下行总流量(上行PDCP PDU数据量+下行PDCP成功发送数据量)-YD(GB)", "Target": "上下行总流量_GB", "Type": "float"}
],
"700MRJGD": [
{"Source": "开始时间", "Target": "开始时间", "Type": "datetime"},
{"Source": "结束时间", "Target": "结束时间", "Type": "datetime"},
{"Source": "gNBId", "Target": "gNBId", "Type": "string"},
{"Source": "cellId", "Target": "cellId", "Type": "string"},
{"Source": "gNBplmn", "Target": "gNBplmn", "Type": "string"},
{"Source": "5G上下行总流量(上行PDCP PDU数据量+下行PDCP成功发送数据量)-GD(GB)", "Target": "上下行总流量_GB", "Type": "float"}
],
"700MRJYD": [
{"Source": "开始时间", "Target": "开始时间", "Type": "datetime"},
{"Source": "结束时间", "Target": "结束时间", "Type": "datetime"},
{"Source": "gNBId", "Target": "gNBId", "Type": "string"},
{"Source": "cellId", "Target": "cellId", "Type": "string"},
{"Source": "gNBplmn", "Target": "gNBplmn", "Type": "string"},
{"Source": "5G上下行总流量(上行PDCP PDU数据量+下行PDCP成功发送数据量)-YD(GB)", "Target": "上下行总流量_GB", "Type": "float"},
{"Source": "5G上下行总流量(上行PDCP PDU数据量+下行PDCP成功发送数据量)(GB)-YD(GB)", "Target": "上下行总流量_GB", "Type": "float"}
] ]
} }
}, },
+5 -1
View File
@@ -7,7 +7,7 @@ from fastapi import APIRouter, Body, HTTPException, UploadFile, File
from fastapi.responses import Response from fastapi.responses import Response
from app import state from app import state
from app.config import HistoryRetentionConfig, RemoteDataConfig from app.config import HistoryRetentionConfig, RJDataConfig, RemoteDataConfig
from app.services.api_tokens import export_tokens, import_tokens from app.services.api_tokens import export_tokens, import_tokens
@@ -132,3 +132,7 @@ def _apply_config_data(data: dict[str, Any]) -> None:
history_retention = data.get("HistoryRetention") history_retention = data.get("HistoryRetention")
if isinstance(history_retention, dict): if isinstance(history_retention, dict):
state.config.history_retention = HistoryRetentionConfig.from_dict(history_retention) state.config.history_retention = HistoryRetentionConfig.from_dict(history_retention)
rj_data = data.get("RJData")
if isinstance(rj_data, dict):
state.config.rj_data = RJDataConfig.from_dict(rj_data)
+6 -1
View File
@@ -525,9 +525,14 @@ class DataProcessor:
# 快速字段匹配(使用预编译的映射表) # 快速字段匹配(使用预编译的映射表)
col_mapping = {} col_mapping = {}
mapped_targets = set()
for col in df.columns: for col in df.columns:
if col in field_map: if col in field_map:
col_mapping[col] = field_map[col] target_col = field_map[col]
if target_col in mapped_targets:
continue
col_mapping[col] = target_col
mapped_targets.add(target_col)
if len(col_mapping) < 1: if len(col_mapping) < 1:
if 'kpis' in str(csv_file).lower(): if 'kpis' in str(csv_file).lower():
+184 -88
View File
@@ -1,7 +1,6 @@
from __future__ import annotations from __future__ import annotations
import json import json
import re
import threading import threading
from collections import defaultdict from collections import defaultdict
from dataclasses import dataclass from dataclasses import dataclass
@@ -11,9 +10,9 @@ from typing import Any
from fastapi import HTTPException from fastapi import HTTPException
from app import state from app import state
from app.config import CACHE_DIR, AutoSchedulerConfig, RemoteDataConfig from app.config import CACHE_DIR, AutoSchedulerConfig
from app.services.remote_download import RemoteDataDownloader, RemoteFileInfo from app.services.remote_download import RemoteDataDownloader, RemoteFileInfo
from app.utils.file_dates import extract_file_date, required_week_days from app.utils.file_dates import parse_file_date_range, required_week_days
READY_DIR = CACHE_DIR / "auto_scheduler" READY_DIR = CACHE_DIR / "auto_scheduler"
@@ -22,29 +21,36 @@ DISABLED_CHECK_SECONDS = 60
STARTUP_CHECK_SECONDS = 5 STARTUP_CHECK_SECONDS = 5
FAILURE_RESULTS = {"scan_failed", "trigger_failed", "source_cleanup_failed", "failed", "invalid_flag"} FAILURE_RESULTS = {"scan_failed", "trigger_failed", "source_cleanup_failed", "failed", "invalid_flag"}
# RJ周文件名模式: CapacityReportData2.6RJ_GD_YYYYMMDDHHMM_YYYYMMDDHHMM.zip
RJ_WEEKLY_FILE_PATTERN = re.compile(
r"CapacityReportData2\.6RJ_(?:GD|YD)_(\d{12})_(\d{12})\.zip$"
)
@dataclass(frozen=True) @dataclass(frozen=True)
class RJWeeklyDirectoryStatus: class RJDirectoryReadyStatus:
"""RJ周数据目录就绪状态""" """RJ 数据目录就绪状态,按最新文件自动识别日粒度或周粒度。"""
directory: str directory: str
ready: bool ready: bool
found: bool = False granularity: str | None = None
found_days: list[date] | None = None
missing_days: list[date] | None = None
file_name: str | None = None file_name: str | None = None
file_count: int = 0 file_count: int = 0
error: str | None = None error: str | None = None
skipped: bool = False
skip_reason: str | None = None
def to_dict(self) -> dict[str, Any]: def to_dict(self) -> dict[str, Any]:
found_days = self.found_days or []
missing_days = self.missing_days or []
return { return {
"ready": self.ready, "ready": self.ready,
"found": self.found, "granularity": self.granularity,
"found_days": [item.isoformat() for item in found_days],
"missing_days": [item.isoformat() for item in missing_days],
"found_count": len(found_days),
"required_count": len(found_days) + len(missing_days),
"file_name": self.file_name, "file_name": self.file_name,
"file_count": self.file_count, "file_count": self.file_count,
"error": self.error, "error": self.error,
"skipped": self.skipped,
"skip_reason": self.skip_reason,
} }
@@ -121,6 +127,7 @@ class AutoScheduler:
"week_offset": scheduler.week_offset, "week_offset": scheduler.week_offset,
"auto_delete_source": config.auto_delete_source, "auto_delete_source": config.auto_delete_source,
"rj_data_enabled": rj_config.enabled, "rj_data_enabled": rj_config.enabled,
"rj_directories": rj_config.weekly_directories,
"rj_weekly_directories": rj_config.weekly_directories, "rj_weekly_directories": rj_config.weekly_directories,
"next_check_at": self._format_dt(self._next_check_at), "next_check_at": self._format_dt(self._next_check_at),
"last_check_at": self._format_dt(self._last_check_at), "last_check_at": self._format_dt(self._last_check_at),
@@ -178,14 +185,22 @@ class AutoScheduler:
target_dates = self._target_dates_from_flag(ready_flag, scheduler) target_dates = self._target_dates_from_flag(ready_flag, scheduler)
return self._trigger_processing(target_dates, ready_flag, manual) return self._trigger_processing(target_dates, ready_flag, manual)
# 检查现有7天目录(4G/5G数据)
target_days = required_week_days(scheduler.week_offset) target_days = required_week_days(scheduler.week_offset)
downloader = RemoteDataDownloader(remote_config) downloader = RemoteDataDownloader(remote_config)
directory_status = self._check_remote_ready(downloader, scheduler, target_days) rj_config = app_config.rj_data.normalized()
self._set_directory_status(directory_status) rj_directories = set(rj_config.weekly_directories) if rj_config.enabled else set()
# 检查现有7天目录(4G/5G数据),RJ 目录单独按粒度判断,避免被普通7天规则误拦。
directory_status = self._check_remote_ready(
downloader,
scheduler,
target_days,
excluded_directories=rj_directories,
)
error_count = sum(1 for item in directory_status.values() if item.error) error_count = sum(1 for item in directory_status.values() if item.error)
if error_count: if error_count:
self._set_combined_directory_status(directory_status, {})
return self._finish_check( return self._finish_check(
"scan_failed", "scan_failed",
f"远程目录扫描失败,{error_count}/{len(directory_status)} 个目录无法访问或扫描失败", f"远程目录扫描失败,{error_count}/{len(directory_status)} 个目录无法访问或扫描失败",
@@ -196,29 +211,32 @@ class AutoScheduler:
skipped_count = len(directory_status) - len(active_status) skipped_count = len(directory_status) - len(active_status)
daily_ready = bool(active_status) and all(item.ready for item in active_status) daily_ready = bool(active_status) and all(item.ready for item in active_status)
# 检查RJ周数据目录(复用同一个 downloader) # 检查 RJ 数据目录,按最新文件自动判断日粒度或周粒度。
rj_weekly_status = self._check_rj_weekly_ready(downloader, scheduler.week_offset) rj_status = self._check_rj_ready(downloader, target_days)
rj_weekly_ready = True # 默认就绪(如果没有配置RJ目录) rj_error_count = sum(1 for item in rj_status.values() if item.error)
self._set_combined_directory_status(directory_status, rj_status)
if rj_error_count:
return self._finish_check(
"scan_failed",
f"RJ 远程目录扫描失败,{rj_error_count}/{len(rj_status)} 个目录无法访问或扫描失败",
manual,
)
active_rj_status = [item for item in rj_status.values() if not item.skipped]
rj_ready = all(item.ready for item in active_rj_status)
rj_status_text = "" rj_status_text = ""
if rj_weekly_status: if active_rj_status:
rj_ready_count = sum(1 for s in rj_weekly_status.values() if s.ready) rj_ready_count = sum(1 for item in active_rj_status if item.ready)
rj_total = len(rj_weekly_status) rj_total = len(active_rj_status)
rj_weekly_ready = rj_ready_count == rj_total if not rj_ready:
rj_status_text = f",RJ 数据 {rj_ready_count}/{rj_total} 个有效目录就绪"
# 保存RJ状态到目录状态中
rj_status_dict = {f"rj_weekly:{name}": status.to_dict() for name, status in rj_weekly_status.items()}
with self._status_lock:
self._directory_status.update(rj_status_dict)
if not rj_weekly_ready:
rj_status_text = f",RJ周数据 {rj_ready_count}/{rj_total} 个目录就绪"
# 两个条件都满足才触发 # 两个条件都满足才触发
if daily_ready and rj_weekly_ready: if daily_ready and rj_ready:
self._mark_ready(target_days, directory_status, rj_weekly_status) self._mark_ready(target_days, directory_status, rj_status)
skipped_text = f",已跳过 {skipped_count} 个停推目录" if skipped_count else "" skipped_text = f",已跳过 {skipped_count} 个停推目录" if skipped_count else ""
rj_text = ",RJ周数据已就绪" if rj_weekly_status else "" rj_text = ",RJ 数据已就绪" if active_rj_status else ""
return self._finish_check( return self._finish_check(
"marked_ready", "marked_ready",
f"远程数据已满足目标周 7 天{skipped_text}{rj_text},已写入就绪标识,下次检查将自动处理", f"远程数据已满足目标周 7 天{skipped_text}{rj_text},已写入就绪标识,下次检查将自动处理",
@@ -232,6 +250,13 @@ class AutoScheduler:
manual, manual,
) )
if not active_status:
return self._finish_check(
"waiting",
f"远程普通日数据未发现有效目录,无法触发处理{rj_status_text}",
manual,
)
ready_count = sum(1 for item in active_status if item.ready) ready_count = sum(1 for item in active_status if item.ready)
skipped_text = f",跳过 {skipped_count} 个停推目录" if skipped_count else "" skipped_text = f",跳过 {skipped_count} 个停推目录" if skipped_count else ""
return self._finish_check( return self._finish_check(
@@ -245,7 +270,9 @@ class AutoScheduler:
downloader: RemoteDataDownloader, downloader: RemoteDataDownloader,
scheduler: AutoSchedulerConfig, scheduler: AutoSchedulerConfig,
target_days: list[date], target_days: list[date],
excluded_directories: set[str] | None = None,
) -> dict[str, DirectoryReadyStatus]: ) -> dict[str, DirectoryReadyStatus]:
excluded = {self._normalize_directory_name(item) for item in (excluded_directories or set())}
expected_directories = scheduler.expected_directories expected_directories = scheduler.expected_directories
if expected_directories: if expected_directories:
return { return {
@@ -255,6 +282,7 @@ class AutoScheduler:
target_days, target_days,
) )
for directory in expected_directories for directory in expected_directories
if not self._is_excluded_directory(directory, excluded)
} }
files = self._safe_list_remote_zip_files(downloader, None) files = self._safe_list_remote_zip_files(downloader, None)
@@ -270,11 +298,7 @@ class AutoScheduler:
) )
} }
grouped: dict[str, list[RemoteFileInfo]] = defaultdict(list) if not files:
for remote_file in files:
grouped[remote_file.parent or "."].append(remote_file)
if not grouped:
return { return {
".": DirectoryReadyStatus( ".": DirectoryReadyStatus(
directory=".", directory=".",
@@ -286,6 +310,16 @@ class AutoScheduler:
) )
} }
grouped: dict[str, list[RemoteFileInfo]] = defaultdict(list)
for remote_file in files:
parent = remote_file.parent or "."
if self._is_excluded_directory(parent, excluded):
continue
grouped[parent].append(remote_file)
if not grouped:
return {}
return { return {
directory: self._directory_ready_status(directory, directory_files, target_days) directory: self._directory_ready_status(directory, directory_files, target_days)
for directory, directory_files in sorted(grouped.items(), key=lambda item: item[0]) for directory, directory_files in sorted(grouped.items(), key=lambda item: item[0])
@@ -330,9 +364,11 @@ class AutoScheduler:
required = set(target_days) required = set(target_days)
found = { found = {
file_date target_day
for remote_file in files for remote_file in files
if (file_date := extract_file_date(remote_file.name)) in required if (date_range := parse_file_date_range(remote_file.name))
for target_day in date_range.covered_days()
if target_day in required
} }
missing = [item for item in target_days if item not in found] missing = [item for item in target_days if item not in found]
return DirectoryReadyStatus( return DirectoryReadyStatus(
@@ -343,79 +379,108 @@ class AutoScheduler:
file_count=len(files), file_count=len(files),
) )
def _check_rj_weekly_ready( def _check_rj_ready(
self, self,
downloader: RemoteDataDownloader, downloader: RemoteDataDownloader,
week_offset: int, target_days: list[date],
) -> dict[str, RJWeeklyDirectoryStatus]: ) -> dict[str, RJDirectoryReadyStatus]:
"""检查RJ周数据目录是否就绪""" """检查 RJ 数据目录是否就绪,自动识别目录最新文件是日粒度还是周粒度。"""
rj_config = state.current_config().rj_data.normalized() rj_config = state.current_config().rj_data.normalized()
if not rj_config.enabled: if not rj_config.enabled:
return {} return {}
target_week_end = self._calculate_week_end_date(week_offset) result: dict[str, RJDirectoryReadyStatus] = {}
result: dict[str, RJWeeklyDirectoryStatus] = {}
for directory in rj_config.weekly_directories: for directory in rj_config.weekly_directories:
result[directory] = self._check_single_rj_directory( result[directory] = self._check_single_rj_directory(downloader, directory, target_days)
downloader, directory, target_week_end
)
return result return result
def _calculate_week_end_date(self, week_offset: int) -> date:
"""计算目标周的结束日期(周日)"""
today = date.today()
# 找到本周的周日
days_since_sunday = today.weekday() + 1 # weekday(): 0=周一, 6=周日
if days_since_sunday == 7:
days_since_sunday = 0
this_sunday = today - timedelta(days=days_since_sunday)
# 根据偏移计算目标周的周日
target_sunday = this_sunday + timedelta(weeks=week_offset)
return target_sunday
def _check_single_rj_directory( def _check_single_rj_directory(
self, self,
downloader: RemoteDataDownloader, downloader: RemoteDataDownloader,
directory: str, directory: str,
target_week_end: date, target_days: list[date],
) -> RJWeeklyDirectoryStatus: ) -> RJDirectoryReadyStatus:
"""检查单个RJ目录是否包含目标周的文件""" """检查单个 RJ 目录是否包含目标周数据。"""
files = self._safe_list_remote_zip_files(downloader, directory) files = self._safe_list_remote_zip_files(downloader, directory)
if files is None: if files is None:
return RJWeeklyDirectoryStatus( return RJDirectoryReadyStatus(
directory=directory, directory=directory,
ready=False, ready=False,
error="远程目录不存在或无法访问", error="远程目录不存在或无法访问",
) )
if not files: if not files:
return RJWeeklyDirectoryStatus( return RJDirectoryReadyStatus(
directory=directory, directory=directory,
ready=False, ready=True,
file_count=0, file_count=0,
skipped=True,
skip_reason="目录为空,视为已停推并跳过",
) )
# 查找匹配目标周的文件 parsed_files = [
target_end_str = target_week_end.strftime("%Y%m%d") + "0000" (remote_file, date_range)
for remote_file in files: for remote_file in files
match = RJ_WEEKLY_FILE_PATTERN.search(remote_file.name) if (date_range := parse_file_date_range(remote_file.name))
if match: ]
file_end_date = match.group(2) if not parsed_files:
if file_end_date == target_end_str: return RJDirectoryReadyStatus(
return RJWeeklyDirectoryStatus( directory=directory,
directory=directory, ready=False,
ready=True, found_days=[],
found=True, missing_days=target_days,
file_name=remote_file.name, file_count=len(files),
file_count=len(files), error="目录中未找到可识别日期的 ZIP 文件",
) )
return RJWeeklyDirectoryStatus( latest_file, latest_range = max(
parsed_files,
key=lambda item: (item[1].start, item[1].end_exclusive, item[0].name),
)
granularity = "daily" if latest_range.span_days <= 1 else "weekly"
required = set(target_days)
found = {
target_day
for _, date_range in parsed_files
for target_day in date_range.covered_days()
if target_day in required
}
missing = [item for item in target_days if item not in found]
if granularity == "daily":
return RJDirectoryReadyStatus(
directory=directory,
ready=not missing,
granularity=granularity,
found_days=sorted(found),
missing_days=missing,
file_name=latest_file.name,
file_count=len(files),
)
for remote_file in files:
date_range = parse_file_date_range(remote_file.name)
if date_range and date_range.covers_all(required):
return RJDirectoryReadyStatus(
directory=directory,
ready=True,
granularity=granularity,
found_days=target_days,
missing_days=[],
file_name=remote_file.name,
file_count=len(files),
)
return RJDirectoryReadyStatus(
directory=directory, directory=directory,
ready=False, ready=False,
found=False, granularity=granularity,
found_days=sorted(found),
missing_days=missing,
file_name=latest_file.name,
file_count=len(files), file_count=len(files),
) )
@@ -457,7 +522,7 @@ class AutoScheduler:
self, self,
target_days: list[date], target_days: list[date],
directory_status: dict[str, DirectoryReadyStatus], directory_status: dict[str, DirectoryReadyStatus],
rj_weekly_status: dict[str, RJWeeklyDirectoryStatus] | None = None, rj_status: dict[str, RJDirectoryReadyStatus] | None = None,
) -> None: ) -> None:
READY_DIR.mkdir(parents=True, exist_ok=True) READY_DIR.mkdir(parents=True, exist_ok=True)
payload = { payload = {
@@ -470,10 +535,14 @@ class AutoScheduler:
for name, item in directory_status.items() for name, item in directory_status.items()
}, },
} }
if rj_weekly_status: if rj_status:
payload["rj_directories"] = {
name: item.to_dict()
for name, item in rj_status.items()
}
payload["rj_weekly_directories"] = { payload["rj_weekly_directories"] = {
name: item.to_dict() name: item.to_dict()
for name, item in rj_weekly_status.items() for name, item in rj_status.items()
} }
READY_FLAG.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") READY_FLAG.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
@@ -546,12 +615,39 @@ class AutoScheduler:
"status": self.get_status(), "status": self.get_status(),
} }
def _set_directory_status(self, directory_status: dict[str, DirectoryReadyStatus]) -> None: def _set_combined_directory_status(
self,
directory_status: dict[str, DirectoryReadyStatus],
rj_status: dict[str, RJDirectoryReadyStatus],
) -> None:
with self._status_lock: with self._status_lock:
self._directory_status = { combined = {
name: item.to_dict() name: item.to_dict()
for name, item in directory_status.items() for name, item in directory_status.items()
} }
combined.update(
{
f"rj:{name}": item.to_dict()
for name, item in rj_status.items()
}
)
self._directory_status = combined
@staticmethod
def _normalize_directory_name(directory: str) -> str:
normalized = str(directory or "").replace("\\", "/").strip().strip("/")
return "." if normalized in {"", "."} else normalized
@classmethod
def _is_excluded_directory(cls, directory: str, excluded: set[str]) -> bool:
if not excluded:
return False
normalized = cls._normalize_directory_name(directory)
return any(
normalized == item or normalized.startswith(f"{item}/")
for item in excluded
if item != "."
)
def _set_next_check(self, value: datetime) -> None: def _set_next_check(self, value: datetime) -> None:
with self._status_lock: with self._status_lock:
+8 -2
View File
@@ -9,7 +9,7 @@ from pathlib import Path
from typing import Callable, Iterable from typing import Callable, Iterable
from app.config import RemoteDataConfig from app.config import RemoteDataConfig
from app.utils.file_dates import extract_file_date, select_recent_items_by_directory from app.utils.file_dates import parse_file_date_range, select_recent_items_by_directory
LogFn = Callable[[str], None] LogFn = Callable[[str], None]
@@ -93,7 +93,13 @@ class RemoteDataDownloader:
selected = [ selected = [
remote_file remote_file
for remote_file in files for remote_file in files
if extract_file_date(remote_file.name) in target_dates if (
date_range := parse_file_date_range(remote_file.name)
) and (
date_range.covers_all(target_dates)
if date_range.span_days > 1
else date_range.covers_any(target_dates)
)
] ]
selected_files.extend(selected) selected_files.extend(selected)
skipped_count = len(files) - len(selected) skipped_count = len(files) - len(selected)
+70 -4
View File
@@ -23,18 +23,84 @@ class DirectoryDateSelection:
start_date: date | None = None start_date: date | None = None
@dataclass(frozen=True)
class FileDateRange:
start: date
end_exclusive: date
timestamp_count: int
@property
def span_days(self) -> int:
return max((self.end_exclusive - self.start).days, 1)
def covered_days(self) -> list[date]:
return [self.start + timedelta(days=offset) for offset in range(self.span_days)]
def covers_any(self, target_days: set[date]) -> bool:
return any(day in target_days for day in self.covered_days())
def covers_all(self, target_days: set[date]) -> bool:
return target_days.issubset(set(self.covered_days()))
def extract_file_date(filename: str | Path) -> date | None: def extract_file_date(filename: str | Path) -> date | None:
"""Extract the first timestamp date from a filename.""" """Extract the first timestamp date from a filename."""
match = FILENAME_DATE_RE.search(Path(str(filename)).name) date_range = parse_file_date_range(filename)
if not match: return date_range.start if date_range else None
def parse_file_date_range(filename: str | Path) -> FileDateRange | None:
"""Extract the covered natural date range from a filename.
Files with one timestamp are treated as one-day data files. Files with two
timestamps use the second timestamp as an exclusive end when it is midnight,
matching names like xxx_202605110000_202605120000.zip.
"""
values = FILENAME_DATE_RE.findall(Path(str(filename)).name)
if not values:
return None return None
return parse_file_timestamp(match.group(1))
start_dt = parse_file_timestamp_datetime(values[0])
if not start_dt:
return None
if len(values) == 1:
return FileDateRange(
start=start_dt.date(),
end_exclusive=start_dt.date() + timedelta(days=1),
timestamp_count=1,
)
end_dt = parse_file_timestamp_datetime(values[1])
if not end_dt or end_dt <= start_dt:
return FileDateRange(
start=start_dt.date(),
end_exclusive=start_dt.date() + timedelta(days=1),
timestamp_count=len(values),
)
end_exclusive = end_dt.date()
if end_dt.time() != datetime.min.time():
end_exclusive += timedelta(days=1)
if end_exclusive <= start_dt.date():
end_exclusive = start_dt.date() + timedelta(days=1)
return FileDateRange(
start=start_dt.date(),
end_exclusive=end_exclusive,
timestamp_count=len(values),
)
def parse_file_timestamp(value: str) -> date | None: def parse_file_timestamp(value: str) -> date | None:
parsed = parse_file_timestamp_datetime(value)
return parsed.date() if parsed else None
def parse_file_timestamp_datetime(value: str) -> datetime | None:
fmt = "%Y%m%d%H%M%S" if len(value) == 14 else "%Y%m%d%H%M" fmt = "%Y%m%d%H%M%S" if len(value) == 14 else "%Y%m%d%H%M"
try: try:
return datetime.strptime(value, fmt).date() return datetime.strptime(value, fmt)
except ValueError: except ValueError:
return None return None
+14 -15
View File
@@ -21,25 +21,22 @@
| 格式 | 示例 | 数据日期 | | 格式 | 示例 | 数据日期 |
|------|------|----------| |------|------|----------|
| `XXX_YYYYMMDDHHMM_YYYYMMDDHHMM` | `CapacityReportData2.6_202605110000_202605120000.zip` | 第一个时间戳 `202605110000` → 2026-05-11 | | `XXX_YYYYMMDDHHMM_YYYYMMDDHHMM` | `CapacityReportData2.6_202605110000_202605120000.zip` | 覆盖起止时间范围,结束零点按右开区间处理 → 2026-05-11 |
| `XXX_YYYYMMDDHHMM` | `CapacityReportData2.6_202605110000.zip` | 时间戳 `202605110000` → 2026-05-11 | | `XXX_YYYYMMDDHHMM` | `CapacityReportData2.6_202605110000.zip` | 视为单日文件 → 2026-05-11 |
### 2.2 解析逻辑 ### 2.2 解析逻辑
复用项目已有的正则 `_ZIP_DATE_RE = re.compile(r"(?<!\d)(20\d{10}(?:\d{2})?)(?!\d)")`,取第一个匹配项作为数据日期。 复用项目已有的正则 `_ZIP_DATE_RE = re.compile(r"(?<!\d)(20\d{10}(?:\d{2})?)(?!\d)")`,优先解析文件覆盖的自然日范围。
```python ```python
def extract_file_date(filename: str) -> date | None: def parse_file_date_range(filename: str) -> FileDateRange | None:
"""从文件名提取数据日期(取第一个匹配的时间戳)""" """从文件名提取数据覆盖范围。"""
match = _ZIP_DATE_RE.search(filename) values = _ZIP_DATE_RE.findall(filename)
if not match: if not values:
return None
timestamp = match.group(1) # 12位: YYYYMMDDHHMM 或 14位: YYYYMMDDHHMMSS
fmt = "%Y%m%d%H%M%S" if len(timestamp) == 14 else "%Y%m%d%H%M"
try:
return datetime.strptime(timestamp, fmt).date()
except ValueError:
return None return None
start = parse_timestamp(values[0])
end = parse_timestamp(values[1]) if len(values) > 1 else start + timedelta(days=1)
return FileDateRange(start=start.date(), end_exclusive=normalize_end(end))
``` ```
--- ---
@@ -130,10 +127,12 @@ def get_target_week_range(week_offset: int = 0) -> tuple[date, date]:
**关键**: **关键**:
1. 不依赖系统日期,而是从文件名中提取日期 1. 不依赖系统日期,而是从文件名中提取日期
2. 文件名只取**第一个**时间戳作为数据日期 2. 文件名带两个时间戳时按覆盖范围判断;只有一个时间戳时视为单日文件
3. 例如文件 `CapacityReportData2.6_202605120000_202605130000.zip` → 数据日期为 2026-05-12 3. 例如文件 `CapacityReportData2.6_202605120000_202605130000.zip` → 覆盖 2026-05-12
4. 如果某个日期没有对应的 ZIP 文件,该目录判定为未就绪 4. 如果某个日期没有对应的 ZIP 文件,该目录判定为未就绪
RJ 目录会额外按最新文件自动判断粒度:最新文件为单日时要求 7 个目标日都齐全;最新文件为多日/周文件时要求存在一个 ZIP 覆盖完整目标自然周。
### 3.4 标识文件 ### 3.4 标识文件
- **位置**:`cache/auto_scheduler/ready.flag` - **位置**:`cache/auto_scheduler/ready.flag`
+10
View File
@@ -1,5 +1,15 @@
# 项目上下文记录 # 项目上下文记录
## 2026-06-01:完善 RJ 自动调度日/周粒度识别
- `app/utils/file_dates.py` 新增文件日期范围解析:`XXX_YYYYMMDDHHMM` 视为单日文件,`XXX_YYYYMMDDHHMM_YYYYMMDDHHMM` 按起止时间展开自然日,结束时间为零点时按右开区间处理。
- `app/services/auto_scheduler.py` 的 RJ 检查改为按目录最新 ZIP 自动识别 `daily` 或 `weekly`:日粒度目录要求目标自然周 7 天都存在,周粒度目录要求有一个 ZIP 覆盖目标自然周;空 RJ 目录继续视为停推并跳过。
- 自动调度普通 4G/5G 目录扫描会排除已配置的 RJ 目录,避免 `expected_directories=[]` 时 RJ 周目录被普通 7 天规则误判阻塞。
- `app/services/remote_download.py` 的调度下载筛选改为使用日期覆盖范围:单日文件只要覆盖目标日即下载,多日/周文件必须覆盖完整目标周才下载,避免 ready 后漏下或误下 RJ 周文件。
- `Configure.json` 将 `RJ/700M/700RJGD`、`RJ/700M/700RJYD` 加入 RJ 数据目录,并补充 `700MRJGD`、`700MRJYD` 字段映射;`processor.py` 避免多个源字段别名映射到同一目标字段时生成重复列。
- 配置上传接口现在会导入/保存 `RJData`,前端类型补充 `rj_data` 和调度状态中的 `granularity` 字段。
- 已验证:后端 AST 语法检查、`Configure.json` JSON 解析、`npm run build`、真实 SFTP 清单识别、MySQL 临时导入 700RJYD 样本并清理测试表均通过;前端构建仅保留既有大 chunk 警告。
## 2026-05-29:新增 RJ 周数据处理功能 ## 2026-05-29:新增 RJ 周数据处理功能
- 新增 `RJData` 配置块到 `app/config.py`,支持 `enabled`、`weekly_directories` 和 `table_field_mappings` 配置项,用于管理 RJ 周数据目录和字段映射。 - 新增 `RJData` 配置块到 `app/config.py`,支持 `enabled`、`weekly_directories` 和 `table_field_mappings` 配置项,用于管理 RJ 周数据目录和字段映射。
+8
View File
@@ -79,10 +79,17 @@ export interface AppConfig {
}; };
remote_data: RemoteDataConfig; remote_data: RemoteDataConfig;
history_retention: HistoryRetentionConfig; history_retention: HistoryRetentionConfig;
rj_data?: RJDataConfig;
sheet_filter: string[]; sheet_filter: string[];
extract_fields: Array<Record<string, unknown>>; extract_fields: Array<Record<string, unknown>>;
} }
export interface RJDataConfig {
enabled: boolean;
weekly_directories: string[];
table_field_mappings: Record<string, Array<Record<string, unknown>>>;
}
export interface RemoteDataConfig { export interface RemoteDataConfig {
enabled: boolean; enabled: boolean;
protocol: 'ftp' | 'sftp'; protocol: 'ftp' | 'sftp';
@@ -134,6 +141,7 @@ export interface RemoteSchedulerStatus {
string, string,
{ {
ready: boolean; ready: boolean;
granularity?: 'daily' | 'weekly' | string | null;
found_days: string[]; found_days: string[];
missing_days: string[]; missing_days: string[];
found_count: number; found_count: number;