feat: 完善RJ数据自动调度
This commit is contained in:
@@ -69,11 +69,13 @@ api_tokens.json
|
|||||||
*.csv
|
*.csv
|
||||||
|
|
||||||
# IDE and OS
|
# IDE and OS
|
||||||
|
.claude/
|
||||||
.vscode/
|
.vscode/
|
||||||
.idea/
|
.idea/
|
||||||
*.swp
|
*.swp
|
||||||
*.swo
|
*.swo
|
||||||
*~
|
*~
|
||||||
|
nul
|
||||||
.DS_Store
|
.DS_Store
|
||||||
Thumbs.db
|
Thumbs.db
|
||||||
desktop.ini
|
desktop.ini
|
||||||
|
|||||||
+21
-2
@@ -33,7 +33,9 @@
|
|||||||
"enabled": true,
|
"enabled": true,
|
||||||
"weekly_directories": [
|
"weekly_directories": [
|
||||||
"RJ/2.6G/2.6RJGD",
|
"RJ/2.6G/2.6RJGD",
|
||||||
"RJ/2.6G/2.6RJYD"
|
"RJ/2.6G/2.6RJYD",
|
||||||
|
"RJ/700M/700RJGD",
|
||||||
|
"RJ/700M/700RJYD"
|
||||||
],
|
],
|
||||||
"table_field_mappings": {
|
"table_field_mappings": {
|
||||||
"2_6GRJGD": [
|
"2_6GRJGD": [
|
||||||
@@ -51,6 +53,23 @@
|
|||||||
{"Source": "cellId", "Target": "cellId", "Type": "string"},
|
{"Source": "cellId", "Target": "cellId", "Type": "string"},
|
||||||
{"Source": "gNBplmn", "Target": "gNBplmn", "Type": "string"},
|
{"Source": "gNBplmn", "Target": "gNBplmn", "Type": "string"},
|
||||||
{"Source": "5G上下行总流量(上行PDCP PDU数据量+下行PDCP成功发送数据量)-YD(GB)", "Target": "上下行总流量_GB", "Type": "float"}
|
{"Source": "5G上下行总流量(上行PDCP PDU数据量+下行PDCP成功发送数据量)-YD(GB)", "Target": "上下行总流量_GB", "Type": "float"}
|
||||||
|
],
|
||||||
|
"700MRJGD": [
|
||||||
|
{"Source": "开始时间", "Target": "开始时间", "Type": "datetime"},
|
||||||
|
{"Source": "结束时间", "Target": "结束时间", "Type": "datetime"},
|
||||||
|
{"Source": "gNBId", "Target": "gNBId", "Type": "string"},
|
||||||
|
{"Source": "cellId", "Target": "cellId", "Type": "string"},
|
||||||
|
{"Source": "gNBplmn", "Target": "gNBplmn", "Type": "string"},
|
||||||
|
{"Source": "5G上下行总流量(上行PDCP PDU数据量+下行PDCP成功发送数据量)-GD(GB)", "Target": "上下行总流量_GB", "Type": "float"}
|
||||||
|
],
|
||||||
|
"700MRJYD": [
|
||||||
|
{"Source": "开始时间", "Target": "开始时间", "Type": "datetime"},
|
||||||
|
{"Source": "结束时间", "Target": "结束时间", "Type": "datetime"},
|
||||||
|
{"Source": "gNBId", "Target": "gNBId", "Type": "string"},
|
||||||
|
{"Source": "cellId", "Target": "cellId", "Type": "string"},
|
||||||
|
{"Source": "gNBplmn", "Target": "gNBplmn", "Type": "string"},
|
||||||
|
{"Source": "5G上下行总流量(上行PDCP PDU数据量+下行PDCP成功发送数据量)-YD(GB)", "Target": "上下行总流量_GB", "Type": "float"},
|
||||||
|
{"Source": "5G上下行总流量(上行PDCP PDU数据量+下行PDCP成功发送数据量)(GB)-YD(GB)", "Target": "上下行总流量_GB", "Type": "float"}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -314,4 +333,4 @@
|
|||||||
"Type": "float"
|
"Type": "float"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ from fastapi import APIRouter, Body, HTTPException, UploadFile, File
|
|||||||
from fastapi.responses import Response
|
from fastapi.responses import Response
|
||||||
|
|
||||||
from app import state
|
from app import state
|
||||||
from app.config import HistoryRetentionConfig, RemoteDataConfig
|
from app.config import HistoryRetentionConfig, RJDataConfig, RemoteDataConfig
|
||||||
from app.services.api_tokens import export_tokens, import_tokens
|
from app.services.api_tokens import export_tokens, import_tokens
|
||||||
|
|
||||||
|
|
||||||
@@ -132,3 +132,7 @@ def _apply_config_data(data: dict[str, Any]) -> None:
|
|||||||
history_retention = data.get("HistoryRetention")
|
history_retention = data.get("HistoryRetention")
|
||||||
if isinstance(history_retention, dict):
|
if isinstance(history_retention, dict):
|
||||||
state.config.history_retention = HistoryRetentionConfig.from_dict(history_retention)
|
state.config.history_retention = HistoryRetentionConfig.from_dict(history_retention)
|
||||||
|
|
||||||
|
rj_data = data.get("RJData")
|
||||||
|
if isinstance(rj_data, dict):
|
||||||
|
state.config.rj_data = RJDataConfig.from_dict(rj_data)
|
||||||
|
|||||||
+6
-1
@@ -525,9 +525,14 @@ class DataProcessor:
|
|||||||
|
|
||||||
# 快速字段匹配(使用预编译的映射表)
|
# 快速字段匹配(使用预编译的映射表)
|
||||||
col_mapping = {}
|
col_mapping = {}
|
||||||
|
mapped_targets = set()
|
||||||
for col in df.columns:
|
for col in df.columns:
|
||||||
if col in field_map:
|
if col in field_map:
|
||||||
col_mapping[col] = field_map[col]
|
target_col = field_map[col]
|
||||||
|
if target_col in mapped_targets:
|
||||||
|
continue
|
||||||
|
col_mapping[col] = target_col
|
||||||
|
mapped_targets.add(target_col)
|
||||||
|
|
||||||
if len(col_mapping) < 1:
|
if len(col_mapping) < 1:
|
||||||
if 'kpis' in str(csv_file).lower():
|
if 'kpis' in str(csv_file).lower():
|
||||||
|
|||||||
+184
-88
@@ -1,7 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import re
|
|
||||||
import threading
|
import threading
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
@@ -11,9 +10,9 @@ from typing import Any
|
|||||||
from fastapi import HTTPException
|
from fastapi import HTTPException
|
||||||
|
|
||||||
from app import state
|
from app import state
|
||||||
from app.config import CACHE_DIR, AutoSchedulerConfig, RemoteDataConfig
|
from app.config import CACHE_DIR, AutoSchedulerConfig
|
||||||
from app.services.remote_download import RemoteDataDownloader, RemoteFileInfo
|
from app.services.remote_download import RemoteDataDownloader, RemoteFileInfo
|
||||||
from app.utils.file_dates import extract_file_date, required_week_days
|
from app.utils.file_dates import parse_file_date_range, required_week_days
|
||||||
|
|
||||||
|
|
||||||
READY_DIR = CACHE_DIR / "auto_scheduler"
|
READY_DIR = CACHE_DIR / "auto_scheduler"
|
||||||
@@ -22,29 +21,36 @@ DISABLED_CHECK_SECONDS = 60
|
|||||||
STARTUP_CHECK_SECONDS = 5
|
STARTUP_CHECK_SECONDS = 5
|
||||||
FAILURE_RESULTS = {"scan_failed", "trigger_failed", "source_cleanup_failed", "failed", "invalid_flag"}
|
FAILURE_RESULTS = {"scan_failed", "trigger_failed", "source_cleanup_failed", "failed", "invalid_flag"}
|
||||||
|
|
||||||
# RJ周文件名模式: CapacityReportData2.6RJ_GD_YYYYMMDDHHMM_YYYYMMDDHHMM.zip
|
|
||||||
RJ_WEEKLY_FILE_PATTERN = re.compile(
|
|
||||||
r"CapacityReportData2\.6RJ_(?:GD|YD)_(\d{12})_(\d{12})\.zip$"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class RJWeeklyDirectoryStatus:
|
class RJDirectoryReadyStatus:
|
||||||
"""RJ周数据目录就绪状态"""
|
"""RJ 数据目录就绪状态,按最新文件自动识别日粒度或周粒度。"""
|
||||||
directory: str
|
directory: str
|
||||||
ready: bool
|
ready: bool
|
||||||
found: bool = False
|
granularity: str | None = None
|
||||||
|
found_days: list[date] | None = None
|
||||||
|
missing_days: list[date] | None = None
|
||||||
file_name: str | None = None
|
file_name: str | None = None
|
||||||
file_count: int = 0
|
file_count: int = 0
|
||||||
error: str | None = None
|
error: str | None = None
|
||||||
|
skipped: bool = False
|
||||||
|
skip_reason: str | None = None
|
||||||
|
|
||||||
def to_dict(self) -> dict[str, Any]:
|
def to_dict(self) -> dict[str, Any]:
|
||||||
|
found_days = self.found_days or []
|
||||||
|
missing_days = self.missing_days or []
|
||||||
return {
|
return {
|
||||||
"ready": self.ready,
|
"ready": self.ready,
|
||||||
"found": self.found,
|
"granularity": self.granularity,
|
||||||
|
"found_days": [item.isoformat() for item in found_days],
|
||||||
|
"missing_days": [item.isoformat() for item in missing_days],
|
||||||
|
"found_count": len(found_days),
|
||||||
|
"required_count": len(found_days) + len(missing_days),
|
||||||
"file_name": self.file_name,
|
"file_name": self.file_name,
|
||||||
"file_count": self.file_count,
|
"file_count": self.file_count,
|
||||||
"error": self.error,
|
"error": self.error,
|
||||||
|
"skipped": self.skipped,
|
||||||
|
"skip_reason": self.skip_reason,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -121,6 +127,7 @@ class AutoScheduler:
|
|||||||
"week_offset": scheduler.week_offset,
|
"week_offset": scheduler.week_offset,
|
||||||
"auto_delete_source": config.auto_delete_source,
|
"auto_delete_source": config.auto_delete_source,
|
||||||
"rj_data_enabled": rj_config.enabled,
|
"rj_data_enabled": rj_config.enabled,
|
||||||
|
"rj_directories": rj_config.weekly_directories,
|
||||||
"rj_weekly_directories": rj_config.weekly_directories,
|
"rj_weekly_directories": rj_config.weekly_directories,
|
||||||
"next_check_at": self._format_dt(self._next_check_at),
|
"next_check_at": self._format_dt(self._next_check_at),
|
||||||
"last_check_at": self._format_dt(self._last_check_at),
|
"last_check_at": self._format_dt(self._last_check_at),
|
||||||
@@ -178,14 +185,22 @@ class AutoScheduler:
|
|||||||
target_dates = self._target_dates_from_flag(ready_flag, scheduler)
|
target_dates = self._target_dates_from_flag(ready_flag, scheduler)
|
||||||
return self._trigger_processing(target_dates, ready_flag, manual)
|
return self._trigger_processing(target_dates, ready_flag, manual)
|
||||||
|
|
||||||
# 检查现有7天目录(4G/5G数据)
|
|
||||||
target_days = required_week_days(scheduler.week_offset)
|
target_days = required_week_days(scheduler.week_offset)
|
||||||
downloader = RemoteDataDownloader(remote_config)
|
downloader = RemoteDataDownloader(remote_config)
|
||||||
directory_status = self._check_remote_ready(downloader, scheduler, target_days)
|
rj_config = app_config.rj_data.normalized()
|
||||||
self._set_directory_status(directory_status)
|
rj_directories = set(rj_config.weekly_directories) if rj_config.enabled else set()
|
||||||
|
|
||||||
|
# 检查现有7天目录(4G/5G数据),RJ 目录单独按粒度判断,避免被普通7天规则误拦。
|
||||||
|
directory_status = self._check_remote_ready(
|
||||||
|
downloader,
|
||||||
|
scheduler,
|
||||||
|
target_days,
|
||||||
|
excluded_directories=rj_directories,
|
||||||
|
)
|
||||||
|
|
||||||
error_count = sum(1 for item in directory_status.values() if item.error)
|
error_count = sum(1 for item in directory_status.values() if item.error)
|
||||||
if error_count:
|
if error_count:
|
||||||
|
self._set_combined_directory_status(directory_status, {})
|
||||||
return self._finish_check(
|
return self._finish_check(
|
||||||
"scan_failed",
|
"scan_failed",
|
||||||
f"远程目录扫描失败,{error_count}/{len(directory_status)} 个目录无法访问或扫描失败",
|
f"远程目录扫描失败,{error_count}/{len(directory_status)} 个目录无法访问或扫描失败",
|
||||||
@@ -196,29 +211,32 @@ class AutoScheduler:
|
|||||||
skipped_count = len(directory_status) - len(active_status)
|
skipped_count = len(directory_status) - len(active_status)
|
||||||
daily_ready = bool(active_status) and all(item.ready for item in active_status)
|
daily_ready = bool(active_status) and all(item.ready for item in active_status)
|
||||||
|
|
||||||
# 检查RJ周数据目录(复用同一个 downloader)
|
# 检查 RJ 数据目录,按最新文件自动判断日粒度或周粒度。
|
||||||
rj_weekly_status = self._check_rj_weekly_ready(downloader, scheduler.week_offset)
|
rj_status = self._check_rj_ready(downloader, target_days)
|
||||||
rj_weekly_ready = True # 默认就绪(如果没有配置RJ目录)
|
rj_error_count = sum(1 for item in rj_status.values() if item.error)
|
||||||
|
self._set_combined_directory_status(directory_status, rj_status)
|
||||||
|
if rj_error_count:
|
||||||
|
return self._finish_check(
|
||||||
|
"scan_failed",
|
||||||
|
f"RJ 远程目录扫描失败,{rj_error_count}/{len(rj_status)} 个目录无法访问或扫描失败",
|
||||||
|
manual,
|
||||||
|
)
|
||||||
|
|
||||||
|
active_rj_status = [item for item in rj_status.values() if not item.skipped]
|
||||||
|
rj_ready = all(item.ready for item in active_rj_status)
|
||||||
rj_status_text = ""
|
rj_status_text = ""
|
||||||
|
|
||||||
if rj_weekly_status:
|
if active_rj_status:
|
||||||
rj_ready_count = sum(1 for s in rj_weekly_status.values() if s.ready)
|
rj_ready_count = sum(1 for item in active_rj_status if item.ready)
|
||||||
rj_total = len(rj_weekly_status)
|
rj_total = len(active_rj_status)
|
||||||
rj_weekly_ready = rj_ready_count == rj_total
|
if not rj_ready:
|
||||||
|
rj_status_text = f",RJ 数据 {rj_ready_count}/{rj_total} 个有效目录就绪"
|
||||||
# 保存RJ状态到目录状态中
|
|
||||||
rj_status_dict = {f"rj_weekly:{name}": status.to_dict() for name, status in rj_weekly_status.items()}
|
|
||||||
with self._status_lock:
|
|
||||||
self._directory_status.update(rj_status_dict)
|
|
||||||
|
|
||||||
if not rj_weekly_ready:
|
|
||||||
rj_status_text = f",RJ周数据 {rj_ready_count}/{rj_total} 个目录就绪"
|
|
||||||
|
|
||||||
# 两个条件都满足才触发
|
# 两个条件都满足才触发
|
||||||
if daily_ready and rj_weekly_ready:
|
if daily_ready and rj_ready:
|
||||||
self._mark_ready(target_days, directory_status, rj_weekly_status)
|
self._mark_ready(target_days, directory_status, rj_status)
|
||||||
skipped_text = f",已跳过 {skipped_count} 个停推目录" if skipped_count else ""
|
skipped_text = f",已跳过 {skipped_count} 个停推目录" if skipped_count else ""
|
||||||
rj_text = ",RJ周数据已就绪" if rj_weekly_status else ""
|
rj_text = ",RJ 数据已就绪" if active_rj_status else ""
|
||||||
return self._finish_check(
|
return self._finish_check(
|
||||||
"marked_ready",
|
"marked_ready",
|
||||||
f"远程数据已满足目标周 7 天{skipped_text}{rj_text},已写入就绪标识,下次检查将自动处理",
|
f"远程数据已满足目标周 7 天{skipped_text}{rj_text},已写入就绪标识,下次检查将自动处理",
|
||||||
@@ -232,6 +250,13 @@ class AutoScheduler:
|
|||||||
manual,
|
manual,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if not active_status:
|
||||||
|
return self._finish_check(
|
||||||
|
"waiting",
|
||||||
|
f"远程普通日数据未发现有效目录,无法触发处理{rj_status_text}",
|
||||||
|
manual,
|
||||||
|
)
|
||||||
|
|
||||||
ready_count = sum(1 for item in active_status if item.ready)
|
ready_count = sum(1 for item in active_status if item.ready)
|
||||||
skipped_text = f",跳过 {skipped_count} 个停推目录" if skipped_count else ""
|
skipped_text = f",跳过 {skipped_count} 个停推目录" if skipped_count else ""
|
||||||
return self._finish_check(
|
return self._finish_check(
|
||||||
@@ -245,7 +270,9 @@ class AutoScheduler:
|
|||||||
downloader: RemoteDataDownloader,
|
downloader: RemoteDataDownloader,
|
||||||
scheduler: AutoSchedulerConfig,
|
scheduler: AutoSchedulerConfig,
|
||||||
target_days: list[date],
|
target_days: list[date],
|
||||||
|
excluded_directories: set[str] | None = None,
|
||||||
) -> dict[str, DirectoryReadyStatus]:
|
) -> dict[str, DirectoryReadyStatus]:
|
||||||
|
excluded = {self._normalize_directory_name(item) for item in (excluded_directories or set())}
|
||||||
expected_directories = scheduler.expected_directories
|
expected_directories = scheduler.expected_directories
|
||||||
if expected_directories:
|
if expected_directories:
|
||||||
return {
|
return {
|
||||||
@@ -255,6 +282,7 @@ class AutoScheduler:
|
|||||||
target_days,
|
target_days,
|
||||||
)
|
)
|
||||||
for directory in expected_directories
|
for directory in expected_directories
|
||||||
|
if not self._is_excluded_directory(directory, excluded)
|
||||||
}
|
}
|
||||||
|
|
||||||
files = self._safe_list_remote_zip_files(downloader, None)
|
files = self._safe_list_remote_zip_files(downloader, None)
|
||||||
@@ -270,11 +298,7 @@ class AutoScheduler:
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
grouped: dict[str, list[RemoteFileInfo]] = defaultdict(list)
|
if not files:
|
||||||
for remote_file in files:
|
|
||||||
grouped[remote_file.parent or "."].append(remote_file)
|
|
||||||
|
|
||||||
if not grouped:
|
|
||||||
return {
|
return {
|
||||||
".": DirectoryReadyStatus(
|
".": DirectoryReadyStatus(
|
||||||
directory=".",
|
directory=".",
|
||||||
@@ -286,6 +310,16 @@ class AutoScheduler:
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
grouped: dict[str, list[RemoteFileInfo]] = defaultdict(list)
|
||||||
|
for remote_file in files:
|
||||||
|
parent = remote_file.parent or "."
|
||||||
|
if self._is_excluded_directory(parent, excluded):
|
||||||
|
continue
|
||||||
|
grouped[parent].append(remote_file)
|
||||||
|
|
||||||
|
if not grouped:
|
||||||
|
return {}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
directory: self._directory_ready_status(directory, directory_files, target_days)
|
directory: self._directory_ready_status(directory, directory_files, target_days)
|
||||||
for directory, directory_files in sorted(grouped.items(), key=lambda item: item[0])
|
for directory, directory_files in sorted(grouped.items(), key=lambda item: item[0])
|
||||||
@@ -330,9 +364,11 @@ class AutoScheduler:
|
|||||||
|
|
||||||
required = set(target_days)
|
required = set(target_days)
|
||||||
found = {
|
found = {
|
||||||
file_date
|
target_day
|
||||||
for remote_file in files
|
for remote_file in files
|
||||||
if (file_date := extract_file_date(remote_file.name)) in required
|
if (date_range := parse_file_date_range(remote_file.name))
|
||||||
|
for target_day in date_range.covered_days()
|
||||||
|
if target_day in required
|
||||||
}
|
}
|
||||||
missing = [item for item in target_days if item not in found]
|
missing = [item for item in target_days if item not in found]
|
||||||
return DirectoryReadyStatus(
|
return DirectoryReadyStatus(
|
||||||
@@ -343,79 +379,108 @@ class AutoScheduler:
|
|||||||
file_count=len(files),
|
file_count=len(files),
|
||||||
)
|
)
|
||||||
|
|
||||||
def _check_rj_weekly_ready(
|
def _check_rj_ready(
|
||||||
self,
|
self,
|
||||||
downloader: RemoteDataDownloader,
|
downloader: RemoteDataDownloader,
|
||||||
week_offset: int,
|
target_days: list[date],
|
||||||
) -> dict[str, RJWeeklyDirectoryStatus]:
|
) -> dict[str, RJDirectoryReadyStatus]:
|
||||||
"""检查RJ周数据目录是否就绪"""
|
"""检查 RJ 数据目录是否就绪,自动识别目录最新文件是日粒度还是周粒度。"""
|
||||||
rj_config = state.current_config().rj_data.normalized()
|
rj_config = state.current_config().rj_data.normalized()
|
||||||
if not rj_config.enabled:
|
if not rj_config.enabled:
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
target_week_end = self._calculate_week_end_date(week_offset)
|
result: dict[str, RJDirectoryReadyStatus] = {}
|
||||||
result: dict[str, RJWeeklyDirectoryStatus] = {}
|
|
||||||
|
|
||||||
for directory in rj_config.weekly_directories:
|
for directory in rj_config.weekly_directories:
|
||||||
result[directory] = self._check_single_rj_directory(
|
result[directory] = self._check_single_rj_directory(downloader, directory, target_days)
|
||||||
downloader, directory, target_week_end
|
|
||||||
)
|
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
def _calculate_week_end_date(self, week_offset: int) -> date:
|
|
||||||
"""计算目标周的结束日期(周日)"""
|
|
||||||
today = date.today()
|
|
||||||
# 找到本周的周日
|
|
||||||
days_since_sunday = today.weekday() + 1 # weekday(): 0=周一, 6=周日
|
|
||||||
if days_since_sunday == 7:
|
|
||||||
days_since_sunday = 0
|
|
||||||
this_sunday = today - timedelta(days=days_since_sunday)
|
|
||||||
# 根据偏移计算目标周的周日
|
|
||||||
target_sunday = this_sunday + timedelta(weeks=week_offset)
|
|
||||||
return target_sunday
|
|
||||||
|
|
||||||
def _check_single_rj_directory(
|
def _check_single_rj_directory(
|
||||||
self,
|
self,
|
||||||
downloader: RemoteDataDownloader,
|
downloader: RemoteDataDownloader,
|
||||||
directory: str,
|
directory: str,
|
||||||
target_week_end: date,
|
target_days: list[date],
|
||||||
) -> RJWeeklyDirectoryStatus:
|
) -> RJDirectoryReadyStatus:
|
||||||
"""检查单个RJ目录是否包含目标周的文件"""
|
"""检查单个 RJ 目录是否包含目标周数据。"""
|
||||||
files = self._safe_list_remote_zip_files(downloader, directory)
|
files = self._safe_list_remote_zip_files(downloader, directory)
|
||||||
if files is None:
|
if files is None:
|
||||||
return RJWeeklyDirectoryStatus(
|
return RJDirectoryReadyStatus(
|
||||||
directory=directory,
|
directory=directory,
|
||||||
ready=False,
|
ready=False,
|
||||||
error="远程目录不存在或无法访问",
|
error="远程目录不存在或无法访问",
|
||||||
)
|
)
|
||||||
|
|
||||||
if not files:
|
if not files:
|
||||||
return RJWeeklyDirectoryStatus(
|
return RJDirectoryReadyStatus(
|
||||||
directory=directory,
|
directory=directory,
|
||||||
ready=False,
|
ready=True,
|
||||||
file_count=0,
|
file_count=0,
|
||||||
|
skipped=True,
|
||||||
|
skip_reason="目录为空,视为已停推并跳过",
|
||||||
)
|
)
|
||||||
|
|
||||||
# 查找匹配目标周的文件
|
parsed_files = [
|
||||||
target_end_str = target_week_end.strftime("%Y%m%d") + "0000"
|
(remote_file, date_range)
|
||||||
for remote_file in files:
|
for remote_file in files
|
||||||
match = RJ_WEEKLY_FILE_PATTERN.search(remote_file.name)
|
if (date_range := parse_file_date_range(remote_file.name))
|
||||||
if match:
|
]
|
||||||
file_end_date = match.group(2)
|
if not parsed_files:
|
||||||
if file_end_date == target_end_str:
|
return RJDirectoryReadyStatus(
|
||||||
return RJWeeklyDirectoryStatus(
|
directory=directory,
|
||||||
directory=directory,
|
ready=False,
|
||||||
ready=True,
|
found_days=[],
|
||||||
found=True,
|
missing_days=target_days,
|
||||||
file_name=remote_file.name,
|
file_count=len(files),
|
||||||
file_count=len(files),
|
error="目录中未找到可识别日期的 ZIP 文件",
|
||||||
)
|
)
|
||||||
|
|
||||||
return RJWeeklyDirectoryStatus(
|
latest_file, latest_range = max(
|
||||||
|
parsed_files,
|
||||||
|
key=lambda item: (item[1].start, item[1].end_exclusive, item[0].name),
|
||||||
|
)
|
||||||
|
granularity = "daily" if latest_range.span_days <= 1 else "weekly"
|
||||||
|
required = set(target_days)
|
||||||
|
|
||||||
|
found = {
|
||||||
|
target_day
|
||||||
|
for _, date_range in parsed_files
|
||||||
|
for target_day in date_range.covered_days()
|
||||||
|
if target_day in required
|
||||||
|
}
|
||||||
|
missing = [item for item in target_days if item not in found]
|
||||||
|
|
||||||
|
if granularity == "daily":
|
||||||
|
return RJDirectoryReadyStatus(
|
||||||
|
directory=directory,
|
||||||
|
ready=not missing,
|
||||||
|
granularity=granularity,
|
||||||
|
found_days=sorted(found),
|
||||||
|
missing_days=missing,
|
||||||
|
file_name=latest_file.name,
|
||||||
|
file_count=len(files),
|
||||||
|
)
|
||||||
|
|
||||||
|
for remote_file in files:
|
||||||
|
date_range = parse_file_date_range(remote_file.name)
|
||||||
|
if date_range and date_range.covers_all(required):
|
||||||
|
return RJDirectoryReadyStatus(
|
||||||
|
directory=directory,
|
||||||
|
ready=True,
|
||||||
|
granularity=granularity,
|
||||||
|
found_days=target_days,
|
||||||
|
missing_days=[],
|
||||||
|
file_name=remote_file.name,
|
||||||
|
file_count=len(files),
|
||||||
|
)
|
||||||
|
|
||||||
|
return RJDirectoryReadyStatus(
|
||||||
directory=directory,
|
directory=directory,
|
||||||
ready=False,
|
ready=False,
|
||||||
found=False,
|
granularity=granularity,
|
||||||
|
found_days=sorted(found),
|
||||||
|
missing_days=missing,
|
||||||
|
file_name=latest_file.name,
|
||||||
file_count=len(files),
|
file_count=len(files),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -457,7 +522,7 @@ class AutoScheduler:
|
|||||||
self,
|
self,
|
||||||
target_days: list[date],
|
target_days: list[date],
|
||||||
directory_status: dict[str, DirectoryReadyStatus],
|
directory_status: dict[str, DirectoryReadyStatus],
|
||||||
rj_weekly_status: dict[str, RJWeeklyDirectoryStatus] | None = None,
|
rj_status: dict[str, RJDirectoryReadyStatus] | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
READY_DIR.mkdir(parents=True, exist_ok=True)
|
READY_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
payload = {
|
payload = {
|
||||||
@@ -470,10 +535,14 @@ class AutoScheduler:
|
|||||||
for name, item in directory_status.items()
|
for name, item in directory_status.items()
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
if rj_weekly_status:
|
if rj_status:
|
||||||
|
payload["rj_directories"] = {
|
||||||
|
name: item.to_dict()
|
||||||
|
for name, item in rj_status.items()
|
||||||
|
}
|
||||||
payload["rj_weekly_directories"] = {
|
payload["rj_weekly_directories"] = {
|
||||||
name: item.to_dict()
|
name: item.to_dict()
|
||||||
for name, item in rj_weekly_status.items()
|
for name, item in rj_status.items()
|
||||||
}
|
}
|
||||||
READY_FLAG.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
READY_FLAG.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||||
|
|
||||||
@@ -546,12 +615,39 @@ class AutoScheduler:
|
|||||||
"status": self.get_status(),
|
"status": self.get_status(),
|
||||||
}
|
}
|
||||||
|
|
||||||
def _set_directory_status(self, directory_status: dict[str, DirectoryReadyStatus]) -> None:
|
def _set_combined_directory_status(
|
||||||
|
self,
|
||||||
|
directory_status: dict[str, DirectoryReadyStatus],
|
||||||
|
rj_status: dict[str, RJDirectoryReadyStatus],
|
||||||
|
) -> None:
|
||||||
with self._status_lock:
|
with self._status_lock:
|
||||||
self._directory_status = {
|
combined = {
|
||||||
name: item.to_dict()
|
name: item.to_dict()
|
||||||
for name, item in directory_status.items()
|
for name, item in directory_status.items()
|
||||||
}
|
}
|
||||||
|
combined.update(
|
||||||
|
{
|
||||||
|
f"rj:{name}": item.to_dict()
|
||||||
|
for name, item in rj_status.items()
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self._directory_status = combined
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _normalize_directory_name(directory: str) -> str:
|
||||||
|
normalized = str(directory or "").replace("\\", "/").strip().strip("/")
|
||||||
|
return "." if normalized in {"", "."} else normalized
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _is_excluded_directory(cls, directory: str, excluded: set[str]) -> bool:
|
||||||
|
if not excluded:
|
||||||
|
return False
|
||||||
|
normalized = cls._normalize_directory_name(directory)
|
||||||
|
return any(
|
||||||
|
normalized == item or normalized.startswith(f"{item}/")
|
||||||
|
for item in excluded
|
||||||
|
if item != "."
|
||||||
|
)
|
||||||
|
|
||||||
def _set_next_check(self, value: datetime) -> None:
|
def _set_next_check(self, value: datetime) -> None:
|
||||||
with self._status_lock:
|
with self._status_lock:
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ from pathlib import Path
|
|||||||
from typing import Callable, Iterable
|
from typing import Callable, Iterable
|
||||||
|
|
||||||
from app.config import RemoteDataConfig
|
from app.config import RemoteDataConfig
|
||||||
from app.utils.file_dates import extract_file_date, select_recent_items_by_directory
|
from app.utils.file_dates import parse_file_date_range, select_recent_items_by_directory
|
||||||
|
|
||||||
|
|
||||||
LogFn = Callable[[str], None]
|
LogFn = Callable[[str], None]
|
||||||
@@ -93,7 +93,13 @@ class RemoteDataDownloader:
|
|||||||
selected = [
|
selected = [
|
||||||
remote_file
|
remote_file
|
||||||
for remote_file in files
|
for remote_file in files
|
||||||
if extract_file_date(remote_file.name) in target_dates
|
if (
|
||||||
|
date_range := parse_file_date_range(remote_file.name)
|
||||||
|
) and (
|
||||||
|
date_range.covers_all(target_dates)
|
||||||
|
if date_range.span_days > 1
|
||||||
|
else date_range.covers_any(target_dates)
|
||||||
|
)
|
||||||
]
|
]
|
||||||
selected_files.extend(selected)
|
selected_files.extend(selected)
|
||||||
skipped_count = len(files) - len(selected)
|
skipped_count = len(files) - len(selected)
|
||||||
|
|||||||
+70
-4
@@ -23,18 +23,84 @@ class DirectoryDateSelection:
|
|||||||
start_date: date | None = None
|
start_date: date | None = None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class FileDateRange:
|
||||||
|
start: date
|
||||||
|
end_exclusive: date
|
||||||
|
timestamp_count: int
|
||||||
|
|
||||||
|
@property
|
||||||
|
def span_days(self) -> int:
|
||||||
|
return max((self.end_exclusive - self.start).days, 1)
|
||||||
|
|
||||||
|
def covered_days(self) -> list[date]:
|
||||||
|
return [self.start + timedelta(days=offset) for offset in range(self.span_days)]
|
||||||
|
|
||||||
|
def covers_any(self, target_days: set[date]) -> bool:
|
||||||
|
return any(day in target_days for day in self.covered_days())
|
||||||
|
|
||||||
|
def covers_all(self, target_days: set[date]) -> bool:
|
||||||
|
return target_days.issubset(set(self.covered_days()))
|
||||||
|
|
||||||
|
|
||||||
def extract_file_date(filename: str | Path) -> date | None:
|
def extract_file_date(filename: str | Path) -> date | None:
|
||||||
"""Extract the first timestamp date from a filename."""
|
"""Extract the first timestamp date from a filename."""
|
||||||
match = FILENAME_DATE_RE.search(Path(str(filename)).name)
|
date_range = parse_file_date_range(filename)
|
||||||
if not match:
|
return date_range.start if date_range else None
|
||||||
|
|
||||||
|
|
||||||
|
def parse_file_date_range(filename: str | Path) -> FileDateRange | None:
|
||||||
|
"""Extract the covered natural date range from a filename.
|
||||||
|
|
||||||
|
Files with one timestamp are treated as one-day data files. Files with two
|
||||||
|
timestamps use the second timestamp as an exclusive end when it is midnight,
|
||||||
|
matching names like xxx_202605110000_202605120000.zip.
|
||||||
|
"""
|
||||||
|
values = FILENAME_DATE_RE.findall(Path(str(filename)).name)
|
||||||
|
if not values:
|
||||||
return None
|
return None
|
||||||
return parse_file_timestamp(match.group(1))
|
|
||||||
|
start_dt = parse_file_timestamp_datetime(values[0])
|
||||||
|
if not start_dt:
|
||||||
|
return None
|
||||||
|
|
||||||
|
if len(values) == 1:
|
||||||
|
return FileDateRange(
|
||||||
|
start=start_dt.date(),
|
||||||
|
end_exclusive=start_dt.date() + timedelta(days=1),
|
||||||
|
timestamp_count=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
end_dt = parse_file_timestamp_datetime(values[1])
|
||||||
|
if not end_dt or end_dt <= start_dt:
|
||||||
|
return FileDateRange(
|
||||||
|
start=start_dt.date(),
|
||||||
|
end_exclusive=start_dt.date() + timedelta(days=1),
|
||||||
|
timestamp_count=len(values),
|
||||||
|
)
|
||||||
|
|
||||||
|
end_exclusive = end_dt.date()
|
||||||
|
if end_dt.time() != datetime.min.time():
|
||||||
|
end_exclusive += timedelta(days=1)
|
||||||
|
if end_exclusive <= start_dt.date():
|
||||||
|
end_exclusive = start_dt.date() + timedelta(days=1)
|
||||||
|
|
||||||
|
return FileDateRange(
|
||||||
|
start=start_dt.date(),
|
||||||
|
end_exclusive=end_exclusive,
|
||||||
|
timestamp_count=len(values),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def parse_file_timestamp(value: str) -> date | None:
|
def parse_file_timestamp(value: str) -> date | None:
|
||||||
|
parsed = parse_file_timestamp_datetime(value)
|
||||||
|
return parsed.date() if parsed else None
|
||||||
|
|
||||||
|
|
||||||
|
def parse_file_timestamp_datetime(value: str) -> datetime | None:
|
||||||
fmt = "%Y%m%d%H%M%S" if len(value) == 14 else "%Y%m%d%H%M"
|
fmt = "%Y%m%d%H%M%S" if len(value) == 14 else "%Y%m%d%H%M"
|
||||||
try:
|
try:
|
||||||
return datetime.strptime(value, fmt).date()
|
return datetime.strptime(value, fmt)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|||||||
+14
-15
@@ -21,25 +21,22 @@
|
|||||||
|
|
||||||
| 格式 | 示例 | 数据日期 |
|
| 格式 | 示例 | 数据日期 |
|
||||||
|------|------|----------|
|
|------|------|----------|
|
||||||
| `XXX_YYYYMMDDHHMM_YYYYMMDDHHMM` | `CapacityReportData2.6_202605110000_202605120000.zip` | 第一个时间戳 `202605110000` → 2026-05-11 |
|
| `XXX_YYYYMMDDHHMM_YYYYMMDDHHMM` | `CapacityReportData2.6_202605110000_202605120000.zip` | 覆盖起止时间范围,结束零点按右开区间处理 → 2026-05-11 |
|
||||||
| `XXX_YYYYMMDDHHMM` | `CapacityReportData2.6_202605110000.zip` | 时间戳 `202605110000` → 2026-05-11 |
|
| `XXX_YYYYMMDDHHMM` | `CapacityReportData2.6_202605110000.zip` | 视为单日文件 → 2026-05-11 |
|
||||||
|
|
||||||
### 2.2 解析逻辑
|
### 2.2 解析逻辑
|
||||||
|
|
||||||
复用项目已有的正则 `_ZIP_DATE_RE = re.compile(r"(?<!\d)(20\d{10}(?:\d{2})?)(?!\d)")`,取第一个匹配项作为数据日期。
|
复用项目已有的正则 `_ZIP_DATE_RE = re.compile(r"(?<!\d)(20\d{10}(?:\d{2})?)(?!\d)")`,优先解析文件覆盖的自然日范围。
|
||||||
|
|
||||||
```python
|
```python
|
||||||
def extract_file_date(filename: str) -> date | None:
|
def parse_file_date_range(filename: str) -> FileDateRange | None:
|
||||||
"""从文件名提取数据日期(取第一个匹配的时间戳)"""
|
"""从文件名提取数据覆盖范围。"""
|
||||||
match = _ZIP_DATE_RE.search(filename)
|
values = _ZIP_DATE_RE.findall(filename)
|
||||||
if not match:
|
if not values:
|
||||||
return None
|
|
||||||
timestamp = match.group(1) # 12位: YYYYMMDDHHMM 或 14位: YYYYMMDDHHMMSS
|
|
||||||
fmt = "%Y%m%d%H%M%S" if len(timestamp) == 14 else "%Y%m%d%H%M"
|
|
||||||
try:
|
|
||||||
return datetime.strptime(timestamp, fmt).date()
|
|
||||||
except ValueError:
|
|
||||||
return None
|
return None
|
||||||
|
start = parse_timestamp(values[0])
|
||||||
|
end = parse_timestamp(values[1]) if len(values) > 1 else start + timedelta(days=1)
|
||||||
|
return FileDateRange(start=start.date(), end_exclusive=normalize_end(end))
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -130,10 +127,12 @@ def get_target_week_range(week_offset: int = 0) -> tuple[date, date]:
|
|||||||
|
|
||||||
**关键**:
|
**关键**:
|
||||||
1. 不依赖系统日期,而是从文件名中提取日期
|
1. 不依赖系统日期,而是从文件名中提取日期
|
||||||
2. 文件名只取**第一个**时间戳作为数据日期
|
2. 文件名带两个时间戳时按覆盖范围判断;只有一个时间戳时视为单日文件
|
||||||
3. 例如文件 `CapacityReportData2.6_202605120000_202605130000.zip` → 数据日期为 2026-05-12
|
3. 例如文件 `CapacityReportData2.6_202605120000_202605130000.zip` → 覆盖 2026-05-12
|
||||||
4. 如果某个日期没有对应的 ZIP 文件,该目录判定为未就绪
|
4. 如果某个日期没有对应的 ZIP 文件,该目录判定为未就绪
|
||||||
|
|
||||||
|
RJ 目录会额外按最新文件自动判断粒度:最新文件为单日时要求 7 个目标日都齐全;最新文件为多日/周文件时要求存在一个 ZIP 覆盖完整目标自然周。
|
||||||
|
|
||||||
### 3.4 标识文件
|
### 3.4 标识文件
|
||||||
|
|
||||||
- **位置**:`cache/auto_scheduler/ready.flag`
|
- **位置**:`cache/auto_scheduler/ready.flag`
|
||||||
|
|||||||
@@ -1,5 +1,15 @@
|
|||||||
# 项目上下文记录
|
# 项目上下文记录
|
||||||
|
|
||||||
|
## 2026-06-01:完善 RJ 自动调度日/周粒度识别
|
||||||
|
|
||||||
|
- `app/utils/file_dates.py` 新增文件日期范围解析:`XXX_YYYYMMDDHHMM` 视为单日文件,`XXX_YYYYMMDDHHMM_YYYYMMDDHHMM` 按起止时间展开自然日,结束时间为零点时按右开区间处理。
|
||||||
|
- `app/services/auto_scheduler.py` 的 RJ 检查改为按目录最新 ZIP 自动识别 `daily` 或 `weekly`:日粒度目录要求目标自然周 7 天都存在,周粒度目录要求有一个 ZIP 覆盖目标自然周;空 RJ 目录继续视为停推并跳过。
|
||||||
|
- 自动调度普通 4G/5G 目录扫描会排除已配置的 RJ 目录,避免 `expected_directories=[]` 时 RJ 周目录被普通 7 天规则误判阻塞。
|
||||||
|
- `app/services/remote_download.py` 的调度下载筛选改为使用日期覆盖范围:单日文件只要覆盖目标日即下载,多日/周文件必须覆盖完整目标周才下载,避免 ready 后漏下或误下 RJ 周文件。
|
||||||
|
- `Configure.json` 将 `RJ/700M/700RJGD`、`RJ/700M/700RJYD` 加入 RJ 数据目录,并补充 `700MRJGD`、`700MRJYD` 字段映射;`processor.py` 避免多个源字段别名映射到同一目标字段时生成重复列。
|
||||||
|
- 配置上传接口现在会导入/保存 `RJData`,前端类型补充 `rj_data` 和调度状态中的 `granularity` 字段。
|
||||||
|
- 已验证:后端 AST 语法检查、`Configure.json` JSON 解析、`npm run build`、真实 SFTP 清单识别、MySQL 临时导入 700RJYD 样本并清理测试表均通过;前端构建仅保留既有大 chunk 警告。
|
||||||
|
|
||||||
## 2026-05-29:新增 RJ 周数据处理功能
|
## 2026-05-29:新增 RJ 周数据处理功能
|
||||||
|
|
||||||
- 新增 `RJData` 配置块到 `app/config.py`,支持 `enabled`、`weekly_directories` 和 `table_field_mappings` 配置项,用于管理 RJ 周数据目录和字段映射。
|
- 新增 `RJData` 配置块到 `app/config.py`,支持 `enabled`、`weekly_directories` 和 `table_field_mappings` 配置项,用于管理 RJ 周数据目录和字段映射。
|
||||||
|
|||||||
@@ -79,10 +79,17 @@ export interface AppConfig {
|
|||||||
};
|
};
|
||||||
remote_data: RemoteDataConfig;
|
remote_data: RemoteDataConfig;
|
||||||
history_retention: HistoryRetentionConfig;
|
history_retention: HistoryRetentionConfig;
|
||||||
|
rj_data?: RJDataConfig;
|
||||||
sheet_filter: string[];
|
sheet_filter: string[];
|
||||||
extract_fields: Array<Record<string, unknown>>;
|
extract_fields: Array<Record<string, unknown>>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface RJDataConfig {
|
||||||
|
enabled: boolean;
|
||||||
|
weekly_directories: string[];
|
||||||
|
table_field_mappings: Record<string, Array<Record<string, unknown>>>;
|
||||||
|
}
|
||||||
|
|
||||||
export interface RemoteDataConfig {
|
export interface RemoteDataConfig {
|
||||||
enabled: boolean;
|
enabled: boolean;
|
||||||
protocol: 'ftp' | 'sftp';
|
protocol: 'ftp' | 'sftp';
|
||||||
@@ -134,6 +141,7 @@ export interface RemoteSchedulerStatus {
|
|||||||
string,
|
string,
|
||||||
{
|
{
|
||||||
ready: boolean;
|
ready: boolean;
|
||||||
|
granularity?: 'daily' | 'weekly' | string | null;
|
||||||
found_days: string[];
|
found_days: string[];
|
||||||
missing_days: string[];
|
missing_days: string[];
|
||||||
found_count: number;
|
found_count: number;
|
||||||
|
|||||||
Reference in New Issue
Block a user