Metrix/InterferenceETL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 自适应等待干扰数据并归档结果

Browse Source
This commit is contained in:
nixevol
2026-08-06 17:02:20 +08:00
parent 1dad8b3465
commit 0c85dd93c2
4 changed files with 521 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_pipeline.py
+184 -25
View File
@@ -15,32 +15,36 @@ from openpyxl import Workbook
COMPLETE_WINDOW = "2026073110001100"
INCOMPLETE_WINDOW = "2026073111001200"
TARGET_KEY = "20260731100000"
class PipelineTest(unittest.TestCase):
def test_latest_complete_window_is_converted_and_merged(self) -> None:
def test_latest_group_is_converted_merged_stored_and_archived(self) -> None:
with tempfile.TemporaryDirectory() as temp:
root = Path(temp) / "source"
output = Path(temp) / "output"
for source_type in main.EXPECTED_TYPES:
create_archive(root, source_type, COMPLETE_WINDOW)
create_archive(root, main.EXPECTED_TYPES[0], INCOMPLETE_WINDOW)
windows = ["2026073110001100", "2026073110141100", "2026073110051100"]
for index, source_type in enumerate(main.EXPECTED_TYPES):
create_archive(root, source_type, windows[index % len(windows)], middle="任意粒度")
create_archive(root, main.EXPECTED_TYPES[0], "2026073110091100", middle="更新版本")
create_cell_data_sources(root)
store = RecordingStore()
history = RecordingHistory()
result = main.process(main.LocalSource(root), output, lookback_days=3, store=store)
result = main.process(main.LocalSource(root), output, lookback_days=3, store=store, history=history)
self.assertEqual(result.name, COMPLETE_WINDOW)
self.assertIsNotNone(result)
assert result is not None
self.assertEqual(result.name, TARGET_KEY)
converted = sorted((result / "converted").glob("*.csv"))
self.assertEqual(len(converted), 7)
with (result / f"interference_summary_{COMPLETE_WINDOW}.csv").open(encoding="utf-8-sig", newline="") as file:
with (result / f"interference_summary_{TARGET_KEY}.csv").open(encoding="utf-8-sig", newline="") as file:
rows = list(csv.DictReader(file))
self.assertEqual(len(rows), 7)
self.assertEqual(
list(rows[0]),
[
"hour_start",
"hour_end",
"metric_time",
"network_type",
"cgi",
"cell_name",
@@ -55,6 +59,7 @@ class PipelineTest(unittest.TestCase):
self.assertEqual(rows_by_type["5G干扰监控"]["network_type"], "2.6G")
self.assertEqual(rows_by_type["700M干扰监控"]["network_type"], "700M")
self.assertEqual(rows_by_type["SDR_FDD干扰监控"]["network_type"], "4G")
self.assertTrue(all(row["metric_time"] == "2026-07-31 10:00:00" for row in rows))
self.assertEqual(rows_by_type["5G干扰监控"]["cgi"], "460-00-200-1")
self.assertEqual(rows_by_type["700M干扰监控"]["cgi"], "460-00-200-1")
for source_type in set(main.EXPECTED_TYPES) - {"5G干扰监控", "700M干扰监控"}:
@@ -76,12 +81,18 @@ class PipelineTest(unittest.TestCase):
self.assertEqual(manifest["threshold_filtered_rows"], 0)
self.assertEqual(manifest["coordinate_matched_rows"], 2)
self.assertEqual(manifest["coordinate_unmatched_rows"], 5)
self.assertEqual(len(manifest["warnings"]), 1)
self.assertIn(INCOMPLETE_WINDOW, manifest["warnings"][0])
self.assertEqual(manifest["metric_time"], "2026-07-31 10:00:00")
self.assertNotIn("hour_start", manifest)
self.assertNotIn("hour_end", manifest)
self.assertNotIn("source_types", manifest)
self.assertTrue(all("source_type" not in item and "source_path" not in item for item in manifest["files"]))
self.assertIn("2026073110091100", [item["source_window"] for item in manifest["files"]])
self.assertEqual(manifest["database"]["metric_time"], "2026-07-31 10:00:00")
self.assertEqual(len(store.rows), 7)
self.assertEqual(history.path, "/history/2026-07-31/干扰数据处理结果_20260731100000.csv")
archived_rows = list(csv.DictReader(io.StringIO(history.payload.decode("utf-8-sig"))))
self.assertEqual(list(archived_rows[0]), list(main.SUMMARY_HEADER))
self.assertEqual(len(archived_rows), 7)
def test_interference_thresholds_remove_only_lower_values(self) -> None:
for network_type, threshold in (("2.6G", "-107"), ("700M", "-110"), ("4G", "-110")):
@@ -105,13 +116,30 @@ class PipelineTest(unittest.TestCase):
self.assertEqual(manifest["summary_rows"], 6)
self.assertEqual(manifest["threshold_filtered_rows"], 1)
def test_requested_incomplete_window_is_rejected(self) -> None:
def test_latest_incomplete_group_waits_without_falling_back(self) -> None:
with tempfile.TemporaryDirectory() as temp:
root = Path(temp) / "source"
for source_type in main.EXPECTED_TYPES:
create_archive(root, source_type, COMPLETE_WINDOW)
create_archive(root, main.EXPECTED_TYPES[0], INCOMPLETE_WINDOW)
store = RecordingStore()
with self.assertRaisesRegex(main.ProcessingError, "Requested window is incomplete"):
main.process(main.LocalSource(root), Path(temp) / "output", 3, INCOMPLETE_WINDOW)
result = main.process(main.LocalSource(root), Path(temp) / "output", 3, store=store)
self.assertIsNone(result)
self.assertEqual(store.latest_calls, 0)
self.assertEqual(store.rows, [])
def test_no_source_files_waits_successfully(self) -> None:
with tempfile.TemporaryDirectory() as temp:
root = Path(temp) / "source"
root.mkdir()
store = RecordingStore()
result = main.process(main.LocalSource(root), Path(temp) / "output", 3, store=store)
self.assertIsNone(result)
self.assertEqual(store.latest_calls, 0)
def test_schema_change_is_rejected(self) -> None:
with tempfile.TemporaryDirectory() as temp:
@@ -122,12 +150,34 @@ class PipelineTest(unittest.TestCase):
with self.assertRaisesRegex(main.ProcessingError, "Unexpected Sheet0 header"):
main.process(main.LocalSource(root), Path(temp) / "output", 3)
def test_cross_midnight_window(self) -> None:
self.assertEqual(
main.window_bounds("2026073023000000"),
("2026-07-30 23:00:00", "2026-07-31 00:00:00"),
def test_workbook_row_outside_selected_quarter_is_rejected(self) -> None:
with tempfile.TemporaryDirectory() as temp:
root = Path(temp) / "source"
for source_type in main.EXPECTED_TYPES:
create_archive(
root,
source_type,
COMPLETE_WINDOW,
row_window="2026073110151100" if source_type == main.EXPECTED_TYPES[0] else "",
)
create_cell_data_sources(root)
with self.assertRaisesRegex(main.ProcessingError, "is outside metric group"):
main.process(main.LocalSource(root), Path(temp) / "output", 3)
def test_file_name_middle_is_flexible_and_time_uses_natural_quarter(self) -> None:
candidate = main.parse_candidate(
"/source/700M干扰监控_任意描述_2026073112141300.zip",
123,
)
self.assertIsNotNone(candidate)
assert candidate is not None
self.assertEqual(candidate.source_type, "700M干扰监控")
self.assertEqual(candidate.size, 123)
self.assertEqual(main.metric_time_for_window(candidate.window), datetime(2026, 7, 31, 12, 0))
self.assertEqual(main.metric_time_for_window("2026073112151300"), datetime(2026, 7, 31, 12, 15))
def test_latest_dated_cell_data_file_is_selected(self) -> None:
entries = [
{"name": "江门5G小区信息表20260727.xlsx", "path": "/old.xlsx", "is_dir": False},
@@ -222,22 +272,93 @@ class PipelineTest(unittest.TestCase):
with self.assertRaisesRegex(main.ProcessingError, "Invalid decimal value for interference_dbm"):
store.replace_latest([row])
def test_api_store_exports_normalized_rows_for_history(self) -> None:
client = FakeApiClient(None)
store = main.ApiSummaryStore(client, "db_share_mysql")
rows = store.rows_for_time(datetime(2026, 7, 31, 10, 0))
self.assertEqual(rows, [database_row()])
export_query = next(
payload for endpoint, payload in client.posts
if endpoint.endswith("/query") and str(payload["sql"]).startswith("SELECT metric_time")
)
self.assertEqual(export_query["page_size"], 1000)
def test_existing_database_time_repairs_missing_history_without_reprocessing(self) -> None:
with tempfile.TemporaryDirectory() as temp:
root = Path(temp) / "source"
for source_type in main.EXPECTED_TYPES:
create_archive(root, source_type, COMPLETE_WINDOW)
stored_row = database_row()
store = RecordingStore(datetime(2026, 7, 31, 10, 0), [stored_row])
history = RecordingHistory(exists=False)
result = main.process(main.LocalSource(root), Path(temp) / "output", 3, store=store, history=history)
self.assertIsNone(result)
self.assertEqual(store.rows_for_time_calls, 1)
self.assertEqual(history.path, "/history/2026-07-31/干扰数据处理结果_20260731100000.csv")
archived_rows = list(csv.DictReader(io.StringIO(history.payload.decode("utf-8-sig"))))
self.assertEqual(archived_rows, [stored_row])
def test_api_history_store_creates_date_directory_and_uploads_csv(self) -> None:
client = FakeHistoryApiClient()
history = main.ApiHistoryStore(client, "stg_test", "/history")
metric_time = datetime(2026, 8, 6, 12, 0)
self.assertFalse(history.exists(metric_time))
path = history.upload(metric_time, b"csv-data")
self.assertEqual(path, "/history/2026-08-06/干扰数据处理结果_20260806120000.csv")
self.assertEqual(client.mkdir_paths, ["/history/2026-08-06"])
self.assertEqual(client.uploads, [("/history/2026-08-06", "干扰数据处理结果_20260806120000.csv", b"csv-data")])
class RecordingStore:
def __init__(self) -> None:
def __init__(self, latest_time: datetime | None = None, stored_rows: list[dict[str, str]] | None = None) -> None:
self.rows: list[dict[str, str]] = []
self.latest_time = latest_time
self.stored_rows = stored_rows or []
self.latest_calls = 0
self.rows_for_time_calls = 0
def latest_metric_time(self) -> datetime | None:
self.latest_calls += 1
return self.latest_time
def replace_latest(self, rows: list[dict[str, str]]) -> dict[str, object]:
self.rows = list(rows)
return {
"enabled": True,
"table": main.DATABASE_TABLE,
"metric_time": rows[0]["hour_start"],
"metric_time": rows[0]["metric_time"],
"inserted_rows": len(rows),
"refreshed_rows": 0,
"old_rows_deleted": 0,
}
def rows_for_time(self, metric_time: datetime) -> list[dict[str, str]]:
self.rows_for_time_calls += 1
self.asserted_metric_time = metric_time
return list(self.stored_rows)
class RecordingHistory:
def __init__(self, exists: bool = False) -> None:
self.exists_value = exists
self.path = ""
self.payload = b""
def exists(self, metric_time: datetime) -> bool:
self.checked_metric_time = metric_time
return self.exists_value
def upload(self, metric_time: datetime, payload: bytes) -> str:
self.payload = payload
self.path = f"/history/{metric_time:%Y-%m-%d}/干扰数据处理结果_{metric_time:%Y%m%d%H%M%S}.csv"
return self.path
class FakeApiClient:
def __init__(self, latest_time: str | None, fail_script: bool = False) -> None:
@@ -258,8 +379,13 @@ class FakeApiClient:
del timeout
self.posts.append((endpoint, payload))
if endpoint.endswith("/query"):
if str(payload["sql"]).lstrip().startswith("SELECT MAX"):
sql = str(payload["sql"]).lstrip()
if sql.startswith("SELECT MAX"):
return {"rows": [{"latest_time": self.latest_time}]}
if sql.startswith("SELECT metric_time"):
row = database_row()
row["metric_time"] = "2026-07-31T10:00:00"
return {"rows": [row], "total": 1}
return {"affected_rows": 0}
if self.fail_script:
return {
@@ -282,10 +408,41 @@ class FakeApiClient:
}
class FakeHistoryApiClient:
def __init__(self) -> None:
self.mkdir_paths: list[str] = []
self.uploads: list[tuple[str, str, bytes]] = []
def get_json(self, endpoint: str, query: dict[str, object] | None = None, timeout: int = 30) -> object:
del endpoint, timeout
path = str((query or {}).get("path") or "")
if path == "/history":
return {"entries": []}
return {"entries": []}
def post_json(self, endpoint: str, payload: dict[str, object], timeout: int = 30) -> object:
del endpoint, timeout
path = str(payload["path"])
self.mkdir_paths.append(path)
return {"path": path}
def post_file(
self,
endpoint: str,
query: dict[str, object],
filename: str,
payload: bytes,
timeout: int = 120,
) -> object:
del endpoint, timeout
directory = str(query["path"])
self.uploads.append((directory, filename, payload))
return {"path": f"{directory}/{filename}"}
def database_row() -> dict[str, str]:
return {
"hour_start": "2026-07-31 10:00:00",
"hour_end": "2026-07-31 11:00:00",
"metric_time": "2026-07-31 10:00:00",
"network_type": "2.6G",
"cgi": "460-00-200-1",
"cell_name": "测试小区",
@@ -311,10 +468,12 @@ def create_archive(
window: str,
bad_header: bool = False,
interference_dbm: float = -100.5,
middle: str = "LWP_每小时_过滤110",
row_window: str = "",
) -> None:
date_dir = root / f"{window[:4]}-{window[4:6]}-{window[6:8]}"
date_dir.mkdir(parents=True, exist_ok=True)
filename = f"{source_type}_LWP_每小时_过滤110_{window}"
filename = f"{source_type}_{middle}_{window}"
workbook = Workbook()
sheet = workbook.active
sheet.title = "Sheet0"
@@ -322,7 +481,7 @@ def create_archive(
if bad_header:
header[-1] = "unexpected"
sheet.append(header)
sheet.append(mock_row(source_type, window, interference_dbm))
sheet.append(mock_row(source_type, row_window or window, interference_dbm))
metadata = workbook.create_sheet("指标(计数器)")
metadata.append(["指标或计数器", "指标或计数器描述", "指标公式", "指标或计数器状态"])
content = io.BytesIO()