777 lines
31 KiB
Python
777 lines
31 KiB
Python
from __future__ import annotations
|
||
|
||
import argparse
|
||
import csv
|
||
import hashlib
|
||
import io
|
||
import json
|
||
import os
|
||
from dataclasses import dataclass
|
||
from datetime import datetime, timedelta, timezone
|
||
from pathlib import Path
|
||
import posixpath
|
||
import re
|
||
import shutil
|
||
import sys
|
||
import time
|
||
from typing import Callable, Protocol
|
||
from urllib.error import HTTPError, URLError
|
||
from urllib.parse import quote, urlencode
|
||
from urllib.request import Request, urlopen
|
||
import warnings
|
||
import zipfile
|
||
|
||
|
||
VENDOR_DIR = Path(__file__).resolve().parent / "vendor"
|
||
sys.path.insert(0, str(VENDOR_DIR))
|
||
|
||
from openpyxl import load_workbook
|
||
|
||
|
||
EXPECTED_TYPES = (
|
||
"5G下FDD干扰监控",
|
||
"5G干扰监控",
|
||
"700M下FDD干扰监控",
|
||
"700M干扰监控",
|
||
"SDR_FDD干扰监控",
|
||
"SDR_TDD干扰监控",
|
||
"反开RD干扰监控",
|
||
)
|
||
|
||
DEFAULT_STORAGE_ID = "stg_4d9a910d72"
|
||
DEFAULT_ROOT = "/网优日常优化数据文档/(勿删)干扰定时小时指标"
|
||
CELL_DATA_DIRECTORIES = (
|
||
"/网优日常优化数据文档/日常性能报表/2026年/5G/5G小区信息表",
|
||
"/网优日常优化数据文档/日常性能报表/2026年/700M/700M小区信息表",
|
||
"/网优日常优化数据文档/日常性能报表/2026年/5G_反开/反开小区信息表",
|
||
"/网优日常优化数据文档/日常性能报表/2026年/TDD_LTE/LTE基础信息数据/LTE_小区信息表",
|
||
"/网优日常优化数据文档/日常性能报表/2026年/FDD_LTE/FDD基础信息数据/FDD小区信息表",
|
||
)
|
||
FILE_RE = re.compile(r"^(?P<source_type>.+)_LWP_每小时_过滤110_(?P<window>\d{16})\.zip$", re.IGNORECASE)
|
||
CELL_DATA_FILE_RE = re.compile(r"(?P<date>20\d{6})\.xlsx$", re.IGNORECASE)
|
||
|
||
LTE_PLMN = "460-00"
|
||
DATABASE_HOST = "172.17.0.1"
|
||
DATABASE_PORT = 3306
|
||
DATABASE_USER = "root"
|
||
DATABASE_PASSWORD = "OSp!jmgm@26"
|
||
DATABASE_NAME = "interference_etl"
|
||
DATABASE_TABLE = "interference_hourly_summary"
|
||
|
||
HEADER_FDD = (
|
||
"开始时间",
|
||
"粒度",
|
||
"子网ID",
|
||
"子网名称",
|
||
"网元ID",
|
||
"管理网元",
|
||
"eNodeB CUID",
|
||
"eNodeB CU名称",
|
||
"LTEID",
|
||
"LTE名称",
|
||
"E-UTRAN FDD小区ID",
|
||
"E-UTRAN FDD小区名称",
|
||
"cellId",
|
||
"eNodeBId",
|
||
"载波平均噪声干扰(dBm)",
|
||
"集团-上下行总业务量（GB）",
|
||
"RRC连接建立最大用户数",
|
||
)
|
||
|
||
HEADER_NR = (
|
||
"开始时间",
|
||
"粒度",
|
||
"子网ID",
|
||
"子网名称",
|
||
"网元ID",
|
||
"管理网元",
|
||
"gNB CU-CP功能配置ID",
|
||
"gNB CU-CP功能配置名称",
|
||
"CU小区配置ID",
|
||
"CU小区配置名称",
|
||
"cellId",
|
||
"duMeMoId",
|
||
"gNBId",
|
||
"gNBIdLength",
|
||
"gNBplmn",
|
||
"masterOperatorId",
|
||
"nrCarrierGroupId",
|
||
"nrPhysicalCellDUId",
|
||
"小区上行平均干扰电平(dBm)",
|
||
"5G上下行总流量（上行PDCP PDU数据量+下行PDCP成功发送数据量）(GB)",
|
||
"RRC连接最大连接用户数",
|
||
)
|
||
|
||
HEADER_SDR = (
|
||
"开始时间",
|
||
"粒度",
|
||
"子网ID",
|
||
"子网名称",
|
||
"网元ID",
|
||
"管理网元",
|
||
"eNodeBID",
|
||
"eNodeB名称",
|
||
"小区ID",
|
||
"小区名称",
|
||
"载波平均噪声干扰(dBm)",
|
||
"集团-上下行总业务量（GB）",
|
||
"RRC连接建立最大用户数",
|
||
)
|
||
|
||
HEADER_RD = (
|
||
"开始时间",
|
||
"粒度",
|
||
"子网ID",
|
||
"子网名称",
|
||
"网元ID",
|
||
"管理网元",
|
||
"eNodeB CUID",
|
||
"eNodeB CU名称",
|
||
"LTEID",
|
||
"LTE名称",
|
||
"E-UTRAN TDD小区ID",
|
||
"E-UTRAN TDD小区名称",
|
||
"cellId",
|
||
"eNodeBId",
|
||
"载波平均噪声干扰(dBm)",
|
||
"上下行总业务量（GB）",
|
||
"RRC连接建立最大用户数",
|
||
)
|
||
|
||
EXPECTED_HEADERS = {
|
||
"5G下FDD干扰监控": HEADER_FDD,
|
||
"5G干扰监控": HEADER_NR,
|
||
"700M下FDD干扰监控": HEADER_FDD,
|
||
"700M干扰监控": HEADER_NR,
|
||
"SDR_FDD干扰监控": HEADER_SDR,
|
||
"SDR_TDD干扰监控": HEADER_SDR,
|
||
"反开RD干扰监控": HEADER_RD,
|
||
}
|
||
|
||
SUMMARY_HEADER = (
|
||
"hour_start",
|
||
"hour_end",
|
||
"cgi",
|
||
"cell_name",
|
||
"interference_dbm",
|
||
"longitude",
|
||
"latitude",
|
||
)
|
||
|
||
class ProcessingError(RuntimeError):
|
||
pass
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Candidate:
|
||
source_type: str
|
||
window: str
|
||
path: str
|
||
size: int = 0
|
||
|
||
|
||
class Source(Protocol):
|
||
def candidates(self, lookback_days: int) -> list[Candidate]: ...
|
||
|
||
def download(self, path: str) -> bytes: ...
|
||
|
||
def cell_data_workbooks(self) -> list[tuple[str, bytes]]: ...
|
||
|
||
|
||
class SummaryStore(Protocol):
|
||
def replace_latest(self, rows: list[dict[str, str]]) -> dict[str, object]: ...
|
||
|
||
|
||
class MySQLSummaryStore:
|
||
def __init__(self, connect_factory: Callable[[], object] | None = None) -> None:
|
||
self.table = DATABASE_TABLE
|
||
self.connect_factory = connect_factory
|
||
|
||
def replace_latest(self, rows: list[dict[str, str]]) -> dict[str, object]:
|
||
if not rows:
|
||
raise ProcessingError("Refusing to replace database data with an empty batch")
|
||
metric_times = {datetime.strptime(row["hour_start"], "%Y-%m-%d %H:%M:%S") for row in rows}
|
||
if len(metric_times) != 1:
|
||
raise ProcessingError("Database batch must contain exactly one metric hour")
|
||
metric_time = next(iter(metric_times))
|
||
table = f"`{self.table}`"
|
||
connection = self._connect()
|
||
cursor = None
|
||
try:
|
||
cursor = connection.cursor()
|
||
cursor.execute(
|
||
f"""
|
||
CREATE TABLE IF NOT EXISTS {table} (
|
||
metric_time DATETIME NOT NULL COMMENT '指标开始时间',
|
||
cgi VARCHAR(128) NOT NULL,
|
||
cell_name VARCHAR(255) NOT NULL,
|
||
interference_dbm DECIMAL(10,3) NOT NULL,
|
||
longitude DECIMAL(10,6) NULL,
|
||
latitude DECIMAL(10,6) NULL,
|
||
PRIMARY KEY (metric_time, cgi)
|
||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||
"""
|
||
)
|
||
cursor.execute(f"SHOW COLUMNS FROM {table}")
|
||
existing_columns = {column[0] for column in cursor.fetchall()}
|
||
for column in ("longitude", "latitude"):
|
||
if column not in existing_columns:
|
||
cursor.execute(f"ALTER TABLE {table} ADD COLUMN `{column}` DECIMAL(10,6) NULL")
|
||
cursor.execute(f"SELECT MAX(metric_time) FROM {table}")
|
||
latest_row = cursor.fetchone()
|
||
latest_time = latest_row[0] if latest_row else None
|
||
if isinstance(latest_time, str):
|
||
latest_time = datetime.fromisoformat(latest_time)
|
||
if latest_time is not None and latest_time > metric_time:
|
||
raise ProcessingError(
|
||
f"Database already contains newer metric time {latest_time:%Y-%m-%d %H:%M:%S}; "
|
||
f"refusing to replace it with {metric_time:%Y-%m-%d %H:%M:%S}"
|
||
)
|
||
|
||
cursor.execute(f"DELETE FROM {table} WHERE metric_time = %s", (metric_time,))
|
||
refreshed_rows = cursor.rowcount
|
||
cursor.executemany(
|
||
f"""
|
||
INSERT INTO {table}
|
||
(metric_time, cgi, cell_name, interference_dbm, longitude, latitude)
|
||
VALUES (%s, %s, %s, %s, %s, %s)
|
||
""",
|
||
[
|
||
(
|
||
metric_time,
|
||
row["cgi"],
|
||
row["cell_name"],
|
||
row["interference_dbm"],
|
||
row["longitude"] or None,
|
||
row["latitude"] or None,
|
||
)
|
||
for row in rows
|
||
],
|
||
)
|
||
cursor.execute(f"DELETE FROM {table} WHERE metric_time <> %s", (metric_time,))
|
||
old_rows_deleted = cursor.rowcount
|
||
connection.commit()
|
||
except ProcessingError:
|
||
connection.rollback()
|
||
raise
|
||
except Exception as exc:
|
||
connection.rollback()
|
||
raise ProcessingError(f"MySQL write failed: {exc}") from exc
|
||
finally:
|
||
if cursor is not None:
|
||
cursor.close()
|
||
connection.close()
|
||
|
||
return {
|
||
"enabled": True,
|
||
"table": self.table,
|
||
"metric_time": metric_time.strftime("%Y-%m-%d %H:%M:%S"),
|
||
"inserted_rows": len(rows),
|
||
"refreshed_rows": refreshed_rows,
|
||
"old_rows_deleted": old_rows_deleted,
|
||
}
|
||
|
||
def _connect(self) -> object:
|
||
if self.connect_factory is not None:
|
||
return self.connect_factory()
|
||
try:
|
||
import pymysql
|
||
except ImportError as exc:
|
||
raise ProcessingError("PyMySQL is required for database output") from exc
|
||
try:
|
||
bootstrap = pymysql.connect(
|
||
host=DATABASE_HOST,
|
||
port=DATABASE_PORT,
|
||
user=DATABASE_USER,
|
||
password=DATABASE_PASSWORD,
|
||
charset="utf8mb4",
|
||
autocommit=True,
|
||
connect_timeout=15,
|
||
read_timeout=60,
|
||
write_timeout=60,
|
||
)
|
||
try:
|
||
cursor = bootstrap.cursor()
|
||
try:
|
||
cursor.execute(
|
||
f"CREATE DATABASE IF NOT EXISTS `{DATABASE_NAME}` "
|
||
"CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"
|
||
)
|
||
finally:
|
||
cursor.close()
|
||
finally:
|
||
bootstrap.close()
|
||
return pymysql.connect(
|
||
host=DATABASE_HOST,
|
||
port=DATABASE_PORT,
|
||
user=DATABASE_USER,
|
||
password=DATABASE_PASSWORD,
|
||
database=DATABASE_NAME,
|
||
charset="utf8mb4",
|
||
autocommit=False,
|
||
connect_timeout=15,
|
||
read_timeout=60,
|
||
write_timeout=60,
|
||
)
|
||
except Exception as exc:
|
||
raise ProcessingError(f"MySQL database setup failed: {exc}") from exc
|
||
|
||
|
||
class ApiSource:
|
||
def __init__(self, base_url: str, token: str, storage_id: str, root: str) -> None:
|
||
if not token:
|
||
raise ProcessingError("METRIX_API_TOKEN is required in API mode")
|
||
self.base_url = base_url.rstrip("/")
|
||
self.token = token
|
||
self.storage_id = storage_id
|
||
self.root = root.rstrip("/") or "/"
|
||
|
||
def candidates(self, lookback_days: int) -> list[Candidate]:
|
||
root_entries = self._list_dir(self.root)
|
||
date_dirs = sorted(
|
||
(item for item in root_entries if item.get("is_dir") and re.fullmatch(r"\d{4}-\d{2}-\d{2}", item.get("name", ""))),
|
||
key=lambda item: item["name"],
|
||
)
|
||
selected_dirs = date_dirs[-max(1, lookback_days) :]
|
||
result: list[Candidate] = []
|
||
for directory in selected_dirs:
|
||
for item in self._list_dir(directory["path"]):
|
||
if item.get("is_dir"):
|
||
continue
|
||
candidate = parse_candidate(item["path"], int(item.get("size") or 0))
|
||
if candidate is not None:
|
||
result.append(candidate)
|
||
return result
|
||
|
||
def download(self, path: str) -> bytes:
|
||
endpoint = f"/api/storages/{quote(self.storage_id, safe='')}/download"
|
||
return self._request(endpoint, {"path": path}, timeout=120)
|
||
|
||
def cell_data_workbooks(self) -> list[tuple[str, bytes]]:
|
||
workbooks: list[tuple[str, bytes]] = []
|
||
for directory in CELL_DATA_DIRECTORIES:
|
||
item = select_latest_cell_data_file(self._list_dir(directory), directory)
|
||
path = str(item["path"])
|
||
workbooks.append((path, self.download(path)))
|
||
return workbooks
|
||
|
||
def _list_dir(self, path: str) -> list[dict[str, object]]:
|
||
endpoint = f"/api/storages/{quote(self.storage_id, safe='')}/files"
|
||
payload = self._request(endpoint, {"path": path, "recursive": "false"})
|
||
return json.loads(payload.decode("utf-8")).get("entries", [])
|
||
|
||
def _request(self, endpoint: str, query: dict[str, object], timeout: int = 30) -> bytes:
|
||
url = f"{self.base_url}{endpoint}?{urlencode(query)}"
|
||
request = Request(url, headers={"Authorization": f"Bearer {self.token}", "Accept": "application/json"})
|
||
last_error: Exception | None = None
|
||
for attempt in range(3):
|
||
try:
|
||
with urlopen(request, timeout=timeout) as response:
|
||
return response.read()
|
||
except HTTPError as exc:
|
||
detail = exc.read().decode("utf-8", "replace")[:1000]
|
||
if exc.code < 500:
|
||
raise ProcessingError(f"Metrix API {exc.code}: {detail}") from exc
|
||
last_error = exc
|
||
except (URLError, TimeoutError) as exc:
|
||
last_error = exc
|
||
if attempt < 2:
|
||
time.sleep(2**attempt)
|
||
raise ProcessingError(f"Metrix API request failed: {last_error}")
|
||
|
||
|
||
class LocalSource:
|
||
def __init__(self, root: Path) -> None:
|
||
self.root = root.resolve()
|
||
if not self.root.is_dir():
|
||
raise ProcessingError(f"Local source directory does not exist: {self.root}")
|
||
|
||
def candidates(self, lookback_days: int) -> list[Candidate]:
|
||
del lookback_days
|
||
result: list[Candidate] = []
|
||
for path in self.root.rglob("*.zip"):
|
||
candidate = parse_candidate(str(path.resolve()), path.stat().st_size)
|
||
if candidate is not None:
|
||
result.append(candidate)
|
||
return result
|
||
|
||
def download(self, path: str) -> bytes:
|
||
return Path(path).read_bytes()
|
||
|
||
def cell_data_workbooks(self) -> list[tuple[str, bytes]]:
|
||
directories = [self.root / Path(directory.lstrip("/")) for directory in CELL_DATA_DIRECTORIES]
|
||
existing = [directory for directory in directories if directory.is_dir()]
|
||
if not existing:
|
||
return []
|
||
if len(existing) != len(directories):
|
||
raise ProcessingError("Local CellData source must contain all five configured directories")
|
||
workbooks: list[tuple[str, bytes]] = []
|
||
for directory in directories:
|
||
entries = [
|
||
{"name": path.name, "path": str(path.resolve()), "is_dir": path.is_dir()}
|
||
for path in directory.iterdir()
|
||
]
|
||
item = select_latest_cell_data_file(entries, str(directory))
|
||
path = str(item["path"])
|
||
workbooks.append((path, self.download(path)))
|
||
return workbooks
|
||
|
||
|
||
def parse_candidate(path: str, size: int = 0) -> Candidate | None:
|
||
name = posixpath.basename(path.replace("\\", "/"))
|
||
match = FILE_RE.fullmatch(name)
|
||
if not match or match.group("source_type") not in EXPECTED_TYPES:
|
||
return None
|
||
return Candidate(match.group("source_type"), match.group("window"), path, size)
|
||
|
||
|
||
def select_latest_cell_data_file(entries: list[dict[str, object]], directory: str) -> dict[str, object]:
|
||
dated_files: list[tuple[datetime, str, dict[str, object]]] = []
|
||
for item in entries:
|
||
if item.get("is_dir"):
|
||
continue
|
||
name = str(item.get("name") or "")
|
||
match = CELL_DATA_FILE_RE.search(name)
|
||
if not match:
|
||
continue
|
||
try:
|
||
file_date = datetime.strptime(match.group("date"), "%Y%m%d")
|
||
except ValueError:
|
||
continue
|
||
dated_files.append((file_date, name, item))
|
||
if not dated_files:
|
||
raise ProcessingError(f"No dated CellData XLSX found in {directory}")
|
||
return max(dated_files, key=lambda entry: (entry[0], entry[1]))[2]
|
||
|
||
|
||
def parse_cell_data_workbook(raw_xlsx: bytes, path: str) -> dict[str, tuple[str, str]]:
|
||
try:
|
||
with warnings.catch_warnings():
|
||
warnings.filterwarnings("ignore", message="Workbook contains no default style")
|
||
workbook = load_workbook(io.BytesIO(raw_xlsx), read_only=True, data_only=True)
|
||
except Exception as exc:
|
||
raise ProcessingError(f"Invalid CellData workbook {path}: {exc}") from exc
|
||
try:
|
||
if "小区信息表" not in workbook.sheetnames:
|
||
raise ProcessingError(f"CellData workbook does not contain 小区信息表: {path}")
|
||
rows = workbook["小区信息表"].iter_rows(values_only=True)
|
||
try:
|
||
header = tuple(normalize_cell(value) for value in next(rows))
|
||
except StopIteration as exc:
|
||
raise ProcessingError(f"CellData workbook is empty: {path}") from exc
|
||
required_columns = ("eNB/gNB", "CI", "经度", "纬度")
|
||
missing = [column for column in required_columns if column not in header]
|
||
if missing:
|
||
raise ProcessingError(f"CellData workbook missing columns {missing}: {path}")
|
||
indexes = {column: header.index(column) for column in required_columns}
|
||
coordinates: dict[str, tuple[str, str]] = {}
|
||
for row_number, row in enumerate(rows, start=2):
|
||
node = normalize_cell(row[indexes["eNB/gNB"]])
|
||
cell = normalize_cell(row[indexes["CI"]])
|
||
longitude = normalize_cell(row[indexes["经度"]])
|
||
latitude = normalize_cell(row[indexes["纬度"]])
|
||
if not node or not cell or not longitude or not latitude:
|
||
continue
|
||
cgi = f"{LTE_PLMN}-{node}-{cell}"
|
||
value = (longitude, latitude)
|
||
existing = coordinates.get(cgi)
|
||
if existing is not None and existing != value:
|
||
raise ProcessingError(f"Conflicting CellData coordinates for {cgi} in {path}, row {row_number}")
|
||
coordinates[cgi] = value
|
||
return coordinates
|
||
finally:
|
||
workbook.close()
|
||
|
||
|
||
def load_cell_coordinates(workbooks: list[tuple[str, bytes]]) -> dict[str, tuple[str, str]]:
|
||
coordinates: dict[str, tuple[str, str]] = {}
|
||
for path, raw_xlsx in workbooks:
|
||
for cgi, value in parse_cell_data_workbook(raw_xlsx, path).items():
|
||
existing = coordinates.get(cgi)
|
||
if existing is not None and existing != value:
|
||
raise ProcessingError(f"Conflicting CellData coordinates for {cgi} across workbooks")
|
||
coordinates[cgi] = value
|
||
return coordinates
|
||
|
||
|
||
def select_window(candidates: list[Candidate], requested: str = "") -> tuple[str, dict[str, Candidate], list[str]]:
|
||
grouped: dict[str, dict[str, Candidate]] = {}
|
||
duplicates: list[str] = []
|
||
for candidate in candidates:
|
||
window_group = grouped.setdefault(candidate.window, {})
|
||
if candidate.source_type in window_group:
|
||
duplicates.append(f"{candidate.window}/{candidate.source_type}")
|
||
window_group[candidate.source_type] = candidate
|
||
if duplicates:
|
||
raise ProcessingError(f"Duplicate source files: {', '.join(sorted(duplicates))}")
|
||
complete = sorted(window for window, items in grouped.items() if all(name in items for name in EXPECTED_TYPES))
|
||
if requested:
|
||
if requested not in complete:
|
||
present = sorted(grouped.get(requested, {}))
|
||
missing = [name for name in EXPECTED_TYPES if name not in present]
|
||
raise ProcessingError(f"Requested window is incomplete: {requested}; missing={missing}")
|
||
selected = requested
|
||
elif complete:
|
||
selected = complete[-1]
|
||
else:
|
||
raise ProcessingError("No hour contains all seven interference source types")
|
||
warnings_out: list[str] = []
|
||
latest_seen = max(grouped) if grouped else ""
|
||
if latest_seen and latest_seen != selected:
|
||
missing = [name for name in EXPECTED_TYPES if name not in grouped[latest_seen]]
|
||
warnings_out.append(f"Latest observed window {latest_seen} is incomplete; using {selected}; missing={missing}")
|
||
return selected, grouped[selected], warnings_out
|
||
|
||
|
||
def parse_workbook(raw_zip: bytes, source_type: str) -> tuple[tuple[str, ...], list[tuple[object, ...]], str]:
|
||
try:
|
||
with zipfile.ZipFile(io.BytesIO(raw_zip)) as archive:
|
||
bad_member = archive.testzip()
|
||
if bad_member:
|
||
raise ProcessingError(f"ZIP CRC check failed: {bad_member}")
|
||
xlsx_members = [item for item in archive.infolist() if not item.is_dir() and item.filename.lower().endswith(".xlsx")]
|
||
if len(xlsx_members) != 1:
|
||
raise ProcessingError(f"Expected one XLSX member, found {len(xlsx_members)}")
|
||
member = xlsx_members[0]
|
||
workbook_bytes = archive.read(member)
|
||
except zipfile.BadZipFile as exc:
|
||
raise ProcessingError("Invalid ZIP archive") from exc
|
||
|
||
with warnings.catch_warnings():
|
||
warnings.filterwarnings("ignore", message="Workbook contains no default style")
|
||
workbook = load_workbook(io.BytesIO(workbook_bytes), read_only=True, data_only=True)
|
||
try:
|
||
if "Sheet0" not in workbook.sheetnames:
|
||
raise ProcessingError("Workbook does not contain Sheet0")
|
||
rows = workbook["Sheet0"].iter_rows(values_only=True)
|
||
try:
|
||
header = tuple(normalize_cell(value) for value in next(rows))
|
||
except StopIteration as exc:
|
||
raise ProcessingError("Sheet0 is empty") from exc
|
||
expected = EXPECTED_HEADERS[source_type]
|
||
if header != expected:
|
||
raise ProcessingError(f"Unexpected Sheet0 header for {source_type}: {header}")
|
||
data = [tuple(row) for row in rows if any(value is not None and normalize_cell(value) != "" for value in row)]
|
||
return header, data, member.filename
|
||
finally:
|
||
workbook.close()
|
||
|
||
|
||
def process(
|
||
source: Source,
|
||
output_root: Path,
|
||
lookback_days: int,
|
||
requested_window: str = "",
|
||
store: SummaryStore | None = None,
|
||
) -> Path:
|
||
candidates = source.candidates(lookback_days)
|
||
window, selected, warnings_out = select_window(candidates, requested_window)
|
||
cell_data_workbooks = source.cell_data_workbooks()
|
||
cell_coordinates = load_cell_coordinates(cell_data_workbooks)
|
||
temp_dir = output_root.resolve() / f".{window}.tmp-{os.getpid()}"
|
||
final_dir = output_root.resolve() / window
|
||
ensure_scoped(output_root.resolve(), temp_dir)
|
||
if temp_dir.exists():
|
||
shutil.rmtree(temp_dir)
|
||
converted_dir = temp_dir / "converted"
|
||
converted_dir.mkdir(parents=True)
|
||
|
||
summary_rows: list[dict[str, str]] = []
|
||
manifest_files: list[dict[str, object]] = []
|
||
database_result: dict[str, object] = {"enabled": False}
|
||
try:
|
||
for source_type in EXPECTED_TYPES:
|
||
candidate = selected[source_type]
|
||
raw_zip = source.download(candidate.path)
|
||
header, rows, member_name = parse_workbook(raw_zip, source_type)
|
||
csv_name = f"{source_type}_{window}.csv"
|
||
write_csv(converted_dir / csv_name, header, rows)
|
||
for row_number, row in enumerate(rows, start=2):
|
||
record = dict(zip(header, row, strict=True))
|
||
summary_rows.append(
|
||
summary_record(record, source_type, candidate.path, window, row_number, cell_coordinates)
|
||
)
|
||
manifest_files.append(
|
||
{
|
||
"source_size": candidate.size,
|
||
"sha256": hashlib.sha256(raw_zip).hexdigest(),
|
||
"xlsx_member": member_name,
|
||
"rows": len(rows),
|
||
"converted_csv": f"converted/{csv_name}",
|
||
}
|
||
)
|
||
|
||
summary_rows.sort(key=lambda item: (item["cgi"], item["cell_name"]))
|
||
summary_name = f"interference_summary_{window}.csv"
|
||
write_dict_csv(temp_dir / summary_name, SUMMARY_HEADER, summary_rows)
|
||
if store is not None:
|
||
database_result = store.replace_latest(summary_rows)
|
||
matched_coordinates = sum(bool(row["longitude"] and row["latitude"]) for row in summary_rows)
|
||
manifest = {
|
||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||
"window": window,
|
||
"hour_start": window_bounds(window)[0],
|
||
"hour_end": window_bounds(window)[1],
|
||
"source_file_count": len(manifest_files),
|
||
"cell_data_file_count": len(cell_data_workbooks),
|
||
"summary_rows": len(summary_rows),
|
||
"coordinate_matched_rows": matched_coordinates,
|
||
"coordinate_unmatched_rows": len(summary_rows) - matched_coordinates,
|
||
"warnings": warnings_out,
|
||
"files": manifest_files,
|
||
"summary_csv": summary_name,
|
||
"database": database_result,
|
||
}
|
||
(temp_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
output_root.resolve().mkdir(parents=True, exist_ok=True)
|
||
if final_dir.exists():
|
||
ensure_scoped(output_root.resolve(), final_dir)
|
||
shutil.rmtree(final_dir)
|
||
temp_dir.replace(final_dir)
|
||
except Exception:
|
||
shutil.rmtree(temp_dir, ignore_errors=True)
|
||
raise
|
||
|
||
print(f"selected_window={window}")
|
||
print(f"source_files={len(manifest_files)}")
|
||
print(f"cell_data_files={len(cell_data_workbooks)}")
|
||
print(f"summary_rows={len(summary_rows)}")
|
||
print(f"coordinate_matched_rows={matched_coordinates}")
|
||
print(f"coordinate_unmatched_rows={len(summary_rows) - matched_coordinates}")
|
||
if database_result["enabled"]:
|
||
print(f"database_metric_time={database_result['metric_time']}")
|
||
print(f"database_inserted_rows={database_result['inserted_rows']}")
|
||
print(f"database_old_rows_deleted={database_result['old_rows_deleted']}")
|
||
for message in warnings_out:
|
||
print(f"warning={message}")
|
||
print(f"output={final_dir}")
|
||
return final_dir
|
||
|
||
|
||
def summary_record(
|
||
record: dict[str, object],
|
||
source_type: str,
|
||
source_path: str,
|
||
window: str,
|
||
row_number: int,
|
||
cell_coordinates: dict[str, tuple[str, str]] | None = None,
|
||
) -> dict[str, str]:
|
||
hour_start = normalize_cell(record["开始时间"])
|
||
expected_start, hour_end = window_bounds(window)
|
||
if hour_start != expected_start:
|
||
raise ProcessingError(f"{source_path}: row {row_number} time {hour_start!r} does not match {expected_start!r}")
|
||
|
||
if source_type in ("5G干扰监控", "700M干扰监控"):
|
||
cgi = build_cgi("", record, ("gNBplmn", "gNBId", "cellId"), source_path, row_number)
|
||
cell_name = required(record, "CU小区配置名称", source_path, row_number)
|
||
interference = required(record, "小区上行平均干扰电平(dBm)", source_path, row_number)
|
||
elif source_type in ("SDR_FDD干扰监控", "SDR_TDD干扰监控"):
|
||
cgi = build_cgi(LTE_PLMN, record, ("eNodeBID", "小区ID"), source_path, row_number)
|
||
cell_name = required(record, "小区名称", source_path, row_number)
|
||
interference = required(record, "载波平均噪声干扰(dBm)", source_path, row_number)
|
||
elif source_type == "反开RD干扰监控":
|
||
cgi = build_cgi(LTE_PLMN, record, ("eNodeBId", "cellId"), source_path, row_number)
|
||
cell_name = required(record, "E-UTRAN TDD小区名称", source_path, row_number)
|
||
interference = required(record, "载波平均噪声干扰(dBm)", source_path, row_number)
|
||
else:
|
||
cgi = build_cgi(LTE_PLMN, record, ("eNodeBId", "cellId"), source_path, row_number)
|
||
cell_name = required(record, "E-UTRAN FDD小区名称", source_path, row_number)
|
||
interference = required(record, "载波平均噪声干扰(dBm)", source_path, row_number)
|
||
|
||
longitude, latitude = (cell_coordinates or {}).get(cgi, ("", ""))
|
||
return {
|
||
"hour_start": hour_start,
|
||
"hour_end": hour_end,
|
||
"cgi": cgi,
|
||
"cell_name": cell_name,
|
||
"interference_dbm": interference,
|
||
"longitude": longitude,
|
||
"latitude": latitude,
|
||
}
|
||
|
||
|
||
def build_cgi(prefix: str, record: dict[str, object], columns: tuple[str, ...], path: str, row: int) -> str:
|
||
parts = [required(record, column, path, row) for column in columns]
|
||
return "-".join(([prefix] if prefix else []) + parts)
|
||
|
||
|
||
def required(record: dict[str, object], column: str, path: str, row: int) -> str:
|
||
value = normalize_cell(record[column])
|
||
if not value:
|
||
raise ProcessingError(f"{path}: row {row} has empty {column}")
|
||
return value
|
||
|
||
|
||
def window_bounds(window: str) -> tuple[str, str]:
|
||
if not re.fullmatch(r"\d{16}", window):
|
||
raise ProcessingError(f"Invalid window: {window}")
|
||
start = datetime.strptime(window[:12], "%Y%m%d%H%M")
|
||
end_hour = int(window[12:14])
|
||
end_minute = int(window[14:16])
|
||
end = start.replace(hour=end_hour, minute=end_minute)
|
||
if end <= start:
|
||
end += timedelta(days=1)
|
||
return start.strftime("%Y-%m-%d %H:%M:%S"), end.strftime("%Y-%m-%d %H:%M:%S")
|
||
|
||
|
||
def normalize_cell(value: object) -> str:
|
||
if value is None:
|
||
return ""
|
||
if isinstance(value, datetime):
|
||
return value.strftime("%Y-%m-%d %H:%M:%S")
|
||
if isinstance(value, float) and value.is_integer():
|
||
return str(int(value))
|
||
return str(value).strip()
|
||
|
||
|
||
def write_csv(path: Path, header: tuple[str, ...], rows: list[tuple[object, ...]]) -> None:
|
||
with path.open("w", encoding="utf-8-sig", newline="") as file:
|
||
writer = csv.writer(file)
|
||
writer.writerow(header)
|
||
for row in rows:
|
||
writer.writerow(normalize_cell(value) for value in row)
|
||
|
||
|
||
def write_dict_csv(path: Path, header: tuple[str, ...], rows: list[dict[str, str]]) -> None:
|
||
with path.open("w", encoding="utf-8-sig", newline="") as file:
|
||
writer = csv.DictWriter(file, fieldnames=header, extrasaction="raise")
|
||
writer.writeheader()
|
||
writer.writerows(rows)
|
||
|
||
|
||
def ensure_scoped(root: Path, target: Path) -> None:
|
||
if target == root or root not in target.parents:
|
||
raise ProcessingError(f"Refusing to modify path outside output root: {target}")
|
||
|
||
|
||
def build_parser() -> argparse.ArgumentParser:
|
||
parser = argparse.ArgumentParser(description="Process the latest complete hour of interference KPI files")
|
||
parser.add_argument("--source-dir", type=Path, help="Use a local source tree instead of the Metrix storage API")
|
||
parser.add_argument("--output-dir", type=Path, default=Path(os.getenv("INTERFERENCE_OUTPUT_DIR", "output")))
|
||
parser.add_argument("--window", default=os.getenv("INTERFERENCE_WINDOW", ""), help="Optional exact 16-digit source window")
|
||
parser.add_argument("--lookback-days", type=int, default=int(os.getenv("INTERFERENCE_LOOKBACK_DAYS", "3")))
|
||
parser.add_argument("--api-base", default=os.getenv("METRIX_API_BASE_URL", "http://172.17.0.1:8000"))
|
||
parser.add_argument("--api-token", default=os.getenv("METRIX_API_TOKEN", ""))
|
||
parser.add_argument("--storage-id", default=os.getenv("METRIX_STORAGE_ID", DEFAULT_STORAGE_ID))
|
||
parser.add_argument("--root", default=os.getenv("INTERFERENCE_SOURCE_ROOT", DEFAULT_ROOT))
|
||
parser.add_argument("--no-database", action="store_true", help="Generate files without writing MySQL")
|
||
return parser
|
||
|
||
|
||
def main(argv: list[str] | None = None) -> int:
|
||
args = build_parser().parse_args(argv)
|
||
if args.lookback_days < 1:
|
||
raise ProcessingError("lookback-days must be at least 1")
|
||
source: Source = LocalSource(args.source_dir) if args.source_dir else ApiSource(args.api_base, args.api_token, args.storage_id, args.root)
|
||
store = None if args.no_database else MySQLSummaryStore()
|
||
process(source, args.output_dir, args.lookback_days, args.window, store=store)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
try:
|
||
raise SystemExit(main())
|
||
except ProcessingError as exc:
|
||
print(f"error={exc}", file=sys.stderr)
|
||
raise SystemExit(1) from exc
|