from __future__ import annotations import csv import re import stat import zipfile from dataclasses import dataclass from datetime import datetime from pathlib import Path from typing import Any, Callable, Iterable import chardet import pymysql from app.config import AppConfig, CellDataConfig from app.services.remote_download import RemoteDataDownloader, RemoteFileInfo LogFn = Callable[[str], None] CELLINFO_COLUMNS = [ "CGI", "eNodeBID", "CellID", "PLMN", "基站名称", "小区名称", "频点", "带宽", "制式", "功率", "网络", ] @dataclass(frozen=True) class SelectedZip: scan_path: str band: str remote_file: RemoteFileInfo timestamp: str @dataclass class CellDataResult: selected_files: int = 0 parsed_rows: int = 0 imported_rows: int = 0 skipped_rows: int = 0 class CellDataProcessor: def __init__(self, app_config: AppConfig, work_dir: Path, logger=None): self.app_config = app_config self.config = app_config.cell_data.normalized() self.work_dir = work_dir self.logger = logger self.downloader = RemoteDataDownloader(self.config.remote_data, self._log) self.year_dir_re = re.compile(self.config.year_dir_regex) self.month_dir_re = re.compile(self.config.month_dir_regex) self.day_dir_re = re.compile(self.config.day_dir_regex) self.file_name_re = re.compile(self.config.file_name_regex) self.file_time_re = re.compile(self.config.file_time_regex) def run(self) -> CellDataResult: if not self.config.remote_data.enabled: self._log("CellData 数据源未启用，跳过") return CellDataResult() self._set_stage("locating") selected = self._select_latest_zip_files() if not selected: raise RuntimeError("未找到可处理的 CellData ZIP 文件") self._log(f"已选择 {len(selected)} 个 CellData ZIP 文件") self._set_stage("downloading") local_files = self._download_selected(selected) self._set_stage("parsing") result = CellDataResult(selected_files=len(selected)) rows = self._parse_zip_files(local_files, result) if not rows: raise RuntimeError("CellData ZIP 中未解析到有效数据") self._set_stage("importing") result.imported_rows = self._replace_cellinfo(rows) self._log(f"CellData 导入完成，共 {result.imported_rows} 行") return result def run_local(self, upload_root: Path) -> CellDataResult: self._set_stage("locating") local_files = self._select_local_zip_files(upload_root) if not local_files: raise RuntimeError("未找到可处理的 CellData ZIP 文件") self._log(f"已选择 {len(local_files)} 个 CellData ZIP 文件") self._set_stage("parsing") result = CellDataResult(selected_files=len(local_files)) rows = self._parse_zip_files(local_files, result) if not rows: raise RuntimeError("CellData ZIP 中未解析到有效数据") self._set_stage("importing") result.imported_rows = self._replace_cellinfo(rows) self._log(f"CellData 导入完成，共 {result.imported_rows} 行") return result def _select_latest_zip_files(self) -> list[SelectedZip]: selected: list[SelectedZip] = [] for template in self.config.scan_paths: scan_path = self._resolve_scan_path(template) band_dirs = [ entry for entry in self._list_dir(scan_path) if entry["type"] == "directory" ] for band_dir in band_dirs: band = str(band_dir["name"]) files = [ entry for entry in self._list_dir(str(band_dir["path"])) if entry["type"] == "file" and self.file_name_re.search(str(entry["name"])) ] latest = self._latest_file(files) if latest: selected.append( SelectedZip( scan_path=scan_path, band=band, remote_file=self._remote_file_info(str(latest["path"]), int(latest.get("size") or 0)), timestamp=str(latest["timestamp"]), ) ) self._log(f"{band}: {latest['name']}") return selected def _resolve_scan_path(self, template: str) -> str: path = self._replace_date_placeholders(str(template).replace("\\", "/").strip()) if not path.startswith("/"): path = self._join_remote_path(self.config.remote_data.remote_dir, path) parts = [part for part in path.split("/") if part] replacements = [ ("{maxyear}", "year", self.year_dir_re, "年份"), ("{maxmonth}", "month", self.month_dir_re, "月份"), ("{maxday}", "day", self.day_dir_re, "日期"), ] for token, group_name, pattern, label in replacements: token_index = next((index for index, part in enumerate(parts) if token in part), -1) if token_index < 0: continue parent = "/" + "/".join(parts[:token_index]) if token_index else "/" values: list[int] = [] for entry in self._list_dir(parent): if entry["type"] != "directory": continue match = pattern.search(str(entry["name"])) if match: values.append(int(match.group(group_name))) if not values: raise RuntimeError(f"未找到{label}目录: {parent}") parts[token_index] = parts[token_index].replace(token, str(max(values))) return "/" + "/".join(parts) def _replace_date_placeholders(self, path: str) -> str: now = datetime.now() return ( path.replace("{yyyy}", now.strftime("%Y")) .replace("{yyyymm}", now.strftime("%Y%m")) .replace("{yyyymmdd}", now.strftime("%Y%m%d")) ) def _latest_file(self, files: list[dict[str, Any]]) -> dict[str, Any] | None: candidates: list[dict[str, Any]] = [] for item in files: match = self.file_time_re.search(str(item["name"])) if not match: continue timestamp = match.group("timestamp") candidates.append({**item, "timestamp": timestamp}) if not candidates: return None return max(candidates, key=lambda item: str(item["timestamp"])) def _download_selected(self, selected: list[SelectedZip]) -> list[tuple[SelectedZip, Path]]: target_dir = self.work_dir / "cell_data" target_dir.mkdir(parents=True, exist_ok=True) local_files: list[tuple[SelectedZip, Path]] = [] if self.config.remote_data.protocol == "ftp": with self.downloader._ftp_client() as ftp: # noqa: SLF001 - reuse existing connection helpers for item in selected: local_path = target_dir / item.band / item.remote_file.name self.downloader._download_ftp_file(ftp, item.remote_file.path, local_path, self._download_result(), item.remote_file.size) # noqa: SLF001 local_files.append((item, local_path)) return local_files ssh = self.downloader._sftp_ssh_client() # noqa: SLF001 try: with ssh.open_sftp() as sftp: for item in selected: local_path = target_dir / item.band / item.remote_file.name self.downloader._download_sftp_file(sftp, item.remote_file.path, local_path, self._download_result(), item.remote_file.size) # noqa: SLF001 local_files.append((item, local_path)) finally: ssh.close() return local_files def _select_local_zip_files(self, upload_root: Path) -> list[tuple[SelectedZip, Path]]: grouped: dict[str, list[Path]] = {} for path in upload_root.rglob("*.zip"): if not self.file_name_re.search(path.name): continue try: parent = str(path.parent.relative_to(upload_root)).replace("\\", "/") except ValueError: parent = "" grouped.setdefault("" if parent == "." else parent, []).append(path) selected: list[tuple[SelectedZip, Path]] = [] for parent, paths in sorted(grouped.items(), key=lambda item: item[0]): candidates = [] for path in paths: match = self.file_time_re.search(path.name) if match: candidates.append((match.group("timestamp"), path)) if not candidates: continue timestamp, path = max(candidates, key=lambda item: item[0]) band = Path(parent).name if parent else "" selected_zip = SelectedZip( scan_path=str(upload_root), band=band, remote_file=RemoteFileInfo( path=str(path), relative_path=str(path.relative_to(upload_root)).replace("\\", "/"), parent=parent, name=path.name, size=path.stat().st_size, ), timestamp=timestamp, ) if not band: self._log(f"未从目录名识别频段: {path.name}") else: self._log(f"{band}: {path.name}") selected.append((selected_zip, path)) return selected @staticmethod def _download_result(): from app.services.remote_download import RemoteDownloadResult return RemoteDownloadResult() def _parse_zip_files(self, local_files: list[tuple[SelectedZip, Path]], result: CellDataResult) -> list[dict[str, str]]: mapping = self.config.mapping key_config = mapping["key"] rows_by_key: dict[str, dict[str, str]] = {} for selected, local_path in local_files: with zipfile.ZipFile(local_path) as zf: sources = list(mapping["sources"]) for info in zf.infolist(): name = Path(info.filename).name if not name.lower().endswith(".csv"): continue matching_sources = [ source for source in sources if name.startswith(source["file_prefix"]) and (not selected.band or source["band"] == selected.band) ] if not matching_sources: continue if not selected.band and len(matching_sources) > 1: self._log(f"跳过无法识别频段的文件: {name}") continue for source in matching_sources: raw = zf.read(info.filename) text = self._decode_csv(raw) reader = csv.DictReader(text.splitlines()) for csv_row in reader: row = self._map_row(source["fields"], csv_row) key = self._render_expr(str(key_config["expr"]), row) if not key or "--" in key: result.skipped_rows += 1 continue row[str(key_config["field"])] = key rows_by_key[key] = {column: row.get(column, "") for column in CELLINFO_COLUMNS} result.parsed_rows += 1 return list(rows_by_key.values()) def _map_row(self, fields: dict[str, Any], csv_row: dict[str, str]) -> dict[str, str]: row: dict[str, str] = {} for target, rule in fields.items(): if isinstance(rule, str): row[target] = str(csv_row.get(rule, "") or "").strip() elif isinstance(rule, dict) and "value" in rule: row[target] = str(rule.get("value", "") or "").strip() else: row[target] = "" return row @staticmethod def _render_expr(expr: str, row: dict[str, str]) -> str: def replace(match): return row.get(match.group(1), "") return re.sub(r"\{([^{}]+)\}", replace, expr).strip() @staticmethod def _decode_csv(raw: bytes) -> str: candidates = ["utf-8-sig", "utf-8", "gb18030", "gbk"] detected = (chardet.detect(raw).get("encoding") or "").lower() if detected and detected not in candidates: candidates.append(detected) for encoding in dict.fromkeys(candidates): try: return raw.decode(encoding).lstrip("\ufeff") except (LookupError, UnicodeDecodeError): pass return raw.decode("utf-8", errors="replace").lstrip("\ufeff") def _replace_cellinfo(self, rows: list[dict[str, str]]) -> int: mysql = self.config.mysql.normalized() table = str(self.config.mapping.get("target_table") or "cellinfo") conn = pymysql.connect( host=mysql.host, port=mysql.port, user=mysql.user, password=mysql.passwd, database=mysql.dbname, charset="utf8mb4", cursorclass=pymysql.cursors.DictCursor, autocommit=False, ) try: with conn.cursor() as cursor: self._ensure_cellinfo_table(cursor, table) cursor.execute(f"TRUNCATE TABLE `{table}`") placeholders = ", ".join(["%s"] * len(CELLINFO_COLUMNS)) columns = ", ".join(f"`{column}`" for column in CELLINFO_COLUMNS) values = [tuple(row.get(column, "") for column in CELLINFO_COLUMNS) for row in rows] for start in range(0, len(values), 1000): cursor.executemany( f"INSERT INTO `{table}` ({columns}) VALUES ({placeholders})", values[start:start + 1000], ) conn.commit() return len(rows) except Exception: conn.rollback() raise finally: conn.close() @staticmethod def _ensure_cellinfo_table(cursor, table: str) -> None: cursor.execute( f""" CREATE TABLE IF NOT EXISTS `{table}` ( `CGI` varchar(120) DEFAULT NULL, `eNodeBID` int DEFAULT NULL, `CellID` int DEFAULT NULL, `PLMN` varchar(100) DEFAULT NULL, `基站名称` varchar(200) DEFAULT NULL, `小区名称` varchar(200) DEFAULT NULL, `频点` varchar(50) DEFAULT NULL, `带宽` varchar(20) DEFAULT NULL, `制式` varchar(50) DEFAULT NULL, `功率` varchar(100) DEFAULT NULL, `网络` varchar(20) DEFAULT NULL, KEY `CGI` (`CGI`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 """ ) def _list_dir(self, remote_path: str) -> list[dict[str, Any]]: if self.config.remote_data.protocol == "ftp": with self.downloader._ftp_client() as ftp: # noqa: SLF001 return [ { "name": name, "type": "directory" if entry_type == "dir" else "file", "size": size, "path": self._join_remote_path(remote_path, name), } for name, entry_type, size in self.downloader._list_ftp_entries(ftp, remote_path) # noqa: SLF001 if name not in {".", ".."} ] ssh = self.downloader._sftp_ssh_client() # noqa: SLF001 try: with ssh.open_sftp() as sftp: return [ { "name": item.filename, "type": "directory" if stat.S_ISDIR(item.st_mode) else "file", "size": int(getattr(item, "st_size", 0) or 0), "path": self._join_remote_path(remote_path, item.filename), } for item in sftp.listdir_attr(remote_path) if item.filename not in {".", ".."} ] finally: ssh.close() def _remote_file_info(self, remote_path: str, size: int) -> RemoteFileInfo: return self.downloader._remote_file_info(remote_path, size) # noqa: SLF001 @staticmethod def _join_remote_path(parent: str, child: str) -> str: parent = parent.replace("\\", "/").rstrip("/") child = child.replace("\\", "/").strip("/") if not parent: return child if parent == "/": return f"/{child}" return f"{parent}/{child}" def _set_stage(self, stage: str) -> None: if hasattr(self.logger, "set_stage"): self.logger.set_stage(stage) def _log(self, message: str) -> None: if self.logger: if hasattr(self.logger, "info"): self.logger.info(message) else: self.logger(message) def refresh_cell_data(app_config: AppConfig, work_dir: Path, logger=None) -> CellDataResult: return CellDataProcessor(app_config, work_dir, logger).run() def execute_celldata_script(script_path: Path, app_config: AppConfig, logger=None) -> None: if not script_path.exists(): if logger: logger.info("CellData 脚本文件不存在，跳过") return sql_text = script_path.read_text(encoding="utf-8").strip() if not sql_text: if logger: logger.info("CellData 脚本为空，跳过") return from app.processor import DataProcessor statements = DataProcessor.parse_sql_script(sql_text) if not statements: if logger: logger.info("CellData 脚本中没有有效语句，跳过") return mysql = app_config.cell_data.mysql.normalized() conn = pymysql.connect( host=mysql.host, port=mysql.port, user=mysql.user, password=mysql.passwd, database=mysql.dbname, charset="utf8mb4", cursorclass=pymysql.cursors.DictCursor, autocommit=False, ) try: with conn.cursor() as cursor: for i, sql in enumerate(statements, 1): preview = sql[:80].replace("\n", " ") if logger: logger.info(f"CellData SQL ({i}/{len(statements)}): {preview}...") cursor.execute(sql) affected = cursor.rowcount if cursor.rowcount >= 0 else 0 if affected > 0 and logger: logger.info(f"完成，影响 {affected} 行") conn.commit() if logger: logger.success(f"CellData 脚本执行完成，共 {len(statements)} 条语句") except Exception: conn.rollback() raise finally: conn.close() def copy_celldata_tables_to_capacity(app_config: AppConfig, logger=None) -> int: cd_mysql = app_config.cell_data.mysql.normalized() cap_mysql = app_config.mysql.normalized() if ( cd_mysql.host == cap_mysql.host and cd_mysql.port == cap_mysql.port and cd_mysql.dbname == cap_mysql.dbname ): if logger: logger.info("CellData 与容量数据库相同，跳过表复制") return 0 cd_conn = pymysql.connect( host=cd_mysql.host, port=cd_mysql.port, user=cd_mysql.user, password=cd_mysql.passwd, database=cd_mysql.dbname, charset="utf8mb4", cursorclass=pymysql.cursors.DictCursor, ) cap_conn = pymysql.connect( host=cap_mysql.host, port=cap_mysql.port, user=cap_mysql.user, password=cap_mysql.passwd, database=cap_mysql.dbname, charset="utf8mb4", cursorclass=pymysql.cursors.DictCursor, local_infile=True, autocommit=False, ) try: tables = _list_tables(cd_conn) if not tables: if logger: logger.info("CellData 数据库中没有表，跳过复制") return 0 copied = 0 for table in tables: rows = _copy_one_table(cd_conn, cap_conn, table, logger) if rows >= 0: copied += 1 cap_conn.commit() if logger: logger.success(f"已将 {copied} 张表从 CellData 复制到容量数据库") return copied except Exception: cap_conn.rollback() raise finally: cd_conn.close() cap_conn.close() def _list_tables(conn) -> list[str]: with conn.cursor() as cur: cur.execute("SHOW TABLES") return [list(row.values())[0] for row in cur.fetchall()] def _copy_one_table(src_conn, dst_conn, table: str, logger=None) -> int: with src_conn.cursor() as cur: cur.execute(f"SHOW CREATE TABLE `{table}`") row = cur.fetchone() create_sql = row.get("Create Table") or list(row.values())[1] with src_conn.cursor() as cur: cur.execute(f"SELECT COUNT(*) AS `cnt` FROM `{table}`") count = cur.fetchone()["cnt"] with dst_conn.cursor() as cur: cur.execute(f"DROP TABLE IF EXISTS `{table}`") cur.execute(create_sql) if count == 0: if logger: logger.info(f"复制表 {table}: 0 行（空表）") return 0 with src_conn.cursor() as src_cur: src_cur.execute(f"SELECT * FROM `{table}`") columns = [desc[0] for desc in src_cur.description] col_list = ", ".join(f"`{c}`" for c in columns) placeholders = ", ".join(["%s"] * len(columns)) insert_sql = f"INSERT INTO `{table}` ({col_list}) VALUES ({placeholders})" batch: list[tuple] = [] inserted = 0 with dst_conn.cursor() as dst_cur: for row in src_cur: batch.append(tuple(row.values())) if len(batch) >= 5000: dst_cur.executemany(insert_sql, batch) inserted += len(batch) batch.clear() if batch: dst_cur.executemany(insert_sql, batch) inserted += len(batch) if logger: logger.info(f"复制表 {table}: {inserted} 行") return inserted