From 2d0d0c181e5be8d7018c2fc58683ced570dac62a Mon Sep 17 00:00:00 2001 From: Nixevol Date: Mon, 3 Aug 2026 09:42:00 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E7=8B=AC=E7=AB=8B=E5=BB=BA=E5=BA=93?= =?UTF-8?q?=E5=B9=B6=E6=95=B4=E7=90=86=E7=A6=BB=E7=BA=BF=E4=BE=9D=E8=B5=96?= =?UTF-8?q?=E7=9B=AE=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 +-- aidocs/project_context.md | 10 +++++-- main.py | 30 +++++++++++++++++++-- scripts/build_offline_package.py | 20 +++++++------- tests/test_pipeline.py | 45 ++++++++++++++++++++++++++++++++ 5 files changed, 92 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index a63ef99..7e35ab5 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ CGI is generated with fixed rules: - NR (`5G干扰监控`, `700M干扰监控`): `{gNBplmn}-{gNBId}-{cellId}`. - 4G (the other five source types): `460-00-{eNodeBID}-{小区ID}`, using each source schema's actual equivalent column names. -MySQL connection values, database `metrix`, and table `interference_hourly_summary` are constants at the top of `main.py`; they are not project environment variables. +MySQL connection values, database `interference_etl`, and table `interference_hourly_summary` are constants at the top of `main.py`; they are not project environment variables. The script creates its database and table automatically when they do not exist. ## Local development @@ -52,7 +52,7 @@ docker build -t interference-etl-runtime:1.1 . python scripts\build_offline_package.py ``` -The generated `dist/InterferenceETL-offline-1.1.zip` contains `main.py`, a ZIP execution entry point, and `openpyxl`, `et_xmlfile`, and `PyMySQL`. The builder verifies both imports and ZIP execution using the standard `python:3.13.11-slim` image, so the uploaded workspace does not need an online `pip install` or a custom runtime image. +The generated `dist/InterferenceETL-offline-1.1.zip` contains `main.py`, a ZIP execution entry point, and a `vendor/` directory containing `openpyxl`, `et_xmlfile`, and `PyMySQL`. The builder verifies both imports and ZIP execution using the standard `python:3.13.11-slim` image, so the uploaded workspace does not need an online `pip install` or a custom runtime image. Create the project, then upload and extract the ZIP in its Script Management workspace. The normal file tree should contain `main.py` at `/workspace/main.py`. Use these project settings: diff --git a/aidocs/project_context.md b/aidocs/project_context.md index 52b3f4b..8444544 100644 --- a/aidocs/project_context.md +++ b/aidocs/project_context.md @@ -16,7 +16,7 @@ ## 2026-07-31: Latest-hour MySQL retention - Scheduled runs write to MySQL by default. File-only development runs must explicitly pass `--no-database`; the MySQL host, port, account, password, database, and table are constants at the top of `main.py`. -- The fixed database is `metrix` and the fixed table is `interference_hourly_summary`. Its only time column is `metric_time DATETIME`, populated from the source KPI `开始时间`, so users can identify the hour represented by every row. +- The fixed script-owned database is `interference_etl` and the fixed table is `interference_hourly_summary`. Its only time column is `metric_time DATETIME`, populated from the source KPI `开始时间`, so users can identify the hour represented by every row. - One transaction deletes the selected hour for idempotent refresh, inserts its complete batch, then deletes every other database hour. Any failure rolls back the data changes, and an older selected hour cannot replace a newer hour already stored. - Database retention never deletes or modifies Metrix Storage/SFTP source ZIP or XLSX files. Generated CSV retention remains a separate pending decision. @@ -31,4 +31,10 @@ - Replaced the NR `masterOperatorId` passthrough with `gNBplmn-gNBId-cellId`. The remaining five 4G sources use the fixed `460-00` prefix plus their schema-specific base-station and cell ID columns. - Removed CGI PLMN and MySQL connection command-line/environment options. Database configuration now lives as a small constant block at the top of `main.py`; the verified bridge-network address is `172.17.0.1:3306` because `ShareMySQL` DNS is unavailable from the default script network. - Read-only online validation selected complete window `2026073115001600` and checked all 2,275 generated CGI values against the seven converted source CSV files. Per-source row counts were `6 / 1,198 / 48 / 91 / 731 / 179 / 22` in `EXPECTED_TYPES` order. -- A connection-only check using the fixed constants returned database `metrix`. It did not create the table, insert rows, or delete existing data; all remote validation files were removed afterward. +- Before the dedicated-database correction, a connection-only check returned the platform database `metrix`; that check did not create a table or modify data. Production use now targets only `interference_etl`. + +## 2026-08-03: Dedicated database initialization + +- InterferenceETL no longer writes into the Metrix platform database. Its fixed database is `interference_etl`, while the table remains `interference_hourly_summary`. +- The default MySQL path first connects without selecting a database, runs `CREATE DATABASE IF NOT EXISTS interference_etl` with `utf8mb4`, then reconnects to that database and creates the table if needed. The configured account therefore needs database creation permission on first run. +- Offline dependencies are stored below the workspace `vendor/` directory instead of separate package directories at the project root. `main.py` prepends this directory to `sys.path`, so the run command remains `python main.py`. diff --git a/main.py b/main.py index a85a23c..e7906eb 100644 --- a/main.py +++ b/main.py @@ -21,6 +21,10 @@ from urllib.request import Request, urlopen import warnings import zipfile + +VENDOR_DIR = Path(__file__).resolve().parent / "vendor" +sys.path.insert(0, str(VENDOR_DIR)) + from openpyxl import load_workbook @@ -43,7 +47,7 @@ DATABASE_HOST = "172.17.0.1" DATABASE_PORT = 3306 DATABASE_USER = "root" DATABASE_PASSWORD = "OSp!jmgm@26" -DATABASE_NAME = "metrix" +DATABASE_NAME = "interference_etl" DATABASE_TABLE = "interference_hourly_summary" HEADER_FDD = ( @@ -260,6 +264,28 @@ class MySQLSummaryStore: except ImportError as exc: raise ProcessingError("PyMySQL is required for database output") from exc try: + bootstrap = pymysql.connect( + host=DATABASE_HOST, + port=DATABASE_PORT, + user=DATABASE_USER, + password=DATABASE_PASSWORD, + charset="utf8mb4", + autocommit=True, + connect_timeout=15, + read_timeout=60, + write_timeout=60, + ) + try: + cursor = bootstrap.cursor() + try: + cursor.execute( + f"CREATE DATABASE IF NOT EXISTS `{DATABASE_NAME}` " + "CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci" + ) + finally: + cursor.close() + finally: + bootstrap.close() return pymysql.connect( host=DATABASE_HOST, port=DATABASE_PORT, @@ -273,7 +299,7 @@ class MySQLSummaryStore: write_timeout=60, ) except Exception as exc: - raise ProcessingError(f"MySQL connection failed: {exc}") from exc + raise ProcessingError(f"MySQL database setup failed: {exc}") from exc class ApiSource: diff --git a/scripts/build_offline_package.py b/scripts/build_offline_package.py index 19ee7dd..b1a638a 100644 --- a/scripts/build_offline_package.py +++ b/scripts/build_offline_package.py @@ -21,7 +21,7 @@ from importlib.metadata import distribution from pathlib import Path import shutil -target = Path('/package') +target = Path('/package/vendor') for name in ('openpyxl', 'et-xmlfile', 'PyMySQL'): package = distribution(name) for item in package.files or (): @@ -76,6 +76,12 @@ def build(output: Path, runtime_image: str, base_image: str) -> Path: ], check=True, ) + if output.exists(): + output.unlink() + with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as archive: + for path in sorted(staging.rglob("*")): + if path.is_file(): + archive.write(path, path.relative_to(staging).as_posix()) subprocess.run( [ "docker", @@ -84,22 +90,14 @@ def build(output: Path, runtime_image: str, base_image: str) -> Path: "-e", "PYTHONDONTWRITEBYTECODE=1", "-v", - f"{staging}:/workspace:ro", - "-w", - "/workspace", + f"{output.parent}:/dist:ro", base_image, "python", - "__main__.py", + f"/dist/{output.name}", "--help", ], check=True, ) - if output.exists(): - output.unlink() - with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as archive: - for path in sorted(staging.rglob("*")): - if path.is_file(): - archive.write(path, path.relative_to(staging).as_posix()) return output diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 87d4804..191bfd6 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -6,7 +6,9 @@ import io import json from pathlib import Path import tempfile +import types import unittest +from unittest.mock import patch import zipfile from openpyxl import Workbook @@ -102,6 +104,25 @@ class PipelineTest(unittest.TestCase): self.assertEqual(connection.cursor_instance.inserted, []) self.assertTrue(connection.closed) + def test_mysql_store_creates_and_connects_to_own_database(self) -> None: + bootstrap = BootstrapConnection() + target = object() + calls: list[dict[str, object]] = [] + + def connect(**kwargs: object) -> object: + calls.append(kwargs) + return bootstrap if len(calls) == 1 else target + + with patch.dict("sys.modules", {"pymysql": types.SimpleNamespace(connect=connect)}): + connection = main.MySQLSummaryStore()._connect() + + self.assertIs(connection, target) + self.assertNotIn("database", calls[0]) + self.assertEqual(calls[1]["database"], "interference_etl") + self.assertIn("CREATE DATABASE IF NOT EXISTS `interference_etl`", bootstrap.cursor_instance.executed[0]) + self.assertTrue(bootstrap.cursor_instance.closed) + self.assertTrue(bootstrap.closed) + class RecordingStore: def __init__(self) -> None: @@ -167,6 +188,30 @@ class FakeConnection: self.closed = True +class BootstrapCursor: + def __init__(self) -> None: + self.executed: list[str] = [] + self.closed = False + + def execute(self, query: str) -> None: + self.executed.append(" ".join(query.split())) + + def close(self) -> None: + self.closed = True + + +class BootstrapConnection: + def __init__(self) -> None: + self.cursor_instance = BootstrapCursor() + self.closed = False + + def cursor(self) -> BootstrapCursor: + return self.cursor_instance + + def close(self) -> None: + self.closed = True + + def database_row() -> dict[str, str]: return { "hour_start": "2026-07-31 10:00:00",