diff --git a/app/processor.py b/app/processor.py index d9e59d2..ff212d8 100644 --- a/app/processor.py +++ b/app/processor.py @@ -105,6 +105,7 @@ class DataProcessor: "numeric": "float", } MYSQL_NUMERIC_PATTERN = r"^-?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))([eE][+-]?[0-9]+)?$" + NUMERIC_NULL_TEXTS = {"", "-", "--", "—", "–", "NA", "N/A", "NULL", "NONE", "NAN", "\\N"} def __init__(self, config: AppConfig, work_dir: Path, logger: ProcessLogger): self.config = config @@ -647,44 +648,37 @@ class DataProcessor: def _convert_int_column(self, series: pd.Series) -> pd.Series: """转换整数列""" try: - # 检测是否包含百分号 - text = series.astype("string") - has_percent = text.str.contains('%', regex=False, na=False) - - # 去除格式字符,保留正常数值含义 - cleaned = self._clean_numeric_text(text) - # 转换为数值 - numeric = pd.to_numeric(cleaned, errors='coerce') - - # 如果有百分号,除以100转换为小数 - numeric[has_percent] = numeric[has_percent] / 100 - - # 四舍五入并转为整数字符串,空值保留为 NULL,正常 0 不再误判为空值 + numeric = self._numeric_series(series) rounded = numeric.round() - return rounded.astype("Int64").astype("string").where(rounded.notna(), None) + return pd.Series( + [None if pd.isna(value) else int(value) for value in rounded], + index=series.index, + dtype=object, + ) except Exception: return series def _convert_float_column(self, series: pd.Series) -> pd.Series: """转换浮点数列""" try: - # 检测是否包含百分号 - text = series.astype("string") - has_percent = text.str.contains('%', regex=False, na=False) - - # 去除格式字符,保留正常数值含义 - cleaned = self._clean_numeric_text(text) - # 转换为数值 - numeric = pd.to_numeric(cleaned, errors='coerce') - - # 如果有百分号,除以100转换为小数(例如:95% → 0.95) - numeric[has_percent] = numeric[has_percent] / 100 - - # 保留小数,空值保留为 NULL,正常 0 不再误判为空值 - return numeric.astype("string").where(numeric.notna(), None) + numeric = self._numeric_series(series) + return pd.Series( + [None if pd.isna(value) else float(value) for value in numeric], + index=series.index, + dtype=object, + ) except Exception: return series + def _numeric_series(self, series: pd.Series) -> pd.Series: + text = series.astype("string") + has_percent = text.str.contains(r"[%%]", regex=True, na=False) + cleaned = self._clean_numeric_text(text) + empty_mask = cleaned.isna() | cleaned.str.upper().isin(self.NUMERIC_NULL_TEXTS) + numeric = pd.to_numeric(cleaned.mask(empty_mask, pd.NA), errors="coerce") + numeric[has_percent & numeric.notna()] = numeric[has_percent & numeric.notna()] / 100 + return numeric + @staticmethod def _clean_numeric_text(series: pd.Series) -> pd.Series: """清理数值文本中的格式字符,不改变正常数字。""" @@ -693,6 +687,7 @@ class DataProcessor: .str.replace(',', '', regex=False) .str.replace(',', '', regex=False) .str.replace('%', '', regex=False) + .str.replace('%', '', regex=False) .str.replace('\t', '', regex=False) .str.replace(' ', '', regex=False) ) diff --git a/docs/project_context.md b/docs/project_context.md index 7c9cad2..597e03f 100644 --- a/docs/project_context.md +++ b/docs/project_context.md @@ -1,5 +1,12 @@ # 项目上下文记录 +## 2026-05-19:修复 CSV 导入阶段数值截断错误 + +- `DataProcessor` 仍会根据 `ReportScript.sql` 的 `MODIFY COLUMN` 提前把业务数值字段建成 `INT/FLOAT`,但数值清洗改为返回真正的 Python `None/int/float`,避免 pandas `` 或异常文本被 PyMySQL 当作字符串写入数值列。 +- 数值字段导入前会把空串、`-`、`--`、长短横线、`NA/N/A/NULL/NONE/NAN/\N` 等源 CSV 占位符转为数据库 `NULL`,正常 `0` 保留为 `0`;逗号/全角逗号、半角/全角百分号和空白仍按数值格式清理。 +- 该问题本质是新版提前按 SQL 类型建表后,MySQL 严格模式会在 CSV 导入阶段拒绝脏数值;旧版多为字符串先落库,所以不会在导入阶段出现 `Data truncated for column`。 +- 已执行数值转换样例验证、`.venv\Scripts\python.exe -m compileall app` 和 `uvx --offline ruff check .`,均通过。 + ## 2026-05-19:优化处理进度阶段显示和日志跟随 - `ProcessLogger` 新增轻量阶段回调,`DataProcessor.process()` 会在远程下载后依次上报 `extracting`、`converting`、`importing`、`scripting`、`completed/failed` 阶段。