fix: 调整数值异常值归零和日志高度
This commit is contained in:
+5
-5
@@ -105,7 +105,7 @@ class DataProcessor:
|
|||||||
"numeric": "float",
|
"numeric": "float",
|
||||||
}
|
}
|
||||||
MYSQL_NUMERIC_PATTERN = r"^-?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))([eE][+-]?[0-9]+)?$"
|
MYSQL_NUMERIC_PATTERN = r"^-?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))([eE][+-]?[0-9]+)?$"
|
||||||
NUMERIC_NULL_TEXTS = {"", "-", "--", "—", "–", "NA", "N/A", "NULL", "NONE", "NAN", "\\N"}
|
NUMERIC_ZERO_TEXTS = {"", "-", "--", "—", "–", "NA", "N/A", "NULL", "NONE", "NAN", "\\N"}
|
||||||
|
|
||||||
def __init__(self, config: AppConfig, work_dir: Path, logger: ProcessLogger):
|
def __init__(self, config: AppConfig, work_dir: Path, logger: ProcessLogger):
|
||||||
self.config = config
|
self.config = config
|
||||||
@@ -651,7 +651,7 @@ class DataProcessor:
|
|||||||
numeric = self._numeric_series(series)
|
numeric = self._numeric_series(series)
|
||||||
rounded = numeric.round()
|
rounded = numeric.round()
|
||||||
return pd.Series(
|
return pd.Series(
|
||||||
[None if pd.isna(value) else int(value) for value in rounded],
|
[int(value) for value in rounded],
|
||||||
index=series.index,
|
index=series.index,
|
||||||
dtype=object,
|
dtype=object,
|
||||||
)
|
)
|
||||||
@@ -663,7 +663,7 @@ class DataProcessor:
|
|||||||
try:
|
try:
|
||||||
numeric = self._numeric_series(series)
|
numeric = self._numeric_series(series)
|
||||||
return pd.Series(
|
return pd.Series(
|
||||||
[None if pd.isna(value) else float(value) for value in numeric],
|
[float(value) for value in numeric],
|
||||||
index=series.index,
|
index=series.index,
|
||||||
dtype=object,
|
dtype=object,
|
||||||
)
|
)
|
||||||
@@ -674,8 +674,8 @@ class DataProcessor:
|
|||||||
text = series.astype("string")
|
text = series.astype("string")
|
||||||
has_percent = text.str.contains(r"[%%]", regex=True, na=False)
|
has_percent = text.str.contains(r"[%%]", regex=True, na=False)
|
||||||
cleaned = self._clean_numeric_text(text)
|
cleaned = self._clean_numeric_text(text)
|
||||||
empty_mask = cleaned.isna() | cleaned.str.upper().isin(self.NUMERIC_NULL_TEXTS)
|
zero_mask = cleaned.isna() | cleaned.str.upper().isin(self.NUMERIC_ZERO_TEXTS)
|
||||||
numeric = pd.to_numeric(cleaned.mask(empty_mask, pd.NA), errors="coerce")
|
numeric = pd.to_numeric(cleaned.mask(zero_mask, "0"), errors="coerce").fillna(0)
|
||||||
numeric[has_percent & numeric.notna()] = numeric[has_percent & numeric.notna()] / 100
|
numeric[has_percent & numeric.notna()] = numeric[has_percent & numeric.notna()] / 100
|
||||||
return numeric
|
return numeric
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,12 @@
|
|||||||
# 项目上下文记录
|
# 项目上下文记录
|
||||||
|
|
||||||
|
## 2026-05-19:调整数值异常值归零和完成后日志高度
|
||||||
|
|
||||||
|
- `DataProcessor` 数值字段清洗策略从异常值写入 `NULL` 改为写入 `0`:空串、`-`、`--`、长短横线、`NA/N/A/NULL/NONE/NAN/\N` 以及其它无法转数值的文本都会归零,正常 `0` 不受影响。
|
||||||
|
- 数值格式继续清理千分位逗号、全角逗号、半角/全角百分号和空白;例如 `12,345.123` 会导入为 `12345.123`,`95%`/`95%` 会导入为 `0.95`。
|
||||||
|
- `frontend/src/components/FileWorkflow.vue` 在任务完成或失败后给处理进度区增加 `finished` 状态,`frontend/src/styles.css` 让完成后的日志框使用自适应最大高度,避免上传区恢复显示后日志仍按运行中高度撑开页面。
|
||||||
|
- 已执行数值转换样例验证、`.venv\Scripts\python.exe -m compileall app`、`uvx --offline ruff check .` 和 `npm run build`,均通过。
|
||||||
|
|
||||||
## 2026-05-19:修复 CSV 导入阶段数值截断错误
|
## 2026-05-19:修复 CSV 导入阶段数值截断错误
|
||||||
|
|
||||||
- `DataProcessor` 仍会根据 `ReportScript.sql` 的 `MODIFY COLUMN` 提前把业务数值字段建成 `INT/FLOAT`,但数值清洗改为返回真正的 Python `None/int/float`,避免 pandas `<NA>` 或异常文本被 PyMySQL 当作字符串写入数值列。
|
- `DataProcessor` 仍会根据 `ReportScript.sql` 的 `MODIFY COLUMN` 提前把业务数值字段建成 `INT/FLOAT`,但数值清洗改为返回真正的 Python `None/int/float`,避免 pandas `<NA>` 或异常文本被 PyMySQL 当作字符串写入数值列。
|
||||||
|
|||||||
@@ -118,7 +118,7 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div v-if="processVisible" class="process-section">
|
<div v-if="processVisible" class="process-section" :class="{ finished: taskFinished }">
|
||||||
<div class="upload-process-card">
|
<div class="upload-process-card">
|
||||||
<div class="card-header">
|
<div class="card-header">
|
||||||
<div class="card-header-left">
|
<div class="card-header-left">
|
||||||
@@ -221,6 +221,9 @@ const taskInProgress = computed(() => {
|
|||||||
}
|
}
|
||||||
return taskStatus.value ? !['completed', 'failed'].includes(taskStatus.value.status) : false;
|
return taskStatus.value ? !['completed', 'failed'].includes(taskStatus.value.status) : false;
|
||||||
});
|
});
|
||||||
|
const taskFinished = computed(() => {
|
||||||
|
return taskStatus.value ? ['completed', 'failed'].includes(taskStatus.value.status) : false;
|
||||||
|
});
|
||||||
const showUploadProgress = computed(() => working.value || uploadProgress.value > 0);
|
const showUploadProgress = computed(() => working.value || uploadProgress.value > 0);
|
||||||
const uploadedCount = computed(() => files.value.filter(item => item.status === 'uploaded').length);
|
const uploadedCount = computed(() => files.value.filter(item => item.status === 'uploaded').length);
|
||||||
const errorCount = computed(() => files.value.filter(item => item.status === 'error').length);
|
const errorCount = computed(() => files.value.filter(item => item.status === 'error').length);
|
||||||
|
|||||||
@@ -906,6 +906,12 @@ select {
|
|||||||
border-radius: var(--td-radius-default);
|
border-radius: var(--td-radius-default);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.process-section.finished .log-container {
|
||||||
|
height: auto;
|
||||||
|
min-height: 180px;
|
||||||
|
max-height: clamp(220px, calc(100vh - 620px), 460px);
|
||||||
|
}
|
||||||
|
|
||||||
.log-content {
|
.log-content {
|
||||||
margin: 0;
|
margin: 0;
|
||||||
color: var(--td-text-color-secondary);
|
color: var(--td-text-color-secondary);
|
||||||
|
|||||||
Reference in New Issue
Block a user