fix: 支持CSV自动识别UTF-8和GBK编码
This commit is contained in:
+24
-9
@@ -182,16 +182,31 @@ class MetrixWarehouse:
|
||||
raise RuntimeError(str(result))
|
||||
return int(result.get("affected_rows", 0)) if isinstance(result, dict) else 0
|
||||
|
||||
def import_csv(self, file_path: str, table_name: str) -> int:
|
||||
def import_csv(self, file_path: str, table_name: str, encoding: str | None = None) -> int:
|
||||
"""走 Metrix 平台导入 API 追加导入到已存在的表(字段映射由平台按列名处理)。"""
|
||||
job_id = self.client.import_csv(
|
||||
self.conn_id, table_name, Path(file_path), mode="append",
|
||||
database=self.database, create_table=False,
|
||||
)
|
||||
job = self.client.wait_job(job_id)
|
||||
if job.get("status") != "success":
|
||||
raise RuntimeError(job.get("error_code") or job.get("status") or "导入失败")
|
||||
return int(job.get("row_count") or 0)
|
||||
source_path = Path(file_path)
|
||||
upload_path = source_path
|
||||
converted_path: Path | None = None
|
||||
if encoding and encoding.lower() not in {"utf-8", "utf-8-sig"}:
|
||||
converted_path = source_path.with_name(f"{source_path.stem}_utf8{source_path.suffix}")
|
||||
with source_path.open("r", encoding=encoding, newline="") as source:
|
||||
with converted_path.open("w", encoding="utf-8-sig", newline="") as target:
|
||||
while chunk := source.read(1024 * 1024):
|
||||
target.write(chunk)
|
||||
upload_path = converted_path
|
||||
|
||||
try:
|
||||
job_id = self.client.import_csv(
|
||||
self.conn_id, table_name, upload_path, mode="append",
|
||||
database=self.database, create_table=False,
|
||||
)
|
||||
job = self.client.wait_job(job_id)
|
||||
if job.get("status") != "success":
|
||||
raise RuntimeError(job.get("error_code") or job.get("status") or "导入失败")
|
||||
return int(job.get("row_count") or 0)
|
||||
finally:
|
||||
if converted_path is not None:
|
||||
converted_path.unlink(missing_ok=True)
|
||||
|
||||
def execute_sql(self, sql: str) -> Tuple[bool, Any]:
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user