from __future__ import annotations import csv import json import shutil import uuid import zipfile from pathlib import Path from urllib.parse import unquote, urlparse from urllib.request import url2pathname import httpx from fastapi import UploadFile from .config import AppConfig, ROOT_DIR from .deepseek import DeepSeekMapper, persist_high_confidence_mappings from .excel_writer import XlsTemplateWriter from .models import JobResult, WeldRecord from .pcf_parser import PcfParser JOBS_DIR = ROOT_DIR / "outputs" / "jobs" UPLOADS_DIR = ROOT_DIR / "outputs" / "uploads" MAX_ATTACHMENT_BYTES = 100 * 1024 * 1024 class JobStore: def __init__(self) -> None: self.results: dict[str, JobResult] = {} def set(self, result: JobResult) -> None: self.results[result.job_id] = result def get(self, job_id: str) -> JobResult | None: return self.results.get(job_id) job_store = JobStore() def safe_filename(name: str) -> str: keep = [] for ch in Path(name).name: if ch.isalnum() or ch in ".-_() ": keep.append(ch) else: keep.append("_") return "".join(keep) or "attachment.pcf" async def process_uploads(files: list[UploadFile]) -> JobResult: job_id = uuid.uuid4().hex[:12] upload_dir = UPLOADS_DIR / job_id upload_dir.mkdir(parents=True, exist_ok=True) saved_files: list[Path] = [] messages: list[str] = [] for upload in files: filename = safe_filename(upload.filename or "upload.pcf") target = upload_dir / filename if target.suffix.lower() != ".pcf": messages.append(f"拒绝非 PCF 文件:{filename}") continue with target.open("wb") as f: shutil.copyfileobj(upload.file, f) saved_files.append(target) return await process_pcf_paths(job_id=job_id, pcf_files=saved_files, initial_messages=messages) async def analyze_attachment_urls(urls: list[str]) -> dict: job_id = uuid.uuid4().hex[:12] download_dir = UPLOADS_DIR / job_id download_dir.mkdir(parents=True, exist_ok=True) pcf_files, messages = await download_attachment_urls(urls, download_dir) analysis = await analyze_pcf_paths(pcf_files=pcf_files, initial_messages=messages) return analysis_to_json_payload(job_id=job_id, input_files=len(pcf_files), analysis=analysis) async def process_pcf_paths( job_id: str, pcf_files: list[Path], initial_messages: list[str] | None = None, ) -> JobResult: job_dir = JOBS_DIR / job_id job_dir.mkdir(parents=True, exist_ok=True) analysis = await analyze_pcf_paths(pcf_files=pcf_files, initial_messages=initial_messages) config = analysis["config"] records = analysis["records"] candidates = analysis["candidates"] decisions = analysis["decisions"] messages = analysis["messages"] import_headers = analysis["import_headers"] import_columns = analysis["import_columns"] import_rows = analysis["import_rows"] issue_rows = analysis["issue_rows"] mapper_enabled = analysis["llm_enabled"] output_xls = job_dir / "焊接数据导入结果.xls" issues_csv = job_dir / "问题清单.csv" report_json = job_dir / "解析报告.json" writer = XlsTemplateWriter(config) writer.write(records, output_xls) write_issues_csv(issue_rows, issues_csv) write_report_json( records=records, candidates=candidates, decisions=decisions, accepted_mappings=analysis["accepted_mappings"], messages=messages, report_path=report_json, llm_enabled=mapper_enabled, ) warning_count = analysis["warning_count"] error_count = analysis["error_count"] result = JobResult( job_id=job_id, input_files=len(pcf_files), weld_count=len(records), warning_count=warning_count, error_count=error_count, output_xls=output_xls, issues_csv=issues_csv, report_json=report_json, messages=messages, llm_enabled=mapper_enabled, llm_mappings=decisions, import_headers=import_headers, import_columns=import_columns, import_rows=import_rows, issue_rows=issue_rows, ) job_store.set(result) return result async def analyze_pcf_paths( pcf_files: list[Path], initial_messages: list[str] | None = None, ) -> dict: messages = list(initial_messages or []) config = AppConfig.load() parser = PcfParser(config) records, candidates, parse_messages = parser.parse_files(pcf_files) messages.extend(parse_messages) mapper = DeepSeekMapper(config) decisions = [] accepted_mappings = {} if mapper.enabled and candidates: try: decisions = await mapper.map_unknown_fields(candidates) accepted_mappings = persist_high_confidence_mappings(config, decisions) if accepted_mappings: config = AppConfig.load() parser = PcfParser(config) records, _, parse_messages = parser.parse_files(pcf_files, accepted_mappings) messages.extend(parse_messages) except Exception as exc: messages.append(f"DeepSeek 字段映射失败,已降级为规则解析:{exc}") elif candidates: messages.append("未在 config/mapping.yaml 配置 deepseek.api_key,已跳过非标准字段智能映射") writer = XlsTemplateWriter(config) import_headers, import_columns, import_rows = writer.build_import_rows(records) issue_rows = build_issue_rows(records, candidates, decisions, config.confidence_threshold) warning_count = sum(len(r.issues) for r in records) + len(messages) error_count = 0 if records else 1 if not records: messages.append("未解析到任何焊口记录") return { "config": config, "records": records, "candidates": candidates, "decisions": decisions, "accepted_mappings": accepted_mappings, "messages": messages, "llm_enabled": mapper.enabled, "import_headers": import_headers, "import_columns": import_columns, "import_rows": import_rows, "issue_rows": issue_rows, "warning_count": warning_count, "error_count": error_count, } async def download_attachment_urls(urls: list[str], target_dir: Path) -> tuple[list[Path], list[str]]: pcf_files: list[Path] = [] messages: list[str] = [] async with httpx.AsyncClient(follow_redirects=True, timeout=60) as client: for index, url in enumerate(urls, start=1): parsed = urlparse(url) if is_local_attachment(url, parsed): local_files, local_messages = collect_local_attachment(url, target_dir, index) pcf_files.extend(local_files) messages.extend(local_messages) continue if parsed.scheme not in {"http", "https"}: messages.append(f"附件引用协议不支持:{url}") continue downloaded, download_messages = await download_remote_attachment(client, url, target_dir, index) pcf_files.extend(downloaded) messages.extend(download_messages) return pcf_files, messages def is_local_attachment(raw: str, parsed) -> bool: if parsed.scheme == "file": return True if parsed.scheme in {"http", "https"}: return False if len(raw) >= 3 and raw[1] == ":" and raw[2] in {"\\", "/"}: return True if raw.startswith("\\\\"): return True return parsed.scheme == "" def collect_local_attachment(raw: str, target_dir: Path, index: int) -> tuple[list[Path], list[str]]: messages: list[str] = [] pcf_files: list[Path] = [] source = local_path_from_reference(raw) if not source.exists() or not source.is_file(): messages.append(f"本地附件不存在或不是文件:{raw}") return pcf_files, messages if source.stat().st_size > MAX_ATTACHMENT_BYTES: messages.append(f"本地附件超过大小限制:{source}") return pcf_files, messages target = target_dir / safe_filename(source.name or f"attachment_{index}.pcf") if source.resolve() != target.resolve(): shutil.copyfile(source, target) else: target = source if target.suffix.lower() == ".pcf": pcf_files.append(target) elif target.suffix.lower() == ".zip": extracted, extract_messages = extract_pcf_zip(target, target_dir / f"zip_{index}") pcf_files.extend(extracted) messages.extend(extract_messages) else: messages.append(f"本地附件不是 PCF 或 ZIP,已跳过:{source.name}") return pcf_files, messages def local_path_from_reference(raw: str) -> Path: parsed = urlparse(raw) if parsed.scheme == "file": if parsed.netloc and parsed.netloc not in {"localhost", "127.0.0.1"}: return Path(f"//{parsed.netloc}{url2pathname(parsed.path)}") return Path(url2pathname(unquote(parsed.path))) return Path(raw) async def download_remote_attachment( client: httpx.AsyncClient, url: str, target_dir: Path, index: int, ) -> tuple[list[Path], list[str]]: pcf_files: list[Path] = [] messages: list[str] = [] try: response = await client.get(url) response.raise_for_status() except Exception as exc: messages.append(f"附件下载失败:{url},原因:{exc}") return pcf_files, messages content = response.content if len(content) > MAX_ATTACHMENT_BYTES: messages.append(f"附件超过大小限制:{url}") return pcf_files, messages filename = filename_from_response(url, response.headers, index) saved = target_dir / filename saved.write_bytes(content) if saved.suffix.lower() == ".pcf": pcf_files.append(saved) elif saved.suffix.lower() == ".zip": extracted, extract_messages = extract_pcf_zip(saved, target_dir / f"zip_{index}") pcf_files.extend(extracted) messages.extend(extract_messages) else: messages.append(f"附件不是 PCF 或 ZIP,已跳过:{filename}") return pcf_files, messages def filename_from_response(url: str, headers: httpx.Headers, index: int) -> str: disposition = headers.get("content-disposition", "") marker = "filename=" if marker in disposition: raw = disposition.split(marker, 1)[1].strip().strip('"') return safe_filename(unquote(raw)) parsed_name = Path(unquote(urlparse(url).path)).name return safe_filename(parsed_name or f"attachment_{index}.pcf") def extract_pcf_zip(zip_path: Path, target_dir: Path) -> tuple[list[Path], list[str]]: target_dir.mkdir(parents=True, exist_ok=True) pcf_files: list[Path] = [] messages: list[str] = [] try: with zipfile.ZipFile(zip_path) as archive: for member in archive.infolist(): if member.is_dir(): continue name = safe_filename(Path(member.filename).name) if not name.lower().endswith(".pcf"): continue target = target_dir / name with archive.open(member) as src, target.open("wb") as dst: shutil.copyfileobj(src, dst) pcf_files.append(target) except zipfile.BadZipFile: messages.append(f"ZIP 附件无法解压:{zip_path.name}") if not pcf_files: messages.append(f"ZIP 附件中未找到 PCF 文件:{zip_path.name}") return pcf_files, messages def analysis_to_json_payload(job_id: str, input_files: int, analysis: dict) -> dict: records: list[WeldRecord] = analysis["records"] return { "job_id": job_id, "input_files": input_files, "weld_count": len(records), "warning_count": analysis["warning_count"], "error_count": analysis["error_count"], "messages": analysis["messages"], "llm_enabled": analysis["llm_enabled"], "tables": { "import_headers": analysis["import_headers"], "import_columns": analysis["import_columns"], "import_rows": analysis["import_rows"], "issue_rows": analysis["issue_rows"], }, "report": build_report_payload( records=records, candidates=analysis["candidates"], decisions=analysis["decisions"], accepted_mappings=analysis["accepted_mappings"], messages=analysis["messages"], llm_enabled=analysis["llm_enabled"], ), } def build_issue_rows( records: list[WeldRecord], candidates, decisions, threshold: float, ) -> list[dict[str, str]]: rows: list[dict[str, str]] = [] for record in records: for issue in record.issues: rows.append( { "类型": "解析警告", "文件": record.source_file, "焊口代号": record.weld_no, "字段": "", "内容": issue, "处理状态": "需复核", } ) high_conf = {d.source_field for d in decisions if d.confidence >= threshold} for candidate in candidates: rows.append( { "类型": "未知字段", "文件": "", "焊口代号": "", "字段": candidate.source_field, "内容": "; ".join(candidate.sample_values), "处理状态": "已自动映射" if candidate.source_field in high_conf else "未映射/需复核", } ) return rows def write_issues_csv(issue_rows: list[dict[str, str]], path: Path) -> None: with path.open("w", encoding="utf-8-sig", newline="") as f: writer = csv.writer(f) headers = ["类型", "文件", "焊口代号", "字段", "内容", "处理状态"] writer.writerow(headers) for row in issue_rows: writer.writerow([row.get(header, "") for header in headers]) def write_report_json( records: list[WeldRecord], candidates, decisions, accepted_mappings: dict[str, str], messages: list[str], report_path: Path, llm_enabled: bool, ) -> None: payload = build_report_payload( records=records, candidates=candidates, decisions=decisions, accepted_mappings=accepted_mappings, messages=messages, llm_enabled=llm_enabled, ) with report_path.open("w", encoding="utf-8") as f: json.dump(payload, f, ensure_ascii=False, indent=2) def build_report_payload( records: list[WeldRecord], candidates, decisions, accepted_mappings: dict[str, str], messages: list[str], llm_enabled: bool, ) -> dict: return { "summary": { "weld_count": len(records), "source_files": sorted({r.source_file for r in records}), "llm_enabled": llm_enabled, "unknown_field_count": len(candidates), }, "messages": messages, "accepted_mappings": accepted_mappings, "llm_mappings": [ { "source_field": d.source_field, "target_field": d.target_field, "confidence": d.confidence, "reason": d.reason, "sample_values": d.sample_values, } for d in decisions ], "records_preview": [ { "source_file": r.source_file, "pipeline_reference": r.pipeline_reference, "line_no": r.line_no, "weld_no": r.weld_no, "diameter": r.diameter, "weld_area": r.weld_area, "issues": r.issues, } for r in records[:20] ], }