refactor: extract atomic JSON helpers to io_utils
Centralize read_json/write_json_atomic used by pipeline meta I/O and worker processed.json bookkeeping. Preserves corrupt-file warning in load_processed(). Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
+12
-17
@@ -25,6 +25,7 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from . import config
|
||||
from .io_utils import read_json, write_json_atomic
|
||||
|
||||
logger = logging.getLogger("livef12.pipeline")
|
||||
|
||||
@@ -236,10 +237,10 @@ def preprocess_original(paths: JobPaths) -> JobPaths:
|
||||
]
|
||||
)
|
||||
try:
|
||||
proc = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
|
||||
proc = subprocess.run(cmd, capture_output=True, text=True, timeout=config.PREPROCESS_TIMEOUT)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
tmp.unlink(missing_ok=True)
|
||||
raise PipelineError("Preprocess Timeout nach 120s") from exc
|
||||
raise PipelineError(f"Preprocess Timeout nach {config.PREPROCESS_TIMEOUT}s") from exc
|
||||
if proc.returncode != 0 or not tmp.exists() or tmp.stat().st_size == 0:
|
||||
tmp.unlink(missing_ok=True)
|
||||
err = strip_ansi((proc.stderr or proc.stdout or "").strip())
|
||||
@@ -346,26 +347,20 @@ def job_paths(stem: str, original_suffix: str = ".jpg") -> JobPaths:
|
||||
)
|
||||
|
||||
|
||||
def _read_meta(stem: str) -> dict[str, Any] | None:
|
||||
def read_meta(stem: str) -> dict[str, Any] | None:
|
||||
"""Return the full meta record for a job, or None if missing/corrupt."""
|
||||
paths = job_paths(stem)
|
||||
if not paths.meta.exists():
|
||||
return None
|
||||
try:
|
||||
return json.loads(paths.meta.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
data = read_json(paths.meta)
|
||||
return data if isinstance(data, dict) else None
|
||||
|
||||
|
||||
def _write_meta(paths: JobPaths, data: dict[str, Any]) -> None:
|
||||
paths.meta.parent.mkdir(parents=True, exist_ok=True)
|
||||
data["updated_at"] = now_iso()
|
||||
tmp = paths.meta.with_suffix(".json.tmp")
|
||||
tmp.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
tmp.replace(paths.meta)
|
||||
write_json_atomic(paths.meta, data)
|
||||
|
||||
|
||||
def read_status(job_id: str) -> dict[str, Any] | None:
|
||||
data = _read_meta(job_id)
|
||||
data = read_meta(job_id)
|
||||
if not data:
|
||||
return None
|
||||
return {
|
||||
@@ -380,7 +375,7 @@ def read_status(job_id: str) -> dict[str, Any] | None:
|
||||
|
||||
|
||||
def write_status(paths: JobPaths, status: str, **extra: Any) -> None:
|
||||
data = _read_meta(paths.stem) or {}
|
||||
data = read_meta(paths.stem) or {}
|
||||
data["job_id"] = paths.stem
|
||||
data["source_stem"] = paths.stem
|
||||
data["status"] = status
|
||||
@@ -390,7 +385,7 @@ def write_status(paths: JobPaths, status: str, **extra: Any) -> None:
|
||||
|
||||
|
||||
def read_manifest(job_id: str) -> dict[str, Any] | None:
|
||||
data = _read_meta(job_id)
|
||||
data = read_meta(job_id)
|
||||
if not data:
|
||||
return None
|
||||
return {
|
||||
@@ -405,7 +400,7 @@ def read_manifest(job_id: str) -> dict[str, Any] | None:
|
||||
|
||||
|
||||
def write_manifest(paths: JobPaths, manifest: dict[str, Any]) -> None:
|
||||
data = _read_meta(paths.stem) or {}
|
||||
data = read_meta(paths.stem) or {}
|
||||
data["job_id"] = paths.stem
|
||||
data["source_stem"] = manifest.get("source_stem") or paths.stem
|
||||
data["original_file"] = manifest.get("original_file")
|
||||
|
||||
Reference in New Issue
Block a user