refactor: extract atomic JSON helpers to io_utils
Centralize read_json/write_json_atomic used by pipeline meta I/O and worker processed.json bookkeeping. Preserves corrupt-file warning in load_processed(). Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
+9
-13
@@ -13,7 +13,6 @@ pipeline is CPU heavy (gmic/rembg) and the worker container is capped at
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import shutil
|
||||
import time
|
||||
@@ -21,11 +20,10 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from . import config, pipeline
|
||||
from .io_utils import read_json, write_json_atomic
|
||||
from .logging_config import configure_logging
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s [%(name)s] %(message)s",
|
||||
)
|
||||
configure_logging()
|
||||
logger = logging.getLogger("livef12.worker")
|
||||
|
||||
|
||||
@@ -42,17 +40,15 @@ def ensure_dirs() -> None:
|
||||
def load_processed() -> dict[str, Any]:
|
||||
if not config.PROCESSED_FILE.exists():
|
||||
return {}
|
||||
try:
|
||||
return json.loads(config.PROCESSED_FILE.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
logger.warning("processed.json unreadable, starting fresh")
|
||||
return {}
|
||||
data = read_json(config.PROCESSED_FILE)
|
||||
if isinstance(data, dict):
|
||||
return data
|
||||
logger.warning("processed.json unreadable, starting fresh")
|
||||
return {}
|
||||
|
||||
|
||||
def save_processed(processed: dict[str, Any]) -> None:
|
||||
tmp = config.PROCESSED_FILE.with_suffix(".json.tmp")
|
||||
tmp.write_text(json.dumps(processed, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
tmp.replace(config.PROCESSED_FILE)
|
||||
write_json_atomic(config.PROCESSED_FILE, processed)
|
||||
|
||||
|
||||
def _file_key(path: Path) -> str:
|
||||
|
||||
Reference in New Issue
Block a user