froxxxy/livef12rocks
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: flatten output into variants/ and intermediates/

Browse Source 
Drop per-job subdirs for phone-friendly Syncthing layout with
stem-suffixed filenames; keep web state in meta/.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Frank Schwenk
2026-07-18 14:52:47 +02:00
parent 9a9a3e3afe
commit b925b151f0
11 changed files with 289 additions and 220 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/pipeline.py
+166 -123
View File
@@ -207,20 +207,20 @@ def run_rembg(input_image: Path, output_image: Path) -> tuple[bool, str]:
def preprocess_original(paths: JobPaths) -> JobPaths:
"""Downscale to MAX_EDGE_PX and normalize to sRGB JPEG as `original.jpg`.
"""Downscale to MAX_EDGE_PX and normalize to sRGB JPEG as `{stem}_original.jpg`.
Replaces any non-jpg original in the job dir. Side-effect free for the
incoming copy — only mutates jobs/<id>/.
Side-effect free for the incoming copy — only mutates variants/.
"""
src = paths.original
if not src.exists():
raise PipelineError(f"Original fehlt: {src}")
dest = paths.root / "original.jpg"
dest = paths.variants / f"{paths.stem}_original.jpg"
paths.variants.mkdir(parents=True, exist_ok=True)
# Write to a temp name first so we can replace an existing file in-place
# without reading/writing the same path.
tmp = paths.variants / f".{paths.stem}_pre_{uuid.uuid4().hex[:8]}.jpg"
resize = f"{config.MAX_EDGE_PX}x{config.MAX_EDGE_PX}>"
# Write to a temp name first so we can replace an existing original.jpg
# in-place without reading/writing the same path.
tmp = paths.root / f".original_pre_{uuid.uuid4().hex[:8]}.jpg"
cmd = _nice(
[
_magick_bin(),
@@ -257,13 +257,12 @@ def preprocess_original(paths: JobPaths) -> JobPaths:
config.MAX_EDGE_PX,
)
return JobPaths(
root=paths.root,
stem=paths.stem,
original=dest,
rembg=paths.rembg,
intermediates=paths.intermediates,
variants=paths.variants,
manifest=paths.manifest,
status=paths.status,
meta=paths.meta,
)
@@ -301,81 +300,11 @@ def now_iso() -> str:
return datetime.now(timezone.utc).isoformat(timespec="seconds")
def new_job_id() -> str:
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
return f"{stamp}-{uuid.uuid4().hex[:6]}"
@dataclass
class JobPaths:
root: Path
original: Path
rembg: Path
intermediates: Path
variants: Path
manifest: Path
status: Path
def job_paths(job_id: str, original_suffix: str = ".jpg") -> JobPaths:
root = config.JOBS_DIR / job_id
return JobPaths(
root=root,
original=root / f"original{original_suffix}",
rembg=root / "rembg.png",
intermediates=root / "intermediates",
variants=root / "variants",
manifest=root / "manifest.json",
status=root / "status.json",
)
def read_status(job_id: str) -> dict[str, Any] | None:
paths = job_paths(job_id)
if not paths.status.exists():
return None
return json.loads(paths.status.read_text(encoding="utf-8"))
def write_status(paths: JobPaths, status: str, **extra: Any) -> None:
data = {}
if paths.status.exists():
try:
data = json.loads(paths.status.read_text(encoding="utf-8"))
except (OSError, ValueError):
data = {}
data["status"] = status
data["updated_at"] = now_iso()
data.setdefault("created_at", data["updated_at"])
data.update(extra)
paths.status.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
def read_manifest(job_id: str) -> dict[str, Any] | None:
paths = job_paths(job_id)
if not paths.manifest.exists():
return None
return json.loads(paths.manifest.read_text(encoding="utf-8"))
def write_manifest(paths: JobPaths, manifest: dict[str, Any]) -> None:
manifest["updated_at"] = now_iso()
paths.manifest.write_text(json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8")
def next_variant_id(manifest: dict[str, Any]) -> str:
existing = {v["id"] for v in manifest.get("variants", [])}
i = len(manifest.get("variants", [])) + 1
while f"v{i}" in existing:
i += 1
return f"v{i}"
_UNSAFE_STEM_RE = re.compile(r"[^\w.\-]+", re.UNICODE)
def sanitize_stem(name: str) -> str:
"""Safe basename stem for variant files (no path separators / junk)."""
"""Safe basename stem for flat files (no path separators / junk)."""
stem = Path(name).stem if name else ""
stem = stem.replace("/", "_").replace("\\", "_").strip().strip(".")
stem = _UNSAFE_STEM_RE.sub("_", stem).strip("._")
@@ -386,13 +315,134 @@ def variant_filename(stem: str, variant_id: str) -> str:
return f"{sanitize_stem(stem)}_{variant_id}.png"
@dataclass
class JobPaths:
"""Flat layout under DATA_DIR — one stem, shared variants/ + intermediates/.
variants/{stem}_original.jpg
variants/{stem}_rembg.png
variants/{stem}_v1.png
intermediates/{stem}_v1_*.png
meta/{stem}.json
"""
stem: str
original: Path
rembg: Path
intermediates: Path
variants: Path
meta: Path
def job_paths(stem: str, original_suffix: str = ".jpg") -> JobPaths:
stem = sanitize_stem(stem)
return JobPaths(
stem=stem,
original=config.VARIANTS_DIR / f"{stem}_original{original_suffix}",
rembg=config.VARIANTS_DIR / f"{stem}_rembg.png",
intermediates=config.INTERMEDIATES_DIR,
variants=config.VARIANTS_DIR,
meta=config.META_DIR / f"{stem}.json",
)
def _read_meta(stem: str) -> dict[str, Any] | None:
paths = job_paths(stem)
if not paths.meta.exists():
return None
try:
return json.loads(paths.meta.read_text(encoding="utf-8"))
except (OSError, ValueError):
return None
def _write_meta(paths: JobPaths, data: dict[str, Any]) -> None:
paths.meta.parent.mkdir(parents=True, exist_ok=True)
data["updated_at"] = now_iso()
tmp = paths.meta.with_suffix(".json.tmp")
tmp.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
tmp.replace(paths.meta)
def read_status(job_id: str) -> dict[str, Any] | None:
data = _read_meta(job_id)
if not data:
return None
return {
"status": data.get("status", "unknown"),
"created_at": data.get("created_at"),
"updated_at": data.get("updated_at"),
"error": data.get("error"),
"variant_errors": data.get("variant_errors"),
"source_file": data.get("source_file"),
"source_stem": data.get("source_stem"),
}
def write_status(paths: JobPaths, status: str, **extra: Any) -> None:
data = _read_meta(paths.stem) or {}
data["job_id"] = paths.stem
data["source_stem"] = paths.stem
data["status"] = status
data.setdefault("created_at", now_iso())
data.update(extra)
_write_meta(paths, data)
def read_manifest(job_id: str) -> dict[str, Any] | None:
data = _read_meta(job_id)
if not data:
return None
return {
"job_id": data.get("job_id", job_id),
"original_file": data.get("original_file"),
"rembg_file": data.get("rembg_file"),
"source_stem": data.get("source_stem", job_id),
"created_at": data.get("created_at"),
"updated_at": data.get("updated_at"),
"variants": data.get("variants") or [],
}
def write_manifest(paths: JobPaths, manifest: dict[str, Any]) -> None:
data = _read_meta(paths.stem) or {}
data["job_id"] = paths.stem
data["source_stem"] = manifest.get("source_stem") or paths.stem
data["original_file"] = manifest.get("original_file")
data["rembg_file"] = manifest.get("rembg_file")
data["variants"] = manifest.get("variants") or []
if "created_at" in manifest:
data.setdefault("created_at", manifest["created_at"])
data.setdefault("status", data.get("status", "processing"))
_write_meta(paths, data)
def list_job_ids() -> list[str]:
if not config.META_DIR.exists():
return []
ids = []
for path in config.META_DIR.glob("*.json"):
if path.name.endswith(".json.tmp"):
continue
ids.append(path.stem)
return ids
def next_variant_id(manifest: dict[str, Any]) -> str:
existing = {v["id"] for v in manifest.get("variants", [])}
i = len(manifest.get("variants", [])) + 1
while f"v{i}" in existing:
i += 1
return f"v{i}"
def resolve_source_stem(manifest: dict[str, Any] | None, paths: JobPaths) -> str:
"""Prefer manifest source_stem; fall back to original path stem."""
"""Prefer manifest source_stem; fall back to JobPaths.stem."""
if manifest:
stored = manifest.get("source_stem")
if isinstance(stored, str) and stored.strip():
return sanitize_stem(stored)
return sanitize_stem(paths.original.stem)
return paths.stem
def compose_variant(
@@ -401,7 +451,7 @@ def compose_variant(
*,
variant_id: str,
source: str,
variant_stem: str,
variant_stem: str | None = None,
bg_name: str | None = None,
bg_mode: str | None = None,
fg_name: str | None = None,
@@ -411,16 +461,15 @@ def compose_variant(
) -> dict[str, Any]:
"""Compose one variant image from the job's cached original + rembg.
If bg_name/fg_name/bg_mode/fg_mode are given (remix path) they are used
directly. Otherwise a random working filter is picked with retries,
exactly like make_random.py's compose_one().
Final file is `variants/{stem}_{variant_id}.png`; manifest id stays `vN`.
Final file is `variants/{stem}_{variant_id}.png`; intermediates share
the same stem prefix under `intermediates/`.
"""
rng = rng or random.Random()
tmp_dir = paths.intermediates
tmp_dir.mkdir(parents=True, exist_ok=True)
stem = sanitize_stem(variant_stem)
paths.variants.mkdir(parents=True, exist_ok=True)
stem = sanitize_stem(variant_stem or paths.stem)
prefix = f"{stem}_{variant_id}"
bg_mode = bg_mode or rng.choice(assets.blend_modes)
fg_mode = fg_mode or rng.choice(assets.blend_modes)
@@ -444,14 +493,13 @@ def compose_variant(
fg_name, fg_command = pick_working_filter(assets.foreground_names, assets.commands, paths.rembg, tmp_dir, "fg", rng)
p = {
"bg_filtered": tmp_dir / f"{variant_id}_bg_filtered.png",
"step1": tmp_dir / f"{variant_id}_bg_blend.png",
"step2": tmp_dir / f"{variant_id}_rembg_alpha.png",
"fg_filtered": tmp_dir / f"{variant_id}_fg_filtered.png",
"composed": tmp_dir / f"{variant_id}_composed.png",
"bg_filtered": tmp_dir / f"{prefix}_bg_filtered.png",
"step1": tmp_dir / f"{prefix}_bg_blend.png",
"step2": tmp_dir / f"{prefix}_rembg_alpha.png",
"fg_filtered": tmp_dir / f"{prefix}_fg_filtered.png",
"composed": tmp_dir / f"{prefix}_composed.png",
"final": paths.variants / variant_filename(stem, variant_id),
}
paths.variants.mkdir(parents=True, exist_ok=True)
ok, err = apply_filter(paths.original, bg_command, p["bg_filtered"])
_raise_on_gmic_failure(ok, err, "Background-Filter fehlgeschlagen")
@@ -468,14 +516,13 @@ def compose_variant(
ok, err = blend_layers_opacity(p["step2"], p["fg_filtered"], fg_mode, opacity, p["composed"])
_raise_on_gmic_failure(ok, err, "Foreground-Blend fehlgeschlagen")
ok, err = apply_filter_chain(p["composed"], config.POST_FILTERS, p["final"], tmp_dir, variant_id)
ok, err = apply_filter_chain(p["composed"], config.POST_FILTERS, p["final"], tmp_dir, prefix)
_raise_on_gmic_failure(ok, err, "Post-Processing fehlgeschlagen")
rel = lambda path: str(path.relative_to(paths.root))
return {
"id": variant_id,
"source": source,
"file": rel(p["final"]),
"file": p["final"].name,
"background_filter": bg_name,
"background_command": bg_command,
"background_blend": bg_mode,
@@ -486,11 +533,11 @@ def compose_variant(
"post_filters": list(config.POST_FILTERS),
"created_at": now_iso(),
"intermediates": {
"bg_filtered": rel(p["bg_filtered"]),
"bg_blend": rel(p["step1"]),
"rembg_alpha": rel(p["step2"]),
"fg_filtered": rel(p["fg_filtered"]),
"composed": rel(p["composed"]),
"bg_filtered": p["bg_filtered"].name,
"bg_blend": p["step1"].name,
"rembg_alpha": p["step2"].name,
"fg_filtered": p["fg_filtered"].name,
"composed": p["composed"].name,
},
}
@@ -503,21 +550,16 @@ def process_job(
source_stem: str | None = None,
) -> None:
"""Full pipeline for a freshly ingested incoming file: preprocess, rembg
once, generate OUTPUT_COUNT variants, write manifest + status.
once, generate OUTPUT_COUNT variants, write meta (status + manifest).
`source_path` must already be a private copy (jobs/<id>/original.*) —
callers (worker.py) are responsible for copying out of incoming first, so
the incoming file itself is never touched here.
Variants that hit FILTER_TIMEOUT during the normal pass are retried
once at the end with FILTER_TIMEOUT_LONG. Filter probes in
pick_working_filter stay on the short timeout (skip, don't escalate).
`job_id` is the sanitized source stem. `source_path` must already be the
private copy at variants/{stem}_original{suffix}.
"""
paths = job_paths(job_id, original_suffix)
paths.root.mkdir(parents=True, exist_ok=True)
paths.intermediates.mkdir(parents=True, exist_ok=True)
stem = sanitize_stem(source_stem or job_id)
paths = job_paths(stem, original_suffix)
paths.variants.mkdir(parents=True, exist_ok=True)
stem = sanitize_stem(source_stem or source_path.stem)
paths.intermediates.mkdir(parents=True, exist_ok=True)
paths.meta.parent.mkdir(parents=True, exist_ok=True)
write_status(paths, "processing", source_file=str(source_path.name), source_stem=stem)
@@ -525,13 +567,14 @@ def process_job(
paths = preprocess_original(paths)
manifest: dict[str, Any] = {
"job_id": job_id,
"job_id": stem,
"original_file": paths.original.name,
"rembg_file": paths.rembg.name,
"source_stem": stem,
"created_at": now_iso(),
"variants": [],
}
write_manifest(paths, manifest)
ok, err = run_rembg(paths.original, paths.rembg)
if not ok:
@@ -556,16 +599,16 @@ def process_job(
manifest["variants"].append(entry)
write_manifest(paths, manifest)
except FilterTimeoutError as exc:
logger.warning("[%s] variant %s timed out (will retry long): %s", job_id, variant_id, exc)
logger.warning("[%s] variant %s timed out (will retry long): %s", stem, variant_id, exc)
timeout_retries.append(variant_id)
except PipelineError as exc:
logger.error("[%s] variant %s failed: %s", job_id, variant_id, exc)
logger.error("[%s] variant %s failed: %s", stem, variant_id, exc)
variant_errors.append(f"{variant_id}: {exc}")
if timeout_retries:
logger.info(
"[%s] long-retry %d variant(s) with timeout=%ds: %s",
job_id,
stem,
len(timeout_retries),
config.FILTER_TIMEOUT_LONG,
", ".join(timeout_retries),
@@ -584,7 +627,7 @@ def process_job(
manifest["variants"].append(entry)
write_manifest(paths, manifest)
except PipelineError as exc:
logger.error("[%s] variant %s long-retry failed: %s", job_id, variant_id, exc)
logger.error("[%s] variant %s long-retry failed: %s", stem, variant_id, exc)
variant_errors.append(f"{variant_id}: {exc}")
if not manifest["variants"]:
@@ -592,8 +635,8 @@ def process_job(
write_status(paths, "done", variant_errors=variant_errors)
except PipelineError as exc:
logger.error("[%s] job failed: %s", job_id, exc)
logger.error("[%s] job failed: %s", stem, exc)
write_status(paths, "error", error=str(exc))
except Exception as exc: # noqa: BLE001 - keep the worker loop alive
logger.exception("[%s] unexpected error", job_id)
logger.exception("[%s] unexpected error", stem)
write_status(paths, "error", error=f"Unerwarteter Fehler: {exc}")