feat: harden pipeline UX — preprocess, rembg alpha, remix/lightbox

Downscale to 2000px JPEG before rembg, random blend opacity, timeout
retries, per-variant remix prefill, lightbox, and longer SFTP idle.

Co-authored-by: Cursor <cursoragent@cursor.com>

app/pipeline.py

@@ -12,14 +12,16 @@ import json
 import logging
 import random
 import re
+import shutil
 import subprocess
 import sys
 import time
 import uuid
-from dataclasses import dataclass, field
+from contextlib import contextmanager
+from dataclasses import dataclass
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Any
+from typing import Any, Iterator
 
 from . import config
 
@@ -27,6 +29,9 @@ logger = logging.getLogger("livef12.pipeline")
 
 _ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
 
+# Optional override for run_gmic timeout (used by the long-retry pass).
+_filter_timeout_override: int | None = None
+
 
 class PipelineError(RuntimeError):
     """Raised when a job (or a single variant) cannot be completed."""
@@ -36,6 +41,10 @@ class FilterNotFoundError(PipelineError):
     """Raised when no working filter could be picked after retries."""
 
 
+class FilterTimeoutError(PipelineError):
+    """Raised when a gmic step hits the active filter timeout."""
+
+
 @dataclass
 class FilterAssets:
     background_names: list[str]
@@ -57,6 +66,31 @@ def _nice(cmd: list[str]) -> list[str]:
     return ["nice", "-n", str(config.NICE_LEVEL), *cmd]
 
 
+def _active_filter_timeout() -> int:
+    return _filter_timeout_override if _filter_timeout_override is not None else config.FILTER_TIMEOUT
+
+
+@contextmanager
+def filter_timeout(seconds: int) -> Iterator[None]:
+    """Temporarily override FILTER_TIMEOUT for run_gmic (long-retry pass)."""
+    global _filter_timeout_override
+    previous = _filter_timeout_override
+    _filter_timeout_override = seconds
+    try:
+        yield
+    finally:
+        _filter_timeout_override = previous
+
+
+def _magick_bin() -> str:
+    if config.MAGICK_BIN:
+        return config.MAGICK_BIN
+    for name in ("magick", "convert"):
+        if shutil.which(name):
+            return name
+    raise PipelineError("ImageMagick nicht installiert (magick/convert fehlt)")
+
+
 def load_lines(path: Path) -> list[str]:
     if not path.exists():
         raise FileNotFoundError(f"Datei nicht gefunden: {path}")
@@ -96,12 +130,13 @@ def load_assets() -> FilterAssets:
 
 def run_gmic(args: list[str], output_image: Path) -> tuple[bool, str, float]:
     output_image.parent.mkdir(parents=True, exist_ok=True)
+    timeout = _active_filter_timeout()
     cmd = _nice([config.GMIC_BIN, *args, "-o", str(output_image)])
     start = time.monotonic()
     try:
-        proc = subprocess.run(cmd, capture_output=True, text=True, timeout=config.FILTER_TIMEOUT)
+        proc = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
     except subprocess.TimeoutExpired:
-        return False, f"Timeout nach {config.FILTER_TIMEOUT}s", time.monotonic() - start
+        return False, f"Timeout nach {timeout}s", time.monotonic() - start
     elapsed = time.monotonic() - start
     if proc.returncode != 0:
         err = strip_ansi((proc.stderr or proc.stdout or "").strip())
@@ -112,6 +147,14 @@ def run_gmic(args: list[str], output_image: Path) -> tuple[bool, str, float]:
     return True, "", elapsed
 
 
+def _raise_on_gmic_failure(ok: bool, err: str, label: str) -> None:
+    if ok:
+        return
+    if err.startswith("Timeout nach"):
+        raise FilterTimeoutError(f"{label}: {err}")
+    raise PipelineError(f"{label}: {err}")
+
+
 def gmic_filter_args(image: Path, full_command: str) -> list[str]:
     if " " in full_command:
         name, args = full_command.split(" ", 1)
@@ -163,6 +206,67 @@ def run_rembg(input_image: Path, output_image: Path) -> tuple[bool, str]:
     return True, ""
 
 
+def preprocess_original(paths: JobPaths) -> JobPaths:
+    """Downscale to MAX_EDGE_PX and normalize to sRGB JPEG as `original.jpg`.
+
+    Replaces any non-jpg original in the job dir. Side-effect free for the
+    inbox copy — only mutates jobs/<id>/.
+    """
+    src = paths.original
+    if not src.exists():
+        raise PipelineError(f"Original fehlt: {src}")
+
+    dest = paths.root / "original.jpg"
+    resize = f"{config.MAX_EDGE_PX}x{config.MAX_EDGE_PX}>"
+    # Write to a temp name first so we can replace an existing original.jpg
+    # in-place without reading/writing the same path.
+    tmp = paths.root / f".original_pre_{uuid.uuid4().hex[:8]}.jpg"
+    cmd = _nice(
+        [
+            _magick_bin(),
+            str(src),
+            "-auto-orient",
+            "-resize",
+            resize,
+            "-colorspace",
+            "sRGB",
+            "-quality",
+            "92",
+            str(tmp),
+        ]
+    )
+    try:
+        proc = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
+    except subprocess.TimeoutExpired as exc:
+        tmp.unlink(missing_ok=True)
+        raise PipelineError("Preprocess Timeout nach 120s") from exc
+    if proc.returncode != 0 or not tmp.exists() or tmp.stat().st_size == 0:
+        tmp.unlink(missing_ok=True)
+        err = strip_ansi((proc.stderr or proc.stdout or "").strip())
+        err = err.splitlines()[-1] if err else f"Exit code {proc.returncode}"
+        raise PipelineError(f"Preprocess fehlgeschlagen: {err}")
+
+    tmp.replace(dest)
+    if src.resolve() != dest.resolve():
+        src.unlink(missing_ok=True)
+
+    logger.info(
+        "preprocessed %s -> %s (max_edge=%d)",
+        src.name,
+        dest.name,
+        config.MAX_EDGE_PX,
+    )
+    return JobPaths(
+        root=paths.root,
+        original=dest,
+        rembg=paths.rembg,
+        intermediates=paths.intermediates,
+        variants=paths.variants,
+        manifest=paths.manifest,
+        status=paths.status,
+    )
+
+
 def pick_working_filter(
     names: list[str],
     commands: dict[str, str],
@@ -292,7 +396,10 @@ def compose_variant(
 
     bg_mode = bg_mode or rng.choice(assets.blend_modes)
     fg_mode = fg_mode or rng.choice(assets.blend_modes)
-    opacity = opacity or config.BLEND_OPACITY
+    if opacity is None:
+        lo = min(config.BLEND_OPACITY_MIN, config.BLEND_OPACITY_MAX)
+        hi = max(config.BLEND_OPACITY_MIN, config.BLEND_OPACITY_MAX)
+        opacity = f"{rng.randint(lo, hi)}%"
 
     if bg_name:
         bg_command = assets.commands.get(bg_name)
@@ -319,28 +426,22 @@ def compose_variant(
     paths.variants.mkdir(parents=True, exist_ok=True)
 
     ok, err = apply_filter(paths.original, bg_command, p["bg_filtered"])
-    if not ok:
-        raise PipelineError(f"Background-Filter fehlgeschlagen: {err}")
+    _raise_on_gmic_failure(ok, err, "Background-Filter fehlgeschlagen")
 
     ok, err = blend_layers_opacity(paths.original, p["bg_filtered"], bg_mode, opacity, p["step1"])
-    if not ok:
-        raise PipelineError(f"Background-Blend fehlgeschlagen: {err}")
+    _raise_on_gmic_failure(ok, err, "Background-Blend fehlgeschlagen")
 
     ok, err = alpha_composite(p["step1"], paths.rembg, p["step2"])
-    if not ok:
-        raise PipelineError(f"Rembg-Alpha fehlgeschlagen: {err}")
+    _raise_on_gmic_failure(ok, err, "Rembg-Alpha fehlgeschlagen")
 
     ok, err = apply_filter(paths.rembg, fg_command, p["fg_filtered"])
-    if not ok:
-        raise PipelineError(f"Foreground-Filter fehlgeschlagen: {err}")
+    _raise_on_gmic_failure(ok, err, "Foreground-Filter fehlgeschlagen")
 
     ok, err = blend_layers_opacity(p["step2"], p["fg_filtered"], fg_mode, opacity, p["composed"])
-    if not ok:
-        raise PipelineError(f"Foreground-Blend fehlgeschlagen: {err}")
+    _raise_on_gmic_failure(ok, err, "Foreground-Blend fehlgeschlagen")
 
     ok, err = apply_filter_chain(p["composed"], config.POST_FILTERS, p["final"], tmp_dir, variant_id)
-    if not ok:
-        raise PipelineError(f"Post-Processing fehlgeschlagen: {err}")
+    _raise_on_gmic_failure(ok, err, "Post-Processing fehlgeschlagen")
 
     rel = lambda path: str(path.relative_to(paths.root))
     return {
@@ -367,12 +468,16 @@ def compose_variant(
 
 
 def process_job(job_id: str, source_path: Path, original_suffix: str) -> None:
-    """Full pipeline for a freshly ingested inbox file: copy, rembg once,
-    generate OUTPUT_COUNT variants, write manifest + status.
+    """Full pipeline for a freshly ingested inbox file: preprocess, rembg
+    once, generate OUTPUT_COUNT variants, write manifest + status.
 
     `source_path` must already be a private copy (jobs/<id>/original.*) —
     callers (worker.py) are responsible for copying out of inbox first, so
     the inbox file itself is never touched here.
+
+    Variants that hit FILTER_TIMEOUT during the normal pass are retried
+    once at the end with FILTER_TIMEOUT_LONG. Filter probes in
+    pick_working_filter stay on the short timeout (skip, don't escalate).
     """
     paths = job_paths(job_id, original_suffix)
     paths.root.mkdir(parents=True, exist_ok=True)
@@ -380,15 +485,18 @@ def process_job(job_id: str, source_path: Path, original_suffix: str) -> None:
     paths.variants.mkdir(parents=True, exist_ok=True)
 
     write_status(paths, "processing", source_file=str(source_path.name))
-    manifest: dict[str, Any] = {
-        "job_id": job_id,
-        "original_file": paths.original.name,
-        "rembg_file": paths.rembg.name,
-        "created_at": now_iso(),
-        "variants": [],
-    }
 
     try:
+        paths = preprocess_original(paths)
+
+        manifest: dict[str, Any] = {
+            "job_id": job_id,
+            "original_file": paths.original.name,
+            "rembg_file": paths.rembg.name,
+            "created_at": now_iso(),
+            "variants": [],
+        }
+
         ok, err = run_rembg(paths.original, paths.rembg)
         if not ok:
             raise PipelineError(f"rembg fehlgeschlagen: {err}")
@@ -396,16 +504,39 @@ def process_job(job_id: str, source_path: Path, original_suffix: str) -> None:
         assets = load_assets()
         rng = random.Random()
         variant_errors: list[str] = []
+        timeout_retries: list[str] = []
+
         for i in range(1, config.OUTPUT_COUNT + 1):
             variant_id = f"v{i}"
             try:
                 entry = compose_variant(paths, assets, variant_id=variant_id, source="auto", rng=rng)
                 manifest["variants"].append(entry)
                 write_manifest(paths, manifest)
+            except FilterTimeoutError as exc:
+                logger.warning("[%s] variant %s timed out (will retry long): %s", job_id, variant_id, exc)
+                timeout_retries.append(variant_id)
             except PipelineError as exc:
                 logger.error("[%s] variant %s failed: %s", job_id, variant_id, exc)
                 variant_errors.append(f"{variant_id}: {exc}")
 
+        if timeout_retries:
+            logger.info(
+                "[%s] long-retry %d variant(s) with timeout=%ds: %s",
+                job_id,
+                len(timeout_retries),
+                config.FILTER_TIMEOUT_LONG,
+                ", ".join(timeout_retries),
+            )
+            with filter_timeout(config.FILTER_TIMEOUT_LONG):
+                for variant_id in timeout_retries:
+                    try:
+                        entry = compose_variant(paths, assets, variant_id=variant_id, source="auto", rng=rng)
+                        manifest["variants"].append(entry)
+                        write_manifest(paths, manifest)
+                    except PipelineError as exc:
+                        logger.error("[%s] variant %s long-retry failed: %s", job_id, variant_id, exc)
+                        variant_errors.append(f"{variant_id}: {exc}")
+
         if not manifest["variants"]:
             raise PipelineError("Keine Variante erfolgreich erzeugt: " + "; ".join(variant_errors))