feat: replace SFTP drop with Syncthing share path

Remove SFTPGo; mount event data from the Syncthing folder, watch
incoming/, and name variants after the source stem.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Frank Schwenk
2026-07-18 14:06:57 +02:00
parent 83d4468b69
commit d5b44b221e
13 changed files with 167 additions and 189 deletions
+1 -1
View File
@@ -1 +1 @@
"""live.f12.rocks — event photo pipeline (SFTP inbox -> gmic/rembg -> web)."""
"""live.f12.rocks — event photo pipeline (Syncthing drop -> gmic/rembg -> web)."""
+1 -1
View File
@@ -26,7 +26,7 @@ def _float_env(name: str, default: float) -> float:
# --- Paths -------------------------------------------------------------
DATA_DIR = Path(os.environ.get("DATA_DIR", "/data"))
INBOX_DIR = DATA_DIR / "inbox"
INBOX_DIR = DATA_DIR / "incoming"
JOBS_DIR = DATA_DIR / "jobs"
ASSETS_DIR = DATA_DIR / "assets"
PROCESSED_FILE = DATA_DIR / "processed.json"
+59 -9
View File
@@ -210,7 +210,7 @@ def preprocess_original(paths: JobPaths) -> JobPaths:
"""Downscale to MAX_EDGE_PX and normalize to sRGB JPEG as `original.jpg`.
Replaces any non-jpg original in the job dir. Side-effect free for the
inbox copy — only mutates jobs/<id>/.
incoming copy — only mutates jobs/<id>/.
"""
src = paths.original
if not src.exists():
@@ -371,12 +371,37 @@ def next_variant_id(manifest: dict[str, Any]) -> str:
return f"v{i}"
_UNSAFE_STEM_RE = re.compile(r"[^\w.\-]+", re.UNICODE)
def sanitize_stem(name: str) -> str:
"""Safe basename stem for variant files (no path separators / junk)."""
stem = Path(name).stem if name else ""
stem = stem.replace("/", "_").replace("\\", "_").strip().strip(".")
stem = _UNSAFE_STEM_RE.sub("_", stem).strip("._")
return stem or "photo"
def variant_filename(stem: str, variant_id: str) -> str:
return f"{sanitize_stem(stem)}_{variant_id}.png"
def resolve_source_stem(manifest: dict[str, Any] | None, paths: JobPaths) -> str:
"""Prefer manifest source_stem; fall back to original path stem."""
if manifest:
stored = manifest.get("source_stem")
if isinstance(stored, str) and stored.strip():
return sanitize_stem(stored)
return sanitize_stem(paths.original.stem)
def compose_variant(
paths: JobPaths,
assets: FilterAssets,
*,
variant_id: str,
source: str,
variant_stem: str,
bg_name: str | None = None,
bg_mode: str | None = None,
fg_name: str | None = None,
@@ -389,10 +414,13 @@ def compose_variant(
If bg_name/fg_name/bg_mode/fg_mode are given (remix path) they are used
directly. Otherwise a random working filter is picked with retries,
exactly like make_random.py's compose_one().
Final file is `variants/{stem}_{variant_id}.png`; manifest id stays `vN`.
"""
rng = rng or random.Random()
tmp_dir = paths.intermediates
tmp_dir.mkdir(parents=True, exist_ok=True)
stem = sanitize_stem(variant_stem)
bg_mode = bg_mode or rng.choice(assets.blend_modes)
fg_mode = fg_mode or rng.choice(assets.blend_modes)
@@ -421,7 +449,7 @@ def compose_variant(
"step2": tmp_dir / f"{variant_id}_rembg_alpha.png",
"fg_filtered": tmp_dir / f"{variant_id}_fg_filtered.png",
"composed": tmp_dir / f"{variant_id}_composed.png",
"final": paths.variants / f"{variant_id}.png",
"final": paths.variants / variant_filename(stem, variant_id),
}
paths.variants.mkdir(parents=True, exist_ok=True)
@@ -467,13 +495,19 @@ def compose_variant(
}
def process_job(job_id: str, source_path: Path, original_suffix: str) -> None:
"""Full pipeline for a freshly ingested inbox file: preprocess, rembg
def process_job(
job_id: str,
source_path: Path,
original_suffix: str,
*,
source_stem: str | None = None,
) -> None:
"""Full pipeline for a freshly ingested incoming file: preprocess, rembg
once, generate OUTPUT_COUNT variants, write manifest + status.
`source_path` must already be a private copy (jobs/<id>/original.*) —
callers (worker.py) are responsible for copying out of inbox first, so
the inbox file itself is never touched here.
callers (worker.py) are responsible for copying out of incoming first, so
the incoming file itself is never touched here.
Variants that hit FILTER_TIMEOUT during the normal pass are retried
once at the end with FILTER_TIMEOUT_LONG. Filter probes in
@@ -483,8 +517,9 @@ def process_job(job_id: str, source_path: Path, original_suffix: str) -> None:
paths.root.mkdir(parents=True, exist_ok=True)
paths.intermediates.mkdir(parents=True, exist_ok=True)
paths.variants.mkdir(parents=True, exist_ok=True)
stem = sanitize_stem(source_stem or source_path.stem)
write_status(paths, "processing", source_file=str(source_path.name))
write_status(paths, "processing", source_file=str(source_path.name), source_stem=stem)
try:
paths = preprocess_original(paths)
@@ -493,6 +528,7 @@ def process_job(job_id: str, source_path: Path, original_suffix: str) -> None:
"job_id": job_id,
"original_file": paths.original.name,
"rembg_file": paths.rembg.name,
"source_stem": stem,
"created_at": now_iso(),
"variants": [],
}
@@ -509,7 +545,14 @@ def process_job(job_id: str, source_path: Path, original_suffix: str) -> None:
for i in range(1, config.OUTPUT_COUNT + 1):
variant_id = f"v{i}"
try:
entry = compose_variant(paths, assets, variant_id=variant_id, source="auto", rng=rng)
entry = compose_variant(
paths,
assets,
variant_id=variant_id,
source="auto",
variant_stem=stem,
rng=rng,
)
manifest["variants"].append(entry)
write_manifest(paths, manifest)
except FilterTimeoutError as exc:
@@ -530,7 +573,14 @@ def process_job(job_id: str, source_path: Path, original_suffix: str) -> None:
with filter_timeout(config.FILTER_TIMEOUT_LONG):
for variant_id in timeout_retries:
try:
entry = compose_variant(paths, assets, variant_id=variant_id, source="auto", rng=rng)
entry = compose_variant(
paths,
assets,
variant_id=variant_id,
source="auto",
variant_stem=stem,
rng=rng,
)
manifest["variants"].append(entry)
write_manifest(paths, manifest)
except PipelineError as exc:
+3
View File
@@ -49,6 +49,8 @@ def create_remix_variant(job_id: str, original_suffix: str, choice: RemixChoice)
"created_at": pipeline.now_iso(),
"variants": [],
}
variant_stem = pipeline.resolve_source_stem(manifest, paths)
manifest.setdefault("source_stem", variant_stem)
variant_id = pipeline.next_variant_id(manifest)
entry = pipeline.compose_variant(
@@ -56,6 +58,7 @@ def create_remix_variant(job_id: str, original_suffix: str, choice: RemixChoice)
assets,
variant_id=variant_id,
source="remix",
variant_stem=variant_stem,
bg_name=choice.bg_filter,
bg_mode=choice.bg_blend,
fg_name=choice.fg_filter,
+1 -1
View File
@@ -2,7 +2,7 @@
{% block title %}{{ site_title }}{% endblock %}
{% block content %}
{% if not jobs %}
<p class="empty-state">Noch keine Jobs. Bild per SFTP hochladen, dann kurz warten.</p>
<p class="empty-state">Noch keine Jobs. Bild in den Sync-Ordner legen, dann kurz warten.</p>
{% else %}
<ul class="job-grid">
{% for job in jobs %}
+21 -8
View File
@@ -1,8 +1,8 @@
"""Inbox watcher + sequential job runner.
"""Incoming watcher + sequential job runner.
Polls `DATA_DIR/inbox` for new, size-stable image files, copies each one
Polls `DATA_DIR/incoming` for new, size-stable image files, copies each one
into its own `jobs/<job_id>/original.*` and runs the compose pipeline on
it. Files are NEVER deleted or moved from the inbox — `processed.json`
it. Files are NEVER deleted or moved from incoming — `processed.json`
tracks what has already been handled (by path + size + mtime) so restarts
don't reprocess everything.
@@ -61,9 +61,18 @@ def _is_already_processed(processed: dict[str, Any], path: Path, stat: Any) -> b
return entry.get("size") == stat.st_size and entry.get("mtime") == stat.st_mtime
def _is_ignored_name(name: str) -> bool:
"""Skip Syncthing metadata/temp files and other dotfiles."""
if name.startswith("."):
return True
if name.startswith(".syncthing.") or ".syncthing." in name:
return True
return False
def _is_stable(path: Path) -> bool:
"""A file is "stable" if its size doesn't change across a short wait —
cheap way to avoid picking up a half-uploaded SFTP transfer."""
cheap way to avoid picking up a half-written Syncthing transfer."""
try:
size_before = path.stat().st_size
except OSError:
@@ -77,12 +86,15 @@ def _is_stable(path: Path) -> bool:
def find_new_files(processed: dict[str, Any]) -> list[Path]:
"""Top-level images under incoming/ only — never recurse into jobs/."""
if not config.INBOX_DIR.exists():
return []
candidates: list[Path] = []
for path in sorted(config.INBOX_DIR.rglob("*")):
for path in sorted(config.INBOX_DIR.iterdir()):
if not path.is_file():
continue
if _is_ignored_name(path.name):
continue
if path.suffix.lower() not in config.SUPPORTED_EXTENSIONS:
continue
try:
@@ -105,13 +117,14 @@ def handle_file(path: Path, processed: dict[str, Any]) -> None:
suffix = path.suffix.lower()
paths = pipeline.job_paths(job_id, suffix)
paths.root.mkdir(parents=True, exist_ok=True)
source_stem = pipeline.sanitize_stem(path.stem)
logger.info("new inbox file %s -> job %s", path.name, job_id)
# Copy (never move) so the inbox stays untouched.
logger.info("new incoming file %s -> job %s (stem=%s)", path.name, job_id, source_stem)
# Copy (never move) so incoming stays untouched.
shutil.copy2(path, paths.original)
try:
pipeline.process_job(job_id, paths.original, suffix)
pipeline.process_job(job_id, paths.original, suffix, source_stem=source_stem)
finally:
# Mark as processed regardless of pipeline outcome so a permanently
# broken image doesn't get retried forever; failures are visible in