feat: replace SFTP drop with Syncthing share path
Remove SFTPGo; mount event data from the Syncthing folder, watch incoming/, and name variants after the source stem. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
+1
-1
@@ -1 +1 @@
|
||||
"""live.f12.rocks — event photo pipeline (SFTP inbox -> gmic/rembg -> web)."""
|
||||
"""live.f12.rocks — event photo pipeline (Syncthing drop -> gmic/rembg -> web)."""
|
||||
|
||||
+1
-1
@@ -26,7 +26,7 @@ def _float_env(name: str, default: float) -> float:
|
||||
|
||||
# --- Paths -------------------------------------------------------------
|
||||
DATA_DIR = Path(os.environ.get("DATA_DIR", "/data"))
|
||||
INBOX_DIR = DATA_DIR / "inbox"
|
||||
INBOX_DIR = DATA_DIR / "incoming"
|
||||
JOBS_DIR = DATA_DIR / "jobs"
|
||||
ASSETS_DIR = DATA_DIR / "assets"
|
||||
PROCESSED_FILE = DATA_DIR / "processed.json"
|
||||
|
||||
+59
-9
@@ -210,7 +210,7 @@ def preprocess_original(paths: JobPaths) -> JobPaths:
|
||||
"""Downscale to MAX_EDGE_PX and normalize to sRGB JPEG as `original.jpg`.
|
||||
|
||||
Replaces any non-jpg original in the job dir. Side-effect free for the
|
||||
inbox copy — only mutates jobs/<id>/.
|
||||
incoming copy — only mutates jobs/<id>/.
|
||||
"""
|
||||
src = paths.original
|
||||
if not src.exists():
|
||||
@@ -371,12 +371,37 @@ def next_variant_id(manifest: dict[str, Any]) -> str:
|
||||
return f"v{i}"
|
||||
|
||||
|
||||
_UNSAFE_STEM_RE = re.compile(r"[^\w.\-]+", re.UNICODE)
|
||||
|
||||
|
||||
def sanitize_stem(name: str) -> str:
|
||||
"""Safe basename stem for variant files (no path separators / junk)."""
|
||||
stem = Path(name).stem if name else ""
|
||||
stem = stem.replace("/", "_").replace("\\", "_").strip().strip(".")
|
||||
stem = _UNSAFE_STEM_RE.sub("_", stem).strip("._")
|
||||
return stem or "photo"
|
||||
|
||||
|
||||
def variant_filename(stem: str, variant_id: str) -> str:
|
||||
return f"{sanitize_stem(stem)}_{variant_id}.png"
|
||||
|
||||
|
||||
def resolve_source_stem(manifest: dict[str, Any] | None, paths: JobPaths) -> str:
|
||||
"""Prefer manifest source_stem; fall back to original path stem."""
|
||||
if manifest:
|
||||
stored = manifest.get("source_stem")
|
||||
if isinstance(stored, str) and stored.strip():
|
||||
return sanitize_stem(stored)
|
||||
return sanitize_stem(paths.original.stem)
|
||||
|
||||
|
||||
def compose_variant(
|
||||
paths: JobPaths,
|
||||
assets: FilterAssets,
|
||||
*,
|
||||
variant_id: str,
|
||||
source: str,
|
||||
variant_stem: str,
|
||||
bg_name: str | None = None,
|
||||
bg_mode: str | None = None,
|
||||
fg_name: str | None = None,
|
||||
@@ -389,10 +414,13 @@ def compose_variant(
|
||||
If bg_name/fg_name/bg_mode/fg_mode are given (remix path) they are used
|
||||
directly. Otherwise a random working filter is picked with retries,
|
||||
exactly like make_random.py's compose_one().
|
||||
|
||||
Final file is `variants/{stem}_{variant_id}.png`; manifest id stays `vN`.
|
||||
"""
|
||||
rng = rng or random.Random()
|
||||
tmp_dir = paths.intermediates
|
||||
tmp_dir.mkdir(parents=True, exist_ok=True)
|
||||
stem = sanitize_stem(variant_stem)
|
||||
|
||||
bg_mode = bg_mode or rng.choice(assets.blend_modes)
|
||||
fg_mode = fg_mode or rng.choice(assets.blend_modes)
|
||||
@@ -421,7 +449,7 @@ def compose_variant(
|
||||
"step2": tmp_dir / f"{variant_id}_rembg_alpha.png",
|
||||
"fg_filtered": tmp_dir / f"{variant_id}_fg_filtered.png",
|
||||
"composed": tmp_dir / f"{variant_id}_composed.png",
|
||||
"final": paths.variants / f"{variant_id}.png",
|
||||
"final": paths.variants / variant_filename(stem, variant_id),
|
||||
}
|
||||
paths.variants.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -467,13 +495,19 @@ def compose_variant(
|
||||
}
|
||||
|
||||
|
||||
def process_job(job_id: str, source_path: Path, original_suffix: str) -> None:
|
||||
"""Full pipeline for a freshly ingested inbox file: preprocess, rembg
|
||||
def process_job(
|
||||
job_id: str,
|
||||
source_path: Path,
|
||||
original_suffix: str,
|
||||
*,
|
||||
source_stem: str | None = None,
|
||||
) -> None:
|
||||
"""Full pipeline for a freshly ingested incoming file: preprocess, rembg
|
||||
once, generate OUTPUT_COUNT variants, write manifest + status.
|
||||
|
||||
`source_path` must already be a private copy (jobs/<id>/original.*) —
|
||||
callers (worker.py) are responsible for copying out of inbox first, so
|
||||
the inbox file itself is never touched here.
|
||||
callers (worker.py) are responsible for copying out of incoming first, so
|
||||
the incoming file itself is never touched here.
|
||||
|
||||
Variants that hit FILTER_TIMEOUT during the normal pass are retried
|
||||
once at the end with FILTER_TIMEOUT_LONG. Filter probes in
|
||||
@@ -483,8 +517,9 @@ def process_job(job_id: str, source_path: Path, original_suffix: str) -> None:
|
||||
paths.root.mkdir(parents=True, exist_ok=True)
|
||||
paths.intermediates.mkdir(parents=True, exist_ok=True)
|
||||
paths.variants.mkdir(parents=True, exist_ok=True)
|
||||
stem = sanitize_stem(source_stem or source_path.stem)
|
||||
|
||||
write_status(paths, "processing", source_file=str(source_path.name))
|
||||
write_status(paths, "processing", source_file=str(source_path.name), source_stem=stem)
|
||||
|
||||
try:
|
||||
paths = preprocess_original(paths)
|
||||
@@ -493,6 +528,7 @@ def process_job(job_id: str, source_path: Path, original_suffix: str) -> None:
|
||||
"job_id": job_id,
|
||||
"original_file": paths.original.name,
|
||||
"rembg_file": paths.rembg.name,
|
||||
"source_stem": stem,
|
||||
"created_at": now_iso(),
|
||||
"variants": [],
|
||||
}
|
||||
@@ -509,7 +545,14 @@ def process_job(job_id: str, source_path: Path, original_suffix: str) -> None:
|
||||
for i in range(1, config.OUTPUT_COUNT + 1):
|
||||
variant_id = f"v{i}"
|
||||
try:
|
||||
entry = compose_variant(paths, assets, variant_id=variant_id, source="auto", rng=rng)
|
||||
entry = compose_variant(
|
||||
paths,
|
||||
assets,
|
||||
variant_id=variant_id,
|
||||
source="auto",
|
||||
variant_stem=stem,
|
||||
rng=rng,
|
||||
)
|
||||
manifest["variants"].append(entry)
|
||||
write_manifest(paths, manifest)
|
||||
except FilterTimeoutError as exc:
|
||||
@@ -530,7 +573,14 @@ def process_job(job_id: str, source_path: Path, original_suffix: str) -> None:
|
||||
with filter_timeout(config.FILTER_TIMEOUT_LONG):
|
||||
for variant_id in timeout_retries:
|
||||
try:
|
||||
entry = compose_variant(paths, assets, variant_id=variant_id, source="auto", rng=rng)
|
||||
entry = compose_variant(
|
||||
paths,
|
||||
assets,
|
||||
variant_id=variant_id,
|
||||
source="auto",
|
||||
variant_stem=stem,
|
||||
rng=rng,
|
||||
)
|
||||
manifest["variants"].append(entry)
|
||||
write_manifest(paths, manifest)
|
||||
except PipelineError as exc:
|
||||
|
||||
@@ -49,6 +49,8 @@ def create_remix_variant(job_id: str, original_suffix: str, choice: RemixChoice)
|
||||
"created_at": pipeline.now_iso(),
|
||||
"variants": [],
|
||||
}
|
||||
variant_stem = pipeline.resolve_source_stem(manifest, paths)
|
||||
manifest.setdefault("source_stem", variant_stem)
|
||||
variant_id = pipeline.next_variant_id(manifest)
|
||||
|
||||
entry = pipeline.compose_variant(
|
||||
@@ -56,6 +58,7 @@ def create_remix_variant(job_id: str, original_suffix: str, choice: RemixChoice)
|
||||
assets,
|
||||
variant_id=variant_id,
|
||||
source="remix",
|
||||
variant_stem=variant_stem,
|
||||
bg_name=choice.bg_filter,
|
||||
bg_mode=choice.bg_blend,
|
||||
fg_name=choice.fg_filter,
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
{% block title %}{{ site_title }}{% endblock %}
|
||||
{% block content %}
|
||||
{% if not jobs %}
|
||||
<p class="empty-state">Noch keine Jobs. Bild per SFTP hochladen, dann kurz warten.</p>
|
||||
<p class="empty-state">Noch keine Jobs. Bild in den Sync-Ordner legen, dann kurz warten.</p>
|
||||
{% else %}
|
||||
<ul class="job-grid">
|
||||
{% for job in jobs %}
|
||||
|
||||
+21
-8
@@ -1,8 +1,8 @@
|
||||
"""Inbox watcher + sequential job runner.
|
||||
"""Incoming watcher + sequential job runner.
|
||||
|
||||
Polls `DATA_DIR/inbox` for new, size-stable image files, copies each one
|
||||
Polls `DATA_DIR/incoming` for new, size-stable image files, copies each one
|
||||
into its own `jobs/<job_id>/original.*` and runs the compose pipeline on
|
||||
it. Files are NEVER deleted or moved from the inbox — `processed.json`
|
||||
it. Files are NEVER deleted or moved from incoming — `processed.json`
|
||||
tracks what has already been handled (by path + size + mtime) so restarts
|
||||
don't reprocess everything.
|
||||
|
||||
@@ -61,9 +61,18 @@ def _is_already_processed(processed: dict[str, Any], path: Path, stat: Any) -> b
|
||||
return entry.get("size") == stat.st_size and entry.get("mtime") == stat.st_mtime
|
||||
|
||||
|
||||
def _is_ignored_name(name: str) -> bool:
|
||||
"""Skip Syncthing metadata/temp files and other dotfiles."""
|
||||
if name.startswith("."):
|
||||
return True
|
||||
if name.startswith(".syncthing.") or ".syncthing." in name:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _is_stable(path: Path) -> bool:
|
||||
"""A file is "stable" if its size doesn't change across a short wait —
|
||||
cheap way to avoid picking up a half-uploaded SFTP transfer."""
|
||||
cheap way to avoid picking up a half-written Syncthing transfer."""
|
||||
try:
|
||||
size_before = path.stat().st_size
|
||||
except OSError:
|
||||
@@ -77,12 +86,15 @@ def _is_stable(path: Path) -> bool:
|
||||
|
||||
|
||||
def find_new_files(processed: dict[str, Any]) -> list[Path]:
|
||||
"""Top-level images under incoming/ only — never recurse into jobs/."""
|
||||
if not config.INBOX_DIR.exists():
|
||||
return []
|
||||
candidates: list[Path] = []
|
||||
for path in sorted(config.INBOX_DIR.rglob("*")):
|
||||
for path in sorted(config.INBOX_DIR.iterdir()):
|
||||
if not path.is_file():
|
||||
continue
|
||||
if _is_ignored_name(path.name):
|
||||
continue
|
||||
if path.suffix.lower() not in config.SUPPORTED_EXTENSIONS:
|
||||
continue
|
||||
try:
|
||||
@@ -105,13 +117,14 @@ def handle_file(path: Path, processed: dict[str, Any]) -> None:
|
||||
suffix = path.suffix.lower()
|
||||
paths = pipeline.job_paths(job_id, suffix)
|
||||
paths.root.mkdir(parents=True, exist_ok=True)
|
||||
source_stem = pipeline.sanitize_stem(path.stem)
|
||||
|
||||
logger.info("new inbox file %s -> job %s", path.name, job_id)
|
||||
# Copy (never move) so the inbox stays untouched.
|
||||
logger.info("new incoming file %s -> job %s (stem=%s)", path.name, job_id, source_stem)
|
||||
# Copy (never move) so incoming stays untouched.
|
||||
shutil.copy2(path, paths.original)
|
||||
|
||||
try:
|
||||
pipeline.process_job(job_id, paths.original, suffix)
|
||||
pipeline.process_job(job_id, paths.original, suffix, source_stem=source_stem)
|
||||
finally:
|
||||
# Mark as processed regardless of pipeline outcome so a permanently
|
||||
# broken image doesn't get retried forever; failures are visible in
|
||||
|
||||
Reference in New Issue
Block a user