From d5b44b221ed9b3f30fb96d8d147df8599c346065 Mon Sep 17 00:00:00 2001 From: Frank Schwenk Date: Sat, 18 Jul 2026 14:06:57 +0200 Subject: [PATCH] feat: replace SFTP drop with Syncthing share path Remove SFTPGo; mount event data from the Syncthing folder, watch incoming/, and name variants after the source stem. Co-authored-by: Cursor --- .env.example | 22 ++++--------- .gitignore | 9 +++++- NOTES.md | 13 ++++++-- README.md | 66 +++++++++++++++----------------------- SOUL.md | 44 +++++++++++++------------- app/__init__.py | 2 +- app/config.py | 2 +- app/pipeline.py | 68 ++++++++++++++++++++++++++++++++++------ app/remix.py | 3 ++ app/templates/index.html | 2 +- app/worker.py | 29 ++++++++++++----- compose.yml | 52 ++++++------------------------ sftpgo/entrypoint.sh | 44 -------------------------- 13 files changed, 167 insertions(+), 189 deletions(-) delete mode 100755 sftpgo/entrypoint.sh diff --git a/.env.example b/.env.example index 37c3da3..e5b694c 100644 --- a/.env.example +++ b/.env.example @@ -1,21 +1,11 @@ # Copy to `.env` and fill in real values. Never commit `.env`. -# --- SFTPGo credentials ----------------------------------------------- -# Whoever runs the event camera/laptop uploads photos here. -SFTP_USER=livef12 -SFTP_PASSWORD=changeme - -# --- Host paths / ports -------------------------------------------------- -# On the server this should be an absolute path, e.g. -# /home/frank/live.f12.rocks/data -DATA_HOST_DIR=./data - -# Spec asked for 121212 which exceeds TCP max (65535). Use 12121. -SFTP_HOST_PORT=12121 - -# Minutes until SFTPGo closes an idle connection. Android/SSHJ clients -# often sit idle after upload; the default 15m shows up as a spurious EOF. -SFTPGO_IDLE_TIMEOUT=120 +# --- Host data path (Syncthing share root) ------------------------------ +# Prod default (compose.yml): /home/frank/sync.schwenk.online/data/livef12 +# Layout under that path: incoming/ jobs/ processed.json +# Local override when the absolute path does not exist: +# DATA_HOST_DIR=./data +# DATA_HOST_DIR=/home/frank/sync.schwenk.online/data/livef12 # --- Pipeline tuning (see make_random.py for background) ----------------- OUTPUT_COUNT=3 diff --git a/.gitignore b/.gitignore index b36b196..b85b0cb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ # Secrets .env -# Runtime data (bind-mounted, never belongs in git) +# Runtime data (bind-mounted / Syncthing share — never in git) /data/ # Local rembg/withoutbg comparison scratch @@ -14,8 +14,15 @@ # Python __pycache__/ *.pyc +*.pyo .venv/ +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ # OS / editor noise .DS_Store *.swp +*~ +.idea/ +.vscode/ diff --git a/NOTES.md b/NOTES.md index 6996423..b97c9a6 100644 --- a/NOTES.md +++ b/NOTES.md @@ -4,7 +4,14 @@ - Live on boka: `/home/frank/live.f12.rocks` - Web: https://live.f12.rocks -- SFTP: `livef12@frank-schwenk.de` port **12121** (spec 121212 is invalid TCP) -- Password in server `.env` only (`chmod 600`); ntfy sent to phone -- Data: `/home/frank/live.f12.rocks/data` (inbox not auto-cleaned) +- Password / secrets in server `.env` only (`chmod 600`) - First rembg job downloads ~176 MB model into docker volume `rembg_cache` + +## Syncthing cutover (2026-07-18) + +- SFTPGo removed; drop path is Syncthing +- Data share: `/home/frank/sync.schwenk.online/data/livef12` + (`incoming/` + `jobs/` + `processed.json`) +- Old path `/home/frank/live.f12.rocks/data` no longer used by compose + (migration/cleanup manual) +- Host port 12121 can be closed after cutover diff --git a/README.md b/README.md index d546d48..747bb7c 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,10 @@ # live.f12.rocks -Event photo pep pipeline: guests upload a photo via SFTP, a worker runs it -through `rembg` (background removal) + `gmic` (random filter/blend -compositing, ported from `make_random.py`), and a small web UI shows the -results with a "remix" option to try different filters on the same photo. +Event photo pep pipeline: guests drop a photo into a Syncthing folder, a +worker runs it through `rembg` (background removal) + `gmic` (random +filter/blend compositing, ported from `make_random.py`), and a small web +UI shows the results with a "remix" option to try different filters on +the same photo. Finished jobs sync back via the same share. No web auth. This is an event tool, not a DAM. @@ -11,25 +12,26 @@ No web auth. This is an event tool, not a DAM. | Service | What | Port | |----------|--------------------------------------------------|--------------------------| -| `sftpgo` | SFTP drop point, `drakkan/sftpgo` | `12121` (host) -> `2022` | -| `worker` | Watches inbox, runs the compose pipeline | none published | +| `worker` | Watches `incoming/`, runs the compose pipeline | none published | | `web` | FastAPI + Jinja UI, browse jobs, remix | via Traefik only | -All three share one host directory (`DATA_HOST_DIR`, default `./data`), -mounted at different paths — see the comment at the top of `compose.yml`. +Both share one host directory (`DATA_HOST_DIR`) mounted at `/data`. +Syncthing runs on the host — not in this compose. -## Data layout (`./data`) +## Data layout (Syncthing share) + +Prod path: `/home/frank/sync.schwenk.online/data/livef12` ``` -inbox/ # SFTP drop — worker only ever reads/copies from here +incoming/ # phone drop — worker only ever reads/copies from here jobs// original. # private copy of the uploaded photo rembg.png # background removed (computed once, reused) intermediates/ # every intermediate step, kept for inspection - variants/ # final composed images + variants/ # final composed images ({stem}_v1.png, …) manifest.json # filter names/commands/blends per variant status.json # pending | processing | done | error -processed.json # worker bookkeeping: which inbox files were handled +processed.json # worker bookkeeping: which incoming files were handled ``` `assets/` (filter lists + trimmed `filters.json`) lives in the repo and is @@ -39,27 +41,20 @@ bind-mounted read-only into `worker`/`web` at `/data/assets`. ```bash cp .env.example .env -# edit .env: set a real SFTP_PASSWORD, and DATA_HOST_DIR on the server +# local: set DATA_HOST_DIR=./data +# prod: leave unset (defaults to the Syncthing share path) docker compose up -d --build ``` -Upload a photo: - -```bash -sftp -P 12121 livef12@ -put photo.jpg -``` - -After a few seconds (poll interval + processing time) it shows up on the -web UI as a new job. +Drop a photo into `incoming/` (via Syncthing or locally). After a few +seconds (poll interval + processing time) it shows up on the web UI as a +new job; the full `jobs//` tree syncs back to the phone. ## Configuration (`.env`, see `.env.example`) | Var | Default | Notes | |-----|---------|-------| -| `SFTP_USER` / `SFTP_PASSWORD` | `livef12` / `changeme` | SFTP login, home dir is locked to the shared inbox | -| `DATA_HOST_DIR` | `./data` | On the server: `/home/frank/live.f12.rocks/data` | -| `SFTP_HOST_PORT` | `12121` | Host-side SFTP port (121212 is invalid TCP) | +| `DATA_HOST_DIR` | `/home/frank/sync.schwenk.online/data/livef12` | Syncthing share root; use `./data` locally | | `OUTPUT_COUNT` | `3` | Variants generated per uploaded photo | | `BLEND_OPACITY` | `30%` | Default blend opacity for auto-generated variants | | `FILTER_TIMEOUT` | `120` | Seconds before a single gmic call is killed | @@ -68,26 +63,17 @@ web UI as a new job. ## Deploy notes / caveats -- **Port note:** Spec originally said `121212`, which exceeds TCP max - (65535). Production default is **`12121`**. -- **Firewall:** open host port `12121` (SFTP) and make sure Traefik - already routes `live.f12.rocks` — this repo only adds the router labels, - it assumes the external `traefik` docker network exists. -- **SFTPGo first run:** the `sftpgo` service auto-creates the SFTP user - from `SFTP_USER`/`SFTP_PASSWORD` on every start via - `sftpgo/entrypoint.sh` (uses `jq`, bundled in the official image, to - build a `loaddata` JSON safely — no manual admin setup needed). Host SSH - keys persist in the `sftpgo_state` named volume, not in `./data`. - The SFTPGo web admin exists internally on port 8080 but is deliberately - **not** published or routed — there's no need for it here. +- **Firewall:** Traefik must already route `live.f12.rocks` — this repo + only adds the router labels; it assumes the external `traefik` docker + network exists. No SFTP port needed. - **rembg model download:** first background-removal call downloads the `u2net` ONNX model (~176 MB) from GitHub. This needs outbound internet on first run and can take a minute or two depending on the link; the model is cached in the `rembg_cache` named volume afterwards, so restarts don't re-download it. -- **Inbox is append-only:** the worker only ever copies out of `inbox/` - and never deletes or moves anything there — plan disk space - accordingly, or clean up `inbox/` manually between events. +- **Incoming is append-only:** the worker only ever copies out of + `incoming/` and never deletes or moves anything there — plan disk space + accordingly, or clean up `incoming/` manually between events. - **Sequential processing:** the worker handles one photo at a time (`cpus: "1.0"`, no concurrency) — fine for an event pace, but a burst of uploads will just queue up and get processed in order. diff --git a/SOUL.md b/SOUL.md index 1c573ef..e4ab434 100644 --- a/SOUL.md +++ b/SOUL.md @@ -6,23 +6,24 @@ ## One-Liner -Event photo pep pipeline: SFTP -> gmic/rembg -> web. Upload a photo at the -event, get back weird/fun filtered variants seconds later. +Event photo pep pipeline: Syncthing drop -> gmic/rembg -> web. Upload a +photo at the event, get back weird/fun filtered variants seconds later. ## Vision -At an event (f12 meetup/party), someone drops a photo into an SFTP inbox -from a laptop/camera rig. A worker rips the background off, runs it +At an event (f12 meetup/party), someone drops a photo into a Syncthing +folder from a phone/camera rig. A worker rips the background off, runs it through a random gmic filter/blend chain, and a handful of variants show up on a shared web page almost immediately — no app install, no login, no waiting for someone to "process the photos later". If a variant is fun, -remix it with different filters right there. +remix it with different filters right there. Finished jobs sync back to +the phone via the same Syncthing share. ## Audience Event organizers and attendees at f12 events, on their phones, often on bad venue wifi. Nobody signs up, nobody logs in. Whoever has the link -(and whoever's uploading from the SFTP side) is the whole audience. +(and whoever's syncing photos in) is the whole audience. ## Tone & Wording @@ -41,7 +42,7 @@ bad venue wifi. Nobody signs up, nobody logs in. Whoever has the link |---------|------|-----| | Success | `Remix erzeugt.` | `Dein Kunstwerk ist fertig! 🎉` | | Error | `Foreground-Filter fehlgeschlagen: Timeout nach 120s` | `Etwas ist schiefgelaufen.` | -| Empty state | `Noch keine Jobs. Bild per SFTP hochladen, dann kurz warten.` | `Hier ist noch nichts los... lade doch was hoch! 😊` | +| Empty state | `Noch keine Jobs. Bild in den Sync-Ordner legen, dann kurz warten.` | `Hier ist noch nichts los... lade doch was hoch! 😊` | ## Design @@ -55,33 +56,32 @@ bad venue wifi. Nobody signs up, nobody logs in. Whoever has the link - No web auth / accounts / login product — anyone with the link sees everything, by design, for this event tool. -- No camera-to-phone client — upload path is SFTP only. +- No camera-to-phone client in this repo — upload path is Syncthing + (phone/folder setup is out of band). - No beamer/projector product built into this repo. - Not a DAM (digital asset manager) — no albums, tagging, search, - retention policies. `inbox/` and `jobs/` are the whole data model. -- No auto-deletion of inbox uploads. Ever. The worker only reads/copies. + retention policies. `incoming/` and `jobs/` are the whole data model. +- No auto-deletion of incoming uploads. Ever. The worker only reads/copies. +- No Syncthing container in this compose — Syncthing runs on the host. ## Infrastructure (project-local) - Workdir on server: `/home/frank/live.f12.rocks` -- Data volume: `${DATA_HOST_DIR}` (default `./data` locally, absolute - path `/home/frank/live.f12.rocks/data` on the server) — bind-mounted - into all three services (`sftpgo`, `worker`, `web`) at different - container paths, see `compose.yml` header comment. -- SFTP: host port `12121` -> `sftpgo:2022` (spec `121212` is not a valid - TCP port). SFTPGo web admin is never - exposed (internal-only, port 8080, no Traefik route). +- Data volume (Syncthing share): `${DATA_HOST_DIR}` — prod default + `/home/frank/sync.schwenk.online/data/livef12`, local override `./data`. + Bind-mounted into `worker` and `web` at `/data` (see `compose.yml`). +- Layout under the share: `incoming/` (phone drop), `jobs/` (full tree + syncs back), `processed.json`. - Domain: `live.f12.rocks`, routed via the shared external `traefik` network, TLS via `myresolver` (see `INFRASTRUCTURE.md`). -- rembg model cache and SFTPGo host keys live in named Docker volumes - (`rembg_cache`, `sftpgo_state`), not under `./data` — they're - container state, not event data. +- rembg model cache lives in the named Docker volume `rembg_cache`, not + under the Syncthing share — container state, not event data. ## Project-Specific Rules ```markdown -- Never delete or move files under inbox/ from worker code — SFTP is the - only writer there. (reason: uploads are the one copy of the original +- Never delete or move files under incoming/ from worker code — Syncthing + is the writer there. (reason: uploads are the one copy of the original that exists outside a phone's camera roll) - Keep every intermediate image in jobs//intermediates/ — don't clean them up automatically. (reason: useful for debugging bad filter picks, diff --git a/app/__init__.py b/app/__init__.py index 99ecf69..3a30041 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1 +1 @@ -"""live.f12.rocks — event photo pipeline (SFTP inbox -> gmic/rembg -> web).""" +"""live.f12.rocks — event photo pipeline (Syncthing drop -> gmic/rembg -> web).""" diff --git a/app/config.py b/app/config.py index 3c095f2..6ccd81b 100644 --- a/app/config.py +++ b/app/config.py @@ -26,7 +26,7 @@ def _float_env(name: str, default: float) -> float: # --- Paths ------------------------------------------------------------- DATA_DIR = Path(os.environ.get("DATA_DIR", "/data")) -INBOX_DIR = DATA_DIR / "inbox" +INBOX_DIR = DATA_DIR / "incoming" JOBS_DIR = DATA_DIR / "jobs" ASSETS_DIR = DATA_DIR / "assets" PROCESSED_FILE = DATA_DIR / "processed.json" diff --git a/app/pipeline.py b/app/pipeline.py index e5e8768..b0f2c83 100644 --- a/app/pipeline.py +++ b/app/pipeline.py @@ -210,7 +210,7 @@ def preprocess_original(paths: JobPaths) -> JobPaths: """Downscale to MAX_EDGE_PX and normalize to sRGB JPEG as `original.jpg`. Replaces any non-jpg original in the job dir. Side-effect free for the - inbox copy — only mutates jobs//. + incoming copy — only mutates jobs//. """ src = paths.original if not src.exists(): @@ -371,12 +371,37 @@ def next_variant_id(manifest: dict[str, Any]) -> str: return f"v{i}" +_UNSAFE_STEM_RE = re.compile(r"[^\w.\-]+", re.UNICODE) + + +def sanitize_stem(name: str) -> str: + """Safe basename stem for variant files (no path separators / junk).""" + stem = Path(name).stem if name else "" + stem = stem.replace("/", "_").replace("\\", "_").strip().strip(".") + stem = _UNSAFE_STEM_RE.sub("_", stem).strip("._") + return stem or "photo" + + +def variant_filename(stem: str, variant_id: str) -> str: + return f"{sanitize_stem(stem)}_{variant_id}.png" + + +def resolve_source_stem(manifest: dict[str, Any] | None, paths: JobPaths) -> str: + """Prefer manifest source_stem; fall back to original path stem.""" + if manifest: + stored = manifest.get("source_stem") + if isinstance(stored, str) and stored.strip(): + return sanitize_stem(stored) + return sanitize_stem(paths.original.stem) + + def compose_variant( paths: JobPaths, assets: FilterAssets, *, variant_id: str, source: str, + variant_stem: str, bg_name: str | None = None, bg_mode: str | None = None, fg_name: str | None = None, @@ -389,10 +414,13 @@ def compose_variant( If bg_name/fg_name/bg_mode/fg_mode are given (remix path) they are used directly. Otherwise a random working filter is picked with retries, exactly like make_random.py's compose_one(). + + Final file is `variants/{stem}_{variant_id}.png`; manifest id stays `vN`. """ rng = rng or random.Random() tmp_dir = paths.intermediates tmp_dir.mkdir(parents=True, exist_ok=True) + stem = sanitize_stem(variant_stem) bg_mode = bg_mode or rng.choice(assets.blend_modes) fg_mode = fg_mode or rng.choice(assets.blend_modes) @@ -421,7 +449,7 @@ def compose_variant( "step2": tmp_dir / f"{variant_id}_rembg_alpha.png", "fg_filtered": tmp_dir / f"{variant_id}_fg_filtered.png", "composed": tmp_dir / f"{variant_id}_composed.png", - "final": paths.variants / f"{variant_id}.png", + "final": paths.variants / variant_filename(stem, variant_id), } paths.variants.mkdir(parents=True, exist_ok=True) @@ -467,13 +495,19 @@ def compose_variant( } -def process_job(job_id: str, source_path: Path, original_suffix: str) -> None: - """Full pipeline for a freshly ingested inbox file: preprocess, rembg +def process_job( + job_id: str, + source_path: Path, + original_suffix: str, + *, + source_stem: str | None = None, +) -> None: + """Full pipeline for a freshly ingested incoming file: preprocess, rembg once, generate OUTPUT_COUNT variants, write manifest + status. `source_path` must already be a private copy (jobs//original.*) — - callers (worker.py) are responsible for copying out of inbox first, so - the inbox file itself is never touched here. + callers (worker.py) are responsible for copying out of incoming first, so + the incoming file itself is never touched here. Variants that hit FILTER_TIMEOUT during the normal pass are retried once at the end with FILTER_TIMEOUT_LONG. Filter probes in @@ -483,8 +517,9 @@ def process_job(job_id: str, source_path: Path, original_suffix: str) -> None: paths.root.mkdir(parents=True, exist_ok=True) paths.intermediates.mkdir(parents=True, exist_ok=True) paths.variants.mkdir(parents=True, exist_ok=True) + stem = sanitize_stem(source_stem or source_path.stem) - write_status(paths, "processing", source_file=str(source_path.name)) + write_status(paths, "processing", source_file=str(source_path.name), source_stem=stem) try: paths = preprocess_original(paths) @@ -493,6 +528,7 @@ def process_job(job_id: str, source_path: Path, original_suffix: str) -> None: "job_id": job_id, "original_file": paths.original.name, "rembg_file": paths.rembg.name, + "source_stem": stem, "created_at": now_iso(), "variants": [], } @@ -509,7 +545,14 @@ def process_job(job_id: str, source_path: Path, original_suffix: str) -> None: for i in range(1, config.OUTPUT_COUNT + 1): variant_id = f"v{i}" try: - entry = compose_variant(paths, assets, variant_id=variant_id, source="auto", rng=rng) + entry = compose_variant( + paths, + assets, + variant_id=variant_id, + source="auto", + variant_stem=stem, + rng=rng, + ) manifest["variants"].append(entry) write_manifest(paths, manifest) except FilterTimeoutError as exc: @@ -530,7 +573,14 @@ def process_job(job_id: str, source_path: Path, original_suffix: str) -> None: with filter_timeout(config.FILTER_TIMEOUT_LONG): for variant_id in timeout_retries: try: - entry = compose_variant(paths, assets, variant_id=variant_id, source="auto", rng=rng) + entry = compose_variant( + paths, + assets, + variant_id=variant_id, + source="auto", + variant_stem=stem, + rng=rng, + ) manifest["variants"].append(entry) write_manifest(paths, manifest) except PipelineError as exc: diff --git a/app/remix.py b/app/remix.py index 32680d7..7516688 100644 --- a/app/remix.py +++ b/app/remix.py @@ -49,6 +49,8 @@ def create_remix_variant(job_id: str, original_suffix: str, choice: RemixChoice) "created_at": pipeline.now_iso(), "variants": [], } + variant_stem = pipeline.resolve_source_stem(manifest, paths) + manifest.setdefault("source_stem", variant_stem) variant_id = pipeline.next_variant_id(manifest) entry = pipeline.compose_variant( @@ -56,6 +58,7 @@ def create_remix_variant(job_id: str, original_suffix: str, choice: RemixChoice) assets, variant_id=variant_id, source="remix", + variant_stem=variant_stem, bg_name=choice.bg_filter, bg_mode=choice.bg_blend, fg_name=choice.fg_filter, diff --git a/app/templates/index.html b/app/templates/index.html index 64cf167..2b791a5 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -2,7 +2,7 @@ {% block title %}{{ site_title }}{% endblock %} {% block content %} {% if not jobs %} -

Noch keine Jobs. Bild per SFTP hochladen, dann kurz warten.

+

Noch keine Jobs. Bild in den Sync-Ordner legen, dann kurz warten.

{% else %}
    {% for job in jobs %} diff --git a/app/worker.py b/app/worker.py index be9c798..546fb81 100644 --- a/app/worker.py +++ b/app/worker.py @@ -1,8 +1,8 @@ -"""Inbox watcher + sequential job runner. +"""Incoming watcher + sequential job runner. -Polls `DATA_DIR/inbox` for new, size-stable image files, copies each one +Polls `DATA_DIR/incoming` for new, size-stable image files, copies each one into its own `jobs//original.*` and runs the compose pipeline on -it. Files are NEVER deleted or moved from the inbox — `processed.json` +it. Files are NEVER deleted or moved from incoming — `processed.json` tracks what has already been handled (by path + size + mtime) so restarts don't reprocess everything. @@ -61,9 +61,18 @@ def _is_already_processed(processed: dict[str, Any], path: Path, stat: Any) -> b return entry.get("size") == stat.st_size and entry.get("mtime") == stat.st_mtime +def _is_ignored_name(name: str) -> bool: + """Skip Syncthing metadata/temp files and other dotfiles.""" + if name.startswith("."): + return True + if name.startswith(".syncthing.") or ".syncthing." in name: + return True + return False + + def _is_stable(path: Path) -> bool: """A file is "stable" if its size doesn't change across a short wait — - cheap way to avoid picking up a half-uploaded SFTP transfer.""" + cheap way to avoid picking up a half-written Syncthing transfer.""" try: size_before = path.stat().st_size except OSError: @@ -77,12 +86,15 @@ def _is_stable(path: Path) -> bool: def find_new_files(processed: dict[str, Any]) -> list[Path]: + """Top-level images under incoming/ only — never recurse into jobs/.""" if not config.INBOX_DIR.exists(): return [] candidates: list[Path] = [] - for path in sorted(config.INBOX_DIR.rglob("*")): + for path in sorted(config.INBOX_DIR.iterdir()): if not path.is_file(): continue + if _is_ignored_name(path.name): + continue if path.suffix.lower() not in config.SUPPORTED_EXTENSIONS: continue try: @@ -105,13 +117,14 @@ def handle_file(path: Path, processed: dict[str, Any]) -> None: suffix = path.suffix.lower() paths = pipeline.job_paths(job_id, suffix) paths.root.mkdir(parents=True, exist_ok=True) + source_stem = pipeline.sanitize_stem(path.stem) - logger.info("new inbox file %s -> job %s", path.name, job_id) - # Copy (never move) so the inbox stays untouched. + logger.info("new incoming file %s -> job %s (stem=%s)", path.name, job_id, source_stem) + # Copy (never move) so incoming stays untouched. shutil.copy2(path, paths.original) try: - pipeline.process_job(job_id, paths.original, suffix) + pipeline.process_job(job_id, paths.original, suffix, source_stem=source_stem) finally: # Mark as processed regardless of pipeline outcome so a permanently # broken image doesn't get retried forever; failures are visible in diff --git a/compose.yml b/compose.yml index dc01d15..8b72685 100644 --- a/compose.yml +++ b/compose.yml @@ -1,51 +1,18 @@ -# live.f12.rocks — event photo pipeline: SFTP inbox -> gmic/rembg -> web. +# live.f12.rocks — event photo pipeline: Syncthing drop -> gmic/rembg -> web. # -# All three services share one host directory (default ./data, see -# .env.example DATA_HOST_DIR) at different mount points: -# sftpgo -> /srv/sftpgo/data (SFTP user home = /srv/sftpgo/data/inbox) -# worker -> /data (watches /data/inbox, writes /data/jobs) -# web -> /data (reads jobs, writes remix variants) +# worker + web share one host directory (Syncthing share root): +# ${DATA_HOST_DIR:-/home/frank/sync.schwenk.online/data/livef12} -> /data +# worker watches /data/incoming, writes /data/jobs +# web reads jobs, writes remix variants # -# Copy .env.example to .env and fill in real SFTP credentials before -# running this. Never commit .env. +# Syncthing itself runs on the host (not in this compose). Copy .env.example +# to .env for local overrides (e.g. DATA_HOST_DIR=./data). Never commit .env. services: - sftpgo: - image: drakkan/sftpgo:v2 - restart: unless-stopped - # The image defaults to a fixed uid 1000 user. worker/web run as root - # (see Dockerfile), and Docker itself creates first-run bind-mount - # directories (./data/inbox, ./data/jobs, ...) as root:root — running - # sftpgo as root too avoids a uid mismatch on the shared ./data tree. - user: root - entrypoint: ["/bin/sh", "/entrypoint.sh"] - environment: - SFTP_USER: ${SFTP_USER:-livef12} - SFTP_PASSWORD: ${SFTP_PASSWORD:-changeme} - # Minutes. Default SFTPGo is 15 — too aggressive for Android clients - # that idle after a successful upload (shows up as EOF in the logs). - SFTPGO_COMMON__IDLE_TIMEOUT: ${SFTPGO_IDLE_TIMEOUT:-120} - volumes: - - ${DATA_HOST_DIR:-./data}:/srv/sftpgo/data - - sftpgo_state:/var/lib/sftpgo - - ./sftpgo/entrypoint.sh:/entrypoint.sh:ro - ports: - # SFTP only. The SFTPGo web admin (container port 8080) is - # intentionally NOT published here and NOT attached to the - # `traefik` network — it stays unreachable from outside docker. - # - # Host SFTP port. Spec asked for 121212 which exceeds TCP max - # (65535); production default is 12121. - - "${SFTP_HOST_PORT:-12121}:2022" - networks: - - internal - worker: build: . restart: unless-stopped command: ["python3", "-m", "app.worker"] - depends_on: - - sftpgo environment: DATA_DIR: /data OUTPUT_COUNT: ${OUTPUT_COUNT:-3} @@ -60,7 +27,7 @@ services: REMBG_ALPHA: ${REMBG_ALPHA:-1} NICE_LEVEL: ${NICE_LEVEL:-18} volumes: - - ${DATA_HOST_DIR:-./data}:/data + - ${DATA_HOST_DIR:-/home/frank/sync.schwenk.online/data/livef12}:/data - ./assets:/data/assets:ro - rembg_cache:/app/.home # Sequential-only by design (no threads/async in worker.py); these @@ -92,7 +59,7 @@ services: NICE_LEVEL: ${NICE_LEVEL:-18} SITE_TITLE: live.f12.rocks volumes: - - ${DATA_HOST_DIR:-./data}:/data + - ${DATA_HOST_DIR:-/home/frank/sync.schwenk.online/data/livef12}:/data - ./assets:/data/assets:ro labels: - "traefik.enable=true" @@ -110,5 +77,4 @@ networks: external: true volumes: - sftpgo_state: rembg_cache: diff --git a/sftpgo/entrypoint.sh b/sftpgo/entrypoint.sh deleted file mode 100755 index d2544d1..0000000 --- a/sftpgo/entrypoint.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh -# Builds a SFTPGo "loaddata" JSON (the format the dumpdata/loaddata REST -# API uses) with exactly one user, then starts sftpgo with -# --loaddata-from that file. Runs on every container start — mode 0 -# (the default) adds new objects and updates existing ones, so re-running -# this is safe and picks up a changed SFTP_PASSWORD on restart. -# -# The user's home_dir is /srv/sftpgo/data/inbox, which — via the -# ./data:/srv/sftpgo/data bind mount in compose.yml — is the same -# directory as the worker's DATA_DIR/inbox. Whatever lands here over SFTP -# is exactly what the worker watches. -# -# Uses `jq` to build the JSON so SFTP_USER/SFTP_PASSWORD are always -# correctly escaped (arbitrary passwords, including quotes/backslashes, -# are safe). jq ships in the official drakkan/sftpgo image; if a future -# image drops it, this script needs an alternative (e.g. python/perl, -# both otherwise present at the time of writing). -set -eu - -OUTPUT="/var/lib/sftpgo/loaddata.json" - -: "${SFTP_USER:?SFTP_USER must be set}" -: "${SFTP_PASSWORD:?SFTP_PASSWORD must be set}" - -jq -n \ - --arg user "$SFTP_USER" \ - --arg pass "$SFTP_PASSWORD" \ - '{ - users: [ - { - status: 1, - username: $user, - password: $pass, - home_dir: "/srv/sftpgo/data/inbox", - permissions: { - "/": ["list", "download", "upload", "overwrite", "create_dirs"] - }, - filesystem: { provider: 0 } - } - ], - version: 15 - }' > "$OUTPUT" - -exec sftpgo serve --loaddata-from "$OUTPUT"