froxxxy/livef12rocks
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: flatten output into variants/ and intermediates/

Browse Source 
Drop per-job subdirs for phone-friendly Syncthing layout with
stem-suffixed filenames; keep web state in meta/.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Frank Schwenk
2026-07-18 14:52:47 +02:00
parent 9a9a3e3afe
commit b925b151f0
11 changed files with 289 additions and 220 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/worker.py
+18 -14
View File
@@ -1,10 +1,10 @@
"""Incoming watcher + sequential job runner.
Polls `DATA_DIR/incoming` for new, size-stable image files, copies each one
into its own `jobs/<job_id>/original.*` and runs the compose pipeline on
it. Files are NEVER deleted or moved from incoming — `processed.json`
tracks what has already been handled (by path + size + mtime) so restarts
don't reprocess everything.
into `variants/{stem}_original.*` and runs the compose pipeline on it.
Files are NEVER deleted or moved from incoming — `processed.json` tracks
what has already been handled (by path + size + mtime) so restarts don't
reprocess everything.
Runs one job at a time (no threading) — this is intentional: the compose
pipeline is CPU heavy (gmic/rembg) and the worker container is capped at
@@ -30,7 +30,12 @@ logger = logging.getLogger("livef12.worker")
def ensure_dirs() -> None:
for path in (config.INBOX_DIR, config.JOBS_DIR):
for path in (
config.INBOX_DIR,
config.VARIANTS_DIR,
config.INTERMEDIATES_DIR,
config.META_DIR,
):
path.mkdir(parents=True, exist_ok=True)
@@ -86,7 +91,7 @@ def _is_stable(path: Path) -> bool:
def find_new_files(processed: dict[str, Any]) -> list[Path]:
"""Top-level images under incoming/ only — never recurse into jobs/."""
"""Top-level images under incoming/ only — never recurse."""
if not config.INBOX_DIR.exists():
return []
candidates: list[Path] = []
@@ -113,26 +118,25 @@ def handle_file(path: Path, processed: dict[str, Any]) -> None:
return
stat = path.stat()
job_id = pipeline.new_job_id()
stem = pipeline.sanitize_stem(path.stem)
suffix = path.suffix.lower()
paths = pipeline.job_paths(job_id, suffix)
paths.root.mkdir(parents=True, exist_ok=True)
source_stem = pipeline.sanitize_stem(path.stem)
paths = pipeline.job_paths(stem, suffix)
paths.variants.mkdir(parents=True, exist_ok=True)
logger.info("new incoming file %s -> job %s (stem=%s)", path.name, job_id, source_stem)
logger.info("new incoming file %s -> stem %s", path.name, stem)
# Copy (never move) so incoming stays untouched.
shutil.copy2(path, paths.original)
try:
pipeline.process_job(job_id, paths.original, suffix, source_stem=source_stem)
pipeline.process_job(stem, paths.original, suffix, source_stem=stem)
finally:
# Mark as processed regardless of pipeline outcome so a permanently
# broken image doesn't get retried forever; failures are visible in
# jobs/<id>/status.json for manual follow-up.
# meta/{stem}.json for manual follow-up.
processed[_file_key(path)] = {
"size": stat.st_size,
"mtime": stat.st_mtime,
"job_id": job_id,
"job_id": stem,
"processed_at": pipeline.now_iso(),
}
save_processed(processed)