feat: replace SFTP drop with Syncthing share path
Remove SFTPGo; mount event data from the Syncthing folder, watch incoming/, and name variants after the source stem. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
+21
-8
@@ -1,8 +1,8 @@
|
||||
"""Inbox watcher + sequential job runner.
|
||||
"""Incoming watcher + sequential job runner.
|
||||
|
||||
Polls `DATA_DIR/inbox` for new, size-stable image files, copies each one
|
||||
Polls `DATA_DIR/incoming` for new, size-stable image files, copies each one
|
||||
into its own `jobs/<job_id>/original.*` and runs the compose pipeline on
|
||||
it. Files are NEVER deleted or moved from the inbox — `processed.json`
|
||||
it. Files are NEVER deleted or moved from incoming — `processed.json`
|
||||
tracks what has already been handled (by path + size + mtime) so restarts
|
||||
don't reprocess everything.
|
||||
|
||||
@@ -61,9 +61,18 @@ def _is_already_processed(processed: dict[str, Any], path: Path, stat: Any) -> b
|
||||
return entry.get("size") == stat.st_size and entry.get("mtime") == stat.st_mtime
|
||||
|
||||
|
||||
def _is_ignored_name(name: str) -> bool:
|
||||
"""Skip Syncthing metadata/temp files and other dotfiles."""
|
||||
if name.startswith("."):
|
||||
return True
|
||||
if name.startswith(".syncthing.") or ".syncthing." in name:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _is_stable(path: Path) -> bool:
|
||||
"""A file is "stable" if its size doesn't change across a short wait —
|
||||
cheap way to avoid picking up a half-uploaded SFTP transfer."""
|
||||
cheap way to avoid picking up a half-written Syncthing transfer."""
|
||||
try:
|
||||
size_before = path.stat().st_size
|
||||
except OSError:
|
||||
@@ -77,12 +86,15 @@ def _is_stable(path: Path) -> bool:
|
||||
|
||||
|
||||
def find_new_files(processed: dict[str, Any]) -> list[Path]:
|
||||
"""Top-level images under incoming/ only — never recurse into jobs/."""
|
||||
if not config.INBOX_DIR.exists():
|
||||
return []
|
||||
candidates: list[Path] = []
|
||||
for path in sorted(config.INBOX_DIR.rglob("*")):
|
||||
for path in sorted(config.INBOX_DIR.iterdir()):
|
||||
if not path.is_file():
|
||||
continue
|
||||
if _is_ignored_name(path.name):
|
||||
continue
|
||||
if path.suffix.lower() not in config.SUPPORTED_EXTENSIONS:
|
||||
continue
|
||||
try:
|
||||
@@ -105,13 +117,14 @@ def handle_file(path: Path, processed: dict[str, Any]) -> None:
|
||||
suffix = path.suffix.lower()
|
||||
paths = pipeline.job_paths(job_id, suffix)
|
||||
paths.root.mkdir(parents=True, exist_ok=True)
|
||||
source_stem = pipeline.sanitize_stem(path.stem)
|
||||
|
||||
logger.info("new inbox file %s -> job %s", path.name, job_id)
|
||||
# Copy (never move) so the inbox stays untouched.
|
||||
logger.info("new incoming file %s -> job %s (stem=%s)", path.name, job_id, source_stem)
|
||||
# Copy (never move) so incoming stays untouched.
|
||||
shutil.copy2(path, paths.original)
|
||||
|
||||
try:
|
||||
pipeline.process_job(job_id, paths.original, suffix)
|
||||
pipeline.process_job(job_id, paths.original, suffix, source_stem=source_stem)
|
||||
finally:
|
||||
# Mark as processed regardless of pipeline outcome so a permanently
|
||||
# broken image doesn't get retried forever; failures are visible in
|
||||
|
||||
Reference in New Issue
Block a user