diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml new file mode 100644 index 0000000..70d6a9c --- /dev/null +++ b/.gitea/workflows/test.yml @@ -0,0 +1,28 @@ +name: Test + +on: + push: + branches: [main, cleanup/**] + pull_request: + +jobs: + pytest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install package and dev dependencies + run: pip install -e ".[dev]" + + - name: Ruff + run: | + pip install ruff + ruff check . + ruff format --check . + + - name: Pytest (unit; skip integration) + run: pytest -m "not integration" diff --git a/.gitignore b/.gitignore index 4417c1f..05e7fe9 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,14 @@ __pycache__/ dist/ build/ .pytest_cache/ +.ruff_cache/ +.mypy_cache/ +.coverage +htmlcov/ .venv/ venv/ .env +*.log + +# Local pipeline run output (if created inside repo) +pipeline_output/ diff --git a/CLEANUP_PLAN.md b/CLEANUP_PLAN.md new file mode 100644 index 0000000..5412203 --- /dev/null +++ b/CLEANUP_PLAN.md @@ -0,0 +1,57 @@ +# Cleanup Plan — imagepipeline quality pass + +Branch: `cleanup/quality-pass` +Started: 2026-07-18 + +## Phase 0 findings (summary) + +| Area | Risk | Notes | +|------|------|-------| +| Core (runner, resume, pipeline) | Medium | Solid tests; `xcf_stack` special-case in runner — defer refactor | +| Modules | Low–Medium | `gmic_grayscale` missing `finalize_gmic_output` (resume parity) | +| Dependencies | High | `dependencies = []` but Pillow/numpy imported at module load | +| Pipelines | Low | Machine-local paths intentional (SOUL); stale `CONTINUE_FROM` left as-is | +| Tests | Medium | 87/88 pass; watermark integration fails on darktable-cli 5.6 | +| Tooling | Safe | No linter/formatter/CI | + +## Roadmap + +### Phase 1 — Low-risk quick wins +- [x] Add Ruff (lint + format) and apply once +- [x] Expand `.gitignore` +- [x] Remove dead duplicate branch in `imagemagick_grayscale` +- [x] Fix flaky `test_watermark_pipeline` (darktable probe skip) + +### Phase 2 — Structure & config +- [x] Explicit `import imagepipeline.modules` in `pipeline.py` +- [x] `gmic_grayscale`: call `finalize_gmic_output` (parity with `gmic`) +- [ ] Pipeline `CONTINUE_FROM` reset — **skipped** (machine-local resume state) +- [ ] `xcf_stack` runner hook — **deferred** (medium refactor risk) + +### Phase 3 — Code quality +- [x] Pytest markers (`integration`, `slow`) +- [ ] Broad dedup of pipeline scripts — **deferred** (domain-specific, RECIPES first) + +### Phase 4 — Dependencies & CI +- [x] Declare Pillow in core; numpy in dev for import-time ai_tone_map +- [x] Add Ruff to dev extras +- [x] Gitea Actions: pytest on push + +### Phase 5 — Docs +- [x] README sync (resume, `.env`, `[ai]`, external tools) +- [x] `docs/ARCHITECTURE.md`, `CONTRIBUTING.md` +- [x] `LICENSE` (MIT, matches pyproject) + +## Known bugs (not fixed — behavior change or out of scope) + +1. **darktable-cli 5.6** — integration test fails locally; style file exists but CLI exits 1 on PNG export (possible upstream CLI change). +2. **Eager module imports** — `ai_tone_map` pulls numpy at import; mitigated via dev dep, not lazy-import refactor. + +## Verification + +```bash +pip install -e ".[dev,ai]" +ruff check . +ruff format --check . +pytest +``` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..1fc5e98 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,36 @@ +# Contributing + +## Setup + +```bash +pip install -e ".[dev,ai]" +``` + +Use `.[dev]` only for core tests; AI module tests need `.[ai]` (torch). OpenRouter/Comfy steps need `OPENROUTER_API_KEY` in `.env` (see `.env.example`). + +## Code style + +- **Ruff** for lint and format (`ruff check .`, `ruff format .`) +- Python **3.11+**, type hints on public APIs +- CLI messages: direct English, no marketing tone (see `SOUL.md`) + +## Tests + +```bash +pytest # full suite (may call local CLIs) +pytest -m "not integration" # fast subset for CI +``` + +Mark new external-tool tests with `@pytest.mark.integration`. + +## Adding a module + +See [MODULE_DEVELOPMENT.md](MODULE_DEVELOPMENT.md). Register in `imagepipeline/modules/__init__.py` and add param/behavior tests. + +## Pipelines + +Scripts in `pipelines/` are examples and shoot-specific recipes. Machine-local `INPUT` paths are intentional. Reset `CONTINUE_FROM` before sharing a script. + +## Commits + +Conventional Commits (`feat:`, `fix:`, `chore:`, …). One logical change per commit. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..6a19352 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Frank + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index ea98af4..6268944 100644 --- a/README.md +++ b/README.md @@ -7,15 +7,19 @@ Each pipeline is a Python script that defines a DAG of processing steps. Every s ## Requirements - Python 3.11+ -- [ImageMagick](https://imagemagick.org/) (`magick` or `convert` on PATH) +- [ImageMagick](https://imagemagick.org/) (`magick` or `convert` on PATH) — most pipelines +- Optional CLIs per module: [G'MIC](https://gmic.eu/), [rembg](https://github.com/danielgatis/rembg), [darktable-cli](https://www.darktable.org/), GIMP (`gimp-console`) ## Installation ```bash cd /path/to/imagepipeline -pip install -e ".[dev]" +pip install -e ".[dev]" # core + tests +pip install -e ".[dev,ai]" # + torch/numpy for AI modules ``` +Copy `.env.example` to `.env` and set `OPENROUTER_API_KEY` when using `openrouter_edit` or Comfy-related workflows. + ## Quick Start Edit the input path in `pipelines/example_grayscale.py`, then run: @@ -35,9 +39,11 @@ with Pipeline(name="my_run", input_dir=Path("/path/to/export")) as p: p.run() ``` +List registered modules: `imagepipeline list-modules` + ## Output Structure -Each run creates a folder like `my_run_20260527143022/`: +Each run creates a folder like `my_run_20260527143022/` under `~/pipeline_output/` (or `output_base`): ``` my_run_20260527143022/ @@ -48,7 +54,26 @@ my_run_20260527143022/ └── ... ``` -Step folders are named `{module_name}_{nn}` (two-digit counter per module name). +Step folders are named `{module_name}_{nn}` by default (two-digit counter per module name). Pass optional `step_id="input_bokeh"` to `p.step()` for a custom folder name and step reference (see [docs/MODULE_DEVELOPMENT.md](docs/MODULE_DEVELOPMENT.md#step-folder-naming)). + +## Resume + +Pipelines support resuming interrupted runs: + +```python +CONTINUE_FROM = Path("~/pipeline_output/my_run_260718120000") +EXISTING_OUTPUTS = {"rembg_01": CONTINUE_FROM / "rembg_01"} + +with Pipeline( + name="my_run", + input_dir=INPUT, + continue_from=CONTINUE_FROM, + existing_outputs=EXISTING_OUTPUTS, +) as p: + ... +``` + +Modules that change file extensions must implement `expected_output_filenames` so skip logic works (e.g. `rembg` → `.png`). ## Writing Pipelines @@ -67,12 +92,27 @@ with Pipeline(name="colorsplash", input_dir=INPUT) as p: - Parameters are passed as kwargs and validated against each module's schema - Multiple uses of the same module get separate numbered folders +Declarative building blocks for agents and humans: [RECIPES.md](RECIPES.md). + ## Adding Modules -See [docs/MODULE_DEVELOPMENT.md](docs/MODULE_DEVELOPMENT.md). +See [docs/MODULE_DEVELOPMENT.md](docs/MODULE_DEVELOPMENT.md). Architecture overview: [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md). + +## Development + +```bash +ruff check . +ruff format . +pytest # full suite (uses local CLIs when present) +pytest -m "not integration" # fast subset (CI default) +``` + +See [CONTRIBUTING.md](CONTRIBUTING.md). ## Tests ```bash pytest ``` + +Optional AI tests require `pip install -e ".[ai]"`. diff --git a/RECIPES.md b/RECIPES.md new file mode 100644 index 0000000..603940b --- /dev/null +++ b/RECIPES.md @@ -0,0 +1,310 @@ +# RECIPES.md + +Reference for agents building `pipelines/.py`. Not executable — expand to `p.step(...)` calls. + +--- + +## Rules + +1. **combine** = one output image. Layers **bottom → top** (first line = background). +2. **composite** takes exactly **2** inputs. For 3+ layers: `composite(A,B)` → `composite(mid, C)` → … +3. **rembg** once per pipeline — reuse the same step ref in every combine. +4. **G'MIC in Python:** `command="-filter args"` (leading `-`). Multi-frame output: framework keeps frame `000001`. +5. **Colors:** hex (`#RRGGBB`) for `imagemagick_fill` / `color_to_alpha`. G'MIC color slots: `R,G,B` ints 0–255 from hex bytes. +6. **Placeholders** at pipeline top: `COLOR1`, `COLOR2`, `R1,G1,B1`, `R2,G2,B2`, `ALPHA1` (default 160), `ALPHA2` (default 110), `STYLE`, `MAX_EDGE`, `PROMPT`, `MODEL`, `TEMPLATE_IMAGE`, `YELLOW`. +7. **xcf_stack:** separate step, not a combine layer. `inputs=[...]` bottom → top; include `"input"` for originals. Needs `gimp` on PATH. +8. Lines starting with `#` in user prompts = human notes — ignore. + +**Hex → RGB:** `#244a89` → `36,74,137`. Strip `#RRGGBBAA` alpha for G'MIC tuples. + +--- + +## Layer lines (example) + +One combine, bottom → top. Other phrases: `grayscale` / `original as greyscale`, `white`/`black background`, `original with gmic: ID`, `#COLOR to alpha`, `resize max edge N`, `crop square`, `darktable style STYLE`, `openrouter edit …`, `xcf stack layers: a, b, c`. + +```text +demo-combine + gradient background COLOR1 COLOR2 radial + original + rembg with gmic: gmic-drop-shadow + make layer 5% bigger then crop to original size + rembg +``` + +--- + +## Modules + +| name | parameters (defaults) | notes | +|------|----------------------|-------| +| `rembg` | `model`=`birefnet-general`, `alpha_matting`=`True` | output `.png` | +| `gmic` | `command` (required) | timeout 300s | +| `gmic_grayscale` | `command`=`-to_gray` | | +| `composite` | `mode`=`over`, `output_ext`=`.png`, `foreground_opacity`=`1.0` | 2 inputs: bg, fg | +| `imagemagick_fill` | `color1`, `color2`=`""`, `gradient`=`False`, `radial`=`False`, `angle`=`None` | sized to input | +| `imagemagick_grayscale` | `colorspace`=`Gray` | | +| `imagemagick_resize` | `max_edge`=`2000` | no upscale | +| `imagemagick_scale_crop` | `scale`=`1.05` | center crop after scale | +| `color_to_alpha` | `color` (required), `fuzz`=`0.0` | output `.png` | +| `crop_square` | — | center square crop | +| `darktable_style` | `style` (required), `style_overwrite`=`True`, `out_ext`=`""`, `config_dir`=`~/.config/darktable` | jpeg/png only | +| `xcf_stack` | `skip_missing`=`False`, `timeout`=`120` | multi-input step | +| `openrouter_edit` | `prompt`, `model`, `strength`=`0.3`, `template_image`=`None`, `api_key_env`=`OPENROUTER_API_KEY` + AI common | needs `.env` | +| `ai_exposure` | `strength`=`1.0` + AI common | `[ai]` extra | +| `ai_tone_map` | `checkpoint`=`""`, `strength`=`1.0`, `net_input_size`=`256` + AI common | `[ai]` extra | +| `comfy_flux_edit` | `prompt`, `denoise`=`0.35`, `seed`=`-1`, `server_url`, `workflow_path`, `poll_interval`=`2.0` + AI common | local ComfyUI | + +**AI common** (`ai_exposure`, `ai_tone_map`, `openrouter_edit`, `comfy_flux_edit`): `skip_existing`=`True`, `max_edge`=`2048`, `device`=`cpu`. + +--- + +## ImageMagick operations + +Used via modules above — not free-form CLI in pipelines. `W×H` = size of reference input image. + +| id | module | operation | +|----|--------|-----------| +| `im-solid` | `imagemagick_fill` | `-size W×H xc:COLOR` | +| `im-gradient-linear` | `imagemagick_fill` | `-size W×H -define gradient:angle=N gradient:COLOR1-COLOR2` | +| `im-gradient-radial` | `imagemagick_fill` | `-size W×H radial-gradient:COLOR1-COLOR2` | +| `im-grayscale` | `imagemagick_grayscale` | `-colorspace Gray` | +| `im-resize-max-edge` | `imagemagick_resize` | `-auto-orient -resize {N}x{N}>` (no upscale) | +| `im-scale-crop` | `imagemagick_scale_crop` | `-resize {w×scale}x{h×scale}! -gravity Center -crop w×h+0+0 +repage` | +| `im-crop-square` | `crop_square` | `-auto-orient -crop S×S+X+Y +repage` (center square) | +| `im-color-to-alpha` | `color_to_alpha` | `-alpha on [-fuzz N%] -transparent COLOR` | +| `im-composite-over` | `composite` | `(bg) (fg) -compose over -composite` | +| `im-composite-multiply` | `composite` | `mode=multiply` | +| `im-composite-screen` | `composite` | `mode=screen` | +| `im-composite-opacity` | `composite` | fg: `-channel A -evaluate multiply {0..1}` then compose | + +**Layer line → id:** `COLOR background` → `im-solid`; `gradient … 45 degree` → `im-gradient-linear`; `gradient … radial` → `im-gradient-radial`; `grayscale` → `im-grayscale`; `resize max edge N` → `im-resize-max-edge`; `make layer 5% bigger …` → `im-scale-crop` (`scale=1.05`); `crop square` → `im-crop-square`; `COLOR to alpha` → `im-color-to-alpha`. + +--- + +## G'MIC commands + +Python string = `-` + filter + args. Placeholders `R1,G1,B1` / `R2,G2,B2` from `COLOR1` / `COLOR2`. + +| id | command | +|----|---------| +| `gmic-tone-mapping` | `fx_map_tones 0.5,0.7,0.1,30,0` | +| `gmic-grayscale` | `to_gray` (via `gmic_grayscale`) | +| `gmic-feltpen` | `fx_feltpen 300,50,1,0.1,20,5` | +| `gmic-edges` | `fx_edges 0,15,0` | +| `gmic-charred-plastic` | `fx_charred_plastic 1,10,40,1,10,0,0,2,6,5,20,0,11` | +| `gmic-neon` | `fx_neon 0,1,1,1,1,0,0.45,40,60,0,1,1.15,2,3,0,3,20,0.4,0.1,1.5,5,0.2,0.1,1,2,1,0,0,1,1,0,50,50` | +| `gmic-cutout` | `fx_cutout 4,0.5,4,1` | +| `gmic-poster` | `fx_poster_hope 0,3` | +| `gmic-color-abstraction` | `fx_color_abstraction 1,10,0.2` | +| `gmic-huffman-glitches` | `fx_huffman_glitches 30,0,25,0,0,0,0,0,50,50` | +| `gmic-queryprimary` | `afre_queryprimary 1,1` | +| `gmic-local-similarity` | `local_similarity_mask 50,50,50,128,4,10` | +| `gmic-luma-invert` | `Lylejk_Luma_Invert 1,0` | +| `gmic-crayongraffiti` | `fx_crayongraffiti2 300,50,1,0.4,12,1,2,2,0` | +| `gmic-anaglyph` | `fx_stereo_to_anaglyph 2,0` | +| `gmic-stereo` | `gcd_stereo_img 0,0,2.028,1,1.714,3.06,4,1,0` | +| `gmic-drop-shadow` | `fx_drop_shadow3d 0,0,0,10,1,1,2,0.5,R2,G2,B2,200,0` | +| `gmic-jpr-smooth` | `jpr_gradient_smooth 0,1.5` | +| `gmic-bokeh` | `fx_bokeh 3,5,0,30,8,4,0.3,0.2,R1,G1,B1,ALPHA1,0.7,30,20,20,1,2,R2,G2,B2,ALPHA2,0.15` | +| `gmic-bwrecolor` | `fx_bwrecolorize 0,0,0,0,0,1,0,2,R2,G2,B2,255,R1,G1,B1,255,158,137,189,255,224,191,228,255,R1,G1,B1,0,255,255,255,255,255,255,255,255,255,R1,G1,B1,0,255` | +| `gmic-custom-gradient-a` | `fx_custom_gradient 0,0,0,1,2,1,0,128,100,100,2,0,1,0,"",1,0,R1,G1,B1,0,255,R2,G2,B2,255,255,255,0,255,255,255,255,255,0,255,255,255,0,255,0,255,0,0,255,255,128,128,128,255,255,0,255,255,0,0,0,0` | +| `gmic-custom-gradient-b` | `fx_custom_gradient 0,0,0,1,2,1,0,128,100,100,2,0,1,0,"",1,0,R2,G2,B2,255,R1,G1,B1,0,255,255,255,0,255,255,255,255,255,0,255,255,255,0,255,0,255,0,0,255,255,128,128,128,255,255,0,255,255,0,0,0,0` | + +**Typical input:** `original with gmic:` = on `"input"`; `rembg with gmic:` = on rembg step. + +--- + +## Combines + +Format: recipe id, then layers bottom → top. `→` = chained composites. + +### 2 layers + +```text +colorsplash + original as greyscale + rembg + +rembg-white-bg + white background + rembg + +rembg-black-bg + black background + rembg + +rembg-gradient-45 + gradient background COLOR1 COLOR2 45 degree + rembg + +rembg-radial-2colors + gradient background COLOR1 COLOR2 radial + rembg + +rembg-custom-gradient-a + rembg with gmic: gmic-custom-gradient-a + rembg + +rembg-custom-gradient-b + rembg with gmic: gmic-custom-gradient-b + rembg + +gmic-feltpen-rembg + rembg with gmic: gmic-feltpen + rembg + +gmic-edges-rembg + rembg with gmic: gmic-edges + rembg + +gmic-charred-plastic-rembg + rembg with gmic: gmic-charred-plastic + rembg + +gmic-neon-rembg + rembg with gmic: gmic-neon + rembg +``` + +### 3 layers + +```text +original-stereo-rembg + original + rembg with gmic: gmic-stereo + rembg +→ composite(original, rembg_stereo) → composite(_, rembg) + +original-drop-shadow-rembg + original + rembg with gmic: gmic-drop-shadow + rembg + +original-bwrecolor-rembg + original + rembg with gmic: gmic-bwrecolor + rembg +→ first composite: foreground_opacity=0.5 + +original-jpr-smooth-rembg + original + rembg with gmic: gmic-jpr-smooth + make layer 5% bigger then crop to original size + rembg +→ gmic → scale_crop on middle layer, then 3-layer chain + +bokeh-oktagon + original + original with gmic: gmic-bokeh + rembg + +color-bg-drop-shadow-rembg + COLOR1 background + rembg with gmic: gmic-drop-shadow + rembg + +yellow-bg-drop-shadow-rembg + YELLOW background + rembg with gmic: gmic-drop-shadow + rembg + +gmic-cutout-rembg + original + original with gmic: gmic-cutout + rembg + +gmic-poster-rembg + original + original with gmic: gmic-poster + rembg + +gmic-color-abstraction-rembg + original + original with gmic: gmic-color-abstraction + rembg + +gmic-huffman-glitches-rembg + original + original with gmic: gmic-huffman-glitches + rembg + +gmic-queryprimary-rembg + original + original with gmic: gmic-queryprimary + rembg + +gmic-local-similarity-rembg + original + original with gmic: gmic-local-similarity + rembg + +gmic-luma-invert-rembg + original + original with gmic: gmic-luma-invert + rembg + +gmic-crayongraffiti-rembg + original + original with gmic: gmic-crayongraffiti + rembg + +gmic-anaglyph-rembg + original + original with gmic: gmic-anaglyph + rembg +``` + +### 4 layers (middle pipeline) + +```text +original-stereo-black-alpha-rembg + original + rembg with gmic: gmic-stereo + #000000 to alpha + rembg + +original-jpr-smooth-grey-alpha-rembg + original + rembg with gmic: gmic-jpr-smooth + #7f7f7f to alpha + make layer 5% bigger then crop to original size + rembg +``` + +### Pipeline chain (not a combine) + +```text +colorsplash-watermark + colorsplash # expand combine first + darktable style STYLE +``` + +--- + +## Agent example + +Request: *pipeline with colorsplash + rembg-radial-2colors, COLOR1=#244a89, COLOR2=#c22518* + +```python +rembg_out = p.step("rembg", inputs="input") +grayscale = p.step("gmic_grayscale", inputs="input") +gradient_bg = p.step("imagemagick_fill", inputs="input", color1=COLOR1, color2=COLOR2, gradient=True, radial=True) + +p.step("composite", inputs=[grayscale, rembg_out]) # colorsplash +p.step("composite", inputs=[gradient_bg, rembg_out]) # rembg-radial-2colors +``` + +3-layer combine (drop shadow on original): + +```python +rembg_shadow = p.step("gmic", inputs=rembg_out, command=GMIC_DROP_SHADOW) +mid = p.step("composite", inputs=["input", rembg_shadow]) +p.step("composite", inputs=[mid, rembg_out]) +``` + +xcf export: + +```python +p.step("xcf_stack", inputs=["input", rembg_out, grayscale, composite_colorsplash, ...]) +``` diff --git a/SOUL.md b/SOUL.md index 2cf829f..656ab37 100644 --- a/SOUL.md +++ b/SOUL.md @@ -12,6 +12,7 @@ What this project is — not chat mood (see `MOOD.md`). - **Resume:** Pipelines support `CONTINUE_FROM` and `EXISTING_OUTPUTS`; modules must declare correct `expected_output_filenames` when output names differ from inputs (e.g. rembg → `.png`). - **Secrets:** `OPENROUTER_API_KEY` in `.env` (see `.env.example`) — never commit. - **Tests:** Run `pytest` before handoff on non-trivial module/resume changes. +- **Recipes:** Declarative pipeline building blocks in `RECIPES.md` — load via `@RECIPES.md` when creating or editing pipelines. - **Commit policy:** No commit unless Fränky asks (this project follows global `AGENTS.md`). --- diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..c8b41c1 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,59 @@ +# Architecture + +Imagepipeline is a small Python framework for defining **batch image pipelines** as DAGs. Each step is a registered module; each run writes numbered folders under a timestamped output root. + +## Layout + +``` +imagepipeline/ +├── core/ # Pipeline, runner, resume, manifest, params, logging +├── modules/ # Processing steps (@register) +├── utils/ # files, subprocess, gmic, gimp helpers +├── ai/ # Optional torch models (HDRNet, Zero-DCE) +└── cli.py # `imagepipeline list-modules` + +pipelines/ # Runnable scripts (machine-local INPUT paths) +tests/ # pytest suite +docs/ # Developer docs +workflows/comfy/ # ComfyUI workflow JSON (optional AI path) +``` + +## Execution flow + +1. **Define** — `Pipeline(name=..., input_dir=...)` collects `step()` calls (`StepDefinition` DAG). +2. **Run** — `PipelineRunner` topologically sorts steps, matches inputs by filename stem, builds `ModuleContext`. +3. **Resume** — `continue_from` / `existing_outputs` reuse prior run folders; modules declare `expected_output_filenames` when extensions change. +4. **Manifest** — `pipeline_manifest.json` records steps, params, and paths. + +## Module contract + +Every module subclasses `BaseModule` or `SubprocessModule`: + +- `name`, `description`, `parameters()` schema +- `run(ctx: ModuleContext)` writes into `ctx.output_dir` +- Optional `expected_output_filenames()` for resume when output names differ from inputs +- `check_dependencies()` for external CLI tools + +Registration happens via `@register` and eager import in `imagepipeline/modules/__init__.py`. + +## External tools + +Many modules shell out to CLIs (not Python packages): + +| Tool | Modules | +|------|---------| +| ImageMagick (`magick`/`convert`) | `imagemagick_*`, `composite`, `color_to_alpha`, `crop_square` | +| G'MIC | `gmic`, `gmic_grayscale` | +| rembg | `rembg` | +| darktable-cli | `darktable_style` | +| GIMP | `xcf_stack` | + +AI modules need `pip install -e ".[ai]"` (torch, numpy) and optionally API keys in `.env`. + +## Design choices + +- **Plain Python pipelines** — no YAML DSL; full control and easy resume constants in script. +- **Stem matching** — multi-input steps align files by basename across step folders. +- **Symlink input** — default run copies/symlinks source images into `input/` for reproducibility. + +See [MODULE_DEVELOPMENT.md](MODULE_DEVELOPMENT.md) for adding modules. diff --git a/docs/MODULE_DEVELOPMENT.md b/docs/MODULE_DEVELOPMENT.md index d354293..35b298b 100644 --- a/docs/MODULE_DEVELOPMENT.md +++ b/docs/MODULE_DEVELOPMENT.md @@ -167,7 +167,7 @@ class MyCliModule(SubprocessModule): ## Step Folder Naming -The runner assigns output folders automatically: `{module_name}_{nn}`. +By default the runner assigns output folders automatically: `{module_name}_{nn}` (two-digit counter per module name). Using the same module twice in one pipeline produces separate folders: @@ -178,7 +178,17 @@ darktable_style_01/ darktable_style_02/ ``` -You do not choose folder names in the module. +In pipeline scripts you can pass an optional `step_id` to `p.step()` for readable folder names and GIMP layer labels (`xcf_stack` uses the step id as the layer name): + +```python +input_bokeh = p.step("gmic", inputs="input", command=GMIC_BOKEH, step_id="input_bokeh") +``` + +`step_id` becomes both the internal step reference and the output subfolder name. It must be unique within the pipeline and must not contain path separators. The per-module counter still increments on every `p.step("gmic", …)` call — a custom id does not reserve or skip a number slot (the next default `gmic` step after `step_id="input_bokeh"` is `gmic_03`, not `gmic_02`, if two prior `gmic` steps ran). + +Use the same ids in `EXISTING_OUTPUTS` when resuming with external folders. + +You do not choose folder names inside the module implementation. ## Format Warnings diff --git a/imagepipeline/ai/cache.py b/imagepipeline/ai/cache.py index 8669057..b32aaaa 100644 --- a/imagepipeline/ai/cache.py +++ b/imagepipeline/ai/cache.py @@ -1,6 +1,5 @@ from __future__ import annotations -import hashlib import urllib.request from pathlib import Path diff --git a/imagepipeline/ai/classical_tone.py b/imagepipeline/ai/classical_tone.py index 3e2eeda..03752f8 100644 --- a/imagepipeline/ai/classical_tone.py +++ b/imagepipeline/ai/classical_tone.py @@ -49,7 +49,7 @@ def _lab_to_rgb(lab: np.ndarray) -> np.ndarray: fz = fy - lab[..., 2] / 200 def finv(t): - t3 = t ** 3 + t3 = t**3 return np.where(t3 > 216 / 24389, t3, (116 * t - 16) / kappa) kappa = 24389 / 27 diff --git a/imagepipeline/ai/hdrnet/model.py b/imagepipeline/ai/hdrnet/model.py index 6b3b3cb..7abaf1c 100644 --- a/imagepipeline/ai/hdrnet/model.py +++ b/imagepipeline/ai/hdrnet/model.py @@ -1,11 +1,8 @@ from __future__ import annotations -import math - import numpy as np import torch import torch.nn as nn -import torch.nn.functional as F from imagepipeline.ai.hdrnet.slice import batch_bilateral_slice @@ -70,22 +67,27 @@ class Slice(nn.Module): class ApplyCoeffs(nn.Module): def forward(self, coeff, full_res_input): - r = torch.sum(full_res_input * coeff[:, 0:3, :, :], dim=1, keepdim=True) + coeff[ - :, 9:10, :, : - ] - g = torch.sum(full_res_input * coeff[:, 3:6, :, :], dim=1, keepdim=True) + coeff[ - :, 10:11, :, : - ] - b = torch.sum(full_res_input * coeff[:, 6:9, :, :], dim=1, keepdim=True) + coeff[ - :, 11:12, :, : - ] + r = ( + torch.sum(full_res_input * coeff[:, 0:3, :, :], dim=1, keepdim=True) + + coeff[:, 9:10, :, :] + ) + g = ( + torch.sum(full_res_input * coeff[:, 3:6, :, :], dim=1, keepdim=True) + + coeff[:, 10:11, :, :] + ) + b = ( + torch.sum(full_res_input * coeff[:, 6:9, :, :], dim=1, keepdim=True) + + coeff[:, 11:12, :, :] + ) return torch.cat([r, g, b], dim=1) class GuideNN(nn.Module): def __init__(self, params) -> None: super().__init__() - self.conv1 = ConvBlock(3, params["guide_complexity"], kernel_size=1, padding=0, batch_norm=True) + self.conv1 = ConvBlock( + 3, params["guide_complexity"], kernel_size=1, padding=0, batch_norm=True + ) self.conv2 = ConvBlock( params["guide_complexity"], 1, kernel_size=1, padding=0, activation=nn.Sigmoid ) @@ -112,9 +114,7 @@ class Coeffs(nn.Module): for index in range(n_layers_splat): use_bn = bn if index > 0 else False out_ch = cm * (2**index) * lb - self.splat_features.append( - ConvBlock(prev_ch, out_ch, 3, stride=2, batch_norm=use_bn) - ) + self.splat_features.append(ConvBlock(prev_ch, out_ch, 3, stride=2, batch_norm=use_bn)) prev_ch = out_ch splat_ch = prev_ch @@ -131,7 +131,9 @@ class Coeffs(nn.Module): prev_ch = int(prev_ch * (nsize / 2**n_total) ** 2) self.global_features_fc.append(FC(prev_ch, 32 * cm * lb, batch_norm=bn)) self.global_features_fc.append(FC(32 * cm * lb, 16 * cm * lb, batch_norm=bn)) - self.global_features_fc.append(FC(16 * cm * lb, 8 * cm * lb, activation=None, batch_norm=bn)) + self.global_features_fc.append( + FC(16 * cm * lb, 8 * cm * lb, activation=None, batch_norm=bn) + ) self.local_features = nn.ModuleList( [ @@ -139,9 +141,7 @@ class Coeffs(nn.Module): ConvBlock(8 * cm * lb, 8 * cm * lb, 3, activation=None, use_bias=False), ] ) - self.conv_out = ConvBlock( - 8 * cm * lb, lb * nout * nin, 1, padding=0, activation=None - ) + self.conv_out = ConvBlock(8 * cm * lb, lb * nout * nin, 1, padding=0, activation=None) self.relu = nn.ReLU() def forward(self, lowres_input): diff --git a/imagepipeline/ai/hdrnet/slice.py b/imagepipeline/ai/hdrnet/slice.py index 6bc964f..0fb24a4 100644 --- a/imagepipeline/ai/hdrnet/slice.py +++ b/imagepipeline/ai/hdrnet/slice.py @@ -82,12 +82,8 @@ def _bilateral_slice(grid, guide): grid_val_110 = grid[gi1c, gj1c, gk0c, :] grid_val_111 = grid[gi1c, gj1c, gk1c, :] - w_000, w_001, w_010, w_011 = map( - torch.atleast_3d, (w_000, w_001, w_010, w_011) - ) - w_100, w_101, w_110, w_111 = map( - torch.atleast_3d, (w_100, w_101, w_110, w_111) - ) + w_000, w_001, w_010, w_011 = map(torch.atleast_3d, (w_000, w_001, w_010, w_011)) + w_100, w_101, w_110, w_111 = map(torch.atleast_3d, (w_100, w_101, w_110, w_111)) return ( torch.multiply(w_000, grid_val_000) diff --git a/imagepipeline/ai/zero_dce.py b/imagepipeline/ai/zero_dce.py index be1c69d..5700e57 100644 --- a/imagepipeline/ai/zero_dce.py +++ b/imagepipeline/ai/zero_dce.py @@ -60,9 +60,7 @@ class EnhanceNetNoPool(nn.Module): if self.scale_factor == 1: x_down = x else: - x_down = F.interpolate( - x, scale_factor=1 / self.scale_factor, mode="bilinear" - ) + x_down = F.interpolate(x, scale_factor=1 / self.scale_factor, mode="bilinear") x1 = self.relu(self.e_conv1(x_down)) x2 = self.relu(self.e_conv2(x1)) @@ -105,7 +103,5 @@ def enhance_image( if strength < 1.0: enhanced = tensor * (1.0 - strength) + enhanced * strength enhanced = torch.clamp(enhanced, 0.0, 1.0) - out = (enhanced.squeeze(0).permute(1, 2, 0).cpu().numpy() * 255.0).astype( - np.uint8 - ) + out = (enhanced.squeeze(0).permute(1, 2, 0).cpu().numpy() * 255.0).astype(np.uint8) return Image.fromarray(out) diff --git a/imagepipeline/cli.py b/imagepipeline/cli.py index 0a841e2..4e380cd 100644 --- a/imagepipeline/cli.py +++ b/imagepipeline/cli.py @@ -2,7 +2,6 @@ from __future__ import annotations import argparse import sys -from pathlib import Path from imagepipeline.modules.registry import list_modules diff --git a/imagepipeline/core/context.py b/imagepipeline/core/context.py index f90e389..1f1f144 100644 --- a/imagepipeline/core/context.py +++ b/imagepipeline/core/context.py @@ -2,7 +2,7 @@ from __future__ import annotations from dataclasses import dataclass, field from pathlib import Path -from typing import Any, TYPE_CHECKING +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from imagepipeline.core.log import PipelineLogger @@ -18,7 +18,8 @@ class ModuleContext: pipeline_output_root: Path step_id: str matched_groups: list[list[Path]] = field(default_factory=list) - prior_steps: list[tuple[str, Path]] = field(default_factory=list) + input_refs: list[str] = field(default_factory=list) + input_layer_dirs: list[tuple[str, Path]] = field(default_factory=list) logger: PipelineLogger | None = None @property diff --git a/imagepipeline/core/log.py b/imagepipeline/core/log.py index c950e48..6ddd318 100644 --- a/imagepipeline/core/log.py +++ b/imagepipeline/core/log.py @@ -36,23 +36,17 @@ class PipelineLogger: inputs: list[str], params: dict, ) -> None: - self.info( - f"Step {step_index}/{step_total}: {step_id} ({module_name})" - ) + self.info(f"Step {step_index}/{step_total}: {step_id} ({module_name})") self.info(f" inputs: {', '.join(inputs)}") if params: rendered = ", ".join(f"{key}={value!r}" for key, value in params.items()) self.info(f" params: {rendered}") def image(self, module_name: str, index: int, total: int, filename: str) -> None: - self.info( - f" Applying module {module_name} to image [{index}/{total}]: {filename}" - ) + self.info(f" Applying module {module_name} to image [{index}/{total}]: {filename}") def skipped(self, module_name: str, index: int, total: int, filename: str) -> None: - self.info( - f" Skipped module {module_name} [{index}/{total}] {filename} (output exists)" - ) + self.info(f" Skipped module {module_name} [{index}/{total}] {filename} (output exists)") def image_done( self, diff --git a/imagepipeline/core/manifest.py b/imagepipeline/core/manifest.py index d49b057..a18e573 100644 --- a/imagepipeline/core/manifest.py +++ b/imagepipeline/core/manifest.py @@ -2,7 +2,7 @@ from __future__ import annotations import json from dataclasses import asdict, dataclass, field -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -39,4 +39,4 @@ def write_manifest(path: Path, manifest: PipelineManifest) -> None: def utc_now_iso() -> str: - return datetime.now(timezone.utc).replace(microsecond=0).isoformat() + return datetime.now(UTC).replace(microsecond=0).isoformat() diff --git a/imagepipeline/core/params.py b/imagepipeline/core/params.py index 4ba2e1c..93ccd78 100644 --- a/imagepipeline/core/params.py +++ b/imagepipeline/core/params.py @@ -1,6 +1,6 @@ from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Any @@ -22,9 +22,7 @@ class Param: if self.choices is not None and value not in self.choices: allowed = ", ".join(repr(c) for c in self.choices) - raise ValueError( - f"Parameter '{name}' must be one of [{allowed}], got {value!r}" - ) + raise ValueError(f"Parameter '{name}' must be one of [{allowed}], got {value!r}") if self.type == "string": if not isinstance(value, str): @@ -56,9 +54,7 @@ class Param: raise ValueError(f"Unknown parameter type '{self.type}' for '{name}'") -def validate_params( - schema: dict[str, Param], raw: dict[str, Any] -) -> dict[str, Any]: +def validate_params(schema: dict[str, Param], raw: dict[str, Any]) -> dict[str, Any]: unknown = set(raw) - set(schema) if unknown: names = ", ".join(sorted(unknown)) diff --git a/imagepipeline/core/pipeline.py b/imagepipeline/core/pipeline.py index dfc3c5a..f841671 100644 --- a/imagepipeline/core/pipeline.py +++ b/imagepipeline/core/pipeline.py @@ -4,14 +4,13 @@ from collections import defaultdict from pathlib import Path from typing import Any +# Register all built-in modules (side effect of submodule imports). +import imagepipeline.modules # noqa: F401 from imagepipeline.core.exceptions import ValidationError from imagepipeline.core.runner import PipelineRunner from imagepipeline.core.step import INPUT_SOURCE, StepDefinition, StepRef from imagepipeline.modules.registry import get_module -# Import built-in modules so they register on package load. -import imagepipeline.modules.imagemagick_grayscale # noqa: F401 - class Pipeline: """Define and run an image processing pipeline.""" @@ -45,20 +44,23 @@ class Pipeline: module_name: str, *, inputs: StepRef | str | list[StepRef | str], + step_id: str | None = None, **params: Any, ) -> StepRef: module_cls = get_module(module_name) self._module_counters[module_name] += 1 counter = self._module_counters[module_name] - output_dir_name = f"{module_name}_{counter:02d}" - step_id = output_dir_name + if step_id is None: + output_dir_name = f"{module_name}_{counter:02d}" + step_id = output_dir_name + else: + self._validate_step_id(step_id) + output_dir_name = step_id input_refs = self._normalize_inputs(inputs) reserved = {"inputs", "input"} if reserved & set(params): - raise ValidationError( - "Do not pass 'inputs' or 'input' as module parameters" - ) + raise ValidationError("Do not pass 'inputs' or 'input' as module parameters") definition = StepDefinition( step_id=step_id, @@ -92,15 +94,21 @@ class Pipeline: def output_root(self) -> Path | None: return self._output_root - def _normalize_inputs( - self, inputs: StepRef | str | list[StepRef | str] - ) -> list[str]: + def _normalize_inputs(self, inputs: StepRef | str | list[StepRef | str]) -> list[str]: if isinstance(inputs, list): if not inputs: raise ValidationError("inputs must not be an empty list") return [self._input_ref(item) for item in inputs] return [self._input_ref(inputs)] + def _validate_step_id(self, step_id: str) -> None: + if not step_id: + raise ValidationError("step_id must not be empty") + if "/" in step_id or "\\" in step_id: + raise ValidationError(f"step_id must not contain path separators: {step_id!r}") + if any(step.step_id == step_id for step in self._steps): + raise ValidationError(f"Duplicate step_id: {step_id!r}") + @staticmethod def _input_ref(value: StepRef | str) -> str: if isinstance(value, StepRef): diff --git a/imagepipeline/core/runner.py b/imagepipeline/core/runner.py index f3f07e5..cf48b80 100644 --- a/imagepipeline/core/runner.py +++ b/imagepipeline/core/runner.py @@ -49,12 +49,9 @@ class PipelineRunner: self.symlink_input = symlink_input self.logger = PipelineLogger(verbose=verbose) self.existing_outputs = { - key: Path(value).resolve() - for key, value in (existing_outputs or {}).items() + key: Path(value).resolve() for key, value in (existing_outputs or {}).items() } - self.continue_from = ( - Path(continue_from).resolve() if continue_from is not None else None - ) + self.continue_from = Path(continue_from).resolve() if continue_from is not None else None self.skip_completed = skip_completed self.output_root = self._build_output_root() self._input_link_dir = self.output_root / "input" @@ -63,9 +60,7 @@ class PipelineRunner: def _build_output_root(self) -> Path: if self.continue_from is not None: if not self.continue_from.is_dir(): - raise ValidationError( - f"continue_from directory not found: {self.continue_from}" - ) + raise ValidationError(f"continue_from directory not found: {self.continue_from}") return self.continue_from timestamp = datetime.now().strftime("%y%m%d%H%M%S") @@ -151,9 +146,23 @@ class PipelineRunner: step.module.check_dependencies() validated = step.module.validate_module_params(step.params) - source_lists = [self._resolve_source_paths(ref) for ref in step.input_refs] - matched_groups = match_by_stem(source_lists) - input_paths = flatten_matched(matched_groups) + if step.module_name == "xcf_stack": + if not step.input_refs: + raise ValidationError("xcf_stack requires at least one input layer") + source_lists = [self._resolve_source_paths(ref) for ref in step.input_refs] + input_paths = source_lists[0] + matched_groups = [[path] for path in input_paths] + input_layer_dirs: list[tuple[str, Path]] = [] + for ref in step.input_refs: + if ref == INPUT_SOURCE: + input_layer_dirs.append(("input", self._input_link_dir)) + else: + input_layer_dirs.append((ref, self._results[ref].output_dir)) + else: + source_lists = [self._resolve_source_paths(ref) for ref in step.input_refs] + matched_groups = match_by_stem(source_lists) + input_paths = flatten_matched(matched_groups) + input_layer_dirs = [] output_dir = self.output_root / step.output_dir_name output_dir.mkdir(parents=True, exist_ok=True) @@ -220,7 +229,8 @@ class PipelineRunner: pipeline_output_root=self.output_root, step_id=step.step_id, matched_groups=matched_groups, - prior_steps=[(sid, res.output_dir) for sid, res in self._results.items()], + input_refs=list(step.input_refs), + input_layer_dirs=input_layer_dirs, logger=self.logger, ) @@ -234,8 +244,7 @@ class PipelineRunner: output_paths = step.module().list_output_images(ctx) if not output_paths: raise StepError( - f"Step '{step.output_dir_name}' ({step.module_name}) " - "produced no output images" + f"Step '{step.output_dir_name}' ({step.module_name}) produced no output images" ) self.logger.step_done(step.output_dir_name, step.output_dir_name, len(output_paths)) @@ -264,9 +273,7 @@ class PipelineRunner: raise ValidationError(f"Step '{step.step_id}' references unknown step '{dep}'") dependents[dep].append(step.step_id) - queue = deque( - step_id for step_id, degree in in_degree.items() if degree == 0 - ) + queue = deque(step_id for step_id, degree in in_degree.items() if degree == 0) ordered_ids: list[str] = [] while queue: diff --git a/imagepipeline/core/step.py b/imagepipeline/core/step.py index 356efc3..2efcab3 100644 --- a/imagepipeline/core/step.py +++ b/imagepipeline/core/step.py @@ -2,7 +2,7 @@ from __future__ import annotations from dataclasses import dataclass, field from pathlib import Path -from typing import Any, TYPE_CHECKING +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from imagepipeline.modules.base import BaseModule diff --git a/imagepipeline/modules/__init__.py b/imagepipeline/modules/__init__.py index fdb9aa5..a01cb2d 100644 --- a/imagepipeline/modules/__init__.py +++ b/imagepipeline/modules/__init__.py @@ -2,8 +2,8 @@ import imagepipeline.modules.ai_exposure # noqa: F401 import imagepipeline.modules.ai_tone_map # noqa: F401 -import imagepipeline.modules.comfy_flux_edit # noqa: F401 import imagepipeline.modules.color_to_alpha # noqa: F401 +import imagepipeline.modules.comfy_flux_edit # noqa: F401 import imagepipeline.modules.composite # noqa: F401 import imagepipeline.modules.crop_square # noqa: F401 import imagepipeline.modules.darktable_style # noqa: F401 diff --git a/imagepipeline/modules/ai_base.py b/imagepipeline/modules/ai_base.py index 885027f..c813d1f 100644 --- a/imagepipeline/modules/ai_base.py +++ b/imagepipeline/modules/ai_base.py @@ -115,9 +115,7 @@ class AIModule(BaseModule): avg = sum(elapsed_times) / len(elapsed_times) eta = avg * remaining if remaining else 0.0 if ctx.logger is not None: - ctx.logger.image_done( - self.name, index, total, dst.name, elapsed, eta_seconds=eta - ) + ctx.logger.image_done(self.name, index, total, dst.name, elapsed, eta_seconds=eta) @staticmethod def _image_size(path: Path) -> tuple[int, int]: diff --git a/imagepipeline/modules/color_to_alpha.py b/imagepipeline/modules/color_to_alpha.py index e696a43..f08dfa0 100644 --- a/imagepipeline/modules/color_to_alpha.py +++ b/imagepipeline/modules/color_to_alpha.py @@ -24,8 +24,7 @@ def build_color_to_alpha_args(*, color: str, fuzz: float) -> list[str]: class ColorToAlphaModule(SubprocessModule): name = "color_to_alpha" description = ( - "Make a solid color transparent (GIMP-style color to alpha). " - "Outputs PNG with alpha." + "Make a solid color transparent (GIMP-style color to alpha). Outputs PNG with alpha." ) command_candidates = ("magick", "convert") @@ -50,10 +49,7 @@ class ColorToAlphaModule(SubprocessModule): "fuzz": Param( "float", default=0.0, - help=( - "Match tolerance in percent (ImageMagick -fuzz); " - "0 = exact color only" - ), + help=("Match tolerance in percent (ImageMagick -fuzz); 0 = exact color only"), ), } diff --git a/imagepipeline/modules/comfy_flux_edit.py b/imagepipeline/modules/comfy_flux_edit.py index c391763..02b0a6f 100644 --- a/imagepipeline/modules/comfy_flux_edit.py +++ b/imagepipeline/modules/comfy_flux_edit.py @@ -76,9 +76,7 @@ class ComfyFluxEditModule(AIModule): server_url = ctx.params["server_url"].rstrip("/") self._ensure_server(server_url) if ctx.logger is not None: - ctx.logger.warning( - "ComfyUI on CPU: expect hours per full-resolution image" - ) + ctx.logger.warning("ComfyUI on CPU: expect hours per full-resolution image") workflow_template = json.loads(workflow_path.read_text(encoding="utf-8")) denoise = ctx.params["denoise"] @@ -98,9 +96,7 @@ class ComfyFluxEditModule(AIModule): seed=seed, ) prompt_id = self._queue_prompt(server_url, workflow) - output_info = self._wait_for_output( - server_url, prompt_id, poll_interval=poll_interval - ) + output_info = self._wait_for_output(server_url, prompt_id, poll_interval=poll_interval) image_bytes = self._download_view(server_url, output_info) dst.write_bytes(image_bytes) @@ -111,9 +107,7 @@ class ComfyFluxEditModule(AIModule): try: urllib.request.urlopen(f"{server_url}/system_stats", timeout=5) except urllib.error.URLError as exc: - raise DependencyError( - f"ComfyUI server not reachable at {server_url}: {exc}" - ) from exc + raise DependencyError(f"ComfyUI server not reachable at {server_url}: {exc}") from exc @staticmethod def _upload_image(server_url: str, src: Path) -> str: diff --git a/imagepipeline/modules/composite.py b/imagepipeline/modules/composite.py index 9a72b67..4e3ac96 100644 --- a/imagepipeline/modules/composite.py +++ b/imagepipeline/modules/composite.py @@ -63,9 +63,7 @@ class CompositeModule(SubprocessModule): for index, group in enumerate(ctx.matched_groups, start=1): if len(group) < 2: - raise ValueError( - "composite requires at least two input sources per image" - ) + raise ValueError("composite requires at least two input sources per image") background, foreground = group[0], group[1] self.log_image(ctx, index, total, foreground) diff --git a/imagepipeline/modules/gmic_grayscale.py b/imagepipeline/modules/gmic_grayscale.py index 3561afe..6a673f6 100644 --- a/imagepipeline/modules/gmic_grayscale.py +++ b/imagepipeline/modules/gmic_grayscale.py @@ -4,7 +4,7 @@ from imagepipeline.core.context import ModuleContext from imagepipeline.core.params import Param from imagepipeline.modules.base import SubprocessModule from imagepipeline.modules.registry import register -from imagepipeline.utils.gmic import split_gmic_command +from imagepipeline.utils.gmic import finalize_gmic_output, split_gmic_command from imagepipeline.utils.subprocess import run_command @@ -35,3 +35,4 @@ class GmicGrayscale(SubprocessModule): dst = ctx.output_dir / src.name cmd = ["gmic", str(src), *split_gmic_command(gmic_command), "-output", str(dst)] run_command(cmd, timeout=self.default_timeout) + finalize_gmic_output(ctx.output_dir, dst) diff --git a/imagepipeline/modules/imagemagick_fill.py b/imagepipeline/modules/imagemagick_fill.py index a237d2a..2b0f4a8 100644 --- a/imagepipeline/modules/imagemagick_fill.py +++ b/imagepipeline/modules/imagemagick_fill.py @@ -50,9 +50,7 @@ def build_fill_arguments( @register class ImageMagickFillModule(SubprocessModule): name = "imagemagick_fill" - description = ( - "Create solid-color or gradient images sized to match each input image" - ) + description = "Create solid-color or gradient images sized to match each input image" command_candidates = ("magick", "convert") @classmethod diff --git a/imagepipeline/modules/imagemagick_grayscale.py b/imagepipeline/modules/imagemagick_grayscale.py index aec0e75..ae741d9 100644 --- a/imagepipeline/modules/imagemagick_grayscale.py +++ b/imagepipeline/modules/imagemagick_grayscale.py @@ -1,7 +1,5 @@ from __future__ import annotations -from pathlib import Path - from imagepipeline.core.context import ModuleContext from imagepipeline.core.params import Param from imagepipeline.modules.base import SubprocessModule @@ -34,8 +32,5 @@ class ImageMagickGrayscale(SubprocessModule): for index, src in enumerate(ctx.input_paths, start=1): self.log_image(ctx, index, total, src) dst = ctx.output_dir / src.name - if command == "magick": - cmd = [command, str(src), "-colorspace", colorspace, str(dst)] - else: - cmd = [command, str(src), "-colorspace", colorspace, str(dst)] + cmd = [command, str(src), "-colorspace", colorspace, str(dst)] run_command(cmd) diff --git a/imagepipeline/modules/imagemagick_scale_crop.py b/imagepipeline/modules/imagemagick_scale_crop.py index 7982621..02a4c11 100644 --- a/imagepipeline/modules/imagemagick_scale_crop.py +++ b/imagepipeline/modules/imagemagick_scale_crop.py @@ -12,9 +12,7 @@ from imagepipeline.utils.subprocess import run_command @register class ImageMagickScaleCropModule(SubprocessModule): name = "imagemagick_scale_crop" - description = ( - "Scale an image then center-crop back to its original dimensions" - ) + description = "Scale an image then center-crop back to its original dimensions" command_candidates = ("magick", "convert") @classmethod diff --git a/imagepipeline/modules/openrouter_edit.py b/imagepipeline/modules/openrouter_edit.py index 22c9046..b73cd2f 100644 --- a/imagepipeline/modules/openrouter_edit.py +++ b/imagepipeline/modules/openrouter_edit.py @@ -70,9 +70,7 @@ class OpenRouterEditModule(AIModule): @classmethod def check_dependencies(cls) -> None: if not os.environ.get("OPENROUTER_API_KEY"): - raise DependencyError( - "OPENROUTER_API_KEY environment variable is not set" - ) + raise DependencyError("OPENROUTER_API_KEY environment variable is not set") def run(self, ctx: ModuleContext) -> None: api_key_env = ctx.params["api_key_env"] @@ -155,17 +153,11 @@ class OpenRouterEditModule(AIModule): *, template_data_url: str | None = None, ) -> dict: - full_prompt = ( - f"{TEMPLATE_PROMPT_PREFIX}{prompt}" if template_data_url else prompt - ) + full_prompt = f"{TEMPLATE_PROMPT_PREFIX}{prompt}" if template_data_url else prompt content: list[dict] = [{"type": "text", "text": full_prompt}] if template_data_url is not None: - content.append( - {"type": "image_url", "image_url": {"url": template_data_url}} - ) - content.append( - {"type": "image_url", "image_url": {"url": source_data_url}} - ) + content.append({"type": "image_url", "image_url": {"url": template_data_url}}) + content.append({"type": "image_url", "image_url": {"url": source_data_url}}) payload: dict = { "model": model, "modalities": cls._modalities_for_model(model), @@ -176,9 +168,7 @@ class OpenRouterEditModule(AIModule): return payload @classmethod - def _save_result_matching_source( - cls, source: Path, result_bytes: bytes, dest: Path - ) -> None: + def _save_result_matching_source(cls, source: Path, result_bytes: bytes, dest: Path) -> None: with Image.open(source) as original: orig_format = original.format orig_size = original.size @@ -229,9 +219,7 @@ class OpenRouterEditModule(AIModule): return json.loads(response.read().decode("utf-8")) except urllib.error.HTTPError as exc: detail = exc.read().decode("utf-8", errors="replace") - raise RuntimeError( - f"OpenRouter API error ({exc.code}): {detail}" - ) from exc + raise RuntimeError(f"OpenRouter API error ({exc.code}): {detail}") from exc @staticmethod def _extract_image_bytes(response: dict) -> bytes: diff --git a/imagepipeline/modules/xcf_stack.py b/imagepipeline/modules/xcf_stack.py index ba8dff5..dc6b454 100644 --- a/imagepipeline/modules/xcf_stack.py +++ b/imagepipeline/modules/xcf_stack.py @@ -4,20 +4,22 @@ from pathlib import Path from imagepipeline.core.context import ModuleContext from imagepipeline.core.params import Param +from imagepipeline.core.step import INPUT_SOURCE from imagepipeline.modules.base import SubprocessModule from imagepipeline.modules.registry import register from imagepipeline.utils.files import find_image_by_stem from imagepipeline.utils.gimp import stack_images_to_xcf +def _layer_name(ref: str) -> str: + return "input" if ref == INPUT_SOURCE else ref + + @register class XcfStackModule(SubprocessModule): name = "xcf_stack" - description = ( - "Stack all prior pipeline step outputs as GIMP layers into one XCF per image" - ) - command_candidates = ("gimp",) - default_timeout = 600.0 + description = "Stack listed pipeline step outputs as GIMP layers into one XCF per image" + command_candidates = ("gimp-console", "gimp") @classmethod def expected_output_filenames( @@ -32,31 +34,26 @@ class XcfStackModule(SubprocessModule): @classmethod def parameters(cls) -> dict[str, Param]: return { - "include_input": Param( - "bool", - default=True, - help="Include pipeline input/ directory as bottom layer", - ), "skip_missing": Param( "bool", default=False, help="Skip missing step outputs instead of failing", ), + "timeout": Param( + "float", + default=120.0, + help="Seconds to wait for gimp-console per image (default 120)", + ), } def run(self, ctx: ModuleContext) -> None: - include_input = ctx.params["include_input"] skip_missing = ctx.params["skip_missing"] + timeout = ctx.params["timeout"] - layer_sources: list[tuple[str, Path]] = [] - if include_input: - layer_sources.append(("input", ctx.pipeline_output_root / "input")) - layer_sources.extend(ctx.prior_steps) - - if not layer_sources: + if not ctx.input_layer_dirs: raise ValueError( - "xcf_stack has no layer sources " - "(include_input=False and no prior_steps)" + "xcf_stack requires at least one input layer " + "(pass step refs in inputs=[...], bottom to top)" ) ctx.output_dir.mkdir(parents=True, exist_ok=True) @@ -67,23 +64,27 @@ class XcfStackModule(SubprocessModule): stem = input_path.stem layers: list[tuple[str, Path]] = [] - for step_id, directory in layer_sources: + for ref, directory in ctx.input_layer_dirs: image_path = find_image_by_stem(directory, stem) if image_path is None: if not skip_missing: raise ValueError( f"xcf_stack: no image with stem '{stem}' in step " - f"'{step_id}' ({directory})" + f"'{_layer_name(ref)}' ({directory})" ) continue - layers.append((step_id, image_path)) + layers.append((_layer_name(ref), image_path)) if not layers: - checked = ", ".join(step_id for step_id, _ in layer_sources) + checked = ", ".join(name for name, _ in ctx.input_layer_dirs) raise ValueError( - f"xcf_stack: no layers found for stem '{stem}' " - f"(checked: {checked})" + f"xcf_stack: no layers found for stem '{stem}' (checked: {checked})" ) dst = ctx.output_dir / f"{stem}.xcf" - stack_images_to_xcf(layers, dst, timeout=self.default_timeout) + stack_images_to_xcf(layers, dst, timeout=timeout) + + def list_output_images(self, ctx: ModuleContext) -> list[Path]: + return sorted( + p for p in ctx.output_dir.iterdir() if p.is_file() and p.suffix.lower() == ".xcf" + ) diff --git a/imagepipeline/utils/files.py b/imagepipeline/utils/files.py index a866ac3..05cf8fe 100644 --- a/imagepipeline/utils/files.py +++ b/imagepipeline/utils/files.py @@ -45,8 +45,7 @@ def match_by_stem(sources: list[list[Path]]) -> list[list[Path]]: key = stem_key(path) if key in mapping: raise ValueError( - f"Duplicate stem '{key}' in {path.parent}: " - f"{mapping[key].name} and {path.name}" + f"Duplicate stem '{key}' in {path.parent}: {mapping[key].name} and {path.name}" ) mapping[key] = path key_maps.append(mapping) diff --git a/imagepipeline/utils/gimp.py b/imagepipeline/utils/gimp.py index b8a469a..0d06ca7 100644 --- a/imagepipeline/utils/gimp.py +++ b/imagepipeline/utils/gimp.py @@ -1,13 +1,18 @@ from __future__ import annotations +import os +import subprocess +import tempfile from pathlib import Path -from imagepipeline.utils.subprocess import require_command, run_command +from imagepipeline.utils.subprocess import require_command + +_GIMP_CANDIDATES = ("gimp-console", "gimp") def require_gimp() -> str: - """Return the GIMP executable name, raising DependencyError if missing.""" - return require_command("gimp") + """Return a GIMP executable for headless batch use.""" + return require_command(*_GIMP_CANDIDATES) def _scheme_string(value: str) -> str: @@ -23,34 +28,35 @@ def _scheme_string(value: str) -> str: def _build_stack_script(layers: list[tuple[str, Path]], outfile: Path) -> str: + """Build a GIMP 3 Script-Fu snippet that stacks images into one XCF.""" first_name, first_path = layers[0] first_path_str = _scheme_string(str(first_path)) outfile_str = _scheme_string(str(outfile)) lines = [ + "(script-fu-use-v3)", "(let* (", - f" (loaded (gimp-file-load RUN-NONINTERACTIVE {first_path_str} {first_path_str}))", - " (image (car loaded))", - " (bottom-layer (cadr loaded))", + f" (image (gimp-file-load RUN-NONINTERACTIVE {first_path_str}))", + " (bottom (vector-ref (gimp-image-get-selected-drawables image) 0))", ")", - f" (gimp-layer-set-name bottom-layer {_scheme_string(first_name)})", + f" (gimp-item-set-name bottom {_scheme_string(first_name)})", ] for layer_name, layer_path in layers[1:]: path_str = _scheme_string(str(layer_path)) lines.extend( [ - " (let ((layer (car (gimp-file-load-layer RUN-NONINTERACTIVE image " - f"{path_str}))))", - " (gimp-image-insert-layer image layer 0 0)", - f" (gimp-layer-set-name layer {_scheme_string(layer_name)})", + " (let ((layer", + f" (gimp-file-load-layer RUN-NONINTERACTIVE image {path_str})))", + " (gimp-image-insert-layer image layer image 0)", + f" (gimp-item-set-name layer {_scheme_string(layer_name)})", " )", ] ) lines.extend( [ - f" (gimp-xcf-save RUN-NONINTERACTIVE image bottom-layer {outfile_str} {outfile_str})", + f" (gimp-file-save RUN-NONINTERACTIVE image {outfile_str})", " (gimp-image-delete image)", ")", ] @@ -58,6 +64,43 @@ def _build_stack_script(layers: list[tuple[str, Path]], outfile: Path) -> str: return "\n".join(lines) +def _run_gimp_batch( + cmd: list[str], + *, + timeout: float | None, + env: dict[str, str], + outfile: Path | None = None, +) -> subprocess.CompletedProcess[str]: + """Run GIMP batch; tolerate exit hang when ``outfile`` was already written.""" + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired as exc: + process.kill() + stdout, stderr = process.communicate() + if outfile is not None and outfile.is_file() and outfile.stat().st_size > 0: + return subprocess.CompletedProcess( + cmd, + returncode=0, + stdout=stdout or "", + stderr=stderr or "", + ) + raise RuntimeError(f"Command timed out after {timeout}s: {' '.join(cmd)}") from exc + + return subprocess.CompletedProcess( + cmd, + returncode=process.returncode if process.returncode is not None else -1, + stdout=stdout or "", + stderr=stderr or "", + ) + + def stack_images_to_xcf( layers: list[tuple[str, Path]], outfile: Path, @@ -67,7 +110,7 @@ def stack_images_to_xcf( """Stack images bottom-to-top into a single GIMP XCF file. ``layers`` is a list of ``(layer_name, image_path)`` tuples in bottom-to-top - order. Invokes GIMP headless via Script-Fu. + order. Invokes GIMP headless via Script-Fu (GIMP 3 compatible). """ if not layers: raise ValueError("stack_images_to_xcf requires at least one layer") @@ -77,35 +120,42 @@ def stack_images_to_xcf( for layer_name, image_path in layers: resolved = image_path.resolve() if not resolved.is_file(): - raise FileNotFoundError( - f"Layer image not found for '{layer_name}': {resolved}" - ) + raise FileNotFoundError(f"Layer image not found for '{layer_name}': {resolved}") resolved_layers.append((layer_name, resolved)) outfile = outfile.resolve() outfile.parent.mkdir(parents=True, exist_ok=True) script = _build_stack_script(resolved_layers, outfile) - cmd = [ - gimp, - "-idf", - "--batch-interpreter", - "plug-in-script-fu-eval", - "-b", - script, - "-b", - "(gimp-quit 0)", - ] + env = {**os.environ, "GIMP_NO_DISPLAY": "1"} + + with tempfile.NamedTemporaryFile( + mode="w", + suffix=".scm", + delete=False, + encoding="utf-8", + ) as handle: + handle.write(script) + script_path = Path(handle.name) try: - run_command(cmd, timeout=timeout) - except RuntimeError as exc: - layer_summary = ", ".join(name for name, _ in resolved_layers) - raise RuntimeError( - f"GIMP failed to stack layers [{layer_summary}] into {outfile}: {exc}" - ) from exc - - if not outfile.is_file(): - raise RuntimeError( - f"GIMP completed but XCF output was not created: {outfile}" - ) + cmd = [ + gimp, + "-d", + "-i", + "--quit", + "--batch-interpreter=plug-in-script-fu-eval", + "--batch", + f"(load {_scheme_string(str(script_path))})", + ] + result = _run_gimp_batch(cmd, timeout=timeout, env=env, outfile=outfile) + if not outfile.is_file(): + stderr = (result.stderr or "").strip() + stdout = (result.stdout or "").strip() + detail = stderr or stdout or f"exit code {result.returncode}" + layer_summary = ", ".join(name for name, _ in resolved_layers) + raise RuntimeError( + f"GIMP failed to stack layers [{layer_summary}] into {outfile}: {detail}" + ) + finally: + script_path.unlink(missing_ok=True) diff --git a/imagepipeline/utils/gmic.py b/imagepipeline/utils/gmic.py index 9e7946f..09b17cd 100644 --- a/imagepipeline/utils/gmic.py +++ b/imagepipeline/utils/gmic.py @@ -38,6 +38,4 @@ def finalize_gmic_output(output_dir: Path, intended: Path) -> Path: frame_000000.rename(intended) return intended - raise FileNotFoundError( - f"G'MIC produced no output for {intended.name} in {output_dir}" - ) + raise FileNotFoundError(f"G'MIC produced no output for {intended.name} in {output_dir}") diff --git a/imagepipeline/utils/subprocess.py b/imagepipeline/utils/subprocess.py index 471c806..6d581b8 100644 --- a/imagepipeline/utils/subprocess.py +++ b/imagepipeline/utils/subprocess.py @@ -16,6 +16,7 @@ def run_command( *, timeout: float | None = None, cwd: Path | None = None, + env: dict[str, str] | None = None, ) -> subprocess.CompletedProcess[str]: try: return subprocess.run( @@ -25,18 +26,15 @@ def run_command( text=True, timeout=timeout, cwd=str(cwd) if cwd else None, + env=env, ) except subprocess.CalledProcessError as exc: stderr = (exc.stderr or "").strip() stdout = (exc.stdout or "").strip() detail = stderr or stdout or str(exc) - raise RuntimeError( - f"Command failed ({exc.returncode}): {' '.join(cmd)}\n{detail}" - ) from exc + raise RuntimeError(f"Command failed ({exc.returncode}): {' '.join(cmd)}\n{detail}") from exc except subprocess.TimeoutExpired as exc: - raise RuntimeError( - f"Command timed out after {timeout}s: {' '.join(cmd)}" - ) from exc + raise RuntimeError(f"Command timed out after {timeout}s: {' '.join(cmd)}") from exc def require_command(*names: str) -> str: diff --git a/pipelines/example_grayscale.py b/pipelines/example_grayscale.py index edc0499..4f77c2a 100644 --- a/pipelines/example_grayscale.py +++ b/pipelines/example_grayscale.py @@ -18,8 +18,9 @@ def main() -> None: input_dir=INPUT, output_base=OUTPUT_BASE, ) as p: - gray = p.step("imagemagick_grayscale", inputs="input") + p.step("imagemagick_grayscale", inputs="input") # Chain another step on the result: + # gray = p.step("imagemagick_grayscale", inputs="input") # p.step("imagemagick_grayscale", inputs=gray, colorspace="Gray") output_root = p.run() diff --git a/pipelines/pipeline_aichelberg_indians.py b/pipelines/pipeline_aichelberg_indians.py new file mode 100644 index 0000000..c879f42 --- /dev/null +++ b/pipelines/pipeline_aichelberg_indians.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +"""Aichelberg Indians pipeline — tone mapping, gmic variants, xcf export.""" + +from pathlib import Path + +from imagepipeline import Pipeline + +INPUT = Path( + "/home/frank/pics/20260712_Aichelberg Indians - Heidelberg Hedgehogs/darktable_exported" +) +OUTPUT_BASE = Path.home() / "pipeline_output" + +EXISTING_OUTPUTS: dict[str, Path] = {} +CONTINUE_FROM = Path("/home/frank/pipeline_output/aichelberg-indians_260713081414") + +COLOR1 = "#244a89" +COLOR2 = "#c22518" + +# COLOR1 = #244a89 -> 36,74,137; COLOR2 = #c22518 -> 194,37,24 +ALPHA1 = 160 +ALPHA2 = 110 + +GMIC_TONE_MAPPING = "-fx_map_tones 0.5,0.7,0.1,30,0" +GMIC_FELTPEN = "-fx_feltpen 300,50,1,0.1,20,5" +GMIC_EDGES = "-fx_edges 0,15,0" +GMIC_CHARRED_PLASTIC = "-fx_charred_plastic 1,10,40,1,10,0,0,2,6,5,20,0,11" +GMIC_NEON = ( + "-fx_neon 0,1,1,1,1,0,0.45,40,60,0,1,1.15,2,3,0,3,20,0.4,0.1,1.5,5,0.2,0.1," + "1,2,1,0,0,1,1,0,50,50" +) +GMIC_CUTOUT = "-fx_cutout 4,0.5,4,1" +GMIC_HUFFMAN_GLITCHES = "-fx_huffman_glitches 30,0,25,0,0,0,0,0,50,50" +GMIC_QUERYPRIMARY = "-afre_queryprimary 1,1" +GMIC_LOCAL_SIMILARITY = "-local_similarity_mask 50,50,50,128,4,10" +GMIC_LUMA_INVERT = "-Lylejk_Luma_Invert 1,0" +GMIC_CRAYONGRAFFITI = "-fx_crayongraffiti2 300,50,1,0.4,12,1,2,2,0" +GMIC_ANAGLYPH = "-fx_stereo_to_anaglyph 2,0" +GMIC_DROP_SHADOW = "-fx_drop_shadow3d 0,0,0,10,1,1,2,0.5,194,37,24,200,0" +GMIC_BOKEH = ( + f"-fx_bokeh 3,5,0,30,8,4,0.3,0.2,36,74,137,{ALPHA1},0.7,30,20,20,1,2,194,37,24,{ALPHA2},0.15" +) + + +def main() -> None: + with Pipeline( + name="aichelberg-indians", + input_dir=INPUT, + output_base=OUTPUT_BASE, + existing_outputs=EXISTING_OUTPUTS or None, + continue_from=CONTINUE_FROM, + ) as p: + rembg_out = p.step("rembg", inputs="input", step_id="rembg_out") + + # recipe: gmic-tone-mapping + input_tone_map = p.step( + "gmic", inputs="input", command=GMIC_TONE_MAPPING, step_id="input_tone_map" + ) + + # recipe: gmic-tone-mapping-rembg + rembg_tone_map = p.step( + "gmic", inputs=rembg_out, command=GMIC_TONE_MAPPING, step_id="rembg_tone_map" + ) + + # recipe: gmic-feltpen-rembg + rembg_feltpen = p.step( + "gmic", inputs=rembg_out, command=GMIC_FELTPEN, step_id="rembg_feltpen" + ) + composite_feltpen = p.step( + "composite", inputs=[rembg_feltpen, rembg_out], step_id="composite_feltpen" + ) + + # recipe: gmic-edges-rembg + rembg_edges = p.step("gmic", inputs=rembg_out, command=GMIC_EDGES, step_id="rembg_edges") + composite_edges = p.step( + "composite", inputs=[rembg_edges, rembg_out], step_id="composite_edges" + ) + + # recipe: gmic-charred-plastic-rembg + rembg_charred_plastic = p.step( + "gmic", + inputs=rembg_out, + command=GMIC_CHARRED_PLASTIC, + step_id="rembg_charred_plastic", + ) + composite_charred_plastic = p.step( + "composite", + inputs=[rembg_charred_plastic, rembg_out], + step_id="composite_charred_plastic", + ) + + # recipe: gmic-neon-rembg + rembg_neon = p.step("gmic", inputs=rembg_out, command=GMIC_NEON, step_id="rembg_neon") + composite_neon = p.step( + "composite", inputs=[rembg_neon, rembg_out], step_id="composite_neon" + ) + + # recipe: gmic-cutout-rembg (tone-mapped input as original) + input_cutout = p.step( + "gmic", inputs=input_tone_map, command=GMIC_CUTOUT, step_id="input_cutout" + ) + cutout_mid = p.step( + "composite", inputs=[input_tone_map, input_cutout], step_id="cutout_mid" + ) + composite_cutout = p.step( + "composite", inputs=[cutout_mid, rembg_out], step_id="composite_cutout" + ) + + # recipe: gmic-huffman-glitches-rembg + input_huffman_glitches = p.step( + "gmic", + inputs=input_tone_map, + command=GMIC_HUFFMAN_GLITCHES, + step_id="input_huffman_glitches", + ) + huffman_glitches_mid = p.step( + "composite", + inputs=[input_tone_map, input_huffman_glitches], + step_id="huffman_glitches_mid", + ) + composite_huffman_glitches = p.step( + "composite", + inputs=[huffman_glitches_mid, rembg_out], + step_id="composite_huffman_glitches", + ) + + # recipe: gmic-queryprimary-rembg + input_queryprimary = p.step( + "gmic", + inputs=input_tone_map, + command=GMIC_QUERYPRIMARY, + step_id="input_queryprimary", + ) + queryprimary_mid = p.step( + "composite", + inputs=[input_tone_map, input_queryprimary], + step_id="queryprimary_mid", + ) + composite_queryprimary = p.step( + "composite", + inputs=[queryprimary_mid, rembg_out], + step_id="composite_queryprimary", + ) + + # recipe: gmic-local-similarity-rembg + input_local_similarity = p.step( + "gmic", + inputs=input_tone_map, + command=GMIC_LOCAL_SIMILARITY, + step_id="input_local_similarity", + ) + local_similarity_mid = p.step( + "composite", + inputs=[input_tone_map, input_local_similarity], + step_id="local_similarity_mid", + ) + composite_local_similarity = p.step( + "composite", + inputs=[local_similarity_mid, rembg_out], + step_id="composite_local_similarity", + ) + + # recipe: gmic-luma-invert-rembg + input_luma_invert = p.step( + "gmic", + inputs=input_tone_map, + command=GMIC_LUMA_INVERT, + step_id="input_luma_invert", + ) + luma_invert_mid = p.step( + "composite", + inputs=[input_tone_map, input_luma_invert], + step_id="luma_invert_mid", + ) + composite_luma_invert = p.step( + "composite", + inputs=[luma_invert_mid, rembg_out], + step_id="composite_luma_invert", + ) + + # recipe: gmic-crayongraffiti-rembg + input_crayongraffiti = p.step( + "gmic", + inputs=input_tone_map, + command=GMIC_CRAYONGRAFFITI, + step_id="input_crayongraffiti", + ) + crayongraffiti_mid = p.step( + "composite", + inputs=[input_tone_map, input_crayongraffiti], + step_id="crayongraffiti_mid", + ) + composite_crayongraffiti = p.step( + "composite", + inputs=[crayongraffiti_mid, rembg_out], + step_id="composite_crayongraffiti", + ) + + # recipe: gmic-anaglyph-rembg + input_anaglyph = p.step( + "gmic", inputs=input_tone_map, command=GMIC_ANAGLYPH, step_id="input_anaglyph" + ) + anaglyph_mid = p.step( + "composite", inputs=[input_tone_map, input_anaglyph], step_id="anaglyph_mid" + ) + composite_anaglyph = p.step( + "composite", inputs=[anaglyph_mid, rembg_out], step_id="composite_anaglyph" + ) + + # recipe: xcf-stack — layers up to composites (excludes bokeh + color-bg-drop-shadow) + p.step( + "xcf_stack", + inputs=[ + "input", + rembg_out, + input_tone_map, + rembg_tone_map, + rembg_feltpen, + rembg_edges, + rembg_charred_plastic, + rembg_neon, + input_cutout, + input_huffman_glitches, + input_queryprimary, + input_local_similarity, + input_luma_invert, + input_crayongraffiti, + input_anaglyph, + composite_feltpen, + composite_edges, + composite_charred_plastic, + composite_neon, + composite_cutout, + composite_huffman_glitches, + composite_queryprimary, + composite_local_similarity, + composite_luma_invert, + composite_crayongraffiti, + composite_anaglyph, + ], + ) + + # recipe: bokeh-oktagon (tone-mapped input as original) + input_bokeh = p.step( + "gmic", inputs=input_tone_map, command=GMIC_BOKEH, step_id="input_bokeh" + ) + bokeh_mid = p.step("composite", inputs=[input_tone_map, input_bokeh], step_id="bokeh_mid") + p.step("composite", inputs=[bokeh_mid, rembg_out], step_id="composite_bokeh") + + # recipe: color-bg-drop-shadow-rembg + color_bg = p.step("imagemagick_fill", inputs="input", color1=COLOR1, step_id="color_bg") + rembg_shadow = p.step( + "gmic", inputs=rembg_out, command=GMIC_DROP_SHADOW, step_id="rembg_shadow" + ) + shadow_color_mid = p.step( + "composite", inputs=[color_bg, rembg_shadow], step_id="shadow_color_mid" + ) + p.step( + "composite", + inputs=[shadow_color_mid, rembg_out], + step_id="composite_color_bg_shadow", + ) + + output_root = p.run() + + print(f"Pipeline finished. Output: {output_root}") + + +if __name__ == "__main__": + main() diff --git a/pipelines/pipeline_baxxter_2.py b/pipelines/pipeline_baxxter_2.py index 9b479d0..f5fad13 100644 --- a/pipelines/pipeline_baxxter_2.py +++ b/pipelines/pipeline_baxxter_2.py @@ -40,14 +40,10 @@ def main() -> None: gmic_shadow = p.step("gmic", inputs=rembg, command=GMIC_DROP_SHADOW) gmic_smooth = p.step("gmic", inputs=rembg, command=GMIC_JPR_SMOOTH) - gmic_stereo_alpha = p.step( - "color_to_alpha", inputs=gmic_stereo, color="#000000" - ) + gmic_stereo_alpha = p.step("color_to_alpha", inputs=gmic_stereo, color="#000000") yellow_bg = p.step("imagemagick_fill", inputs="input", color1=YELLOW) - gmic_smooth_alpha = p.step( - "color_to_alpha", inputs=gmic_smooth, color="#7f7f7f" - ) + gmic_smooth_alpha = p.step("color_to_alpha", inputs=gmic_smooth, color="#7f7f7f") gmic_smooth_sized = p.step( "imagemagick_scale_crop", inputs=gmic_smooth_alpha, diff --git a/pipelines/pipeline_colorsplash_watermark_f12.py b/pipelines/pipeline_colorsplash_watermark_f12.py index d710750..e729d2e 100644 --- a/pipelines/pipeline_colorsplash_watermark_f12.py +++ b/pipelines/pipeline_colorsplash_watermark_f12.py @@ -6,7 +6,9 @@ from pathlib import Path from imagepipeline import Pipeline # Darktable export folder. -INPUT = Path("/home/frank/pics/20260517_Albershausen Crusaders - Biberach Beavers/darktable_exported/png") +INPUT = Path( + "/home/frank/pics/20260517_Albershausen Crusaders - Biberach Beavers/darktable_exported/png" +) # Where timestamped run folders are created. OUTPUT_BASE = Path.home() / "pipeline_output" diff --git a/pipelines/pipeline_crusaders.py b/pipelines/pipeline_crusaders.py index cd99773..2b37634 100644 --- a/pipelines/pipeline_crusaders.py +++ b/pipelines/pipeline_crusaders.py @@ -5,7 +5,9 @@ from pathlib import Path from imagepipeline import Pipeline -INPUT = Path("/home/frank/pics/20260620_Albershausen Crusaders - Montabaur Fighting Farmers/darktable_exported") +INPUT = Path( + "/home/frank/pics/20260620_Albershausen Crusaders - Montabaur Fighting Farmers/darktable_exported" +) OUTPUT_BASE = Path.home() / "pipeline_output" # Reuse outputs from a previous run or external folder (key = step id, e.g. rembg_01). @@ -80,13 +82,9 @@ def main() -> None: scale=1.05, ) - rembg_stereo_alpha = p.step( - "color_to_alpha", inputs=rembg_stereo, color="#000000" - ) + rembg_stereo_alpha = p.step("color_to_alpha", inputs=rembg_stereo, color="#000000") color_bg = p.step("imagemagick_fill", inputs="input", color1=COLOR1) - rembg_smooth_alpha = p.step( - "color_to_alpha", inputs=rembg_jpr_smooth, color="#7f7f7f" - ) + rembg_smooth_alpha = p.step("color_to_alpha", inputs=rembg_jpr_smooth, color="#7f7f7f") rembg_smooth_sized = p.step( "imagemagick_scale_crop", inputs=rembg_smooth_alpha, diff --git a/pipelines/pipeline_orange.py b/pipelines/pipeline_orange.py index 44687c7..26cebf4 100644 --- a/pipelines/pipeline_orange.py +++ b/pipelines/pipeline_orange.py @@ -35,6 +35,7 @@ GMIC_GRADIENT_B = ( "255,255,128,128,128,255,255,0,255,255,0,0,0,0" ) + def main() -> None: with Pipeline( name="orange", diff --git a/pipelines/pipeline_rezepttest.py b/pipelines/pipeline_rezepttest.py new file mode 100644 index 0000000..6485708 --- /dev/null +++ b/pipelines/pipeline_rezepttest.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Rezepttest pipeline — RECIPES.md smoke test (Indians Aichelberg logo colors).""" + +from pathlib import Path + +from imagepipeline import Pipeline + +# Change to your Darktable export folder. +INPUT = Path("/home/frank/tmp/bar") +OUTPUT_BASE = Path.home() / "pipeline_output" + +EXISTING_OUTPUTS: dict[str, Path] = {} +CONTINUE_FROM: Path | None = None + +# Colors from Indians Aichelberg logo (orange text + red stitching). +COLOR1 = "#f0b020" # 240, 176, 32 +COLOR2 = "#f01000" # 240, 16, 0 + +ALPHA1 = 160 +ALPHA2 = 110 + +GMIC_DROP_SHADOW = "-fx_drop_shadow3d 0,0,0,10,1,1,2,0.5,240,16,0,200,0" +GMIC_JPR_SMOOTH = "-jpr_gradient_smooth 0,1.5" +GMIC_BOKEH = ( + f"-fx_bokeh 3,5,0,30,8,4,0.3,0.2,240,176,32,{ALPHA1},0.7,30,20,20,1,2,240,16,0,{ALPHA2},0.15" +) + + +def main() -> None: + with Pipeline( + name="rezepttest", + input_dir=INPUT, + output_base=OUTPUT_BASE, + existing_outputs=EXISTING_OUTPUTS or None, + continue_from=CONTINUE_FROM, + ) as p: + rembg_out = p.step("rembg", inputs="input", step_id="rembg_out") + grayscale = p.step("gmic_grayscale", inputs="input", step_id="grayscale") + gradient_radial_bg = p.step( + "imagemagick_fill", + inputs="input", + color1=COLOR1, + color2=COLOR2, + gradient=True, + radial=True, + step_id="gradient_radial_bg", + ) + + rembg_shadow = p.step( + "gmic", inputs=rembg_out, command=GMIC_DROP_SHADOW, step_id="rembg_shadow" + ) + rembg_jpr_smooth = p.step( + "gmic", inputs=rembg_out, command=GMIC_JPR_SMOOTH, step_id="rembg_jpr_smooth" + ) + rembg_jpr_smooth_sized = p.step( + "imagemagick_scale_crop", + inputs=rembg_jpr_smooth, + scale=1.05, + step_id="rembg_jpr_smooth_sized", + ) + input_bokeh = p.step("gmic", inputs="input", command=GMIC_BOKEH, step_id="input_bokeh") + + # recipe: colorsplash + composite_colorsplash = p.step( + "composite", inputs=[grayscale, rembg_out], step_id="composite_colorsplash" + ) + + # recipe: rembg-radial-2colors (gradient-radial) + composite_radial = p.step( + "composite", inputs=[gradient_radial_bg, rembg_out], step_id="composite_radial" + ) + + # recipe: original-drop-shadow-rembg + shadow_mid = p.step("composite", inputs=["input", rembg_shadow], step_id="shadow_mid") + composite_shadow = p.step( + "composite", inputs=[shadow_mid, rembg_out], step_id="composite_shadow" + ) + + # recipe: original-jpr-smooth-rembg + smooth_mid = p.step( + "composite", inputs=["input", rembg_jpr_smooth_sized], step_id="smooth_mid" + ) + composite_smooth = p.step( + "composite", inputs=[smooth_mid, rembg_out], step_id="composite_smooth" + ) + + # recipe: bokeh-oktagon + bokeh_mid = p.step("composite", inputs=["input", input_bokeh], step_id="bokeh_mid") + composite_bokeh = p.step( + "composite", inputs=[bokeh_mid, rembg_out], step_id="composite_bokeh" + ) + + # recipe: xcf-stack — explicit layer list, bottom to top + p.step( + "xcf_stack", + inputs=[ + "input", + rembg_out, + grayscale, + gradient_radial_bg, + rembg_shadow, + rembg_jpr_smooth_sized, + input_bokeh, + composite_colorsplash, + composite_radial, + composite_shadow, + composite_smooth, + composite_bokeh, + ], + ) + + output_root = p.run() + + print(f"Pipeline finished. Output: {output_root}") + + +if __name__ == "__main__": + main() diff --git a/pipelines/pipeline_watermark_f12_2000px.py b/pipelines/pipeline_watermark_f12_2000px.py new file mode 100644 index 0000000..f92b36d --- /dev/null +++ b/pipelines/pipeline_watermark_f12_2000px.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +"""Resize to 2000px max edge, then colorsplash + F12 watermark darktable style.""" + +from pathlib import Path + +from imagepipeline import Pipeline + +INPUT = Path( + "/home/frank/pics/20260712_Aichelberg Indians - Heidelberg Hedgehogs/darktable_exported/edits" +) +OUTPUT_BASE = Path.home() / "pipeline_output" + +MAX_EDGE = 2000 +STYLE = "Watermark F12.rocks" + + +def main() -> None: + with Pipeline( + name="watermark_f12_2000px", + input_dir=INPUT, + output_base=OUTPUT_BASE, + ) as p: + resized = p.step("imagemagick_resize", inputs="input", max_edge=MAX_EDGE) + p.step("darktable_style", inputs=resized, style=STYLE) + output_root = p.run() + + print(f"Pipeline finished. Output: {output_root}") + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index bbb0508..f77c4bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,11 +10,17 @@ readme = "README.md" requires-python = ">=3.11" license = { text = "MIT" } authors = [{ name = "Frank" }] -dependencies = [] +dependencies = [ + "Pillow>=10.0", +] [project.optional-dependencies] -ai = ["numpy>=1.26", "Pillow>=10.0", "torch>=2.0"] -dev = ["pytest>=8.0"] +ai = ["numpy>=1.26", "torch>=2.0"] +dev = [ + "pytest>=8.0", + "numpy>=1.26", + "ruff>=0.8", +] [project.scripts] imagepipeline = "imagepipeline.cli:main" @@ -25,3 +31,28 @@ include = ["imagepipeline*"] [tool.pytest.ini_options] testpaths = ["tests"] +markers = [ + "integration: tests that call external CLI tools (ImageMagick, G'MIC, rembg, darktable, GIMP)", + "slow: tests that take more than a few seconds", +] + +[tool.ruff] +target-version = "py311" +line-length = 100 +src = ["imagepipeline", "tests", "pipelines"] + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "F", # pyflakes + "I", # isort + "UP", # pyupgrade + "B", # flake8-bugbear +] +ignore = [ + "E501", # line length handled by formatter + "B027", # optional empty hooks on BaseModule +] + +[tool.ruff.lint.isort] +known-first-party = ["imagepipeline"] diff --git a/tests/conftest.py b/tests/conftest.py index 8c8fa42..a59461d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,10 +20,7 @@ def make_png( crc = zlib.crc32(tag + data) & 0xFFFFFFFF return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", crc) - raw = b"".join( - b"\x00" + bytes([r, g, b] * width) - for _ in range(height) - ) + raw = b"".join(b"\x00" + bytes([r, g, b] * width) for _ in range(height)) compressed = zlib.compress(raw, 9) ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) png = ( @@ -37,7 +34,7 @@ def make_png( @pytest.fixture(autouse=True) def _ensure_builtin_modules() -> None: - import imagepipeline.modules # noqa: F401 + pass # noqa: F401 @pytest.fixture diff --git a/tests/integration_helpers.py b/tests/integration_helpers.py new file mode 100644 index 0000000..af9fdb1 --- /dev/null +++ b/tests/integration_helpers.py @@ -0,0 +1,53 @@ +"""Shared helpers for integration tests.""" + +from __future__ import annotations + +import shutil +import subprocess +import tempfile +from pathlib import Path + +from tests.conftest import make_png + + +def has_darktable_style(style_name: str, config_dir: Path | None = None) -> bool: + """Return True if darktable-cli can apply ``style_name`` to a tiny PNG.""" + if not shutil.which("darktable-cli"): + return False + + config = config_dir or Path.home() / ".config" / "darktable" + styles_dir = config / "styles" + if not styles_dir.is_dir(): + return False + if not any(p.stem == style_name for p in styles_dir.glob("*.dtstyle")): + return False + + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + src = tmp_path / "probe.png" + out_dir = tmp_path / "out" + out_dir.mkdir() + make_png(src) + cmd = [ + "darktable-cli", + str(src), + str(out_dir), + "--style", + style_name, + "--out-ext", + "png", + "--core", + "--configdir", + str(config), + "--style-overwrite", + ] + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + return False + return any(out_dir.iterdir()) + + +def has_workflow_tools() -> bool: + """True when rembg, gmic, ImageMagick, and darktable-cli are on PATH.""" + has_magick = bool(shutil.which("magick") or shutil.which("convert")) + return all(shutil.which(name) for name in ("rembg", "gmic", "darktable-cli")) and has_magick diff --git a/tests/test_ai_modules.py b/tests/test_ai_modules.py index 4c3d522..2738fb0 100644 --- a/tests/test_ai_modules.py +++ b/tests/test_ai_modules.py @@ -110,9 +110,7 @@ class TestAIParameters: assert payload["modalities"] == ["image"] assert payload["image_config"] == {"strength": 0.25} - def test_save_result_matching_source_preserves_png_size( - self, tmp_path: Path - ) -> None: + def test_save_result_matching_source_preserves_png_size(self, tmp_path: Path) -> None: try: from PIL import Image except ImportError: diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 5a60bba..4c52dd9 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -113,9 +113,11 @@ class TestPipelineRunner: for src in ctx.input_paths: shutil.copy2(src, ctx.output_dir / src.name) - with Pipeline(name="order_test", input_dir=input_dir, output_base=output_base, verbose=False) as p: + with Pipeline( + name="order_test", input_dir=input_dir, output_base=output_base, verbose=False + ) as p: step_b = p.step("order_tracker", inputs="input") - step_a = p.step("order_tracker", inputs=step_b) + p.step("order_tracker", inputs=step_b) p.run() assert order == ["order_tracker_01", "order_tracker_02"] @@ -132,7 +134,9 @@ class TestPipelineRunner: for src in ctx.input_paths: shutil.copy2(src, ctx.output_dir / src.name) - with Pipeline(name="dup_test", input_dir=input_dir, output_base=output_base, verbose=False) as p: + with Pipeline( + name="dup_test", input_dir=input_dir, output_base=output_base, verbose=False + ) as p: first = p.step("number_tracker", inputs="input") p.step("number_tracker", inputs=first) root = p.run() @@ -159,6 +163,74 @@ class TestManifest: assert data["finished_at"] is not None +class TestCustomStepId: + def test_custom_step_id_output_folder(self, input_dir: Path, output_base: Path) -> None: + @register + class NamedModule(BaseModule): + name = "named_tracker" + + def run(self, ctx) -> None: + ctx.output_dir.mkdir(parents=True, exist_ok=True) + for src in ctx.input_paths: + shutil.copy2(src, ctx.output_dir / src.name) + + with Pipeline( + name="named_test", input_dir=input_dir, output_base=output_base, verbose=False + ) as p: + ref = p.step("named_tracker", inputs="input", step_id="input_bokeh") + root = p.run() + + assert ref.step_id == "input_bokeh" + assert ref.output_dir_name == "input_bokeh" + assert (root / "input_bokeh").is_dir() + assert not (root / "named_tracker_01").exists() + + unregister("named_tracker") + + def test_module_counter_independent_of_custom_step_id( + self, input_dir: Path, output_base: Path + ) -> None: + @register + class CounterModule(BaseModule): + name = "counter_tracker" + + def run(self, ctx) -> None: + ctx.output_dir.mkdir(parents=True, exist_ok=True) + for src in ctx.input_paths: + shutil.copy2(src, ctx.output_dir / src.name) + + with Pipeline( + name="counter_test", input_dir=input_dir, output_base=output_base, verbose=False + ) as p: + p.step("counter_tracker", inputs="input") + p.step("counter_tracker", inputs="input", step_id="custom_mid") + p.step("counter_tracker", inputs="input") + root = p.run() + + assert (root / "counter_tracker_01").is_dir() + assert (root / "custom_mid").is_dir() + assert (root / "counter_tracker_03").is_dir() + assert not (root / "counter_tracker_02").exists() + + unregister("counter_tracker") + + def test_duplicate_step_id_rejected(self, input_dir: Path) -> None: + with Pipeline(name="dup_id", input_dir=input_dir, verbose=False) as p: + p.step("imagemagick_grayscale", inputs="input", step_id="my_step") + with pytest.raises(ValidationError, match="Duplicate step_id"): + p.step("imagemagick_grayscale", inputs="input", step_id="my_step") + + def test_empty_step_id_rejected(self, input_dir: Path) -> None: + with Pipeline(name="empty_id", input_dir=input_dir, verbose=False) as p: + with pytest.raises(ValidationError, match="must not be empty"): + p.step("imagemagick_grayscale", inputs="input", step_id="") + + def test_step_id_with_slash_rejected(self, input_dir: Path) -> None: + with Pipeline(name="slash_id", input_dir=input_dir, verbose=False) as p: + with pytest.raises(ValidationError, match="path separators"): + p.step("imagemagick_grayscale", inputs="input", step_id="bad/name") + + class TestPipelineValidation: def test_empty_pipeline_rejected(self, input_dir: Path) -> None: with Pipeline(name="empty", input_dir=input_dir, verbose=False) as p: diff --git a/tests/test_resume.py b/tests/test_resume.py index 74ea0c4..951ab60 100644 --- a/tests/test_resume.py +++ b/tests/test_resume.py @@ -13,7 +13,6 @@ from imagepipeline.core.resume import ( ) from imagepipeline.core.step import StepDefinition from imagepipeline.modules.imagemagick_grayscale import ImageMagickGrayscale -from imagepipeline.modules.registry import get_module from imagepipeline.modules.rembg import RembgModule from imagepipeline.utils.gmic import finalize_gmic_output, split_gmic_command from tests.conftest import make_png @@ -124,7 +123,9 @@ class TestExpectedOutputFilenames: class TestPipelineResume: @pytest.mark.skipif(not shutil.which("magick"), reason="ImageMagick not installed") - def test_continue_skips_completed_steps(self, input_dir: Path, output_base: Path, capsys) -> None: + def test_continue_skips_completed_steps( + self, input_dir: Path, output_base: Path, capsys + ) -> None: with Pipeline( name="resume_test", input_dir=input_dir, @@ -178,6 +179,32 @@ class TestPipelineResume: assert "Reused external output for imagemagick_grayscale_01" in output assert (root / "imagemagick_grayscale_01" / "photo_a.png").exists() + @pytest.mark.skipif(not shutil.which("magick"), reason="ImageMagick not installed") + def test_existing_outputs_with_custom_step_id( + self, input_dir: Path, output_base: Path, tmp_path: Path, capsys + ) -> None: + external = tmp_path / "external_gray" + external.mkdir() + for src in input_dir.iterdir(): + if src.is_file(): + make_png(external / src.name, width=4, height=4, rgb=(10, 20, 30)) + + with Pipeline( + name="custom_id_external", + input_dir=input_dir, + output_base=output_base, + verbose=True, + existing_outputs={"input_bokeh": external}, + ) as p: + reused = p.step("imagemagick_grayscale", inputs="input", step_id="input_bokeh") + p.step("imagemagick_grayscale", inputs=reused) + root = p.run() + + output = capsys.readouterr().out + assert "Reused external output for input_bokeh" in output + assert (root / "input_bokeh" / "photo_a.png").exists() + assert not (root / "imagemagick_grayscale_01").exists() + class TestMaterializeExternal: def test_links_files_by_stem(self, tmp_path: Path) -> None: diff --git a/tests/test_workflow_modules.py b/tests/test_workflow_modules.py index 7126a10..4c10b6b 100644 --- a/tests/test_workflow_modules.py +++ b/tests/test_workflow_modules.py @@ -5,7 +5,6 @@ from pathlib import Path import pytest -from imagepipeline.core.params import validate_params from imagepipeline.modules.color_to_alpha import ( ColorToAlphaModule, build_color_to_alpha_args, @@ -13,18 +12,19 @@ from imagepipeline.modules.color_to_alpha import ( from imagepipeline.modules.composite import CompositeModule from imagepipeline.modules.crop_square import CropSquareModule from imagepipeline.modules.darktable_style import DarktableStyleModule +from imagepipeline.modules.gmic_grayscale import GmicGrayscale from imagepipeline.modules.imagemagick_fill import ( ImageMagickFillModule, build_fill_arguments, ) -from imagepipeline.modules.imagemagick_grayscale import ImageMagickGrayscale from imagepipeline.modules.imagemagick_resize import ( ImageMagickResizeModule, build_resize_arguments, ) -from imagepipeline.modules.gmic_grayscale import GmicGrayscale from imagepipeline.modules.registry import get_module, list_modules from imagepipeline.modules.rembg import RembgModule +from tests.integration_helpers import has_darktable_style +from tests.integration_helpers import has_workflow_tools as check_workflow_tools has_magick = bool(shutil.which("magick") or shutil.which("convert")) @@ -184,7 +184,6 @@ class TestImageMagickFill: def test_output_matches_input_size(self, tmp_path: Path) -> None: from imagepipeline.core.context import ModuleContext from imagepipeline.utils.subprocess import run_command - from tests.conftest import make_png src = tmp_path / "ref.png" @@ -221,7 +220,6 @@ class TestCropSquare: def test_center_crops_to_square(self, tmp_path: Path) -> None: from imagepipeline.core.context import ModuleContext from imagepipeline.utils.subprocess import run_command - from tests.conftest import make_png src = tmp_path / "wide.png" @@ -255,9 +253,7 @@ class TestModuleParameters: DarktableStyleModule.validate_module_params({}) def test_darktable_style_accepts_style(self) -> None: - params = DarktableStyleModule.validate_module_params( - {"style": "Watermark F12.rocks"} - ) + params = DarktableStyleModule.validate_module_params({"style": "Watermark F12.rocks"}) assert params["style"] == "Watermark F12.rocks" assert params["style_overwrite"] is True @@ -273,9 +269,7 @@ class TestModuleParameters: def test_darktable_export_conf_jpeg(self) -> None: from imagepipeline.modules.darktable_style import export_conf_options - assert export_conf_options("jpeg") == [ - "plugins/imageio/format/jpeg/quality=90" - ] + assert export_conf_options("jpeg") == ["plugins/imageio/format/jpeg/quality=90"] def test_darktable_export_conf_png(self) -> None: from imagepipeline.modules.darktable_style import export_conf_options @@ -308,7 +302,6 @@ class TestCompositeColor: def test_preserves_color_over_grayscale_background(self, tmp_path: Path) -> None: from imagepipeline.core.context import ModuleContext from imagepipeline.utils.subprocess import run_command - from tests.conftest import make_png src = tmp_path / "src.png" @@ -347,19 +340,22 @@ class TestCompositeColor: output = output_dir / "fg.png" assert output.is_file() - result = run_command( - [magick, "identify", "-format", "%[type]", str(output)] - ) - assert result.stdout.strip() != "Grayscale" + from PIL import Image + + with Image.open(output) as img: + assert img.mode != "L" -has_workflow_tools = all( - shutil.which(name) - for name in ("rembg", "gmic", "magick", "darktable-cli") -) or all(shutil.which(name) for name in ("rembg", "gmic", "convert", "darktable-cli")) +has_workflow_tools = check_workflow_tools() +@pytest.mark.integration +@pytest.mark.slow @pytest.mark.skipif(not has_workflow_tools, reason="Workflow CLI tools not installed") +@pytest.mark.skipif( + not has_darktable_style("Watermark F12.rocks"), + reason="darktable style 'Watermark F12.rocks' not available or darktable-cli export failed", +) class TestWorkflowIntegration: def test_watermark_pipeline(self, input_dir, output_base) -> None: from imagepipeline import Pipeline diff --git a/tests/test_xcf_stack.py b/tests/test_xcf_stack.py index d5bf6d9..c4ccd7b 100644 --- a/tests/test_xcf_stack.py +++ b/tests/test_xcf_stack.py @@ -1,6 +1,7 @@ from __future__ import annotations import shutil +import subprocess from pathlib import Path from unittest.mock import patch @@ -12,26 +13,10 @@ from imagepipeline.modules.xcf_stack import XcfStackModule from imagepipeline.utils.files import find_image_by_stem from tests.conftest import make_png -has_gimp = bool(shutil.which("gimp")) +has_gimp = bool(shutil.which("gimp-console") or shutil.which("gimp")) has_magick = bool(shutil.which("magick") or shutil.which("convert")) -def _gimp_headless_works() -> bool: - gimp = shutil.which("gimp") - if not gimp: - return False - try: - from imagepipeline.utils.subprocess import run_command - - run_command([gimp, "-idf", "-b", "(gimp-quit 0)"], timeout=10) - return True - except RuntimeError: - return False - - -has_working_gimp = _gimp_headless_works() - - class TestFindImageByStem: def test_finds_matching_stem_case_insensitively(self, tmp_path: Path) -> None: make_png(tmp_path / "Photo.PNG") @@ -56,6 +41,41 @@ class TestModuleRegistration: def test_get_module_returns_xcf_stack_class(self) -> None: assert get_module("xcf_stack") is XcfStackModule + def test_xcf_stack_waits_for_layer_inputs(self) -> None: + from imagepipeline.core.runner import PipelineRunner + from imagepipeline.core.step import StepDefinition + from imagepipeline.modules.imagemagick_grayscale import ImageMagickGrayscale + + steps = [ + StepDefinition( + step_id="xcf_stack_01", + module_name="xcf_stack", + module=XcfStackModule, + input_refs=["input", "imagemagick_grayscale_01"], + params={}, + output_dir_name="xcf_stack_01", + ), + StepDefinition( + step_id="imagemagick_grayscale_01", + module_name="imagemagick_grayscale", + module=ImageMagickGrayscale, + input_refs=["input"], + params={}, + output_dir_name="imagemagick_grayscale_01", + ), + ] + runner = PipelineRunner( + name="order", + input_dir=Path("/tmp/unused"), + output_base=Path("/tmp/unused"), + steps=steps, + ) + ordered = runner._topological_sort() + assert [step.step_id for step in ordered] == [ + "imagemagick_grayscale_01", + "xcf_stack_01", + ] + class TestExpectedOutputFilenames: def test_returns_xcf_for_jpg_input(self) -> None: @@ -66,6 +86,14 @@ class TestExpectedOutputFilenames: ) assert names == ["photo.xcf"] + def test_default_timeout(self) -> None: + params = XcfStackModule.validate_module_params({}) + assert params["timeout"] == 120.0 + + def test_custom_timeout(self) -> None: + params = XcfStackModule.validate_module_params({"timeout": 45}) + assert params["timeout"] == 45.0 + def _make_stack_fixture(tmp_path: Path) -> dict[str, Path]: root = tmp_path @@ -91,9 +119,7 @@ def _make_stack_fixture(tmp_path: Path) -> dict[str, Path]: class TestXcfStackRun: @patch("imagepipeline.modules.xcf_stack.stack_images_to_xcf") - def test_collects_layers_from_input_and_prior_steps( - self, mock_stack: object, tmp_path: Path - ) -> None: + def test_collects_layers_from_explicit_inputs(self, mock_stack: object, tmp_path: Path) -> None: paths = _make_stack_fixture(tmp_path) input_path = paths["root"] / "refs" / "photo.jpg" input_path.parent.mkdir() @@ -105,7 +131,8 @@ class TestXcfStackRun: params=XcfStackModule.validate_module_params({}), pipeline_output_root=paths["root"], step_id="xcf_stack_01", - prior_steps=[ + input_layer_dirs=[ + ("input", paths["input_dir"]), ("step_a", paths["step_a"]), ("step_b", paths["step_b"]), ], @@ -123,9 +150,7 @@ class TestXcfStackRun: ] @patch("imagepipeline.modules.xcf_stack.stack_images_to_xcf") - def test_skip_missing_true_skips_missing_step( - self, mock_stack: object, tmp_path: Path - ) -> None: + def test_skip_missing_true_skips_missing_step(self, mock_stack: object, tmp_path: Path) -> None: paths = _make_stack_fixture(tmp_path) (paths["step_b"] / "photo.png").unlink() input_path = paths["root"] / "photo.jpg" @@ -136,7 +161,8 @@ class TestXcfStackRun: params=XcfStackModule.validate_module_params({"skip_missing": True}), pipeline_output_root=paths["root"], step_id="xcf_stack_01", - prior_steps=[ + input_layer_dirs=[ + ("input", paths["input_dir"]), ("step_a", paths["step_a"]), ("step_b", paths["step_b"]), ], @@ -162,7 +188,8 @@ class TestXcfStackRun: params=XcfStackModule.validate_module_params({"skip_missing": False}), pipeline_output_root=paths["root"], step_id="xcf_stack_01", - prior_steps=[ + input_layer_dirs=[ + ("input", paths["input_dir"]), ("step_a", paths["step_a"]), ("step_b", paths["step_b"]), ], @@ -174,7 +201,7 @@ class TestXcfStackRun: mock_stack.assert_not_called() -@pytest.mark.skipif(not has_working_gimp, reason="GIMP headless not available") +@pytest.mark.skipif(not has_gimp, reason="GIMP not installed") @pytest.mark.skipif(not has_magick, reason="ImageMagick not installed") class TestStackImagesToXcfIntegration: def test_stacks_pngs_into_xcf(self, tmp_path: Path) -> None: @@ -198,3 +225,33 @@ class TestStackImagesToXcfIntegration: assert outfile.is_file() assert outfile.stat().st_size > 0 + names = outfile.read_bytes() + assert b"bottom" in names + assert b"top" in names + + +class TestGimpTimeoutHandling: + def test_accepts_outfile_when_gimp_hangs_on_exit(self, tmp_path: Path) -> None: + from unittest.mock import MagicMock, patch + + from imagepipeline.utils.gimp import stack_images_to_xcf + + layer = tmp_path / "layer.png" + layer.write_bytes(b"png") + outfile = tmp_path / "stack.xcf" + outfile.write_bytes(b"xcf-data") + + def fake_popen(*_args, **_kwargs): + process = MagicMock() + process.communicate.side_effect = [ + subprocess.TimeoutExpired(cmd="gimp", timeout=1), + ("", ""), + ] + process.kill = MagicMock() + return process + + with patch("imagepipeline.utils.gimp.subprocess.Popen", side_effect=fake_popen): + with patch("imagepipeline.utils.gimp.require_gimp", return_value="gimp-console"): + stack_images_to_xcf([("layer", layer)], outfile, timeout=1.0) + + assert outfile.read_bytes() == b"xcf-data"