From 765ebbe3da6266ade61f948a9dc78b257f01b4b4 Mon Sep 17 00:00:00 2001 From: Frank Schwenk Date: Sun, 12 Jul 2026 22:21:01 +0200 Subject: [PATCH] feat: optional step_id for custom output folder names Allow readable step folders and xcf_stack layer labels via p.step(..., step_id=...) while keeping the per-module auto counter unchanged. Co-authored-by: Cursor --- README.md | 2 +- docs/MODULE_DEVELOPMENT.md | 14 ++++++- imagepipeline/core/pipeline.py | 19 +++++++++- pipelines/pipeline_rezepttest.py | 48 +++++++++++++++++------- tests/test_pipeline.py | 64 ++++++++++++++++++++++++++++++++ tests/test_resume.py | 28 ++++++++++++++ 6 files changed, 157 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index ea98af4..05410de 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ my_run_20260527143022/ └── ... ``` -Step folders are named `{module_name}_{nn}` (two-digit counter per module name). +Step folders are named `{module_name}_{nn}` by default (two-digit counter per module name). Pass optional `step_id="input_bokeh"` to `p.step()` for a custom folder name and step reference (see [docs/MODULE_DEVELOPMENT.md](docs/MODULE_DEVELOPMENT.md#step-folder-naming)). ## Writing Pipelines diff --git a/docs/MODULE_DEVELOPMENT.md b/docs/MODULE_DEVELOPMENT.md index d354293..35b298b 100644 --- a/docs/MODULE_DEVELOPMENT.md +++ b/docs/MODULE_DEVELOPMENT.md @@ -167,7 +167,7 @@ class MyCliModule(SubprocessModule): ## Step Folder Naming -The runner assigns output folders automatically: `{module_name}_{nn}`. +By default the runner assigns output folders automatically: `{module_name}_{nn}` (two-digit counter per module name). Using the same module twice in one pipeline produces separate folders: @@ -178,7 +178,17 @@ darktable_style_01/ darktable_style_02/ ``` -You do not choose folder names in the module. +In pipeline scripts you can pass an optional `step_id` to `p.step()` for readable folder names and GIMP layer labels (`xcf_stack` uses the step id as the layer name): + +```python +input_bokeh = p.step("gmic", inputs="input", command=GMIC_BOKEH, step_id="input_bokeh") +``` + +`step_id` becomes both the internal step reference and the output subfolder name. It must be unique within the pipeline and must not contain path separators. The per-module counter still increments on every `p.step("gmic", …)` call — a custom id does not reserve or skip a number slot (the next default `gmic` step after `step_id="input_bokeh"` is `gmic_03`, not `gmic_02`, if two prior `gmic` steps ran). + +Use the same ids in `EXISTING_OUTPUTS` when resuming with external folders. + +You do not choose folder names inside the module implementation. ## Format Warnings diff --git a/imagepipeline/core/pipeline.py b/imagepipeline/core/pipeline.py index dfc3c5a..8bcf516 100644 --- a/imagepipeline/core/pipeline.py +++ b/imagepipeline/core/pipeline.py @@ -45,13 +45,18 @@ class Pipeline: module_name: str, *, inputs: StepRef | str | list[StepRef | str], + step_id: str | None = None, **params: Any, ) -> StepRef: module_cls = get_module(module_name) self._module_counters[module_name] += 1 counter = self._module_counters[module_name] - output_dir_name = f"{module_name}_{counter:02d}" - step_id = output_dir_name + if step_id is None: + output_dir_name = f"{module_name}_{counter:02d}" + step_id = output_dir_name + else: + self._validate_step_id(step_id) + output_dir_name = step_id input_refs = self._normalize_inputs(inputs) reserved = {"inputs", "input"} @@ -101,6 +106,16 @@ class Pipeline: return [self._input_ref(item) for item in inputs] return [self._input_ref(inputs)] + def _validate_step_id(self, step_id: str) -> None: + if not step_id: + raise ValidationError("step_id must not be empty") + if "/" in step_id or "\\" in step_id: + raise ValidationError( + f"step_id must not contain path separators: {step_id!r}" + ) + if any(step.step_id == step_id for step in self._steps): + raise ValidationError(f"Duplicate step_id: {step_id!r}") + @staticmethod def _input_ref(value: StepRef | str) -> str: if isinstance(value, StepRef): diff --git a/pipelines/pipeline_rezepttest.py b/pipelines/pipeline_rezepttest.py index 4751801..3f16e1e 100644 --- a/pipelines/pipeline_rezepttest.py +++ b/pipelines/pipeline_rezepttest.py @@ -35,8 +35,8 @@ def main() -> None: existing_outputs=EXISTING_OUTPUTS or None, continue_from=CONTINUE_FROM, ) as p: - rembg_out = p.step("rembg", inputs="input") - grayscale = p.step("gmic_grayscale", inputs="input") + rembg_out = p.step("rembg", inputs="input", step_id="rembg_out") + grayscale = p.step("gmic_grayscale", inputs="input", step_id="grayscale") gradient_radial_bg = p.step( "imagemagick_fill", inputs="input", @@ -44,34 +44,56 @@ def main() -> None: color2=COLOR2, gradient=True, radial=True, + step_id="gradient_radial_bg", ) - rembg_shadow = p.step("gmic", inputs=rembg_out, command=GMIC_DROP_SHADOW) - rembg_jpr_smooth = p.step("gmic", inputs=rembg_out, command=GMIC_JPR_SMOOTH) + rembg_shadow = p.step( + "gmic", inputs=rembg_out, command=GMIC_DROP_SHADOW, step_id="rembg_shadow" + ) + rembg_jpr_smooth = p.step( + "gmic", inputs=rembg_out, command=GMIC_JPR_SMOOTH, step_id="rembg_jpr_smooth" + ) rembg_jpr_smooth_sized = p.step( "imagemagick_scale_crop", inputs=rembg_jpr_smooth, scale=1.05, + step_id="rembg_jpr_smooth_sized", + ) + input_bokeh = p.step( + "gmic", inputs="input", command=GMIC_BOKEH, step_id="input_bokeh" ) - input_bokeh = p.step("gmic", inputs="input", command=GMIC_BOKEH) # recipe: colorsplash - composite_colorsplash = p.step("composite", inputs=[grayscale, rembg_out]) + composite_colorsplash = p.step( + "composite", inputs=[grayscale, rembg_out], step_id="composite_colorsplash" + ) # recipe: rembg-radial-2colors (gradient-radial) - composite_radial = p.step("composite", inputs=[gradient_radial_bg, rembg_out]) + composite_radial = p.step( + "composite", inputs=[gradient_radial_bg, rembg_out], step_id="composite_radial" + ) # recipe: original-drop-shadow-rembg - shadow_mid = p.step("composite", inputs=["input", rembg_shadow]) - composite_shadow = p.step("composite", inputs=[shadow_mid, rembg_out]) + shadow_mid = p.step( + "composite", inputs=["input", rembg_shadow], step_id="shadow_mid" + ) + composite_shadow = p.step( + "composite", inputs=[shadow_mid, rembg_out], step_id="composite_shadow" + ) # recipe: original-jpr-smooth-rembg - smooth_mid = p.step("composite", inputs=["input", rembg_jpr_smooth_sized]) - composite_smooth = p.step("composite", inputs=[smooth_mid, rembg_out]) + smooth_mid = p.step( + "composite", inputs=["input", rembg_jpr_smooth_sized], step_id="smooth_mid" + ) + composite_smooth = p.step( + "composite", inputs=[smooth_mid, rembg_out], step_id="composite_smooth" + ) # recipe: bokeh-oktagon - bokeh_mid = p.step("composite", inputs=["input", input_bokeh]) - composite_bokeh = p.step("composite", inputs=[bokeh_mid, rembg_out]) + bokeh_mid = p.step("composite", inputs=["input", input_bokeh], step_id="bokeh_mid") + composite_bokeh = p.step( + "composite", inputs=[bokeh_mid, rembg_out], step_id="composite_bokeh" + ) # recipe: xcf-stack — explicit layer list, bottom to top p.step( diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 5a60bba..715bd30 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -159,6 +159,70 @@ class TestManifest: assert data["finished_at"] is not None +class TestCustomStepId: + def test_custom_step_id_output_folder(self, input_dir: Path, output_base: Path) -> None: + @register + class NamedModule(BaseModule): + name = "named_tracker" + + def run(self, ctx) -> None: + ctx.output_dir.mkdir(parents=True, exist_ok=True) + for src in ctx.input_paths: + shutil.copy2(src, ctx.output_dir / src.name) + + with Pipeline(name="named_test", input_dir=input_dir, output_base=output_base, verbose=False) as p: + ref = p.step("named_tracker", inputs="input", step_id="input_bokeh") + root = p.run() + + assert ref.step_id == "input_bokeh" + assert ref.output_dir_name == "input_bokeh" + assert (root / "input_bokeh").is_dir() + assert not (root / "named_tracker_01").exists() + + unregister("named_tracker") + + def test_module_counter_independent_of_custom_step_id( + self, input_dir: Path, output_base: Path + ) -> None: + @register + class CounterModule(BaseModule): + name = "counter_tracker" + + def run(self, ctx) -> None: + ctx.output_dir.mkdir(parents=True, exist_ok=True) + for src in ctx.input_paths: + shutil.copy2(src, ctx.output_dir / src.name) + + with Pipeline(name="counter_test", input_dir=input_dir, output_base=output_base, verbose=False) as p: + p.step("counter_tracker", inputs="input") + p.step("counter_tracker", inputs="input", step_id="custom_mid") + p.step("counter_tracker", inputs="input") + root = p.run() + + assert (root / "counter_tracker_01").is_dir() + assert (root / "custom_mid").is_dir() + assert (root / "counter_tracker_03").is_dir() + assert not (root / "counter_tracker_02").exists() + + unregister("counter_tracker") + + def test_duplicate_step_id_rejected(self, input_dir: Path) -> None: + with Pipeline(name="dup_id", input_dir=input_dir, verbose=False) as p: + p.step("imagemagick_grayscale", inputs="input", step_id="my_step") + with pytest.raises(ValidationError, match="Duplicate step_id"): + p.step("imagemagick_grayscale", inputs="input", step_id="my_step") + + def test_empty_step_id_rejected(self, input_dir: Path) -> None: + with Pipeline(name="empty_id", input_dir=input_dir, verbose=False) as p: + with pytest.raises(ValidationError, match="must not be empty"): + p.step("imagemagick_grayscale", inputs="input", step_id="") + + def test_step_id_with_slash_rejected(self, input_dir: Path) -> None: + with Pipeline(name="slash_id", input_dir=input_dir, verbose=False) as p: + with pytest.raises(ValidationError, match="path separators"): + p.step("imagemagick_grayscale", inputs="input", step_id="bad/name") + + class TestPipelineValidation: def test_empty_pipeline_rejected(self, input_dir: Path) -> None: with Pipeline(name="empty", input_dir=input_dir, verbose=False) as p: diff --git a/tests/test_resume.py b/tests/test_resume.py index 74ea0c4..3132671 100644 --- a/tests/test_resume.py +++ b/tests/test_resume.py @@ -178,6 +178,34 @@ class TestPipelineResume: assert "Reused external output for imagemagick_grayscale_01" in output assert (root / "imagemagick_grayscale_01" / "photo_a.png").exists() + @pytest.mark.skipif(not shutil.which("magick"), reason="ImageMagick not installed") + def test_existing_outputs_with_custom_step_id( + self, input_dir: Path, output_base: Path, tmp_path: Path, capsys + ) -> None: + external = tmp_path / "external_gray" + external.mkdir() + for src in input_dir.iterdir(): + if src.is_file(): + make_png(external / src.name, width=4, height=4, rgb=(10, 20, 30)) + + with Pipeline( + name="custom_id_external", + input_dir=input_dir, + output_base=output_base, + verbose=True, + existing_outputs={"input_bokeh": external}, + ) as p: + reused = p.step( + "imagemagick_grayscale", inputs="input", step_id="input_bokeh" + ) + p.step("imagemagick_grayscale", inputs=reused) + root = p.run() + + output = capsys.readouterr().out + assert "Reused external output for input_bokeh" in output + assert (root / "input_bokeh" / "photo_a.png").exists() + assert not (root / "imagemagick_grayscale_01").exists() + class TestMaterializeExternal: def test_links_files_by_stem(self, tmp_path: Path) -> None: