Cleanup/quality pass #1
@@ -48,7 +48,7 @@ my_run_20260527143022/
|
|||||||
└── ...
|
└── ...
|
||||||
```
|
```
|
||||||
|
|
||||||
Step folders are named `{module_name}_{nn}` (two-digit counter per module name).
|
Step folders are named `{module_name}_{nn}` by default (two-digit counter per module name). Pass optional `step_id="input_bokeh"` to `p.step()` for a custom folder name and step reference (see [docs/MODULE_DEVELOPMENT.md](docs/MODULE_DEVELOPMENT.md#step-folder-naming)).
|
||||||
|
|
||||||
## Writing Pipelines
|
## Writing Pipelines
|
||||||
|
|
||||||
|
|||||||
@@ -167,7 +167,7 @@ class MyCliModule(SubprocessModule):
|
|||||||
|
|
||||||
## Step Folder Naming
|
## Step Folder Naming
|
||||||
|
|
||||||
The runner assigns output folders automatically: `{module_name}_{nn}`.
|
By default the runner assigns output folders automatically: `{module_name}_{nn}` (two-digit counter per module name).
|
||||||
|
|
||||||
Using the same module twice in one pipeline produces separate folders:
|
Using the same module twice in one pipeline produces separate folders:
|
||||||
|
|
||||||
@@ -178,7 +178,17 @@ darktable_style_01/
|
|||||||
darktable_style_02/
|
darktable_style_02/
|
||||||
```
|
```
|
||||||
|
|
||||||
You do not choose folder names in the module.
|
In pipeline scripts you can pass an optional `step_id` to `p.step()` for readable folder names and GIMP layer labels (`xcf_stack` uses the step id as the layer name):
|
||||||
|
|
||||||
|
```python
|
||||||
|
input_bokeh = p.step("gmic", inputs="input", command=GMIC_BOKEH, step_id="input_bokeh")
|
||||||
|
```
|
||||||
|
|
||||||
|
`step_id` becomes both the internal step reference and the output subfolder name. It must be unique within the pipeline and must not contain path separators. The per-module counter still increments on every `p.step("gmic", …)` call — a custom id does not reserve or skip a number slot (the next default `gmic` step after `step_id="input_bokeh"` is `gmic_03`, not `gmic_02`, if two prior `gmic` steps ran).
|
||||||
|
|
||||||
|
Use the same ids in `EXISTING_OUTPUTS` when resuming with external folders.
|
||||||
|
|
||||||
|
You do not choose folder names inside the module implementation.
|
||||||
|
|
||||||
## Format Warnings
|
## Format Warnings
|
||||||
|
|
||||||
|
|||||||
@@ -45,13 +45,18 @@ class Pipeline:
|
|||||||
module_name: str,
|
module_name: str,
|
||||||
*,
|
*,
|
||||||
inputs: StepRef | str | list[StepRef | str],
|
inputs: StepRef | str | list[StepRef | str],
|
||||||
|
step_id: str | None = None,
|
||||||
**params: Any,
|
**params: Any,
|
||||||
) -> StepRef:
|
) -> StepRef:
|
||||||
module_cls = get_module(module_name)
|
module_cls = get_module(module_name)
|
||||||
self._module_counters[module_name] += 1
|
self._module_counters[module_name] += 1
|
||||||
counter = self._module_counters[module_name]
|
counter = self._module_counters[module_name]
|
||||||
|
if step_id is None:
|
||||||
output_dir_name = f"{module_name}_{counter:02d}"
|
output_dir_name = f"{module_name}_{counter:02d}"
|
||||||
step_id = output_dir_name
|
step_id = output_dir_name
|
||||||
|
else:
|
||||||
|
self._validate_step_id(step_id)
|
||||||
|
output_dir_name = step_id
|
||||||
|
|
||||||
input_refs = self._normalize_inputs(inputs)
|
input_refs = self._normalize_inputs(inputs)
|
||||||
reserved = {"inputs", "input"}
|
reserved = {"inputs", "input"}
|
||||||
@@ -101,6 +106,16 @@ class Pipeline:
|
|||||||
return [self._input_ref(item) for item in inputs]
|
return [self._input_ref(item) for item in inputs]
|
||||||
return [self._input_ref(inputs)]
|
return [self._input_ref(inputs)]
|
||||||
|
|
||||||
|
def _validate_step_id(self, step_id: str) -> None:
|
||||||
|
if not step_id:
|
||||||
|
raise ValidationError("step_id must not be empty")
|
||||||
|
if "/" in step_id or "\\" in step_id:
|
||||||
|
raise ValidationError(
|
||||||
|
f"step_id must not contain path separators: {step_id!r}"
|
||||||
|
)
|
||||||
|
if any(step.step_id == step_id for step in self._steps):
|
||||||
|
raise ValidationError(f"Duplicate step_id: {step_id!r}")
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _input_ref(value: StepRef | str) -> str:
|
def _input_ref(value: StepRef | str) -> str:
|
||||||
if isinstance(value, StepRef):
|
if isinstance(value, StepRef):
|
||||||
|
|||||||
@@ -35,8 +35,8 @@ def main() -> None:
|
|||||||
existing_outputs=EXISTING_OUTPUTS or None,
|
existing_outputs=EXISTING_OUTPUTS or None,
|
||||||
continue_from=CONTINUE_FROM,
|
continue_from=CONTINUE_FROM,
|
||||||
) as p:
|
) as p:
|
||||||
rembg_out = p.step("rembg", inputs="input")
|
rembg_out = p.step("rembg", inputs="input", step_id="rembg_out")
|
||||||
grayscale = p.step("gmic_grayscale", inputs="input")
|
grayscale = p.step("gmic_grayscale", inputs="input", step_id="grayscale")
|
||||||
gradient_radial_bg = p.step(
|
gradient_radial_bg = p.step(
|
||||||
"imagemagick_fill",
|
"imagemagick_fill",
|
||||||
inputs="input",
|
inputs="input",
|
||||||
@@ -44,34 +44,56 @@ def main() -> None:
|
|||||||
color2=COLOR2,
|
color2=COLOR2,
|
||||||
gradient=True,
|
gradient=True,
|
||||||
radial=True,
|
radial=True,
|
||||||
|
step_id="gradient_radial_bg",
|
||||||
)
|
)
|
||||||
|
|
||||||
rembg_shadow = p.step("gmic", inputs=rembg_out, command=GMIC_DROP_SHADOW)
|
rembg_shadow = p.step(
|
||||||
rembg_jpr_smooth = p.step("gmic", inputs=rembg_out, command=GMIC_JPR_SMOOTH)
|
"gmic", inputs=rembg_out, command=GMIC_DROP_SHADOW, step_id="rembg_shadow"
|
||||||
|
)
|
||||||
|
rembg_jpr_smooth = p.step(
|
||||||
|
"gmic", inputs=rembg_out, command=GMIC_JPR_SMOOTH, step_id="rembg_jpr_smooth"
|
||||||
|
)
|
||||||
rembg_jpr_smooth_sized = p.step(
|
rembg_jpr_smooth_sized = p.step(
|
||||||
"imagemagick_scale_crop",
|
"imagemagick_scale_crop",
|
||||||
inputs=rembg_jpr_smooth,
|
inputs=rembg_jpr_smooth,
|
||||||
scale=1.05,
|
scale=1.05,
|
||||||
|
step_id="rembg_jpr_smooth_sized",
|
||||||
|
)
|
||||||
|
input_bokeh = p.step(
|
||||||
|
"gmic", inputs="input", command=GMIC_BOKEH, step_id="input_bokeh"
|
||||||
)
|
)
|
||||||
input_bokeh = p.step("gmic", inputs="input", command=GMIC_BOKEH)
|
|
||||||
|
|
||||||
# recipe: colorsplash
|
# recipe: colorsplash
|
||||||
composite_colorsplash = p.step("composite", inputs=[grayscale, rembg_out])
|
composite_colorsplash = p.step(
|
||||||
|
"composite", inputs=[grayscale, rembg_out], step_id="composite_colorsplash"
|
||||||
|
)
|
||||||
|
|
||||||
# recipe: rembg-radial-2colors (gradient-radial)
|
# recipe: rembg-radial-2colors (gradient-radial)
|
||||||
composite_radial = p.step("composite", inputs=[gradient_radial_bg, rembg_out])
|
composite_radial = p.step(
|
||||||
|
"composite", inputs=[gradient_radial_bg, rembg_out], step_id="composite_radial"
|
||||||
|
)
|
||||||
|
|
||||||
# recipe: original-drop-shadow-rembg
|
# recipe: original-drop-shadow-rembg
|
||||||
shadow_mid = p.step("composite", inputs=["input", rembg_shadow])
|
shadow_mid = p.step(
|
||||||
composite_shadow = p.step("composite", inputs=[shadow_mid, rembg_out])
|
"composite", inputs=["input", rembg_shadow], step_id="shadow_mid"
|
||||||
|
)
|
||||||
|
composite_shadow = p.step(
|
||||||
|
"composite", inputs=[shadow_mid, rembg_out], step_id="composite_shadow"
|
||||||
|
)
|
||||||
|
|
||||||
# recipe: original-jpr-smooth-rembg
|
# recipe: original-jpr-smooth-rembg
|
||||||
smooth_mid = p.step("composite", inputs=["input", rembg_jpr_smooth_sized])
|
smooth_mid = p.step(
|
||||||
composite_smooth = p.step("composite", inputs=[smooth_mid, rembg_out])
|
"composite", inputs=["input", rembg_jpr_smooth_sized], step_id="smooth_mid"
|
||||||
|
)
|
||||||
|
composite_smooth = p.step(
|
||||||
|
"composite", inputs=[smooth_mid, rembg_out], step_id="composite_smooth"
|
||||||
|
)
|
||||||
|
|
||||||
# recipe: bokeh-oktagon
|
# recipe: bokeh-oktagon
|
||||||
bokeh_mid = p.step("composite", inputs=["input", input_bokeh])
|
bokeh_mid = p.step("composite", inputs=["input", input_bokeh], step_id="bokeh_mid")
|
||||||
composite_bokeh = p.step("composite", inputs=[bokeh_mid, rembg_out])
|
composite_bokeh = p.step(
|
||||||
|
"composite", inputs=[bokeh_mid, rembg_out], step_id="composite_bokeh"
|
||||||
|
)
|
||||||
|
|
||||||
# recipe: xcf-stack — explicit layer list, bottom to top
|
# recipe: xcf-stack — explicit layer list, bottom to top
|
||||||
p.step(
|
p.step(
|
||||||
|
|||||||
@@ -159,6 +159,70 @@ class TestManifest:
|
|||||||
assert data["finished_at"] is not None
|
assert data["finished_at"] is not None
|
||||||
|
|
||||||
|
|
||||||
|
class TestCustomStepId:
|
||||||
|
def test_custom_step_id_output_folder(self, input_dir: Path, output_base: Path) -> None:
|
||||||
|
@register
|
||||||
|
class NamedModule(BaseModule):
|
||||||
|
name = "named_tracker"
|
||||||
|
|
||||||
|
def run(self, ctx) -> None:
|
||||||
|
ctx.output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
for src in ctx.input_paths:
|
||||||
|
shutil.copy2(src, ctx.output_dir / src.name)
|
||||||
|
|
||||||
|
with Pipeline(name="named_test", input_dir=input_dir, output_base=output_base, verbose=False) as p:
|
||||||
|
ref = p.step("named_tracker", inputs="input", step_id="input_bokeh")
|
||||||
|
root = p.run()
|
||||||
|
|
||||||
|
assert ref.step_id == "input_bokeh"
|
||||||
|
assert ref.output_dir_name == "input_bokeh"
|
||||||
|
assert (root / "input_bokeh").is_dir()
|
||||||
|
assert not (root / "named_tracker_01").exists()
|
||||||
|
|
||||||
|
unregister("named_tracker")
|
||||||
|
|
||||||
|
def test_module_counter_independent_of_custom_step_id(
|
||||||
|
self, input_dir: Path, output_base: Path
|
||||||
|
) -> None:
|
||||||
|
@register
|
||||||
|
class CounterModule(BaseModule):
|
||||||
|
name = "counter_tracker"
|
||||||
|
|
||||||
|
def run(self, ctx) -> None:
|
||||||
|
ctx.output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
for src in ctx.input_paths:
|
||||||
|
shutil.copy2(src, ctx.output_dir / src.name)
|
||||||
|
|
||||||
|
with Pipeline(name="counter_test", input_dir=input_dir, output_base=output_base, verbose=False) as p:
|
||||||
|
p.step("counter_tracker", inputs="input")
|
||||||
|
p.step("counter_tracker", inputs="input", step_id="custom_mid")
|
||||||
|
p.step("counter_tracker", inputs="input")
|
||||||
|
root = p.run()
|
||||||
|
|
||||||
|
assert (root / "counter_tracker_01").is_dir()
|
||||||
|
assert (root / "custom_mid").is_dir()
|
||||||
|
assert (root / "counter_tracker_03").is_dir()
|
||||||
|
assert not (root / "counter_tracker_02").exists()
|
||||||
|
|
||||||
|
unregister("counter_tracker")
|
||||||
|
|
||||||
|
def test_duplicate_step_id_rejected(self, input_dir: Path) -> None:
|
||||||
|
with Pipeline(name="dup_id", input_dir=input_dir, verbose=False) as p:
|
||||||
|
p.step("imagemagick_grayscale", inputs="input", step_id="my_step")
|
||||||
|
with pytest.raises(ValidationError, match="Duplicate step_id"):
|
||||||
|
p.step("imagemagick_grayscale", inputs="input", step_id="my_step")
|
||||||
|
|
||||||
|
def test_empty_step_id_rejected(self, input_dir: Path) -> None:
|
||||||
|
with Pipeline(name="empty_id", input_dir=input_dir, verbose=False) as p:
|
||||||
|
with pytest.raises(ValidationError, match="must not be empty"):
|
||||||
|
p.step("imagemagick_grayscale", inputs="input", step_id="")
|
||||||
|
|
||||||
|
def test_step_id_with_slash_rejected(self, input_dir: Path) -> None:
|
||||||
|
with Pipeline(name="slash_id", input_dir=input_dir, verbose=False) as p:
|
||||||
|
with pytest.raises(ValidationError, match="path separators"):
|
||||||
|
p.step("imagemagick_grayscale", inputs="input", step_id="bad/name")
|
||||||
|
|
||||||
|
|
||||||
class TestPipelineValidation:
|
class TestPipelineValidation:
|
||||||
def test_empty_pipeline_rejected(self, input_dir: Path) -> None:
|
def test_empty_pipeline_rejected(self, input_dir: Path) -> None:
|
||||||
with Pipeline(name="empty", input_dir=input_dir, verbose=False) as p:
|
with Pipeline(name="empty", input_dir=input_dir, verbose=False) as p:
|
||||||
|
|||||||
@@ -178,6 +178,34 @@ class TestPipelineResume:
|
|||||||
assert "Reused external output for imagemagick_grayscale_01" in output
|
assert "Reused external output for imagemagick_grayscale_01" in output
|
||||||
assert (root / "imagemagick_grayscale_01" / "photo_a.png").exists()
|
assert (root / "imagemagick_grayscale_01" / "photo_a.png").exists()
|
||||||
|
|
||||||
|
@pytest.mark.skipif(not shutil.which("magick"), reason="ImageMagick not installed")
|
||||||
|
def test_existing_outputs_with_custom_step_id(
|
||||||
|
self, input_dir: Path, output_base: Path, tmp_path: Path, capsys
|
||||||
|
) -> None:
|
||||||
|
external = tmp_path / "external_gray"
|
||||||
|
external.mkdir()
|
||||||
|
for src in input_dir.iterdir():
|
||||||
|
if src.is_file():
|
||||||
|
make_png(external / src.name, width=4, height=4, rgb=(10, 20, 30))
|
||||||
|
|
||||||
|
with Pipeline(
|
||||||
|
name="custom_id_external",
|
||||||
|
input_dir=input_dir,
|
||||||
|
output_base=output_base,
|
||||||
|
verbose=True,
|
||||||
|
existing_outputs={"input_bokeh": external},
|
||||||
|
) as p:
|
||||||
|
reused = p.step(
|
||||||
|
"imagemagick_grayscale", inputs="input", step_id="input_bokeh"
|
||||||
|
)
|
||||||
|
p.step("imagemagick_grayscale", inputs=reused)
|
||||||
|
root = p.run()
|
||||||
|
|
||||||
|
output = capsys.readouterr().out
|
||||||
|
assert "Reused external output for input_bokeh" in output
|
||||||
|
assert (root / "input_bokeh" / "photo_a.png").exists()
|
||||||
|
assert not (root / "imagemagick_grayscale_01").exists()
|
||||||
|
|
||||||
|
|
||||||
class TestMaterializeExternal:
|
class TestMaterializeExternal:
|
||||||
def test_links_files_by_stem(self, tmp_path: Path) -> None:
|
def test_links_files_by_stem(self, tmp_path: Path) -> None:
|
||||||
|
|||||||
Reference in New Issue
Block a user