Cleanup/quality pass #1

Merged
froxxxy merged 14 commits from cleanup/quality-pass into main 2026-07-18 17:53:57 +02:00
39 changed files with 150 additions and 202 deletions
Showing only changes of commit a60a18a253 - Show all commits
+8
View File
@@ -5,6 +5,14 @@ __pycache__/
dist/ dist/
build/ build/
.pytest_cache/ .pytest_cache/
.ruff_cache/
.mypy_cache/
.coverage
htmlcov/
.venv/ .venv/
venv/ venv/
.env .env
*.log
# Local pipeline run output (if created inside repo)
pipeline_output/
-1
View File
@@ -1,6 +1,5 @@
from __future__ import annotations from __future__ import annotations
import hashlib
import urllib.request import urllib.request
from pathlib import Path from pathlib import Path
+20 -20
View File
@@ -1,11 +1,8 @@
from __future__ import annotations from __future__ import annotations
import math
import numpy as np import numpy as np
import torch import torch
import torch.nn as nn import torch.nn as nn
import torch.nn.functional as F
from imagepipeline.ai.hdrnet.slice import batch_bilateral_slice from imagepipeline.ai.hdrnet.slice import batch_bilateral_slice
@@ -70,22 +67,27 @@ class Slice(nn.Module):
class ApplyCoeffs(nn.Module): class ApplyCoeffs(nn.Module):
def forward(self, coeff, full_res_input): def forward(self, coeff, full_res_input):
r = torch.sum(full_res_input * coeff[:, 0:3, :, :], dim=1, keepdim=True) + coeff[ r = (
:, 9:10, :, : torch.sum(full_res_input * coeff[:, 0:3, :, :], dim=1, keepdim=True)
] + coeff[:, 9:10, :, :]
g = torch.sum(full_res_input * coeff[:, 3:6, :, :], dim=1, keepdim=True) + coeff[ )
:, 10:11, :, : g = (
] torch.sum(full_res_input * coeff[:, 3:6, :, :], dim=1, keepdim=True)
b = torch.sum(full_res_input * coeff[:, 6:9, :, :], dim=1, keepdim=True) + coeff[ + coeff[:, 10:11, :, :]
:, 11:12, :, : )
] b = (
torch.sum(full_res_input * coeff[:, 6:9, :, :], dim=1, keepdim=True)
+ coeff[:, 11:12, :, :]
)
return torch.cat([r, g, b], dim=1) return torch.cat([r, g, b], dim=1)
class GuideNN(nn.Module): class GuideNN(nn.Module):
def __init__(self, params) -> None: def __init__(self, params) -> None:
super().__init__() super().__init__()
self.conv1 = ConvBlock(3, params["guide_complexity"], kernel_size=1, padding=0, batch_norm=True) self.conv1 = ConvBlock(
3, params["guide_complexity"], kernel_size=1, padding=0, batch_norm=True
)
self.conv2 = ConvBlock( self.conv2 = ConvBlock(
params["guide_complexity"], 1, kernel_size=1, padding=0, activation=nn.Sigmoid params["guide_complexity"], 1, kernel_size=1, padding=0, activation=nn.Sigmoid
) )
@@ -112,9 +114,7 @@ class Coeffs(nn.Module):
for index in range(n_layers_splat): for index in range(n_layers_splat):
use_bn = bn if index > 0 else False use_bn = bn if index > 0 else False
out_ch = cm * (2**index) * lb out_ch = cm * (2**index) * lb
self.splat_features.append( self.splat_features.append(ConvBlock(prev_ch, out_ch, 3, stride=2, batch_norm=use_bn))
ConvBlock(prev_ch, out_ch, 3, stride=2, batch_norm=use_bn)
)
prev_ch = out_ch prev_ch = out_ch
splat_ch = prev_ch splat_ch = prev_ch
@@ -131,7 +131,9 @@ class Coeffs(nn.Module):
prev_ch = int(prev_ch * (nsize / 2**n_total) ** 2) prev_ch = int(prev_ch * (nsize / 2**n_total) ** 2)
self.global_features_fc.append(FC(prev_ch, 32 * cm * lb, batch_norm=bn)) self.global_features_fc.append(FC(prev_ch, 32 * cm * lb, batch_norm=bn))
self.global_features_fc.append(FC(32 * cm * lb, 16 * cm * lb, batch_norm=bn)) self.global_features_fc.append(FC(32 * cm * lb, 16 * cm * lb, batch_norm=bn))
self.global_features_fc.append(FC(16 * cm * lb, 8 * cm * lb, activation=None, batch_norm=bn)) self.global_features_fc.append(
FC(16 * cm * lb, 8 * cm * lb, activation=None, batch_norm=bn)
)
self.local_features = nn.ModuleList( self.local_features = nn.ModuleList(
[ [
@@ -139,9 +141,7 @@ class Coeffs(nn.Module):
ConvBlock(8 * cm * lb, 8 * cm * lb, 3, activation=None, use_bias=False), ConvBlock(8 * cm * lb, 8 * cm * lb, 3, activation=None, use_bias=False),
] ]
) )
self.conv_out = ConvBlock( self.conv_out = ConvBlock(8 * cm * lb, lb * nout * nin, 1, padding=0, activation=None)
8 * cm * lb, lb * nout * nin, 1, padding=0, activation=None
)
self.relu = nn.ReLU() self.relu = nn.ReLU()
def forward(self, lowres_input): def forward(self, lowres_input):
+2 -6
View File
@@ -82,12 +82,8 @@ def _bilateral_slice(grid, guide):
grid_val_110 = grid[gi1c, gj1c, gk0c, :] grid_val_110 = grid[gi1c, gj1c, gk0c, :]
grid_val_111 = grid[gi1c, gj1c, gk1c, :] grid_val_111 = grid[gi1c, gj1c, gk1c, :]
w_000, w_001, w_010, w_011 = map( w_000, w_001, w_010, w_011 = map(torch.atleast_3d, (w_000, w_001, w_010, w_011))
torch.atleast_3d, (w_000, w_001, w_010, w_011) w_100, w_101, w_110, w_111 = map(torch.atleast_3d, (w_100, w_101, w_110, w_111))
)
w_100, w_101, w_110, w_111 = map(
torch.atleast_3d, (w_100, w_101, w_110, w_111)
)
return ( return (
torch.multiply(w_000, grid_val_000) torch.multiply(w_000, grid_val_000)
+2 -6
View File
@@ -60,9 +60,7 @@ class EnhanceNetNoPool(nn.Module):
if self.scale_factor == 1: if self.scale_factor == 1:
x_down = x x_down = x
else: else:
x_down = F.interpolate( x_down = F.interpolate(x, scale_factor=1 / self.scale_factor, mode="bilinear")
x, scale_factor=1 / self.scale_factor, mode="bilinear"
)
x1 = self.relu(self.e_conv1(x_down)) x1 = self.relu(self.e_conv1(x_down))
x2 = self.relu(self.e_conv2(x1)) x2 = self.relu(self.e_conv2(x1))
@@ -105,7 +103,5 @@ def enhance_image(
if strength < 1.0: if strength < 1.0:
enhanced = tensor * (1.0 - strength) + enhanced * strength enhanced = tensor * (1.0 - strength) + enhanced * strength
enhanced = torch.clamp(enhanced, 0.0, 1.0) enhanced = torch.clamp(enhanced, 0.0, 1.0)
out = (enhanced.squeeze(0).permute(1, 2, 0).cpu().numpy() * 255.0).astype( out = (enhanced.squeeze(0).permute(1, 2, 0).cpu().numpy() * 255.0).astype(np.uint8)
np.uint8
)
return Image.fromarray(out) return Image.fromarray(out)
-1
View File
@@ -2,7 +2,6 @@ from __future__ import annotations
import argparse import argparse
import sys import sys
from pathlib import Path
from imagepipeline.modules.registry import list_modules from imagepipeline.modules.registry import list_modules
+1 -1
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
from dataclasses import dataclass, field from dataclasses import dataclass, field
from pathlib import Path from pathlib import Path
from typing import Any, TYPE_CHECKING from typing import TYPE_CHECKING, Any
if TYPE_CHECKING: if TYPE_CHECKING:
from imagepipeline.core.log import PipelineLogger from imagepipeline.core.log import PipelineLogger
+3 -9
View File
@@ -36,23 +36,17 @@ class PipelineLogger:
inputs: list[str], inputs: list[str],
params: dict, params: dict,
) -> None: ) -> None:
self.info( self.info(f"Step {step_index}/{step_total}: {step_id} ({module_name})")
f"Step {step_index}/{step_total}: {step_id} ({module_name})"
)
self.info(f" inputs: {', '.join(inputs)}") self.info(f" inputs: {', '.join(inputs)}")
if params: if params:
rendered = ", ".join(f"{key}={value!r}" for key, value in params.items()) rendered = ", ".join(f"{key}={value!r}" for key, value in params.items())
self.info(f" params: {rendered}") self.info(f" params: {rendered}")
def image(self, module_name: str, index: int, total: int, filename: str) -> None: def image(self, module_name: str, index: int, total: int, filename: str) -> None:
self.info( self.info(f" Applying module {module_name} to image [{index}/{total}]: {filename}")
f" Applying module {module_name} to image [{index}/{total}]: {filename}"
)
def skipped(self, module_name: str, index: int, total: int, filename: str) -> None: def skipped(self, module_name: str, index: int, total: int, filename: str) -> None:
self.info( self.info(f" Skipped module {module_name} [{index}/{total}] {filename} (output exists)")
f" Skipped module {module_name} [{index}/{total}] {filename} (output exists)"
)
def image_done( def image_done(
self, self,
+2 -2
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
import json import json
from dataclasses import asdict, dataclass, field from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone from datetime import UTC, datetime
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
@@ -39,4 +39,4 @@ def write_manifest(path: Path, manifest: PipelineManifest) -> None:
def utc_now_iso() -> str: def utc_now_iso() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat() return datetime.now(UTC).replace(microsecond=0).isoformat()
+3 -7
View File
@@ -1,6 +1,6 @@
from __future__ import annotations from __future__ import annotations
from dataclasses import dataclass, field from dataclasses import dataclass
from typing import Any from typing import Any
@@ -22,9 +22,7 @@ class Param:
if self.choices is not None and value not in self.choices: if self.choices is not None and value not in self.choices:
allowed = ", ".join(repr(c) for c in self.choices) allowed = ", ".join(repr(c) for c in self.choices)
raise ValueError( raise ValueError(f"Parameter '{name}' must be one of [{allowed}], got {value!r}")
f"Parameter '{name}' must be one of [{allowed}], got {value!r}"
)
if self.type == "string": if self.type == "string":
if not isinstance(value, str): if not isinstance(value, str):
@@ -56,9 +54,7 @@ class Param:
raise ValueError(f"Unknown parameter type '{self.type}' for '{name}'") raise ValueError(f"Unknown parameter type '{self.type}' for '{name}'")
def validate_params( def validate_params(schema: dict[str, Param], raw: dict[str, Any]) -> dict[str, Any]:
schema: dict[str, Param], raw: dict[str, Any]
) -> dict[str, Any]:
unknown = set(raw) - set(schema) unknown = set(raw) - set(schema)
if unknown: if unknown:
names = ", ".join(sorted(unknown)) names = ", ".join(sorted(unknown))
+5 -13
View File
@@ -49,12 +49,9 @@ class PipelineRunner:
self.symlink_input = symlink_input self.symlink_input = symlink_input
self.logger = PipelineLogger(verbose=verbose) self.logger = PipelineLogger(verbose=verbose)
self.existing_outputs = { self.existing_outputs = {
key: Path(value).resolve() key: Path(value).resolve() for key, value in (existing_outputs or {}).items()
for key, value in (existing_outputs or {}).items()
} }
self.continue_from = ( self.continue_from = Path(continue_from).resolve() if continue_from is not None else None
Path(continue_from).resolve() if continue_from is not None else None
)
self.skip_completed = skip_completed self.skip_completed = skip_completed
self.output_root = self._build_output_root() self.output_root = self._build_output_root()
self._input_link_dir = self.output_root / "input" self._input_link_dir = self.output_root / "input"
@@ -63,9 +60,7 @@ class PipelineRunner:
def _build_output_root(self) -> Path: def _build_output_root(self) -> Path:
if self.continue_from is not None: if self.continue_from is not None:
if not self.continue_from.is_dir(): if not self.continue_from.is_dir():
raise ValidationError( raise ValidationError(f"continue_from directory not found: {self.continue_from}")
f"continue_from directory not found: {self.continue_from}"
)
return self.continue_from return self.continue_from
timestamp = datetime.now().strftime("%y%m%d%H%M%S") timestamp = datetime.now().strftime("%y%m%d%H%M%S")
@@ -249,8 +244,7 @@ class PipelineRunner:
output_paths = step.module().list_output_images(ctx) output_paths = step.module().list_output_images(ctx)
if not output_paths: if not output_paths:
raise StepError( raise StepError(
f"Step '{step.output_dir_name}' ({step.module_name}) " f"Step '{step.output_dir_name}' ({step.module_name}) produced no output images"
"produced no output images"
) )
self.logger.step_done(step.output_dir_name, step.output_dir_name, len(output_paths)) self.logger.step_done(step.output_dir_name, step.output_dir_name, len(output_paths))
@@ -279,9 +273,7 @@ class PipelineRunner:
raise ValidationError(f"Step '{step.step_id}' references unknown step '{dep}'") raise ValidationError(f"Step '{step.step_id}' references unknown step '{dep}'")
dependents[dep].append(step.step_id) dependents[dep].append(step.step_id)
queue = deque( queue = deque(step_id for step_id, degree in in_degree.items() if degree == 0)
step_id for step_id, degree in in_degree.items() if degree == 0
)
ordered_ids: list[str] = [] ordered_ids: list[str] = []
while queue: while queue:
+1 -1
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
from dataclasses import dataclass, field from dataclasses import dataclass, field
from pathlib import Path from pathlib import Path
from typing import Any, TYPE_CHECKING from typing import TYPE_CHECKING, Any
if TYPE_CHECKING: if TYPE_CHECKING:
from imagepipeline.modules.base import BaseModule from imagepipeline.modules.base import BaseModule
+1 -1
View File
@@ -2,8 +2,8 @@
import imagepipeline.modules.ai_exposure # noqa: F401 import imagepipeline.modules.ai_exposure # noqa: F401
import imagepipeline.modules.ai_tone_map # noqa: F401 import imagepipeline.modules.ai_tone_map # noqa: F401
import imagepipeline.modules.comfy_flux_edit # noqa: F401
import imagepipeline.modules.color_to_alpha # noqa: F401 import imagepipeline.modules.color_to_alpha # noqa: F401
import imagepipeline.modules.comfy_flux_edit # noqa: F401
import imagepipeline.modules.composite # noqa: F401 import imagepipeline.modules.composite # noqa: F401
import imagepipeline.modules.crop_square # noqa: F401 import imagepipeline.modules.crop_square # noqa: F401
import imagepipeline.modules.darktable_style # noqa: F401 import imagepipeline.modules.darktable_style # noqa: F401
+1 -3
View File
@@ -115,9 +115,7 @@ class AIModule(BaseModule):
avg = sum(elapsed_times) / len(elapsed_times) avg = sum(elapsed_times) / len(elapsed_times)
eta = avg * remaining if remaining else 0.0 eta = avg * remaining if remaining else 0.0
if ctx.logger is not None: if ctx.logger is not None:
ctx.logger.image_done( ctx.logger.image_done(self.name, index, total, dst.name, elapsed, eta_seconds=eta)
self.name, index, total, dst.name, elapsed, eta_seconds=eta
)
@staticmethod @staticmethod
def _image_size(path: Path) -> tuple[int, int]: def _image_size(path: Path) -> tuple[int, int]:
+2 -6
View File
@@ -24,8 +24,7 @@ def build_color_to_alpha_args(*, color: str, fuzz: float) -> list[str]:
class ColorToAlphaModule(SubprocessModule): class ColorToAlphaModule(SubprocessModule):
name = "color_to_alpha" name = "color_to_alpha"
description = ( description = (
"Make a solid color transparent (GIMP-style color to alpha). " "Make a solid color transparent (GIMP-style color to alpha). Outputs PNG with alpha."
"Outputs PNG with alpha."
) )
command_candidates = ("magick", "convert") command_candidates = ("magick", "convert")
@@ -50,10 +49,7 @@ class ColorToAlphaModule(SubprocessModule):
"fuzz": Param( "fuzz": Param(
"float", "float",
default=0.0, default=0.0,
help=( help=("Match tolerance in percent (ImageMagick -fuzz); 0 = exact color only"),
"Match tolerance in percent (ImageMagick -fuzz); "
"0 = exact color only"
),
), ),
} }
+3 -9
View File
@@ -76,9 +76,7 @@ class ComfyFluxEditModule(AIModule):
server_url = ctx.params["server_url"].rstrip("/") server_url = ctx.params["server_url"].rstrip("/")
self._ensure_server(server_url) self._ensure_server(server_url)
if ctx.logger is not None: if ctx.logger is not None:
ctx.logger.warning( ctx.logger.warning("ComfyUI on CPU: expect hours per full-resolution image")
"ComfyUI on CPU: expect hours per full-resolution image"
)
workflow_template = json.loads(workflow_path.read_text(encoding="utf-8")) workflow_template = json.loads(workflow_path.read_text(encoding="utf-8"))
denoise = ctx.params["denoise"] denoise = ctx.params["denoise"]
@@ -98,9 +96,7 @@ class ComfyFluxEditModule(AIModule):
seed=seed, seed=seed,
) )
prompt_id = self._queue_prompt(server_url, workflow) prompt_id = self._queue_prompt(server_url, workflow)
output_info = self._wait_for_output( output_info = self._wait_for_output(server_url, prompt_id, poll_interval=poll_interval)
server_url, prompt_id, poll_interval=poll_interval
)
image_bytes = self._download_view(server_url, output_info) image_bytes = self._download_view(server_url, output_info)
dst.write_bytes(image_bytes) dst.write_bytes(image_bytes)
@@ -111,9 +107,7 @@ class ComfyFluxEditModule(AIModule):
try: try:
urllib.request.urlopen(f"{server_url}/system_stats", timeout=5) urllib.request.urlopen(f"{server_url}/system_stats", timeout=5)
except urllib.error.URLError as exc: except urllib.error.URLError as exc:
raise DependencyError( raise DependencyError(f"ComfyUI server not reachable at {server_url}: {exc}") from exc
f"ComfyUI server not reachable at {server_url}: {exc}"
) from exc
@staticmethod @staticmethod
def _upload_image(server_url: str, src: Path) -> str: def _upload_image(server_url: str, src: Path) -> str:
+1 -3
View File
@@ -63,9 +63,7 @@ class CompositeModule(SubprocessModule):
for index, group in enumerate(ctx.matched_groups, start=1): for index, group in enumerate(ctx.matched_groups, start=1):
if len(group) < 2: if len(group) < 2:
raise ValueError( raise ValueError("composite requires at least two input sources per image")
"composite requires at least two input sources per image"
)
background, foreground = group[0], group[1] background, foreground = group[0], group[1]
self.log_image(ctx, index, total, foreground) self.log_image(ctx, index, total, foreground)
+1 -3
View File
@@ -50,9 +50,7 @@ def build_fill_arguments(
@register @register
class ImageMagickFillModule(SubprocessModule): class ImageMagickFillModule(SubprocessModule):
name = "imagemagick_fill" name = "imagemagick_fill"
description = ( description = "Create solid-color or gradient images sized to match each input image"
"Create solid-color or gradient images sized to match each input image"
)
command_candidates = ("magick", "convert") command_candidates = ("magick", "convert")
@classmethod @classmethod
@@ -12,9 +12,7 @@ from imagepipeline.utils.subprocess import run_command
@register @register
class ImageMagickScaleCropModule(SubprocessModule): class ImageMagickScaleCropModule(SubprocessModule):
name = "imagemagick_scale_crop" name = "imagemagick_scale_crop"
description = ( description = "Scale an image then center-crop back to its original dimensions"
"Scale an image then center-crop back to its original dimensions"
)
command_candidates = ("magick", "convert") command_candidates = ("magick", "convert")
@classmethod @classmethod
+6 -18
View File
@@ -70,9 +70,7 @@ class OpenRouterEditModule(AIModule):
@classmethod @classmethod
def check_dependencies(cls) -> None: def check_dependencies(cls) -> None:
if not os.environ.get("OPENROUTER_API_KEY"): if not os.environ.get("OPENROUTER_API_KEY"):
raise DependencyError( raise DependencyError("OPENROUTER_API_KEY environment variable is not set")
"OPENROUTER_API_KEY environment variable is not set"
)
def run(self, ctx: ModuleContext) -> None: def run(self, ctx: ModuleContext) -> None:
api_key_env = ctx.params["api_key_env"] api_key_env = ctx.params["api_key_env"]
@@ -155,17 +153,11 @@ class OpenRouterEditModule(AIModule):
*, *,
template_data_url: str | None = None, template_data_url: str | None = None,
) -> dict: ) -> dict:
full_prompt = ( full_prompt = f"{TEMPLATE_PROMPT_PREFIX}{prompt}" if template_data_url else prompt
f"{TEMPLATE_PROMPT_PREFIX}{prompt}" if template_data_url else prompt
)
content: list[dict] = [{"type": "text", "text": full_prompt}] content: list[dict] = [{"type": "text", "text": full_prompt}]
if template_data_url is not None: if template_data_url is not None:
content.append( content.append({"type": "image_url", "image_url": {"url": template_data_url}})
{"type": "image_url", "image_url": {"url": template_data_url}} content.append({"type": "image_url", "image_url": {"url": source_data_url}})
)
content.append(
{"type": "image_url", "image_url": {"url": source_data_url}}
)
payload: dict = { payload: dict = {
"model": model, "model": model,
"modalities": cls._modalities_for_model(model), "modalities": cls._modalities_for_model(model),
@@ -176,9 +168,7 @@ class OpenRouterEditModule(AIModule):
return payload return payload
@classmethod @classmethod
def _save_result_matching_source( def _save_result_matching_source(cls, source: Path, result_bytes: bytes, dest: Path) -> None:
cls, source: Path, result_bytes: bytes, dest: Path
) -> None:
with Image.open(source) as original: with Image.open(source) as original:
orig_format = original.format orig_format = original.format
orig_size = original.size orig_size = original.size
@@ -229,9 +219,7 @@ class OpenRouterEditModule(AIModule):
return json.loads(response.read().decode("utf-8")) return json.loads(response.read().decode("utf-8"))
except urllib.error.HTTPError as exc: except urllib.error.HTTPError as exc:
detail = exc.read().decode("utf-8", errors="replace") detail = exc.read().decode("utf-8", errors="replace")
raise RuntimeError( raise RuntimeError(f"OpenRouter API error ({exc.code}): {detail}") from exc
f"OpenRouter API error ({exc.code}): {detail}"
) from exc
@staticmethod @staticmethod
def _extract_image_bytes(response: dict) -> bytes: def _extract_image_bytes(response: dict) -> bytes:
+2 -5
View File
@@ -18,9 +18,7 @@ def _layer_name(ref: str) -> str:
@register @register
class XcfStackModule(SubprocessModule): class XcfStackModule(SubprocessModule):
name = "xcf_stack" name = "xcf_stack"
description = ( description = "Stack listed pipeline step outputs as GIMP layers into one XCF per image"
"Stack listed pipeline step outputs as GIMP layers into one XCF per image"
)
command_candidates = ("gimp-console", "gimp") command_candidates = ("gimp-console", "gimp")
@classmethod @classmethod
@@ -80,8 +78,7 @@ class XcfStackModule(SubprocessModule):
if not layers: if not layers:
checked = ", ".join(name for name, _ in ctx.input_layer_dirs) checked = ", ".join(name for name, _ in ctx.input_layer_dirs)
raise ValueError( raise ValueError(
f"xcf_stack: no layers found for stem '{stem}' " f"xcf_stack: no layers found for stem '{stem}' (checked: {checked})"
f"(checked: {checked})"
) )
dst = ctx.output_dir / f"{stem}.xcf" dst = ctx.output_dir / f"{stem}.xcf"
+1 -2
View File
@@ -45,8 +45,7 @@ def match_by_stem(sources: list[list[Path]]) -> list[list[Path]]:
key = stem_key(path) key = stem_key(path)
if key in mapping: if key in mapping:
raise ValueError( raise ValueError(
f"Duplicate stem '{key}' in {path.parent}: " f"Duplicate stem '{key}' in {path.parent}: {mapping[key].name} and {path.name}"
f"{mapping[key].name} and {path.name}"
) )
mapping[key] = path mapping[key] = path
key_maps.append(mapping) key_maps.append(mapping)
+3 -8
View File
@@ -91,9 +91,7 @@ def _run_gimp_batch(
stdout=stdout or "", stdout=stdout or "",
stderr=stderr or "", stderr=stderr or "",
) )
raise RuntimeError( raise RuntimeError(f"Command timed out after {timeout}s: {' '.join(cmd)}") from exc
f"Command timed out after {timeout}s: {' '.join(cmd)}"
) from exc
return subprocess.CompletedProcess( return subprocess.CompletedProcess(
cmd, cmd,
@@ -122,9 +120,7 @@ def stack_images_to_xcf(
for layer_name, image_path in layers: for layer_name, image_path in layers:
resolved = image_path.resolve() resolved = image_path.resolve()
if not resolved.is_file(): if not resolved.is_file():
raise FileNotFoundError( raise FileNotFoundError(f"Layer image not found for '{layer_name}': {resolved}")
f"Layer image not found for '{layer_name}': {resolved}"
)
resolved_layers.append((layer_name, resolved)) resolved_layers.append((layer_name, resolved))
outfile = outfile.resolve() outfile = outfile.resolve()
@@ -159,8 +155,7 @@ def stack_images_to_xcf(
detail = stderr or stdout or f"exit code {result.returncode}" detail = stderr or stdout or f"exit code {result.returncode}"
layer_summary = ", ".join(name for name, _ in resolved_layers) layer_summary = ", ".join(name for name, _ in resolved_layers)
raise RuntimeError( raise RuntimeError(
f"GIMP failed to stack layers [{layer_summary}] into {outfile}: " f"GIMP failed to stack layers [{layer_summary}] into {outfile}: {detail}"
f"{detail}"
) )
finally: finally:
script_path.unlink(missing_ok=True) script_path.unlink(missing_ok=True)
+1 -3
View File
@@ -38,6 +38,4 @@ def finalize_gmic_output(output_dir: Path, intended: Path) -> Path:
frame_000000.rename(intended) frame_000000.rename(intended)
return intended return intended
raise FileNotFoundError( raise FileNotFoundError(f"G'MIC produced no output for {intended.name} in {output_dir}")
f"G'MIC produced no output for {intended.name} in {output_dir}"
)
+2 -6
View File
@@ -32,13 +32,9 @@ def run_command(
stderr = (exc.stderr or "").strip() stderr = (exc.stderr or "").strip()
stdout = (exc.stdout or "").strip() stdout = (exc.stdout or "").strip()
detail = stderr or stdout or str(exc) detail = stderr or stdout or str(exc)
raise RuntimeError( raise RuntimeError(f"Command failed ({exc.returncode}): {' '.join(cmd)}\n{detail}") from exc
f"Command failed ({exc.returncode}): {' '.join(cmd)}\n{detail}"
) from exc
except subprocess.TimeoutExpired as exc: except subprocess.TimeoutExpired as exc:
raise RuntimeError( raise RuntimeError(f"Command timed out after {timeout}s: {' '.join(cmd)}") from exc
f"Command timed out after {timeout}s: {' '.join(cmd)}"
) from exc
def require_command(*names: str) -> str: def require_command(*names: str) -> str:
+2 -1
View File
@@ -18,8 +18,9 @@ def main() -> None:
input_dir=INPUT, input_dir=INPUT,
output_base=OUTPUT_BASE, output_base=OUTPUT_BASE,
) as p: ) as p:
gray = p.step("imagemagick_grayscale", inputs="input") p.step("imagemagick_grayscale", inputs="input")
# Chain another step on the result: # Chain another step on the result:
# gray = p.step("imagemagick_grayscale", inputs="input")
# p.step("imagemagick_grayscale", inputs=gray, colorspace="Gray") # p.step("imagemagick_grayscale", inputs=gray, colorspace="Gray")
output_root = p.run() output_root = p.run()
+5 -14
View File
@@ -37,8 +37,7 @@ GMIC_CRAYONGRAFFITI = "-fx_crayongraffiti2 300,50,1,0.4,12,1,2,2,0"
GMIC_ANAGLYPH = "-fx_stereo_to_anaglyph 2,0" GMIC_ANAGLYPH = "-fx_stereo_to_anaglyph 2,0"
GMIC_DROP_SHADOW = "-fx_drop_shadow3d 0,0,0,10,1,1,2,0.5,194,37,24,200,0" GMIC_DROP_SHADOW = "-fx_drop_shadow3d 0,0,0,10,1,1,2,0.5,194,37,24,200,0"
GMIC_BOKEH = ( GMIC_BOKEH = (
f"-fx_bokeh 3,5,0,30,8,4,0.3,0.2,36,74,137,{ALPHA1},0.7,30,20,20,1,2," f"-fx_bokeh 3,5,0,30,8,4,0.3,0.2,36,74,137,{ALPHA1},0.7,30,20,20,1,2,194,37,24,{ALPHA2},0.15"
f"194,37,24,{ALPHA2},0.15"
) )
@@ -71,9 +70,7 @@ def main() -> None:
) )
# recipe: gmic-edges-rembg # recipe: gmic-edges-rembg
rembg_edges = p.step( rembg_edges = p.step("gmic", inputs=rembg_out, command=GMIC_EDGES, step_id="rembg_edges")
"gmic", inputs=rembg_out, command=GMIC_EDGES, step_id="rembg_edges"
)
composite_edges = p.step( composite_edges = p.step(
"composite", inputs=[rembg_edges, rembg_out], step_id="composite_edges" "composite", inputs=[rembg_edges, rembg_out], step_id="composite_edges"
) )
@@ -92,9 +89,7 @@ def main() -> None:
) )
# recipe: gmic-neon-rembg # recipe: gmic-neon-rembg
rembg_neon = p.step( rembg_neon = p.step("gmic", inputs=rembg_out, command=GMIC_NEON, step_id="rembg_neon")
"gmic", inputs=rembg_out, command=GMIC_NEON, step_id="rembg_neon"
)
composite_neon = p.step( composite_neon = p.step(
"composite", inputs=[rembg_neon, rembg_out], step_id="composite_neon" "composite", inputs=[rembg_neon, rembg_out], step_id="composite_neon"
) )
@@ -248,15 +243,11 @@ def main() -> None:
input_bokeh = p.step( input_bokeh = p.step(
"gmic", inputs=input_tone_map, command=GMIC_BOKEH, step_id="input_bokeh" "gmic", inputs=input_tone_map, command=GMIC_BOKEH, step_id="input_bokeh"
) )
bokeh_mid = p.step( bokeh_mid = p.step("composite", inputs=[input_tone_map, input_bokeh], step_id="bokeh_mid")
"composite", inputs=[input_tone_map, input_bokeh], step_id="bokeh_mid"
)
p.step("composite", inputs=[bokeh_mid, rembg_out], step_id="composite_bokeh") p.step("composite", inputs=[bokeh_mid, rembg_out], step_id="composite_bokeh")
# recipe: color-bg-drop-shadow-rembg # recipe: color-bg-drop-shadow-rembg
color_bg = p.step( color_bg = p.step("imagemagick_fill", inputs="input", color1=COLOR1, step_id="color_bg")
"imagemagick_fill", inputs="input", color1=COLOR1, step_id="color_bg"
)
rembg_shadow = p.step( rembg_shadow = p.step(
"gmic", inputs=rembg_out, command=GMIC_DROP_SHADOW, step_id="rembg_shadow" "gmic", inputs=rembg_out, command=GMIC_DROP_SHADOW, step_id="rembg_shadow"
) )
+2 -6
View File
@@ -40,14 +40,10 @@ def main() -> None:
gmic_shadow = p.step("gmic", inputs=rembg, command=GMIC_DROP_SHADOW) gmic_shadow = p.step("gmic", inputs=rembg, command=GMIC_DROP_SHADOW)
gmic_smooth = p.step("gmic", inputs=rembg, command=GMIC_JPR_SMOOTH) gmic_smooth = p.step("gmic", inputs=rembg, command=GMIC_JPR_SMOOTH)
gmic_stereo_alpha = p.step( gmic_stereo_alpha = p.step("color_to_alpha", inputs=gmic_stereo, color="#000000")
"color_to_alpha", inputs=gmic_stereo, color="#000000"
)
yellow_bg = p.step("imagemagick_fill", inputs="input", color1=YELLOW) yellow_bg = p.step("imagemagick_fill", inputs="input", color1=YELLOW)
gmic_smooth_alpha = p.step( gmic_smooth_alpha = p.step("color_to_alpha", inputs=gmic_smooth, color="#7f7f7f")
"color_to_alpha", inputs=gmic_smooth, color="#7f7f7f"
)
gmic_smooth_sized = p.step( gmic_smooth_sized = p.step(
"imagemagick_scale_crop", "imagemagick_scale_crop",
inputs=gmic_smooth_alpha, inputs=gmic_smooth_alpha,
@@ -6,7 +6,9 @@ from pathlib import Path
from imagepipeline import Pipeline from imagepipeline import Pipeline
# Darktable export folder. # Darktable export folder.
INPUT = Path("/home/frank/pics/20260517_Albershausen Crusaders - Biberach Beavers/darktable_exported/png") INPUT = Path(
"/home/frank/pics/20260517_Albershausen Crusaders - Biberach Beavers/darktable_exported/png"
)
# Where timestamped run folders are created. # Where timestamped run folders are created.
OUTPUT_BASE = Path.home() / "pipeline_output" OUTPUT_BASE = Path.home() / "pipeline_output"
+5 -7
View File
@@ -5,7 +5,9 @@ from pathlib import Path
from imagepipeline import Pipeline from imagepipeline import Pipeline
INPUT = Path("/home/frank/pics/20260620_Albershausen Crusaders - Montabaur Fighting Farmers/darktable_exported") INPUT = Path(
"/home/frank/pics/20260620_Albershausen Crusaders - Montabaur Fighting Farmers/darktable_exported"
)
OUTPUT_BASE = Path.home() / "pipeline_output" OUTPUT_BASE = Path.home() / "pipeline_output"
# Reuse outputs from a previous run or external folder (key = step id, e.g. rembg_01). # Reuse outputs from a previous run or external folder (key = step id, e.g. rembg_01).
@@ -80,13 +82,9 @@ def main() -> None:
scale=1.05, scale=1.05,
) )
rembg_stereo_alpha = p.step( rembg_stereo_alpha = p.step("color_to_alpha", inputs=rembg_stereo, color="#000000")
"color_to_alpha", inputs=rembg_stereo, color="#000000"
)
color_bg = p.step("imagemagick_fill", inputs="input", color1=COLOR1) color_bg = p.step("imagemagick_fill", inputs="input", color1=COLOR1)
rembg_smooth_alpha = p.step( rembg_smooth_alpha = p.step("color_to_alpha", inputs=rembg_jpr_smooth, color="#7f7f7f")
"color_to_alpha", inputs=rembg_jpr_smooth, color="#7f7f7f"
)
rembg_smooth_sized = p.step( rembg_smooth_sized = p.step(
"imagemagick_scale_crop", "imagemagick_scale_crop",
inputs=rembg_smooth_alpha, inputs=rembg_smooth_alpha,
+1
View File
@@ -35,6 +35,7 @@ GMIC_GRADIENT_B = (
"255,255,128,128,128,255,255,0,255,255,0,0,0,0" "255,255,128,128,128,255,255,0,255,255,0,0,0,0"
) )
def main() -> None: def main() -> None:
with Pipeline( with Pipeline(
name="orange", name="orange",
+3 -8
View File
@@ -22,8 +22,7 @@ ALPHA2 = 110
GMIC_DROP_SHADOW = "-fx_drop_shadow3d 0,0,0,10,1,1,2,0.5,240,16,0,200,0" GMIC_DROP_SHADOW = "-fx_drop_shadow3d 0,0,0,10,1,1,2,0.5,240,16,0,200,0"
GMIC_JPR_SMOOTH = "-jpr_gradient_smooth 0,1.5" GMIC_JPR_SMOOTH = "-jpr_gradient_smooth 0,1.5"
GMIC_BOKEH = ( GMIC_BOKEH = (
f"-fx_bokeh 3,5,0,30,8,4,0.3,0.2,240,176,32,{ALPHA1},0.7,30,20,20,1,2," f"-fx_bokeh 3,5,0,30,8,4,0.3,0.2,240,176,32,{ALPHA1},0.7,30,20,20,1,2,240,16,0,{ALPHA2},0.15"
f"240,16,0,{ALPHA2},0.15"
) )
@@ -59,9 +58,7 @@ def main() -> None:
scale=1.05, scale=1.05,
step_id="rembg_jpr_smooth_sized", step_id="rembg_jpr_smooth_sized",
) )
input_bokeh = p.step( input_bokeh = p.step("gmic", inputs="input", command=GMIC_BOKEH, step_id="input_bokeh")
"gmic", inputs="input", command=GMIC_BOKEH, step_id="input_bokeh"
)
# recipe: colorsplash # recipe: colorsplash
composite_colorsplash = p.step( composite_colorsplash = p.step(
@@ -74,9 +71,7 @@ def main() -> None:
) )
# recipe: original-drop-shadow-rembg # recipe: original-drop-shadow-rembg
shadow_mid = p.step( shadow_mid = p.step("composite", inputs=["input", rembg_shadow], step_id="shadow_mid")
"composite", inputs=["input", rembg_shadow], step_id="shadow_mid"
)
composite_shadow = p.step( composite_shadow = p.step(
"composite", inputs=[shadow_mid, rembg_out], step_id="composite_shadow" "composite", inputs=[shadow_mid, rembg_out], step_id="composite_shadow"
) )
+34 -3
View File
@@ -10,11 +10,17 @@ readme = "README.md"
requires-python = ">=3.11" requires-python = ">=3.11"
license = { text = "MIT" } license = { text = "MIT" }
authors = [{ name = "Frank" }] authors = [{ name = "Frank" }]
dependencies = [] dependencies = [
"Pillow>=10.0",
]
[project.optional-dependencies] [project.optional-dependencies]
ai = ["numpy>=1.26", "Pillow>=10.0", "torch>=2.0"] ai = ["numpy>=1.26", "torch>=2.0"]
dev = ["pytest>=8.0"] dev = [
"pytest>=8.0",
"numpy>=1.26",
"ruff>=0.8",
]
[project.scripts] [project.scripts]
imagepipeline = "imagepipeline.cli:main" imagepipeline = "imagepipeline.cli:main"
@@ -25,3 +31,28 @@ include = ["imagepipeline*"]
[tool.pytest.ini_options] [tool.pytest.ini_options]
testpaths = ["tests"] testpaths = ["tests"]
markers = [
"integration: tests that call external CLI tools (ImageMagick, G'MIC, rembg, darktable, GIMP)",
"slow: tests that take more than a few seconds",
]
[tool.ruff]
target-version = "py311"
line-length = 100
src = ["imagepipeline", "tests", "pipelines"]
[tool.ruff.lint]
select = [
"E", # pycodestyle errors
"F", # pyflakes
"I", # isort
"UP", # pyupgrade
"B", # flake8-bugbear
]
ignore = [
"E501", # line length handled by formatter
"B027", # optional empty hooks on BaseModule
]
[tool.ruff.lint.isort]
known-first-party = ["imagepipeline"]
+2 -5
View File
@@ -20,10 +20,7 @@ def make_png(
crc = zlib.crc32(tag + data) & 0xFFFFFFFF crc = zlib.crc32(tag + data) & 0xFFFFFFFF
return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", crc) return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", crc)
raw = b"".join( raw = b"".join(b"\x00" + bytes([r, g, b] * width) for _ in range(height))
b"\x00" + bytes([r, g, b] * width)
for _ in range(height)
)
compressed = zlib.compress(raw, 9) compressed = zlib.compress(raw, 9)
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
png = ( png = (
@@ -37,7 +34,7 @@ def make_png(
@pytest.fixture(autouse=True) @pytest.fixture(autouse=True)
def _ensure_builtin_modules() -> None: def _ensure_builtin_modules() -> None:
import imagepipeline.modules # noqa: F401 pass # noqa: F401
@pytest.fixture @pytest.fixture
+1 -3
View File
@@ -110,9 +110,7 @@ class TestAIParameters:
assert payload["modalities"] == ["image"] assert payload["modalities"] == ["image"]
assert payload["image_config"] == {"strength": 0.25} assert payload["image_config"] == {"strength": 0.25}
def test_save_result_matching_source_preserves_png_size( def test_save_result_matching_source_preserves_png_size(self, tmp_path: Path) -> None:
self, tmp_path: Path
) -> None:
try: try:
from PIL import Image from PIL import Image
except ImportError: except ImportError:
+13 -5
View File
@@ -113,9 +113,11 @@ class TestPipelineRunner:
for src in ctx.input_paths: for src in ctx.input_paths:
shutil.copy2(src, ctx.output_dir / src.name) shutil.copy2(src, ctx.output_dir / src.name)
with Pipeline(name="order_test", input_dir=input_dir, output_base=output_base, verbose=False) as p: with Pipeline(
name="order_test", input_dir=input_dir, output_base=output_base, verbose=False
) as p:
step_b = p.step("order_tracker", inputs="input") step_b = p.step("order_tracker", inputs="input")
step_a = p.step("order_tracker", inputs=step_b) p.step("order_tracker", inputs=step_b)
p.run() p.run()
assert order == ["order_tracker_01", "order_tracker_02"] assert order == ["order_tracker_01", "order_tracker_02"]
@@ -132,7 +134,9 @@ class TestPipelineRunner:
for src in ctx.input_paths: for src in ctx.input_paths:
shutil.copy2(src, ctx.output_dir / src.name) shutil.copy2(src, ctx.output_dir / src.name)
with Pipeline(name="dup_test", input_dir=input_dir, output_base=output_base, verbose=False) as p: with Pipeline(
name="dup_test", input_dir=input_dir, output_base=output_base, verbose=False
) as p:
first = p.step("number_tracker", inputs="input") first = p.step("number_tracker", inputs="input")
p.step("number_tracker", inputs=first) p.step("number_tracker", inputs=first)
root = p.run() root = p.run()
@@ -170,7 +174,9 @@ class TestCustomStepId:
for src in ctx.input_paths: for src in ctx.input_paths:
shutil.copy2(src, ctx.output_dir / src.name) shutil.copy2(src, ctx.output_dir / src.name)
with Pipeline(name="named_test", input_dir=input_dir, output_base=output_base, verbose=False) as p: with Pipeline(
name="named_test", input_dir=input_dir, output_base=output_base, verbose=False
) as p:
ref = p.step("named_tracker", inputs="input", step_id="input_bokeh") ref = p.step("named_tracker", inputs="input", step_id="input_bokeh")
root = p.run() root = p.run()
@@ -193,7 +199,9 @@ class TestCustomStepId:
for src in ctx.input_paths: for src in ctx.input_paths:
shutil.copy2(src, ctx.output_dir / src.name) shutil.copy2(src, ctx.output_dir / src.name)
with Pipeline(name="counter_test", input_dir=input_dir, output_base=output_base, verbose=False) as p: with Pipeline(
name="counter_test", input_dir=input_dir, output_base=output_base, verbose=False
) as p:
p.step("counter_tracker", inputs="input") p.step("counter_tracker", inputs="input")
p.step("counter_tracker", inputs="input", step_id="custom_mid") p.step("counter_tracker", inputs="input", step_id="custom_mid")
p.step("counter_tracker", inputs="input") p.step("counter_tracker", inputs="input")
+4 -5
View File
@@ -13,7 +13,6 @@ from imagepipeline.core.resume import (
) )
from imagepipeline.core.step import StepDefinition from imagepipeline.core.step import StepDefinition
from imagepipeline.modules.imagemagick_grayscale import ImageMagickGrayscale from imagepipeline.modules.imagemagick_grayscale import ImageMagickGrayscale
from imagepipeline.modules.registry import get_module
from imagepipeline.modules.rembg import RembgModule from imagepipeline.modules.rembg import RembgModule
from imagepipeline.utils.gmic import finalize_gmic_output, split_gmic_command from imagepipeline.utils.gmic import finalize_gmic_output, split_gmic_command
from tests.conftest import make_png from tests.conftest import make_png
@@ -124,7 +123,9 @@ class TestExpectedOutputFilenames:
class TestPipelineResume: class TestPipelineResume:
@pytest.mark.skipif(not shutil.which("magick"), reason="ImageMagick not installed") @pytest.mark.skipif(not shutil.which("magick"), reason="ImageMagick not installed")
def test_continue_skips_completed_steps(self, input_dir: Path, output_base: Path, capsys) -> None: def test_continue_skips_completed_steps(
self, input_dir: Path, output_base: Path, capsys
) -> None:
with Pipeline( with Pipeline(
name="resume_test", name="resume_test",
input_dir=input_dir, input_dir=input_dir,
@@ -195,9 +196,7 @@ class TestPipelineResume:
verbose=True, verbose=True,
existing_outputs={"input_bokeh": external}, existing_outputs={"input_bokeh": external},
) as p: ) as p:
reused = p.step( reused = p.step("imagemagick_grayscale", inputs="input", step_id="input_bokeh")
"imagemagick_grayscale", inputs="input", step_id="input_bokeh"
)
p.step("imagemagick_grayscale", inputs=reused) p.step("imagemagick_grayscale", inputs=reused)
root = p.run() root = p.run()
+2 -6
View File
@@ -119,9 +119,7 @@ def _make_stack_fixture(tmp_path: Path) -> dict[str, Path]:
class TestXcfStackRun: class TestXcfStackRun:
@patch("imagepipeline.modules.xcf_stack.stack_images_to_xcf") @patch("imagepipeline.modules.xcf_stack.stack_images_to_xcf")
def test_collects_layers_from_explicit_inputs( def test_collects_layers_from_explicit_inputs(self, mock_stack: object, tmp_path: Path) -> None:
self, mock_stack: object, tmp_path: Path
) -> None:
paths = _make_stack_fixture(tmp_path) paths = _make_stack_fixture(tmp_path)
input_path = paths["root"] / "refs" / "photo.jpg" input_path = paths["root"] / "refs" / "photo.jpg"
input_path.parent.mkdir() input_path.parent.mkdir()
@@ -152,9 +150,7 @@ class TestXcfStackRun:
] ]
@patch("imagepipeline.modules.xcf_stack.stack_images_to_xcf") @patch("imagepipeline.modules.xcf_stack.stack_images_to_xcf")
def test_skip_missing_true_skips_missing_step( def test_skip_missing_true_skips_missing_step(self, mock_stack: object, tmp_path: Path) -> None:
self, mock_stack: object, tmp_path: Path
) -> None:
paths = _make_stack_fixture(tmp_path) paths = _make_stack_fixture(tmp_path)
(paths["step_b"] / "photo.png").unlink() (paths["step_b"] / "photo.png").unlink()
input_path = paths["root"] / "photo.jpg" input_path = paths["root"] / "photo.jpg"