feat(preprocessing): Pillow resize and augmentation drivers

docs/content/api.md

@@ -568,7 +568,7 @@ sentence. The real message and traceback go to the server log under the same id
 greps one string, and a response body never becomes a channel for filesystem paths, SQL text, or
 a stack trace.
 
-Eight errors opt out and expose their real message, each because that message *is* the remedy:
+Nine errors opt out and expose their real message, each because that message *is* the remedy:
 
 | Code | Why the message is published |
 | --- | --- |
@@ -580,6 +580,7 @@ Eight errors opt out and expose their real message, each because that message *i
 | `INFERENCE_CONNECTION_NOT_RUNNABLE` | Says which of the two things nothing installed here can run - a recorded driver that is not installed, or a model family (declared by the config, or by the endpoint) no installed driver serves - and lists what is installed instead. A fact about the installation rather than about the request, and one that changes when a driver is installed. |
 | `INFERENCE_OUT_OF_MEMORY` | Names which memory ran out - the device's or the machine's - and the ways off it, which are not the same ways: a full device can be answered by moving the connection to the CPU, and a full machine is only made worse by it. No generic sentence can carry that. |
 | `INFERENCE_ENDPOINT_UNAVAILABLE` | Names the endpoint an `http` connection points at and what it did - unreachable, timed out, a bad status, or a body outside the contract - which is the whole remedy: look at the endpoint, not at this connection or this machine. |
+| `PREPROCESSING_DRIVER_NOT_FOUND` | Names the recipe step kind no installed driver applies and lists the kinds that are installed. A fact about the installation rather than about the request: the recipe grammar admits only the kinds this distribution ships drivers for, so reaching it means a plugin the build was made with is missing. |
 
 A **mapped** 5xx keeps its own code (`WORKSPACE_CORRUPT`, `CONSTRAINT_VIOLATED`). An exception no
 rule covers - a bug - gets `INTERNAL_ERROR`. That difference is how the two are told apart in a
@@ -614,7 +615,7 @@ argument for branching on `code`.
 | **422** | `VALIDATION_ERROR` · `ASSET_NOT_IN_BATCH` · `ANNOTATION_NOT_FROM_MODEL` · `INVALID_NAME` · `INFERENCE_CONNECTION_INVALID` · `INVALID_SCHEMA` · `UNSUPPORTED_GEOMETRY` · `INVALID_ANNOTATION` · `LABEL_CLASS_NOT_IN_SCHEMA` · `DISALLOWED_GEOMETRY` · `ANNOTATION_GEOMETRY_OUT_OF_BOUNDS` · `DUPLICATE_CLASSIFICATION_TAG` · `MISSING_REQUIRED_ATTRIBUTE` · `UNKNOWN_ATTRIBUTE` · `INVALID_ATTRIBUTE_VALUE` · `INVALID_PARTITION` · `UNKNOWN_JOB_TYPE` · `MEDIA_ERROR` · `UNSUPPORTED_MEDIA` · `CORRUPT_MEDIA` · `UNSUPPORTED_PROMPT` · `PROMPT_POINT_OUT_OF_BOUNDS` · `GEOMETRY_NOT_PRODUCED` |
 | **502** | `INFERENCE_ENDPOINT_UNAVAILABLE` |
 | **503** | `WORKSPACE_BUSY` |
-| **500** | `WORKSPACE_CORRUPT` · `NOT_A_WORKSPACE` · `WORKSPACE_FORMAT_TOO_NEW` · `WORKSPACE_SCHEMA_MISMATCH` · `ENTITY_NOT_FOUND` · `ENTITY_ALREADY_EXISTS` · `CONSTRAINT_VIOLATED` · `MEDIA_TOOL_UNAVAILABLE` · `LOCAL_INFERENCE_UNAVAILABLE` · `INFERENCE_CONNECTION_NOT_RUNNABLE` · `INFERENCE_OUT_OF_MEMORY` · `EXPORT_TARGET_CONFLICT` · `INVALID_EXPORT_TARGET` · `INTERNAL_ERROR` |
+| **500** | `WORKSPACE_CORRUPT` · `NOT_A_WORKSPACE` · `WORKSPACE_FORMAT_TOO_NEW` · `WORKSPACE_SCHEMA_MISMATCH` · `ENTITY_NOT_FOUND` · `ENTITY_ALREADY_EXISTS` · `CONSTRAINT_VIOLATED` · `MEDIA_TOOL_UNAVAILABLE` · `LOCAL_INFERENCE_UNAVAILABLE` · `INFERENCE_CONNECTION_NOT_RUNNABLE` · `INFERENCE_OUT_OF_MEMORY` · `EXPORT_TARGET_CONFLICT` · `INVALID_EXPORT_TARGET` · `PREPROCESSING_DRIVER_NOT_FOUND` · `INTERNAL_ERROR` |
 
 Every row but `VALIDATION_ERROR`, `NOT_FOUND`, `METHOD_NOT_ALLOWED`, `UNAUTHORIZED` and
 `INTERNAL_ERROR` — the five the framework and the auth guard raise — comes from `ERROR_RULES`

pyproject.toml

@@ -136,6 +136,13 @@ openlane-2d = "visionset.formats.lanes:OpenLane2dExporter"
 # Inference driver discovery. A third-party distribution registers its own
 # provider in this same group, and pins the VisionSet it was built for in its
 # ordinary `dependencies` — discovery reads that pin and skips a mismatch.
+# The pixel side of a pre-processing recipe, one driver per step kind. Discovered
+# the way exporters are, so a distribution can register a driver for a kind the
+# built-ins do not apply.
+[project.entry-points."visionset.preprocessing"]
+pillow-resize = "visionset.preprocessing.pillow:PillowResizeDriver"
+pillow-augment = "visionset.preprocessing.pillow:PillowAugmentDriver"
+
 [project.entry-points."visionset.providers"]
 sam = "visionset.inference.sam_provider:SamProvider"
 grounding-dino = "visionset.inference.transformers_provider:GroundingDinoProvider"
@@ -257,6 +264,10 @@ forbidden_modules = [
     # implementation. The kernel knows the configuration and the protocol; what
     # runs is on the other side of this line.
     "visionset.inference",
+    # The pre-processing driver registry and the Pillow drivers behind the
+    # `PreprocessingDriver` port. Same direction as `visionset.formats`: the kernel
+    # takes driver instances, and discovery stays on the other side of the line.
+    "visionset.preprocessing",
     "fastapi",
     "typer",
     "mcp",

src/visionset/kernel/__init__.py

@@ -68,6 +68,7 @@
     MissingRequiredAttribute,
     NoSplitRecipe,
     NotAWorkspace,
+    PreprocessingDriverNotFound,
     PreprocessingStepUnsupportedGeometry,
     ProjectNameTaken,
     ProjectNotFound,
@@ -152,6 +153,7 @@
     "MissingRequiredAttribute",
     "NoSplitRecipe",
     "NotAWorkspace",
+    "PreprocessingDriverNotFound",
     "PreprocessingStepUnsupportedGeometry",
     "ProjectNameTaken",
     "ProjectNotFound",

src/visionset/kernel/errors.py

@@ -1345,3 +1345,33 @@ def __init__(
             self.geometry = geometry
         if asset_id is not None:
             self.asset_id = asset_id
+
+
+class PreprocessingDriverNotFound(VisionSetError):
+    """No installed driver applies steps of that kind.
+
+    ``ExportFormatNotFound``'s shape for the pre-processing registry, which
+    lives in ``visionset.preprocessing`` for the same reason the format
+    registry lives outside the kernel: the kernel takes driver *instances* and
+    may not scan entry points itself. The class lives here anyway, with every
+    other refusal, so that no surface invents a second error shape for it.
+
+    Unlike a format, a step kind is not something a caller can mistype — the
+    recipe grammar admits only the kinds this distribution ships drivers for —
+    so reaching this means the installation is missing a plugin it was built
+    with. What is wrong is the machine, on ``MediaToolUnavailable``'s terms,
+    and the message names the kind and lists the drivers that are installed.
+    """
+
+    installed: tuple[str, ...] | None = None
+    """Every step kind an installed driver applies, sorted.
+
+    A class attribute with a ``None`` default and not a constructor parameter,
+    as ``ExportTargetNotFound.installed`` is, so this error stays constructible
+    from one message. The registry sets it as a keyword.
+    """
+
+    def __init__(self, message: str, *, installed: tuple[str, ...] | None = None) -> None:
+        super().__init__(message)
+        if installed is not None:
+            self.installed = installed

src/visionset/preprocessing/__init__.py

@@ -0,0 +1,7 @@
+"""Pre-processing drivers: the pixel side of a recipe, outside the kernel.
+
+The kernel owns the geometry of every recipe step and takes a
+:class:`~visionset.kernel.ports.PreprocessingDriver` *instance* per step kind;
+it may not scan entry points, so discovery lives here, beside the built-in
+Pillow drivers, the way ``visionset.formats`` sits beside its plugins.
+"""

src/visionset/preprocessing/pillow/__init__.py

@@ -0,0 +1,201 @@
+# usage: from visionset.preprocessing.pillow import PillowResizeDriver, PillowAugmentDriver
+"""The built-in drivers: Pillow doing the pixels for every v1 recipe step.
+
+Both drivers decode, orient, transform and re-encode; they never move a
+coordinate. The orientation step mirrors ingest — ``PillowImageProcessor``
+bakes the EXIF turn in before it measures, so the manifest's width and height
+already describe the oriented picture, and a driver that skipped the turn
+would resize a picture the kernel's geometry never saw. Where a letterbox
+lands and what a variant draws are read from the kernel, never recomputed.
+
+Re-encoding keeps the source's format: a JPEG comes back a JPEG at quality 95
+with its chroma subsampling kept, a PNG comes back a lossless PNG, and any
+other encoding comes back a PNG. No metadata travels — the EXIF that named
+the orientation is gone because the orientation is now in the pixels, and an
+ICC profile or a text chunk would be a second input to bytes that should
+depend on the pixels alone. Byte stability is promised within one environment
+only: the same Pillow, the same codecs.
+"""
+
+from __future__ import annotations
+
+import io
+from dataclasses import dataclass
+from typing import Final
+
+from PIL import Image, ImageEnhance, ImageOps
+from PIL.JpegImagePlugin import get_sampling
+
+from visionset.kernel.domain import (
+    AugmentOp,
+    AugmentStep,
+    ResizeStep,
+    ResizeStrategy,
+    Step,
+    brightness_contrast_factors,
+    hflip_applied,
+    letterbox_fit,
+    rot90_quarter_turns,
+)
+from visionset.kernel.errors import UnsupportedMedia
+
+JPEG_QUALITY: Final = 95
+
+#: The bytes a driver returns are always in one of these, keyed by Pillow's name.
+#: Anything else the decoder can open — an MPO is a JPEG container — is written
+#: as a PNG, the one lossless encoding every reader has.
+MEDIA_TYPES: Final[dict[str, str]] = {"JPEG": "image/jpeg", "PNG": "image/png"}
+
+#: Modes the transforms keep as they are. Everything else — palette, bilevel,
+#: 16-bit, CMYK — is converted first: Pillow silently resamples a palette image
+#: nearest-neighbour, and a letterbox canvas needs a mode a grey can be spelled in.
+_KEPT_MODES: Final = frozenset({"L", "LA", "RGB", "RGBA"})
+_ALPHA_MODES: Final = frozenset({"RGBA", "LA", "PA"})
+
+_QUARTER_TURNS: Final = {
+    1: Image.Transpose.ROTATE_90,
+    2: Image.Transpose.ROTATE_180,
+    3: Image.Transpose.ROTATE_270,
+}
+
+
+@dataclass(frozen=True)
+class _Decoded:
+    """An oriented, transform-ready image and what its bytes were encoded as."""
+
+    image: Image.Image
+    pillow_format: str
+    jpeg_sampling: int
+
+
+def _decode(data: bytes) -> _Decoded:
+    with Image.open(io.BytesIO(data)) as opened:
+        pillow_format = "JPEG" if opened.format == "MPO" else (opened.format or "")
+        opened.load()
+        sampling = get_sampling(opened) if pillow_format == "JPEG" else -1
+        ImageOps.exif_transpose(opened, in_place=True)
+        image = opened if opened.mode in _KEPT_MODES else _converted(opened)
+        image = image.copy() if image is opened else image
+    return _Decoded(image=image, pillow_format=pillow_format, jpeg_sampling=sampling)
+
+
+def _converted(image: Image.Image) -> Image.Image:
+    has_alpha = image.mode in _ALPHA_MODES or "transparency" in image.info
+    return image.convert("RGBA" if has_alpha else "RGB")
+
+
+def _encode(image: Image.Image, decoded: _Decoded) -> bytes:
+    buffer = io.BytesIO()
+    if decoded.pillow_format == "JPEG":
+        options: dict[str, object] = {"quality": JPEG_QUALITY}
+        if decoded.jpeg_sampling >= 0:
+            options["subsampling"] = decoded.jpeg_sampling
+        image.convert("RGB").save(buffer, format="JPEG", **options)
+    else:
+        image.save(buffer, format="PNG")
+    return buffer.getvalue()
+
+
+def media_type(data: bytes) -> str:
+    """The media type of bytes a driver returned, or would return for this source."""
+    with Image.open(io.BytesIO(data)) as opened:
+        pillow_format = "JPEG" if opened.format == "MPO" else (opened.format or "")
+    return MEDIA_TYPES.get(pillow_format, MEDIA_TYPES["PNG"])
+
+
+def _pad_colour(mode: str, pad_value: int) -> tuple[int, ...]:
+    bands = Image.getmodebands(mode)
+    if mode in _ALPHA_MODES:
+        return (pad_value,) * (bands - 1) + (255,)
+    return (pad_value,) * bands
+
+
+def _resample(source: tuple[int, int], target: tuple[int, int]) -> Image.Resampling:
+    downscaling = target[0] * target[1] < source[0] * source[1]
+    return Image.Resampling.LANCZOS if downscaling else Image.Resampling.BICUBIC
+
+
+def _resized(image: Image.Image, size: tuple[int, int]) -> Image.Image:
+    if size[0] < 1 or size[1] < 1:
+        raise UnsupportedMedia(
+            f"letterboxing a {image.width}×{image.height} image leaves no pixels on one "
+            f"side; choose a canvas closer to its aspect ratio or use the stretch strategy"
+        )
+    return image.resize(size, _resample(image.size, size), reducing_gap=None)
+
+
+class PillowResizeDriver:
+    """``resize`` steps: stretch to the size, or letterbox onto a padded canvas.
+
+    Stretch is one ``Image.resize`` per axis, LANCZOS when the pixel count
+    shrinks and BICUBIC when it grows. Letterbox reads ``letterbox_fit`` for
+    the content size and offset and pastes the resized content there on a
+    canvas filled with ``pad_value`` — the same numbers the kernel used to
+    place the annotations, so pixels and labels cannot disagree by a rounding.
+    Neither reads the seed or the variant: a resize is the same for every
+    variant of an image.
+    """
+
+    step_kinds: frozenset[str] = frozenset({"resize"})
+
+    def apply(self, step: Step, image: bytes, *, seed: bytes, variant: int) -> bytes:
+        """The image at ``step.width × step.height``, re-encoded in its own format.
+
+        Raises:
+            UnsupportedMedia: a letterbox whose aspect ratio rounds one side of
+                the content to zero pixels.
+        """
+        if not isinstance(step, ResizeStep):
+            raise TypeError(f"{type(self).__name__} applies resize steps, not {step.kind!r}")
+        decoded = _decode(image)
+        with decoded.image as source:
+            if step.strategy is ResizeStrategy.STRETCH:
+                result = _resized(source, (step.width, step.height))
+            else:
+                result = _letterboxed(source, step)
+        with result:
+            return _encode(result, decoded)
+
+
+def _letterboxed(source: Image.Image, step: ResizeStep) -> Image.Image:
+    fit = letterbox_fit(
+        source.width, source.height, target_width=step.width, target_height=step.height
+    )
+    canvas = Image.new(
+        source.mode, (step.width, step.height), _pad_colour(source.mode, step.pad_value)
+    )
+    with _resized(source, (fit.content_width, fit.content_height)) as content:
+        canvas.paste(content, (fit.offset_x, fit.offset_y))
+    return canvas
+
+
+class PillowAugmentDriver:
+    """``augment`` steps: mirror, brightness then contrast, or a quarter turn.
+
+    Every draw comes from the kernel — ``hflip_applied``,
+    ``brightness_contrast_factors``, ``rot90_quarter_turns`` — over the seed
+    the caller passes, so the pixels land where the geometry transform put the
+    labels. Variant 0 is the base image and is returned untouched apart from
+    orientation and re-encoding, which is what a step applied to it means.
+    """
+
+    step_kinds: frozenset[str] = frozenset({"augment"})
+
+    def apply(self, step: Step, image: bytes, *, seed: bytes, variant: int) -> bytes:
+        """The variant this seed draws for one augmentation, in the source's format."""
+        if not isinstance(step, AugmentStep):
+            raise TypeError(f"{type(self).__name__} applies augment steps, not {step.kind!r}")
+        decoded = _decode(image)
+        with decoded.image as source:
+            result = source if variant == 0 else _augmented(source, step, seed)
+            return _encode(result, decoded)
+
+
+def _augmented(source: Image.Image, step: AugmentStep, seed: bytes) -> Image.Image:
+    if step.op is AugmentOp.HFLIP:
+        return ImageOps.mirror(source) if hflip_applied(seed) else source
+    if step.op is AugmentOp.BRIGHTNESS_CONTRAST:
+        brightness, contrast = brightness_contrast_factors(seed, step.amount)
+        brightened = ImageEnhance.Brightness(source).enhance(brightness)
+        return ImageEnhance.Contrast(brightened).enhance(contrast)
+    return source.transpose(_QUARTER_TURNS[rot90_quarter_turns(seed)])

src/visionset/preprocessing/registry.py

@@ -0,0 +1,86 @@
+# usage: from visionset.preprocessing.registry import driver, drivers, driver_for
+"""Finding the driver that applies a recipe step.
+
+``visionset.formats.registry``'s shape, one port over. The kernel takes
+``PreprocessingDriver`` instances because import-linter forbids it from
+importing this package, so resolving a step *kind* to an implementation is
+work for whoever composed the call, and every surface reaches it here rather
+than keeping its own map.
+
+Discovery is ``importlib.metadata`` over the ``visionset.preprocessing``
+entry-point group, never a hardcoded dict: a third-party distribution registers
+into the same group and is indistinguishable from a built-in. What comes out
+is filtered by the port itself — ``isinstance`` on an instance, because
+``PreprocessingDriver`` is a ``@runtime_checkable`` protocol with a data
+member — and keyed by every step kind the driver declares.
+
+Nothing is cached. Entry points are read from installed metadata, so the cost
+is one scan and the alternative is a process that must be restarted after an
+install.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from importlib.metadata import entry_points
+
+from visionset.kernel.domain import Step
+from visionset.kernel.errors import PreprocessingDriverNotFound
+from visionset.kernel.ports import PreprocessingDriver
+
+GROUP = "visionset.preprocessing"
+
+
+def drivers() -> dict[str, PreprocessingDriver]:
+    """Every installed driver, keyed by each step kind it applies.
+
+    A driver declaring two kinds appears under both. Two drivers claiming one
+    kind resolve to whichever the scan met last, as two exporters sharing a
+    ``format_name`` do; the built-in pair claims one kind each.
+    """
+    found: dict[str, PreprocessingDriver] = {}
+    for entry_point in entry_points(group=GROUP):
+        plugin = entry_point.load()()
+        if isinstance(plugin, PreprocessingDriver):
+            for kind in plugin.step_kinds:
+                found[kind] = plugin
+    return found
+
+
+def pick(installed: Mapping[str, PreprocessingDriver], step_kind: str) -> PreprocessingDriver:
+    """One driver out of a set already in hand, or say none applies that kind.
+
+    Split from :func:`driver` so the refusal has one wording no matter who
+    scanned the entry points. A caller holding the mapping must not index it
+    directly: a ``KeyError`` is outside the ``VisionSetError`` tree and would
+    answer 500 with no message to a request the installation cannot serve.
+
+    Raises:
+        PreprocessingDriverNotFound: no installed driver applies ``step_kind``.
+    """
+    if step_kind not in installed:
+        known = tuple(sorted(installed))
+        raise PreprocessingDriverNotFound(
+            f"no pre-processing driver is installed for step kind {step_kind!r}; "
+            f"installed step kinds: {', '.join(known) or 'none'}",
+            installed=known,
+        )
+    return installed[step_kind]
+
+
+def driver_for(installed: Mapping[str, PreprocessingDriver], step: Step) -> PreprocessingDriver:
+    """The driver that applies this step, out of a set already in hand.
+
+    Raises:
+        PreprocessingDriverNotFound: no installed driver applies the step's kind.
+    """
+    return pick(installed, step.kind)
+
+
+def driver(step_kind: str) -> PreprocessingDriver:
+    """The installed driver for that step kind.
+
+    Raises:
+        PreprocessingDriverNotFound: no installed driver applies it.
+    """
+    return pick(drivers(), step_kind)