feat(generator): support arbitrary ucp refs and regenerate c5c6139 models

.github/workflows/test.yml

@@ -0,0 +1,38 @@
+#   Copyright 2026 Google LLC
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+name: Test
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  pytest:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+    steps:
+    - uses: actions/checkout@v4
+    - uses: astral-sh/setup-uv@v3
+    - uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: uv sync
+    - name: Run tests
+      run: uv run pytest -v

generate_models.sh

@@ -1,30 +1,65 @@
 #!/bin/bash
 # Generate Pydantic models from UCP JSON Schemas
 
+set -euo pipefail
+
 # Ensure we are in the script's directory
-cd "$(dirname "$0")" || exit
+cd "$(dirname "$0")"
 
 # Add ~/.local/bin to PATH for uv
 export PATH="$HOME/.local/bin:$PATH"
 
 # Check if git is installed
 if ! command -v git &> /dev/null; then
-    echo "Error: git not found. Please install git."
+    echo "Error: git not found. Please install git." >&2
     exit 1
 fi
 
-# UCP Version to use (if provided, use release/$1 branch; otherwise, use main)
-if [ -z "$1" ]; then
-    BRANCH="main"
+# UCP source selection — three modes:
+#   ./generate_models.sh                  # main branch
+#   ./generate_models.sh 2026-04-08       # release/2026-04-08 branch
+#   ./generate_models.sh --ref <sha|ref>  # arbitrary commit/branch/tag
+#
+# Identity-linking, info_code, and warning_code schemas merged after the
+# 2026-04-08 release branch was cut, so consumers needing those must pin
+# an explicit ref via --ref.
+REF_MODE=""
+REF_VALUE=""
+if [ "${1:-}" = "--ref" ]; then
+    if [ -z "${2:-}" ]; then
+        echo "Error: --ref requires a commit SHA, branch, or tag." >&2
+        exit 1
+    fi
+    REF_MODE="ref"
+    REF_VALUE="$2"
+    echo "Cloning ucp at ref $REF_VALUE..."
+elif [ -z "${1:-}" ]; then
+    REF_MODE="branch"
+    REF_VALUE="main"
     echo "No version specified, cloning main branch..."
 else
-    BRANCH="release/$1"
-    echo "Cloning version $1 (branch: $BRANCH)..."
+    REF_MODE="branch"
+    REF_VALUE="release/$1"
+    echo "Cloning version $1 (branch: $REF_VALUE)..."
 fi
 
 # Ensure ucp directory is clean before cloning
 rm -rf ucp
-git clone -b "$BRANCH" --depth 1 https://github.com/Universal-Commerce-Protocol/ucp ucp
+if [ "$REF_MODE" = "ref" ]; then
+    # Arbitrary ref may not be reachable via shallow clone of a branch;
+    # do a full clone, then check out the ref. With set -e, a failed
+    # checkout (bad SHA, missing tag, network blip) aborts the script
+    # before any regen runs.
+    git clone https://github.com/Universal-Commerce-Protocol/ucp ucp
+    git -C ucp checkout "$REF_VALUE"
+    git -C ucp rev-parse --verify HEAD >/dev/null
+else
+    git clone -b "$REF_VALUE" --depth 1 https://github.com/Universal-Commerce-Protocol/ucp ucp
+fi
+
+# Resolve the actual commit SHA we ended up on (for provenance manifest below)
+UCP_SCHEMA_REF="$(git -C ucp rev-parse HEAD)"
+echo "ucp HEAD: $UCP_SCHEMA_REF"
 
 # Output directory
 OUTPUT_DIR="src/ucp_sdk/models/schemas"
@@ -39,8 +74,8 @@ echo "Generating Pydantic models from preprocessed schemas..."
 
 # Check if uv is installed
 if ! command -v uv &> /dev/null; then
-    echo "Error: uv not found."
-    echo "Please install uv: curl -LsSf https://astral.sh/uv/install.sh | sh"
+    echo "Error: uv not found." >&2
+    echo "Please install uv: curl -LsSf https://astral.sh/uv/install.sh | sh" >&2
     exit 1
 fi
 
@@ -75,7 +110,50 @@ uv run \
 
 echo "Formatting generated models..."
 uv run ruff format
-uv run ruff check --fix "$OUTPUT_DIR"
+# Match the project's pre-commit ignore list (`.pre-commit-config.yaml`):
+# auto-generated docstrings legitimately exceed line length and miss
+# Google-style sections. Without these ignores, set -e would abort the
+# whole regen on a benign lint diff.
+uv run ruff check --fix --ignore "D,E501" "$OUTPUT_DIR"
+
+# Normalize end-of-file on every generated module: codegen emits a
+# trailing blank line that pre-commit's end-of-file-fixer strips on the
+# next run, dirtying the working tree on every regen. Collapse trailing
+# blank lines to exactly one newline up front.
+uv run python <<'PY'
+from pathlib import Path
+
+for path in Path("src/ucp_sdk/models/schemas").rglob("*.py"):
+    text = path.read_text(encoding="utf-8")
+    stripped = text.rstrip("\n")
+    if stripped + "\n" != text:
+        path.write_text(stripped + "\n", encoding="utf-8")
+PY
+
+
+# Record the source ref that produced this generated tree, so downstream
+# consumers can verify provenance after upstream main moves. Format below
+# is chosen to round-trip through ruff format / ruff check unchanged.
+GEN_CMD="$0 $*"
+if [ -z "$*" ]; then
+    GEN_CMD="$0"
+fi
+cat > "src/ucp_sdk/_schema_ref.py" <<EOF
+# Generated by generate_models.sh. Do not edit by hand.
+"""Provenance metadata for the regenerated UCP schema models."""
+
+# Commit SHA on Universal-Commerce-Protocol/ucp the models were generated from.
+UCP_SCHEMA_REF = "$UCP_SCHEMA_REF"
+
+# generate_models.sh invocation that produced the current tree.
+GENERATE_COMMAND = "$GEN_CMD"
+EOF
+# Re-format _schema_ref.py so the committed file matches what ruff would
+# do on its own — without this, a long --ref or invocation can produce a
+# line that ruff format would later wrap, putting the working tree out
+# of sync with future regens.
+uv run ruff format src/ucp_sdk/_schema_ref.py
+echo "Wrote provenance to src/ucp_sdk/_schema_ref.py: $UCP_SCHEMA_REF"
 
 
 echo "Done. Models generated in $OUTPUT_DIR"

preprocess_schemas.py

@@ -399,6 +399,63 @@ def generate_variants(path, schema, ops, global_variant_requirements):
 # --- Global Normalization ---
 
 
+def lift_capability_extension_defs(schema, file_stem):
+    """Lift `$defs[<reverse.domain.key>]/{platform,business}_schema` to top-level.
+
+    Capability-extension files (e.g. identity_linking.json, fulfillment.json)
+    nest their typed schemas under a reverse-domain `$defs` key, e.g.
+
+        $defs:
+          dev.ucp.common.identity_linking:
+            platform_schema: {...}
+            business_schema: {...}
+
+    The dotted key is not a valid Python identifier, so datamodel-codegen
+    can't reach the inner schemas and emits only a `RootModel[Any]` wrapper
+    for the file root. This loses the typed `config.scopes` / `scope_policy`
+    surface that downstream consumers need.
+
+    Lift each `platform_schema` / `business_schema` child into top-level
+    `$defs` with a `{file_stem}_{child}` name, then remove the dotted-key
+    holder, then add a root-level `oneOf` so codegen anchors on something
+    walkable. Capability-extension schemas have no external `$ref` pointing
+    into the dotted-key path (verified across `ucp/source/schemas`), so
+    removing the holder is safe.
+
+    Schema-extension entries with `allOf`-style payload (e.g. fulfillment's
+    `dev.ucp.shopping.checkout`) are left untouched — codegen already
+    produces typed classes for those via the deeply-nested module path.
+    """
+    defs = schema.get("$defs")
+    if not isinstance(defs, dict):
+        return
+    # Iterate over a fixed tuple so generated diffs are stable across
+    # Python hash seeds. set ordering used to leak the seed into codegen.
+    lifted = {}
+    drop_keys = []
+    for key, value in defs.items():
+        if "." not in key or not isinstance(value, dict):
+            continue
+        matched = False
+        for child_name in ("platform_schema", "business_schema"):
+            child_value = value.get(child_name)
+            if not isinstance(child_value, dict):
+                continue
+            new_name = f"{file_stem}_{child_name}"
+            lifted[new_name] = child_value
+            matched = True
+        if matched:
+            drop_keys.append(key)
+    if not lifted:
+        return
+    for key in drop_keys:
+        defs.pop(key, None)
+    defs.update(lifted)
+    # Anchor the file root on the lifted schemas so codegen produces
+    # typed classes rather than RootModel[Any].
+    schema["oneOf"] = [{"$ref": f"#/$defs/{name}"} for name in lifted]
+
+
 def normalize_metadata_schemas(schemas, target_dir):
     """
     Ensures ucp.json has a root union and other files point to it generically.
@@ -416,6 +473,7 @@ def normalize_metadata_schemas(schemas, target_dir):
                 "response_checkout_schema",
                 "response_order_schema",
                 "response_cart_schema",
+                "response_catalog_schema",
             ]
         ]
 
@@ -530,6 +588,7 @@ def main():
     for p_abs, s in schemas.items():
         if "ucp.json" in p_abs or "_request.json" in p_abs:
             continue
+        lift_capability_extension_defs(s, Path(p_abs).stem)
         preprocess_full_schema(s, entity_def)
         # Write back the flattened core schema
         save_json(s, Path(p_abs))

pyproject.toml

@@ -28,8 +28,12 @@ dependencies = [
 [dependency-groups]
 dev = [
     "datamodel-code-generator[http,ruff]>=0.50.0",
+    "pytest>=8.0",
 ]
 
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
 [build-system]
 requires = ["hatchling"]
 build-backend = "hatchling.build"

src/ucp_sdk/_schema_ref.py

@@ -0,0 +1,10 @@
+# Generated by generate_models.sh. Do not edit by hand.
+"""Provenance metadata for the regenerated UCP schema models."""
+
+# Commit SHA on Universal-Commerce-Protocol/ucp the models were generated from.
+UCP_SCHEMA_REF = "c5c61396ebf9a5afb9c4169bfa3af1409daa6fd6"
+
+# generate_models.sh invocation that produced the current tree.
+GENERATE_COMMAND = (
+    "generate_models.sh --ref c5c61396ebf9a5afb9c4169bfa3af1409daa6fd6"
+)