feat(index): validate search-index against the mirrored federation contract (content#175)

.github/workflows/generate-index.yml

@@ -40,7 +40,7 @@ jobs:
           python-version: "3.11"
 
       - name: Install dependencies
-        run: pip install pyyaml
+        run: pip install pyyaml "jsonschema>=4,<5"
 
       - name: Regenerate search-index.json
         run: python scripts/generate_search_index.py

.github/workflows/schema-drift.yml

@@ -42,3 +42,11 @@ jobs:
           python-version: "3.12"
       - name: Check schema mirror against the pinned engine release
         run: python scripts/check_schema_drift.py
+      - name: Check search-index schema mirror against adaptive-learner-content
+        # The federation contract (adaptive-learner-content#175) is owned by
+        # the official content repo; this mirror must match its main. Drift
+        # goes red until the mirror is refreshed - same consequence rule as
+        # the engine mirror above.
+        run: |
+          curl -fsSL https://raw.githubusercontent.com/astrapi69/adaptive-learner-content/main/schema/search-index.schema.json -o /tmp/upstream-search-index.schema.json
+          diff -u /tmp/upstream-search-index.schema.json schema/search-index.schema.json

schema/search-index.schema.json

@@ -0,0 +1,123 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://github.com/astrapi69/adaptive-learner-content/schema/search-index.schema.json",
+  "title": "SearchIndex",
+  "description": "The per-repo discovery feed (search-index.json) that the app's cross-repo search federates over. Every registered content repo must publish an index matching this shape at its pinned commit. Extra fields are tolerated for forward-compatibility; the required fields are the federation contract.",
+  "type": "object",
+  "required": [
+    "repo",
+    "schema_version",
+    "sets"
+  ],
+  "properties": {
+    "repo": {
+      "type": "string",
+      "description": "owner/name slug of the publishing repo.",
+      "pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"
+    },
+    "generated": {
+      "type": "string"
+    },
+    "schema_version": {
+      "type": "string",
+      "minLength": 1
+    },
+    "sets": {
+      "type": "array",
+      "items": {
+        "$ref": "#/$defs/SetEntry"
+      }
+    },
+    "total_lessons": {
+      "type": "integer",
+      "minimum": 0
+    },
+    "total_cards": {
+      "type": "integer",
+      "minimum": 0
+    }
+  },
+  "$defs": {
+    "SetEntry": {
+      "type": "object",
+      "required": [
+        "id",
+        "name",
+        "source_language",
+        "target_language",
+        "level",
+        "domain"
+      ],
+      "properties": {
+        "id": {
+          "type": "string",
+          "minLength": 1
+        },
+        "name": {
+          "type": "string",
+          "minLength": 1
+        },
+        "description": {
+          "type": "string"
+        },
+        "source_language": {
+          "type": "string",
+          "minLength": 1
+        },
+        "target_language": {
+          "type": "string",
+          "minLength": 1
+        },
+        "level": {
+          "type": "string",
+          "minLength": 1
+        },
+        "domain": {
+          "type": "string",
+          "minLength": 1
+        },
+        "lesson_count": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "card_count": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "tags": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "visibility": {
+          "type": "string",
+          "enum": [
+            "visible",
+            "hidden"
+          ],
+          "description": "Consumer-display hint from the manifest set entry (engine schema 1.8). \"hidden\" asks a consumer app not to surface the set to learners; the generator emits a concrete value on every entry (absent or unknown manifest values normalize to \"visible\"). Optional here so older published indexes stay valid."
+        },
+        "review_status": {
+          "type": "string",
+          "enum": [
+            "authored",
+            "generated",
+            "reviewed"
+          ],
+          "description": "Three-state review standing from the manifest set entry (engine schema 1.9), derived from ORIGIN: 'authored' = hand-written by a speaker or domain expert (also the meaning of an absent manifest field), 'generated' = machine-generated with review pending, 'reviewed' = machine-generated and reviewed. Consumers derive 'advertisable as reviewed' as review_status != 'generated'. Optional here so older published indexes stay valid."
+        },
+        "ai_validated": {
+          "type": "boolean"
+        },
+        "trust_level": {
+          "type": "integer"
+        },
+        "book": {},
+        "updated_at": {
+          "type": "string"
+        }
+      }
+    }
+  }
+}

scripts/generate_search_index.py

@@ -288,6 +288,9 @@ def _now_iso() -> str:
 )
 
 
+SEARCH_INDEX_SCHEMA = Path(__file__).resolve().parents[1] / "schema" / "search-index.schema.json"
+
+
 def validate_index(index: dict) -> list[str]:
     errors: list[str] = []
     for key in ("repo", "generated", "schema_version", "sets", "total_lessons", "total_cards"):
@@ -304,9 +307,29 @@ def validate_index(index: dict) -> list[str]:
         errors.append("total_lessons does not match sum of set lesson_count")
     if sum(e.get("card_count", 0) for e in index.get("sets", [])) != index.get("total_cards"):
         errors.append("total_cards does not match sum of set card_count")
+    errors.extend(_schema_errors(index))
     return errors
 
 
+def _schema_errors(index: dict) -> list[str]:
+    """Validate against the mirrored federation contract
+    (adaptive-learner-content#175, ``schema/search-index.schema.json``).
+
+    The hand checks above stay as this variant's floor; the schema is the
+    contract every writing repo shares, so the two can no longer disagree
+    silently. Requires ``jsonschema`` (installed by validate-content.yml
+    and generate-index.yml)."""
+    import jsonschema
+
+    contract = json.loads(SEARCH_INDEX_SCHEMA.read_text(encoding="utf-8"))
+    validator = jsonschema.Draft202012Validator(contract)
+    violations: list[str] = []
+    for violation in sorted(validator.iter_errors(index), key=lambda v: list(v.absolute_path)):
+        location = "/" + "/".join(str(step) for step in violation.absolute_path)
+        violations.append(f"schema: {location}: {violation.message}")
+    return violations
+
+
 def _comparable(index: dict) -> dict:
     """Strip the volatile ``generated`` timestamp for staleness checks."""
     clean = dict(index)

tests/test_search_index_schema.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+"""Schema validation of the search index against the mirrored federation
+contract (adaptive-learner-content#175).
+
+``schema/search-index.schema.json`` is a mirror of the contract owned by
+adaptive-learner-content (the same mirroring relationship the engine
+schemas have). ``validate_index`` must validate against it IN ADDITION to
+the hand-maintained checks: the hand checks are this variant's floor, the
+schema is the contract every writing repo shares. Before this test the
+two could disagree silently - a ``lesson_count`` of ``"5"`` (string)
+passed the hand check (truthy, not empty) while violating the contract.
+
+Runs under pytest (``python -m pytest tests -q``).
+"""
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+SCRIPTS_DIR = REPO_ROOT / "scripts"
+
+if str(SCRIPTS_DIR) not in sys.path:
+    sys.path.insert(0, str(SCRIPTS_DIR))
+
+import generate_search_index as gsi  # noqa: E402
+
+
+def minimal_index() -> dict:
+    """The smallest index that satisfies both the hand checks and the
+    mirrored schema."""
+    return {
+        "repo": "astrapi69/example-repo",
+        "generated": "2026-08-05T00:00:00Z",
+        "schema_version": "1.0",
+        "sets": [
+            {
+                "id": "example-set",
+                "name": "Example",
+                "source_language": "de",
+                "target_language": "en",
+                "level": "A1",
+                "domain": "language",
+                "lesson_count": 2,
+                "card_count": 10,
+                "visibility": "visible",
+                "review_status": "authored",
+            }
+        ],
+        "total_lessons": 2,
+        "total_cards": 10,
+    }
+
+
+def test_mirror_exists_and_is_draft_2020_12() -> None:
+    schema_path = REPO_ROOT / "schema" / "search-index.schema.json"
+    assert schema_path.is_file(), "search-index.schema.json mirror is missing"
+    schema = json.loads(schema_path.read_text(encoding="utf-8"))
+    assert schema.get("$schema") == "https://json-schema.org/draft/2020-12/schema"
+
+
+def test_conforming_index_passes() -> None:
+    assert gsi.validate_index(minimal_index()) == []
+
+
+def test_schema_catches_what_the_hand_check_cannot() -> None:
+    """Discriminating case: an integer ``level`` is truthy and non-empty, so
+    the hand-maintained REQUIRED_SET_FIELDS loop is silent - only the
+    schema's ``"type": "string"`` on the contract side rejects it."""
+    index = minimal_index()
+    index["sets"][0]["level"] = 123
+    violations = [error for error in gsi.validate_index(index) if error.startswith("schema:")]
+    assert violations, "schema violation must be reported"
+    assert any("level" in error for error in violations)
+
+
+def test_schema_catches_a_missing_contract_field() -> None:
+    """``domain`` is required by the federation contract."""
+    index = minimal_index()
+    del index["sets"][0]["domain"]
+    violations = [error for error in gsi.validate_index(index) if error.startswith("schema:")]
+    assert any("domain" in error for error in violations)