chore(engine): pin 0.19.1, mirror the Python helper, collect manifest warnings

.github/workflows/generate-index.yml

@@ -40,7 +40,7 @@ jobs:
           python-version: "3.11"
 
       - name: Install dependencies
-        run: pip install pyyaml "jsonschema>=4,<5"
+        run: pip install pyyaml "jsonschema>=4,<5" regex
 
       - name: Regenerate search-index.json
         run: python scripts/generate_search_index.py

.github/workflows/validate-content.yml

@@ -31,7 +31,7 @@ jobs:
         with:
           python-version: "3.12"
       - name: Install validator dependencies
-        run: pip install "pyyaml>=6,<7" "jsonschema>=4,<5" "pytest>=8,<9"
+        run: pip install "pyyaml>=6,<7" "jsonschema>=4,<5" "pytest>=8,<9" regex
       - name: Validate content tree
         run: python scripts/validate_content.py
       - name: Repo test suite (shape parity #1208, drift-gate units)

schema/content-manifest.schema.json

@@ -348,7 +348,7 @@
     "metadata": {
       "additionalProperties": true,
       "default": {},
-      "description": "Free-form repo-level metadata (license, author, homepage URL, contact). The loader does not interpret these fields — they surface as-is in the Set Browser's 'About this source' panel.",
+      "description": "Free-form metadata (license, author, homepage URL, contact), surfaced as-is in the Set Browser's 'About this source' panel. One key IS interpreted: on a SET-level manifest, ``metadata.lessons`` (a list of lesson filenames) steers which files the reference consumer downloads. It does NOT control display order - consumers sort lesson ids lexicographically (engine#106).",
       "title": "Metadata",
       "type": "object"
     },
@@ -380,5 +380,5 @@
   ],
   "title": "ContentManifest",
   "type": "object",
-  "x-schema-version": "1.9"
+  "x-schema-version": "1.11"
 }

schema/engine-version.txt

@@ -1 +1 @@
-0.17.0
+0.19.1

schema/lesson.schema.json

@@ -104,10 +104,8 @@
         },
         "id": {
           "description": "Slug-safe id. Unique within the parent lesson. SRS reviews this id, not the surface term.",
-          "maxLength": 120,
-          "minLength": 1,
           "title": "Id",
-          "type": "string"
+          "$ref": "#/$defs/SlugId"
         },
         "stable_id": {
           "anyOf": [
@@ -121,7 +119,7 @@
           ],
           "default": null,
           "title": "Stable Id",
-          "description": "engine#90 - schema 1.9 (additive). Author-owned, version-stable identity for progress/SRS joins: once published it NEVER changes, set-wide unique (cross-lesson uniqueness is checked by the repo gate via collectStableIds; the schema sees one document). Opaque mint-once value (lowercase slug, 8-64 chars), NOT derived from content, so answer-text fixes do not move it. Optional: pre-1.9 content validates unchanged. SCOPE: this closes orphaning by slug rename or position shift on the exercise/card level; it does NOT close the element-level case (an answer correction inside a surviving exercise still moves the content-derived element key, engine#91)."
+          "description": "engine#90 - schema 1.9 (additive). Author-owned, version-stable identity for progress/SRS joins: once published it NEVER changes, set-wide unique (cross-lesson uniqueness is checked by the repo gate via collectStableIds; the schema sees one document). Opaque mint-once value (lowercase slug, 8-64 chars), NOT derived from content, so answer-text fixes do not move it. Optional: pre-1.9 content validates unchanged. SCOPE: this closes orphaning by slug rename or position shift on the exercise/card level; it does NOT close the element-level case (an answer correction inside a surviving exercise still moves the content-derived element key, engine#91). COMPAT NOTE (engine#105): this pattern predates $defs/SlugId and is deliberately NOT tightened - stable_ids are immutable once published, so the underscore stays allowed here even though SlugId forbids it. For NEW mints prefer the stricter SlugId shape (hyphens only); the bundled mint-stable-ids minter already emits only [a-z0-9-]."
         },
         "image": {
           "anyOf": [
@@ -171,9 +169,9 @@
         },
         "tags": {
           "default": [],
-          "description": "Slug-safe tags for SRS filtering ('greeting', 'verb-present', 'irregular').",
+          "description": "Tags for SRS filtering ('greeting', 'verb-present', 'irregular'). Each tag must match $defs/SlugId - the reference consumer checks tags with the same regex it applies to ids and skips lessons whose tags fail (engine#108, hard since schema 1.11 after the published corpus was cleaned).",
           "items": {
-            "type": "string"
+            "$ref": "#/$defs/SlugId"
           },
           "maxItems": 20,
           "title": "Tags",
@@ -425,10 +423,8 @@
         },
         "id": {
           "description": "Slug-safe id, unique within the lesson.",
-          "maxLength": 120,
-          "minLength": 1,
           "title": "Id",
-          "type": "string"
+          "$ref": "#/$defs/SlugId"
         },
         "stable_id": {
           "anyOf": [
@@ -442,7 +438,7 @@
           ],
           "default": null,
           "title": "Stable Id",
-          "description": "engine#90 - schema 1.9 (additive). Author-owned, version-stable identity for progress/SRS joins: once published it NEVER changes, set-wide unique (cross-lesson uniqueness is checked by the repo gate via collectStableIds; the schema sees one document). Opaque mint-once value (lowercase slug, 8-64 chars), NOT derived from content, so answer-text fixes do not move it. Optional: pre-1.9 content validates unchanged. SCOPE: this closes orphaning by slug rename or position shift on the exercise/card level; it does NOT close the element-level case (an answer correction inside a surviving exercise still moves the content-derived element key, engine#91)."
+          "description": "engine#90 - schema 1.9 (additive). Author-owned, version-stable identity for progress/SRS joins: once published it NEVER changes, set-wide unique (cross-lesson uniqueness is checked by the repo gate via collectStableIds; the schema sees one document). Opaque mint-once value (lowercase slug, 8-64 chars), NOT derived from content, so answer-text fixes do not move it. Optional: pre-1.9 content validates unchanged. SCOPE: this closes orphaning by slug rename or position shift on the exercise/card level; it does NOT close the element-level case (an answer correction inside a surviving exercise still moves the content-derived element key, engine#91). COMPAT NOTE (engine#105): this pattern predates $defs/SlugId and is deliberately NOT tightened - stable_ids are immutable once published, so the underscore stays allowed here even though SlugId forbids it. For NEW mints prefer the stricter SlugId shape (hyphens only); the bundled mint-stable-ids minter already emits only [a-z0-9-]."
         },
         "images": {
           "anyOf": [
@@ -836,10 +832,8 @@
         },
         "id": {
           "description": "Slug-safe id, unique within the lesson.",
-          "maxLength": 120,
-          "minLength": 1,
           "title": "Id",
-          "type": "string"
+          "$ref": "#/$defs/SlugId"
         },
         "review_lesson_id": {
           "anyOf": [
@@ -994,6 +988,14 @@
       "title": "PictureImage",
       "type": "object"
     },
+    "SlugId": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 120,
+      "pattern": "^[\\p{Ll}\\p{Nd}]+(-[\\p{Ll}\\p{Nd}]+)*$",
+      "title": "SlugId",
+      "description": "Slug id: lowercase Unicode letters and digits in hyphen-separated runs (no leading/trailing/double hyphen, no underscore, no uppercase, no whitespace). Exactly the rule the reference consumer (adaptive-learner) enforces on import - an id that fails it is silently skipped there, so the engine rejects it up front (engine#105)."
+    },
     "StepType": {
       "description": "Closed enum for top-level step kinds.\n\nTHEORY = Markdown content step. EXERCISE = one of the\nExerciseType variants. The viewer (Phase 44) branches\non this; no other step kinds are valid in v1.0.",
       "enum": [
@@ -1096,11 +1098,9 @@
       "type": "integer"
     },
     "id": {
-      "description": "Slug-safe id, unique within the parent set. Convention: ``NN-slug`` (e.g. ``01-greetings``) for deterministic ordering, though the loader does not enforce ordering — it reads the set's manifest for the lesson sequence.",
-      "maxLength": 120,
-      "minLength": 1,
+      "description": "Slug id (see $defs/SlugId), unique within the parent set. The display order of a set's lessons is the LEXICOGRAPHIC sort of these ids: consumers sort the stored ``lessons/<id>.json`` filenames (the set manifest's ``metadata.lessons`` list only steers download discovery, never display order). The ``NN-slug`` prefix (e.g. ``01-greetings``) is therefore the ordering mechanism, not cosmetics - zero-pad it to one fixed width per set, or ``10-`` sorts before ``2-`` (engine#106).",
       "title": "Id",
-      "type": "string"
+      "$ref": "#/$defs/SlugId"
     },
     "requires_extensions": {
       "description": "Extensions this lesson needs, each ``ext:<vendor>-<name>@<major>`` (e.g. ``ext:acme-ordering@1``). A consumer that has not registered a declared extension refuses the lesson loudly (E-EXT-UNSUPPORTED) rather than mis-rendering. Absent / empty on core lessons; additive, so pre-1.7 content validates unchanged.",
@@ -1205,5 +1205,5 @@
   ],
   "title": "Lesson",
   "type": "object",
-  "x-schema-version": "1.9"
+  "x-schema-version": "1.11"
 }

schema/quality-rules.json

@@ -1,5 +1,5 @@
 {
-  "$schema-version": "1.9",
+  "$schema-version": "1.11",
   "_comment": "GENERATED from scripts/generate_lesson_schema.py (EXP-039). Do not edit. Shared quality minimums for the content quality gate (app + content repo).",
   "rules": {
     "minExerciseTypes": 2,

scripts/check_schema_drift.py

@@ -63,6 +63,15 @@
         "package/schema/content-manifest.schema.json"
     ),
     "schema/quality-rules.json": "package/schema/quality-rules.json",
+    # The engine also ships the Python-side validator helper
+    # (learn-content-engine#115). Schema 1.10+ uses Unicode property escapes
+    # (``\p{Ll}``) in the slug pattern, which Python's built-in ``re``
+    # cannot compile - without this helper ``validate_content.py`` and the
+    # whole pytest suite die on every document. Mirrored rather than
+    # imported from node_modules so validation stays Python-only, and
+    # byte-compared against the pinned tarball like every other mirror, so
+    # it cannot drift here unnoticed.
+    "scripts/lce_schema.py": "package/python/lce_schema.py",
 }
 
 

scripts/lce_schema.py

@@ -0,0 +1,90 @@
+"""Python-side JSON-Schema validation for the canonical lesson schema.
+
+WHY THIS FILE EXISTS (engine#115)
+---------------------------------
+The ecosystem validates the SAME schema with two engines: the TypeScript
+engine (ajv, ECMA-262 regexes with the ``u`` flag) and every content repo's
+``scripts/validate_content.py`` (the ``jsonschema`` library, Python ``re``).
+
+Since schema 1.10 the canonical slug rule uses Unicode property escapes::
+
+    ^[\\p{Ll}\\p{Nd}]+(-[\\p{Ll}\\p{Nd}]+)*$
+
+``\\p{...}`` is valid ECMA-262 and NOT valid Python ``re``. The consequence
+is not a rule that quietly stops working - it is total: ``check_schema``
+rejects the whole schema, and instance validation raises
+``re.PatternError``, so the Python validator dies on every document.
+
+Disabling the ``format`` check is a HALF fix that looks green: it silences
+the metaschema rejection while instance validation still raises. This module
+takes the other route - it swaps the ``pattern`` keyword for an
+implementation backed by the ``regex`` package, which understands
+``\\p{...}``. Both engines then APPLY the same rule instead of merely
+surviving it.
+
+This file ships INSIDE the npm package (``python/`` is in package ``files``)
+so all consumers read one pinned source instead of eleven copies that drift
+independently.
+
+USAGE::
+
+    from lce_schema import build_validator
+    validator = build_validator(json.loads(schema_path.read_text()))
+    errors = list(validator.iter_errors(lesson))
+
+Requires ``jsonschema>=4`` and ``regex``.
+"""
+from __future__ import annotations
+
+from typing import Any
+
+import jsonschema
+from jsonschema import validators
+
+try:
+    import regex as _regex
+except ImportError as _import_error:  # pragma: no cover - environment guard
+    # Loud on purpose. Falling back to ``re`` would make every pattern with a
+    # Unicode property escape raise, and silently skipping the pattern keyword
+    # would leave a rule that can never fail - worse than no rule at all.
+    raise SystemExit(
+        "FATAL: the 'regex' package is required to validate this schema "
+        "(the slug pattern uses Unicode property escapes, which Python's "
+        "built-in 're' cannot compile). Install it with: pip install regex"
+    ) from _import_error
+
+
+def _pattern_with_unicode_properties(
+    validator: Any, patternValue: str, instance: Any, schemaNode: Any
+):
+    """``pattern`` keyword backed by ``regex`` instead of ``re``.
+
+    Mirrors the JSON-Schema contract: non-strings are ignored, and the match
+    is unanchored (the schema's own ``^``/``$`` do the anchoring), matching
+    ajv's behaviour on the engine side.
+    """
+    if not isinstance(instance, str):
+        return
+    if _regex.search(patternValue, instance) is None:
+        yield jsonschema.ValidationError(f"{instance!r} does not match {patternValue!r}")
+
+
+#: Draft 2020-12 validator whose ``pattern`` keyword understands ``\p{...}``.
+UnicodeAwareValidator = validators.extend(
+    jsonschema.Draft202012Validator,
+    {"pattern": _pattern_with_unicode_properties},
+)
+
+
+def build_validator(schema: dict) -> Any:
+    """Return a validator for ``schema`` that can read Unicode property
+    escapes in ``pattern``.
+
+    ``check_schema`` runs with ``format_checker=None``: the metaschema's
+    ``format: regex`` assertion is a Python-``re`` compile check, which is
+    exactly the thing that does not apply here. Every other structural
+    assertion of the metaschema still runs, so a genuinely malformed schema
+    is still rejected.
+    """
+    jsonschema.Draft202012Validator.check_schema(schema, format_checker=None)
+    return UnicodeAwareValidator(schema, format_checker=None)

scripts/validate_content.py

@@ -44,7 +44,7 @@
 from pathlib import Path
 
 import yaml
-from jsonschema import Draft202012Validator
+from lce_schema import build_validator
 
 import generate_search_index
 
@@ -96,15 +96,18 @@ def hint_states_answer_length(hint: object) -> bool:
 }
 
 
-def _load_lesson_schema() -> Draft202012Validator:
+def _load_lesson_schema():
     if not LESSON_SCHEMA_PATH.is_file():
         raise SystemExit(
             f"FATAL: missing mirrored schema {LESSON_SCHEMA_PATH.relative_to(REPO_ROOT)} "
             "(run scripts/check_schema_drift.py --update)"
         )
     schema = json.loads(LESSON_SCHEMA_PATH.read_text(encoding="utf-8"))
-    Draft202012Validator.check_schema(schema)
-    return Draft202012Validator(schema)
+    # Built through the engine's shipped helper (scripts/lce_schema.py,
+    # mirrored from the pinned release): the slug pattern uses Unicode
+    # property escapes that Python's built-in ``re`` cannot compile, so a
+    # plain Draft202012Validator dies on the schema itself.
+    return build_validator(schema)
 
 
 def _load_quality_rules() -> dict:

scripts/validate_with_engine.mjs

@@ -265,6 +265,11 @@ function validateAll(repoRoot, { showWarnings = false } = {}) {
       manifests += 1;
       const res = validateManifest(parseYaml(readFileSync(file, "utf8")));
       if (!res.valid) report(rel, res.errors);
+      // Manifest warnings were collected for lessons but DROPPED here, so the
+      // set-level ordering gate (learn-content-engine#110, W-SET-ORDER-*)
+      // reached this runner and reported nothing. A connected gate that
+      // stays silent is indistinguishable from one that was never connected.
+      if (showWarnings && res.warnings.length) warned.push({ file: rel, warnings: res.warnings });
     }
   }
 
@@ -274,6 +279,9 @@ function validateAll(repoRoot, { showWarnings = false } = {}) {
     parseYaml(readFileSync(join(repoRoot, "manifest.yaml"), "utf8")),
   );
   if (!rootRes.valid) report("manifest.yaml", rootRes.errors);
+  if (showWarnings && rootRes.warnings.length) {
+    warned.push({ file: "manifest.yaml", warnings: rootRes.warnings });
+  }
 
   const totalWarnings = warned.reduce((sum, w) => sum + w.warnings.length, 0);
   console.log(

tests/test_check_schema_drift.py

@@ -43,13 +43,17 @@ def make_tarball(files: dict[str, bytes]) -> bytes:
 SCHEMA_BYTES = json.dumps({"$id": "lesson", "x-schema-version": "9.9"}).encode()
 MANIFEST_BYTES = json.dumps({"$id": "content-manifest"}).encode()
 QUALITY_BYTES = json.dumps({"rules": {"minExercisesPerLesson": 5}}).encode()
+# The engine also ships the Python validator helper (engine#115); the mirror
+# carries it, so the synthetic tarball must contain it too.
+HELPER_BYTES = b"# stand-in for python/lce_schema.py\n"
 
 
 def engine_tarball(**overrides: bytes) -> bytes:
     files = {
         "package/schema/lesson.schema.json": SCHEMA_BYTES,
         "package/schema/content-manifest.schema.json": MANIFEST_BYTES,
         "package/schema/quality-rules.json": QUALITY_BYTES,
+        "package/python/lce_schema.py": HELPER_BYTES,
     }
     files.update(overrides)
     return make_tarball(files)
@@ -60,11 +64,14 @@ def write_mirror(
     lesson: bytes = SCHEMA_BYTES,
     manifest: bytes = MANIFEST_BYTES,
     quality: bytes = QUALITY_BYTES,
+    helper: bytes = HELPER_BYTES,
 ) -> None:
     (root / "schema").mkdir(parents=True, exist_ok=True)
     (root / "schema" / "lesson.schema.json").write_bytes(lesson)
     (root / "schema" / "content-manifest.schema.json").write_bytes(manifest)
     (root / "schema" / "quality-rules.json").write_bytes(quality)
+    (root / "scripts").mkdir(parents=True, exist_ok=True)
+    (root / "scripts" / "lce_schema.py").write_bytes(helper)
 
 
 def test_pin_is_read_from_engine_version_file() -> None: