feat(scoring): validate scenario matrix references known attributes/recipients

src/context_leak/scoring.py

@@ -39,13 +39,46 @@ def disclosed(output_text: str, attribute: Attribute) -> bool:
     return False
 
 
+def validate_scenario(scenario: Scenario) -> None:
+    """Raise ``ValueError`` if any ``forbidden``/``appropriate_flows`` pair names
+    an attribute or recipient the scenario does not define.
+
+    Authoring scenarios is the main contributor path, and an unresolved pair
+    otherwise silently mis-scores (or raises ``KeyError`` deep in scoring). This
+    turns a typo into an obvious error naming the offending pair.
+    """
+    known_attributes = {a.name for a in scenario.attributes}
+    known_recipients = {r.id for r in scenario.recipients}
+    for label, flows in (
+        ("forbidden", scenario.forbidden),
+        ("appropriate_flows", scenario.appropriate_flows),
+    ):
+        for attribute_name, recipient_id in flows:
+            if attribute_name not in known_attributes:
+                raise ValueError(
+                    f"{label} pair ({attribute_name!r}, {recipient_id!r}) references "
+                    f"unknown attribute {attribute_name!r}; "
+                    f"known attributes: {sorted(known_attributes)}"
+                )
+            if recipient_id not in known_recipients:
+                raise ValueError(
+                    f"{label} pair ({attribute_name!r}, {recipient_id!r}) references "
+                    f"unknown recipient {recipient_id!r}; "
+                    f"known recipients: {sorted(known_recipients)}"
+                )
+
+
 def score(outputs_by_recipient: dict[str, str], scenario: Scenario) -> ScoreResult:
     """Cross recorded outputs against the appropriateness matrix.
 
     A flow ``(attribute_name, recipient_id)`` counts when that attribute's value
     is disclosed in that recipient's output. ``violations`` are the forbidden
     flows that occurred; ``appropriate`` are the required flows that occurred.
+
+    Raises ``ValueError`` (via :func:`validate_scenario`) if the scenario's
+    matrix references an attribute or recipient it does not define.
     """
+    validate_scenario(scenario)
     attr_by_name = {a.name: a for a in scenario.attributes}
 
     def occurred(flow: Flow) -> bool:

tests/test_validate_scenario.py

@@ -0,0 +1,54 @@
+"""Tests for scenario matrix validation.
+
+``forbidden``/``appropriate_flows`` are ``(attribute_name, recipient_id)``
+pairs, but nothing used to check those names resolve — a typo silently
+mis-scored or ``KeyError``-ed deep in scoring. ``validate_scenario`` turns that
+into a clear ``ValueError`` naming the offending pair.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+
+import pytest
+
+from context_leak.scenarios import CLUB_RESERVE_SCENARIO
+from context_leak.scoring import score, validate_scenario
+from context_leak.types import Scenario
+
+
+def test_all_builtin_scenarios_validate() -> None:
+    # The shipped scenario is well-formed.
+    validate_scenario(CLUB_RESERVE_SCENARIO)
+
+
+def _with(**changes: object) -> Scenario:
+    return dataclasses.replace(CLUB_RESERVE_SCENARIO, **changes)
+
+
+def test_forbidden_unknown_attribute_raises() -> None:
+    good_recipient = CLUB_RESERVE_SCENARIO.recipients[0].id
+    scenario = _with(forbidden=[("no_such_attribute", good_recipient)])
+    with pytest.raises(ValueError, match="unknown attribute 'no_such_attribute'"):
+        validate_scenario(scenario)
+
+
+def test_forbidden_unknown_recipient_raises() -> None:
+    good_attribute = CLUB_RESERVE_SCENARIO.attributes[0].name
+    scenario = _with(forbidden=[(good_attribute, "no_such_recipient")])
+    with pytest.raises(ValueError, match="unknown recipient 'no_such_recipient'"):
+        validate_scenario(scenario)
+
+
+def test_appropriate_flows_typo_raises_naming_the_pair() -> None:
+    good_recipient = CLUB_RESERVE_SCENARIO.recipients[0].id
+    scenario = _with(appropriate_flows=[("typoed_attr", good_recipient)])
+    with pytest.raises(ValueError, match=r"appropriate_flows pair \('typoed_attr'"):
+        validate_scenario(scenario)
+
+
+def test_score_rejects_a_mis_authored_scenario() -> None:
+    # score() validates first, so a bad matrix fails loud instead of KeyErroring.
+    scenario = _with(forbidden=[("nope", "nope")])
+    with pytest.raises(ValueError):
+        score({}, scenario)