diff --git a/conformance/fixtures/invalid/verbose-prediction-too-confident.kpack/PACK.yaml b/conformance/fixtures/invalid/verbose-prediction-too-confident.kpack/PACK.yaml new file mode 100644 index 0000000..cc489c3 --- /dev/null +++ b/conformance/fixtures/invalid/verbose-prediction-too-confident.kpack/PACK.yaml @@ -0,0 +1,4 @@ +name: verbose-prediction-too-confident +version: 2026.05.09 +domain: test/conformance +author: KP Conformance Suite diff --git a/conformance/fixtures/invalid/verbose-prediction-too-confident.kpack/claims.md b/conformance/fixtures/invalid/verbose-prediction-too-confident.kpack/claims.md new file mode 100644 index 0000000..35da932 --- /dev/null +++ b/conformance/fixtures/invalid/verbose-prediction-too-confident.kpack/claims.md @@ -0,0 +1,30 @@ + +--- +pack: verbose-prediction-too-confident | v: 2026.05.09 | domain: test/conformance +confidence: simple | normalized +--- + +# Verbose Prediction Too Confident Fixture + +> INVALID: C001 uses the verbose named-field syntax with nature=prediction and +> confidence 0.97, exceeding the 0.95 cap. Violates SC-12. This is the verbose +> counterpart of prediction-too-confident.kpack (dense syntax), so SC-12 is +> exercised on both metadata forms. + +## Claims + +- **[C001]** The widget market will reach $5B in fiscal 2027 + `confidence: 0.97 | type: inferred | evidence: E001 | since: 2026-05-09 | depth: investigated | nature: prediction` + A future-tense claim with confidence above the 0.95 prediction cap. + +- **[C002]** Plausibly-bounded prediction (sibling control) + `confidence: 0.80 | type: inferred | evidence: E001 | since: 2026-05-09 | depth: investigated | nature: prediction` + This one is within the 0.95 cap and is therefore valid. diff --git a/conformance/fixtures/invalid/verbose-prediction-too-confident.kpack/evidence.md b/conformance/fixtures/invalid/verbose-prediction-too-confident.kpack/evidence.md new file mode 100644 index 0000000..69ac730 --- /dev/null +++ b/conformance/fixtures/invalid/verbose-prediction-too-confident.kpack/evidence.md @@ -0,0 +1,8 @@ +# Evidence — Verbose Prediction Too Confident Fixture + +## E001 +> **type:** document | **captured:** 2026-05-09 +> **source:** Test data + +Single evidence record; the violation is in the prediction confidence cap +(SC-12), here expressed via the verbose named-field syntax. diff --git a/conformance/run.py b/conformance/run.py index be287f2..ab1793e 100644 --- a/conformance/run.py +++ b/conformance/run.py @@ -61,6 +61,7 @@ "dangling-relation-target.kpack", "wrong-pack-name.kpack", "prediction-too-confident.kpack", + "verbose-prediction-too-confident.kpack", ] # Expected error categories for invalid fixtures. @@ -75,6 +76,7 @@ "dangling-relation-target.kpack": "SC-05", "wrong-pack-name.kpack": "SC-07", "prediction-too-confident.kpack": "SC-12", + "verbose-prediction-too-confident.kpack": "SC-12", } EXAMPLE_ORDER = [ "solar-energy-market.kpack", @@ -310,6 +312,20 @@ def validate_pack(pack_dir: Path) -> list[Err]: if type_s not in {"o", "r", "c", "i"}: errs.append(Err("parse", f"invalid claim type '{type_s}' for {cid} (expected o/r/c/i)")) + # Position 4 (date): ISODate. Grammar is YYYY-MM-DD; validate the + # format when the slot is non-empty (empty interior slots are valid). + date_s = parts[3].strip() + if date_s and not re.fullmatch(r"\d{4}-\d{2}-\d{2}", date_s): + errs.append(Err("parse", f"invalid date '{date_s}' for {cid} (expected YYYY-MM-DD)")) + + # Position 5 (depth) intentionally NOT enum-validated here. The spec + # (CORE.md) lists assumed/investigated/exhaustive, but a large share + # of the shipped corpus (system + grounding packs) uses additional + # depth values such as `practitioner`/`confirmed`. Enforcing the + # closed set would flag hundreds of production claims — that is a + # spec-vs-corpus reconciliation, not a parser quick-win, so it is + # left for a dedicated decision rather than enforced here. + # Empty evidence position → syntactic error if not ev_s: errs.append(Err("parse", f"empty evidence ref list for {cid}")) @@ -350,6 +366,13 @@ def validate_pack(pack_dir: Path) -> list[Err]: if cm: confidences.append((cid, float(cm.group(1)), cm.group(1))) + # SC-12: a prediction-nature claim must keep confidence ≤0.95. + # The dense branch enforces this at the position-6 nature slot; the + # verbose branch previously skipped it entirely. + nm = re.search(r"nature:\s*(\w+)", meta) + if cm and nm and nm.group(1) == "prediction" and float(cm.group(1)) > 0.95: + predictions_high_conf.append((cid, float(cm.group(1)))) + # Evidence refs em = re.search(r"evidence:\s*([^|]+)", meta) if em: