feat: reproducible evaluation dataset pipeline with provenance

.github/workflows/ci.yml

@@ -26,6 +26,50 @@ jobs:
           node-version: 20
       - run: npm run validate
 
+  verify-manifest:
+    name: Verify provenance manifest
+    needs: secret-check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+      - name: Verify evaluation-manifest.json
+        run: npm run verify-manifest
+      # Catch a manifest that was committed without being regenerated after an
+      # input file changed. Regenerate, then compare only hash-bearing fields
+      # (excluding generatedAt and sourceCommit which legitimately differ on
+      # every regeneration). Any diff means the manifest is stale.
+      - name: Detect stale manifest (inputs changed without regenerating)
+        run: |
+          # Capture the committed manifest for comparison.
+          node -e "
+            const fs = require('fs');
+            const m = JSON.parse(fs.readFileSync('evaluation-manifest.json','utf-8'));
+            // Extract only the fields that must match: inputs hashes, expectedCounts,
+            // fixtureRelease, manifestVersion, mappingVersion, tierThresholds.
+            const sig = { inputs: m.inputs, expectedCounts: m.expectedCounts,
+                          fixtureRelease: m.fixtureRelease, manifestVersion: m.manifestVersion,
+                          mappingVersion: m.mappingVersion, tierThresholds: m.tierThresholds };
+            fs.writeFileSync('manifest-sig-before.json', JSON.stringify(sig, null, 2) + '\n');
+          "
+          npm run generate-manifest
+          node -e "
+            const fs = require('fs');
+            const m = JSON.parse(fs.readFileSync('evaluation-manifest.json','utf-8'));
+            const sig = { inputs: m.inputs, expectedCounts: m.expectedCounts,
+                          fixtureRelease: m.fixtureRelease, manifestVersion: m.manifestVersion,
+                          mappingVersion: m.mappingVersion, tierThresholds: m.tierThresholds };
+            fs.writeFileSync('manifest-sig-after.json', JSON.stringify(sig, null, 2) + '\n');
+          "
+          if ! diff -q manifest-sig-before.json manifest-sig-after.json > /dev/null; then
+            echo "::error file=evaluation-manifest.json::evaluation-manifest.json is stale — an input file changed but the manifest was not regenerated. Run 'npm run generate-manifest' and commit the result."
+            diff manifest-sig-before.json manifest-sig-after.json
+            exit 1
+          fi
+          echo "Manifest signature unchanged — manifest is up to date."
+
   validate-changelog:
     name: Changelog updated
     runs-on: ubuntu-latest
@@ -58,4 +102,3 @@ jobs:
             echo "::error file=CHANGELOG.md::This PR modifies fixture files but the [Unreleased] section of CHANGELOG.md is empty."
             exit 1
           fi
-          echo "CHANGELOG.md [Unreleased] section is populated — check passed."

EVALUATION_RESULT_SCHEMA.json

@@ -0,0 +1,188 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://github.com/Gryd-lock/grydlock-testkit/blob/main/EVALUATION_RESULT_SCHEMA.json",
+  "title": "Gryd Lock Evaluation Result",
+  "description": "Schema for the result document that grydlock-research must produce when evaluating a testkit fixture release. Every evaluation run must identify its dataset, implementation, and mapping versions so results can be compared across runs without ambiguity.",
+  "type": "object",
+  "required": [
+    "schemaVersion",
+    "fixtureRelease",
+    "sourceCommit",
+    "evaluatedAt",
+    "evaluatorVersion",
+    "mappingVersion",
+    "tierThresholds",
+    "summary",
+    "perDestination"
+  ],
+  "additionalProperties": false,
+  "properties": {
+    "schemaVersion": {
+      "type": "string",
+      "description": "Version of this result schema (semver). Must match the manifestVersion in the evaluation-manifest.json used for this run.",
+      "pattern": "^\\d+\\.\\d+\\.\\d+$",
+      "examples": ["1.0.0"]
+    },
+    "fixtureRelease": {
+      "type": "string",
+      "description": "The grydlock-testkit release tag evaluated (semver). Must match evaluation-manifest.json fixtureRelease.",
+      "pattern": "^\\d+\\.\\d+\\.\\d+$",
+      "examples": ["0.1.0"]
+    },
+    "sourceCommit": {
+      "type": "string",
+      "description": "Full SHA-1 git commit of grydlock-testkit at evaluation time. Must match evaluation-manifest.json sourceCommit.",
+      "pattern": "^[0-9a-f]{40}$",
+      "examples": ["4707242b30d5c2c3539d8ec90cd3612779c8b3b2"]
+    },
+    "evaluatedAt": {
+      "type": "string",
+      "description": "ISO-8601 datetime when this evaluation run was performed.",
+      "format": "date-time",
+      "examples": ["2026-08-18T00:00:00.000Z"]
+    },
+    "evaluatorVersion": {
+      "type": "string",
+      "description": "Version of grydlock-research that produced this result (semver).",
+      "pattern": "^\\d+\\.\\d+\\.\\d+$",
+      "examples": ["0.1.0"]
+    },
+    "mappingVersion": {
+      "type": "string",
+      "description": "Version of the score-to-tier mapping used (semver). Must match evaluation-manifest.json mappingVersion. Bump this whenever tier threshold boundaries change.",
+      "pattern": "^\\d+\\.\\d+\\.\\d+$",
+      "examples": ["1.0.0"]
+    },
+    "tierThresholds": {
+      "type": "object",
+      "description": "The exact tier threshold configuration applied during scoring. Copied from evaluation-manifest.json so the result is self-contained.",
+      "required": ["clean", "suspicious", "malicious"],
+      "additionalProperties": false,
+      "properties": {
+        "clean": {
+          "type": "object",
+          "required": ["max"],
+          "properties": {
+            "max": { "type": "integer", "minimum": 0, "maximum": 100 }
+          }
+        },
+        "suspicious": {
+          "type": "object",
+          "required": ["min", "max"],
+          "properties": {
+            "min": { "type": "integer", "minimum": 0, "maximum": 100 },
+            "max": { "type": "integer", "minimum": 0, "maximum": 100 }
+          }
+        },
+        "malicious": {
+          "type": "object",
+          "required": ["min"],
+          "properties": {
+            "min": { "type": "integer", "minimum": 0, "maximum": 100 }
+          }
+        }
+      }
+    },
+    "summary": {
+      "type": "object",
+      "description": "Aggregate accuracy metrics across the full dataset.",
+      "required": ["total", "correct", "accuracy", "byLabel"],
+      "additionalProperties": false,
+      "properties": {
+        "total": {
+          "type": "integer",
+          "description": "Total number of destinations evaluated. Must match expectedCounts.total in the manifest.",
+          "minimum": 1
+        },
+        "correct": {
+          "type": "integer",
+          "description": "Number of destinations where the assigned tier matched the label.",
+          "minimum": 0
+        },
+        "accuracy": {
+          "type": "number",
+          "description": "Fraction of correct tier assignments (correct / total). Range [0, 1].",
+          "minimum": 0,
+          "maximum": 1
+        },
+        "byLabel": {
+          "type": "object",
+          "description": "Per-label breakdown of evaluation counts and accuracy.",
+          "required": ["clean", "suspicious", "malicious"],
+          "additionalProperties": false,
+          "properties": {
+            "clean": { "$ref": "#/$defs/labelMetrics" },
+            "suspicious": { "$ref": "#/$defs/labelMetrics" },
+            "malicious": { "$ref": "#/$defs/labelMetrics" }
+          }
+        }
+      }
+    },
+    "perDestination": {
+      "type": "array",
+      "description": "One entry per evaluated destination, ordered by destination ID.",
+      "minItems": 1,
+      "items": { "$ref": "#/$defs/destinationResult" }
+    }
+  },
+  "$defs": {
+    "labelMetrics": {
+      "type": "object",
+      "required": ["total", "correct", "accuracy"],
+      "additionalProperties": false,
+      "properties": {
+        "total": {
+          "type": "integer",
+          "description": "Number of destinations with this label.",
+          "minimum": 0
+        },
+        "correct": {
+          "type": "integer",
+          "description": "Number correctly classified.",
+          "minimum": 0
+        },
+        "accuracy": {
+          "type": "number",
+          "description": "Fraction correct for this label.",
+          "minimum": 0,
+          "maximum": 1
+        }
+      }
+    },
+    "destinationResult": {
+      "type": "object",
+      "required": ["id", "label", "score", "assignedTier", "correct"],
+      "additionalProperties": false,
+      "properties": {
+        "id": {
+          "type": "string",
+          "description": "Destination ID, matching destinations.json."
+        },
+        "label": {
+          "type": "string",
+          "description": "Ground-truth label from destinations.json.",
+          "enum": ["clean", "suspicious", "malicious"]
+        },
+        "score": {
+          "type": "integer",
+          "description": "Stub score from scores.json.",
+          "minimum": 0,
+          "maximum": 100
+        },
+        "assignedTier": {
+          "type": "string",
+          "description": "Tier computed by applying tierThresholds to score.",
+          "enum": ["clean", "suspicious", "malicious"]
+        },
+        "correct": {
+          "type": "boolean",
+          "description": "Whether assignedTier matches label."
+        },
+        "notes": {
+          "type": "string",
+          "description": "Optional free-text annotation for this result (e.g. edge-case explanation)."
+        }
+      }
+    }
+  }
+}

README.md

@@ -24,7 +24,10 @@ grydlock-testkit/
   README.md
   LICENSE
   package.json
+  evaluation-manifest.json
+  EVALUATION_RESULT_SCHEMA.json
   .github/workflows/ci.yml
+  .github/workflows/consumer-contract-test.yml
   destinations.json
   scores.json
   scripts/