From ec723a31ab594e5151fbf295af3fa9e0a66e4bb8 Mon Sep 17 00:00:00 2001
From: ci <ci@example.com>
Date: Wed, 2 Sep 2026 15:01:19 -0400
Subject: [PATCH 1/6] Add inactive hermetic eval framework

---
 README.md                           |  15 ++
 RESTORE.md                          |  14 ++
 ci/required-files.txt               |   5 +
 evals/v1/framework.jq               | 222 ++++++++++++++++++++++++++++
 evals/v1/run.sh                     | 146 ++++++++++++++++++
 scripts/test/eval-framework.test.sh | 222 ++++++++++++++++++++++++++++
 6 files changed, 624 insertions(+)
 create mode 100644 evals/v1/framework.jq
 create mode 100755 evals/v1/run.sh
 create mode 100755 scripts/test/eval-framework.test.sh

diff --git a/README.md b/README.md
index 78006e5c..c56654eb 100644
--- a/README.md
+++ b/README.md
@@ -230,6 +230,21 @@ main. The payload is offline and unqualified. It does not call GitHub or a CLI,
 credential, change a repository or request, grant authority or qualification, or
 activate a profile.
 
+## Inactive hermetic eval framework
+
+`evals/v1/run.sh evaluate BUNDLE.json` validates a bounded canonical eval suite,
+its cases, trials, and grades, then emits one deterministic report. Every record
+is content-bound. The suite pins its scope and framework version. Model cases
+require multiple trials, while deterministic, model, and human graders remain
+immutable data references rather than commands the runner can invoke.
+
+The runner snapshots a fixed jq program and the selected public-core schema into
+a private directory. It accepts only canonical JSON, rejects stale links,
+tampering, duplicates, and non-canonical order, and keeps `unavailable` distinct
+from `inconclusive`. It has no adapter or arbitrary-command seam, uses no network
+or credential, grants no authority, and makes no activation or qualification
+claim.
+
 ## The current default team
 
 You talk **only** to yshifu, in a Claude Code session. yshifu orchestrates the other roles
diff --git a/RESTORE.md b/RESTORE.md
index e872d00e..ba08fa8a 100644
--- a/RESTORE.md
+++ b/RESTORE.md
@@ -464,6 +464,20 @@ durable main commit and add default-set wiring. The pure jq payload is offline a
 unqualified. It does not call GitHub, use a credential, change a repository or
 request, grant authority or qualification, or activate a profile.
 
+Restore the three paths in the manifest's inactive hermetic eval framework block,
+then run:
+
+```sh
+bash scripts/test/eval-framework.test.sh
+```
+
+This checks canonical suite, case, trial, and grade records; immutable framework
+and scope references; deterministic and stochastic multi-trial results; explicit
+`unavailable` and `inconclusive` states; and stale, tampered, duplicate, missing,
+or out-of-order input rejection. The runner invokes no grader, model, adapter, or
+arbitrary command. It uses no network or credential and grants no authority,
+activation, qualification, or external effect.
+
 ---
 
 ## 5. Smoke test — prove the rebuilt team is alive
diff --git a/ci/required-files.txt b/ci/required-files.txt
index 64bb6b8b..e04a13c5 100644
--- a/ci/required-files.txt
+++ b/ci/required-files.txt
@@ -232,3 +232,8 @@ scripts/test/orchestrator-reconciliation-plan.test.sh
 # Inactive GitHub forge normalizer payload
 adapters/github-forge/v1/normalize.jq
 scripts/test/default-github-forge-adapter.test.sh
+
+# Inactive hermetic eval framework
+evals/v1/framework.jq
+evals/v1/run.sh
+scripts/test/eval-framework.test.sh
diff --git a/evals/v1/framework.jq b/evals/v1/framework.jq
new file mode 100644
index 00000000..8ddb3e9a
--- /dev/null
+++ b/evals/v1/framework.jq
@@ -0,0 +1,222 @@
+import "schema" as schema;
+
+def record_media_type: "application/vnd.ystack.eval-record+json";
+def bundle_media_type: "application/vnd.ystack.eval-bundle+json";
+
+def present_ref_ok:
+  (schema::exact_fields(["state"];[]) and .state == "absent") or
+  (schema::exact_fields(["state","value"];[]) and .state == "present" and
+   (.value | schema::content_ref_ok));
+
+def present_reason_ok:
+  (schema::exact_fields(["state"];[]) and .state == "absent") or
+  (schema::exact_fields(["state","value"];[]) and .state == "present" and
+   (.value | schema::id_ok));
+
+def envelope_ok($kind):
+  schema::exact_fields(["schema_version","kind","id","body"];[]) and
+  .schema_version == 1 and .kind == $kind and
+  (.id | schema::id_ok) and (.body | type == "object");
+
+def grader_ok:
+  schema::exact_fields(
+    ["grader_id","grader_kind","implementation_ref","instructions_ref"];[]) and
+  (.grader_id | schema::id_ok) and
+  (.grader_kind == "deterministic" or .grader_kind == "model" or
+   .grader_kind == "human") and
+  (.implementation_ref | schema::content_ref_ok) and
+  (.instructions_ref | schema::content_ref_ok);
+
+def scope_ok:
+  schema::exact_fields(
+    ["scope_id","scope_version","definition_ref","scope_sha256"];[]) and
+  (.scope_id | schema::id_ok) and (.scope_version | schema::version_ok) and
+  (.definition_ref | schema::content_ref_ok) and
+  (.scope_sha256 | schema::sha256_ok) and
+  .scope_sha256 == .definition_ref.sha256;
+
+def suite_ok:
+  envelope_ok("eval_suite") and
+  (.body |
+   schema::exact_fields(
+     ["suite_version","framework_ref","scope","case_ids"];[]) and
+   .suite_version == "v1" and
+   (.framework_ref | schema::content_ref_ok) and
+   (.scope | scope_ok) and
+   (.case_ids | schema::bounded_set(1;32;schema::id_ok;.)));
+
+def case_ok:
+  envelope_ok("eval_case") and
+  (.body |
+   schema::exact_fields(
+     ["case_version","suite_ref","execution_kind","input_ref","expected_ref",
+      "trial_count","trial_ids","graders"];[]) and
+   .case_version == "v1" and
+   (.suite_ref | schema::content_ref_ok) and
+   (.execution_kind == "deterministic" or .execution_kind == "model") and
+   (.input_ref | schema::content_ref_ok) and
+   (.expected_ref | schema::content_ref_ok) and
+   (.trial_count | schema::int_ok) and .trial_count >= 1 and .trial_count <= 16 and
+   (if .execution_kind == "model" then .trial_count >= 2 else true end) and
+   (.trial_ids | schema::bounded_set(1;16;schema::id_ok;.)) and
+   .trial_count == (.trial_ids | length) and
+   (.graders | schema::bounded_set(1;8;grader_ok;.grader_id)));
+
+def trial_ok:
+  envelope_ok("eval_trial") and
+  (.body |
+   schema::exact_fields(
+     ["trial_version","case_ref","trial_index","status","output_ref","reason"];[]) and
+   .trial_version == "v1" and
+   (.case_ref | schema::content_ref_ok) and
+   (.trial_index | schema::int_ok) and .trial_index >= 1 and .trial_index <= 16 and
+   (.output_ref | present_ref_ok) and (.reason | present_reason_ok) and
+   ((.status == "completed" and .output_ref.state == "present" and
+     .reason.state == "absent") or
+    (.status == "unavailable" and .output_ref.state == "absent" and
+     .reason.state == "present")));
+
+def grade_ok:
+  envelope_ok("eval_grade") and
+  (.body |
+   schema::exact_fields(
+     ["grade_version","trial_ref","grader_id","grader_kind","grader_ref",
+      "status","evidence_ref","reason"];[]) and
+   .grade_version == "v1" and
+   (.trial_ref | schema::content_ref_ok) and
+   (.grader_id | schema::id_ok) and
+   (.grader_kind == "deterministic" or .grader_kind == "model" or
+    .grader_kind == "human") and
+   (.grader_ref | schema::content_ref_ok) and
+   (.evidence_ref | present_ref_ok) and (.reason | present_reason_ok) and
+   (((.status == "passed" or .status == "failed") and
+     .evidence_ref.state == "present" and .reason.state == "absent") or
+    (.status == "inconclusive" and .reason.state == "present") or
+    (.status == "unavailable" and .evidence_ref.state == "absent" and
+     .reason.state == "present")));
+
+def measured_sha($section;$index):
+  [$measured_docs[0][] | select(.section == $section and .index == $index)] |
+  if length == 1 then .[0].sha256 else null end;
+
+def wrapper_shape_ok(value_ok):
+  schema::exact_fields(["ref","value"];[]) and
+  (.ref | schema::content_ref_ok) and .ref.media_type == record_media_type and
+  (.value | value_ok) and .ref.content_id == .value.id;
+
+def wrapper_hash_ok($section;$index):
+  .ref.sha256 == measured_sha($section;$index);
+
+def bundle_shape_ok:
+  . as $bundle |
+  schema::parsed_limits_ok and
+  schema::exact_fields(["schema_version","kind","id","body"];[]) and
+  .schema_version == 1 and .kind == "eval_bundle" and (.id | schema::id_ok) and
+  (.body | schema::exact_fields(["suite","cases","trials","grades"];[])) and
+  ($bundle.body.suite | wrapper_shape_ok(suite_ok)) and
+  ($bundle.body.cases | type == "array" and length >= 1 and length <= 32) and
+  all($bundle.body.cases[]; wrapper_shape_ok(case_ok)) and
+  ($bundle.body.trials | type == "array" and length >= 1 and length <= 128) and
+  all($bundle.body.trials[]; wrapper_shape_ok(trial_ok)) and
+  ($bundle.body.grades | type == "array" and length >= 1 and length <= 256) and
+  all($bundle.body.grades[]; wrapper_shape_ok(grade_ok));
+
+def bundle_hashes_ok($program_sha256):
+  . as $bundle |
+  ($bundle.body.suite | wrapper_hash_ok("suite";0)) and
+  all(range(0;($bundle.body.cases | length)); . as $index |
+      $bundle.body.cases[$index] | wrapper_hash_ok("cases";$index)) and
+  all(range(0;($bundle.body.trials | length)); . as $index |
+      $bundle.body.trials[$index] | wrapper_hash_ok("trials";$index)) and
+  all(range(0;($bundle.body.grades | length)); . as $index |
+      $bundle.body.grades[$index] | wrapper_hash_ok("grades";$index)) and
+  $bundle.body.suite.value.body.framework_ref == {
+    content_id:"eval-framework.v1",
+    media_type:"application/vnd.ystack.eval-framework+jq",
+    sha256:$program_sha256
+  };
+
+def bundle_order_ok:
+  .body as $body |
+  ([$body.suite] + $body.cases + $body.trials + $body.grades |
+   map(.value.id)) as $all_ids |
+  ($all_ids | length) == ($all_ids | unique | length) and
+  ($body.cases | map(.value.id)) as $case_ids |
+  $case_ids == ($case_ids | sort) and
+  ($body.trials | map([.value.body.case_ref.content_id,.value.body.trial_index])) as $trial_keys |
+  $trial_keys == ($trial_keys | sort) and
+  ($trial_keys | length) == ($trial_keys | unique | length) and
+  ($body.grades | map([.value.body.trial_ref.content_id,.value.body.grader_id])) as $grade_keys |
+  $grade_keys == ($grade_keys | sort) and
+  ($grade_keys | length) == ($grade_keys | unique | length);
+
+def bundle_relations_ok:
+  .body as $body |
+  $body.suite.ref as $suite_ref |
+  ($body.cases | map(.value.id)) as $case_ids |
+  $body.suite.value.body.case_ids == $case_ids and
+  all($body.cases[]; .value.body.suite_ref == $suite_ref) and
+  all($body.cases[]; . as $case |
+    [$body.trials[] | select(.value.body.case_ref == $case.ref)] as $trials |
+    ($trials | map(.value.id)) == $case.value.body.trial_ids and
+    ($trials | map(.value.body.trial_index)) ==
+      [range(1;($case.value.body.trial_count + 1))]) and
+  all($body.trials[]; . as $trial |
+    [$body.cases[] | select(.ref == $trial.value.body.case_ref)] as $cases |
+    ($cases | length) == 1 and
+    ($cases[0].value.body.graders) as $graders |
+    [$body.grades[] | select(.value.body.trial_ref == $trial.ref)] as $grades |
+    ($grades | map(.value.body.grader_id)) == ($graders | map(.grader_id)) and
+    all($grades[]; . as $grade |
+      [$graders[] | select(.grader_id == $grade.value.body.grader_id)] as $matches |
+      ($matches | length) == 1 and
+      $grade.value.body.grader_kind == $matches[0].grader_kind and
+      $grade.value.body.grader_ref == $matches[0].implementation_ref)) and
+  all($body.grades[]; . as $grade |
+    any($body.trials[]; .ref == $grade.value.body.trial_ref));
+
+def fold_status($statuses):
+  if any($statuses[]; . == "failed") then "failed"
+  elif any($statuses[]; . == "unavailable") then "unavailable"
+  elif any($statuses[]; . == "inconclusive") then "inconclusive"
+  else "passed"
+  end;
+
+def trial_summary($body;$trial):
+  [$body.grades[] | select(.value.body.trial_ref == $trial.ref) |
+   .value.body.status] as $grade_statuses |
+  (if $trial.value.body.status == "unavailable" then "unavailable"
+   else fold_status($grade_statuses)
+   end) as $status |
+  {trial_ref:$trial.ref,trial_index:$trial.value.body.trial_index,status:$status};
+
+def case_summary($body;$case):
+  [$body.trials[] | select(.value.body.case_ref == $case.ref) |
+   trial_summary($body;.)] as $trials |
+  {case_ref:$case.ref,execution_kind:$case.value.body.execution_kind,
+   status:fold_status($trials | map(.status)),trials:$trials};
+
+def build_report($bundle_sha256):
+  . as $bundle |
+  [$bundle.body.cases[] | case_summary($bundle.body;.)] as $cases |
+  {
+    schema_version:1,
+    kind:"eval_report",
+    id:$bundle.id,
+    body:{
+      mode:"evaluation-only",
+      bundle_ref:{content_id:$bundle.id,media_type:bundle_media_type,sha256:$bundle_sha256},
+      framework_ref:$bundle.body.suite.value.body.framework_ref,
+      suite_ref:$bundle.body.suite.ref,
+      status:fold_status($cases | map(.status)),
+      cases:$cases
+    }
+  };
+
+. as $input |
+if ($input | bundle_shape_ok | not) then error("E_SHAPE")
+elif ($input | bundle_hashes_ok($program_sha256) | not) then error("E_STALE")
+elif ($input | bundle_order_ok | not) then error("E_RELATION")
+elif ($input | bundle_relations_ok | not) then error("E_RELATION")
+else $input | build_report($bundle_sha256)
+end
diff --git a/evals/v1/run.sh b/evals/v1/run.sh
new file mode 100755
index 00000000..3f2928e3
--- /dev/null
+++ b/evals/v1/run.sh
@@ -0,0 +1,146 @@
+#!/bin/bash
+# shellcheck disable=SC2016
+set -euo pipefail
+export LC_ALL=C
+umask 077
+
+emit_error() {
+  local token=${1:-E_RUNTIME}
+  case "$token" in
+    E_USAGE|E_RUNTIME|E_LIMIT|E_PARSE|E_CANONICAL|E_SHAPE|E_RELATION|E_STALE) ;;
+    *) token=E_RUNTIME ;;
+  esac
+  /usr/bin/printf '%s\n' "$token" >&2
+  exit 1
+}
+
+sha_file() { /usr/bin/shasum -a 256 "$1" | /usr/bin/awk '{print $1}'; }
+sha_line() {
+  builtin printf '%s\n' "$1" | /usr/bin/shasum -a 256 | /usr/bin/awk '{print $1}'
+}
+snapshot_file() {
+  /usr/bin/perl -MFcntl=:DEFAULT,:mode -e '
+    my ($source,$target,$limit,$mode)=@ARGV;
+    sysopen(my $in,$source,O_RDONLY|O_NOFOLLOW) or exit 40;
+    my @stat=stat($in); @stat && S_ISREG($stat[2]) or exit 40;
+    $stat[7] <= $limit or exit 42;
+    sysopen(my $out,$target,O_WRONLY|O_CREAT|O_EXCL|O_NOFOLLOW,oct($mode)) or exit 40;
+    my $total=0;
+    while (1) {
+      my $read=sysread($in,my $buffer,65536); defined($read) or exit 40;
+      last if $read==0; $total += $read; $total <= $limit or exit 42;
+      my $offset=0;
+      while ($offset < $read) {
+        my $written=syswrite($out,$buffer,$read-$offset,$offset);
+        defined($written) && $written>0 or exit 40; $offset += $written;
+      }
+    }
+    close($in) or exit 40; close($out) or exit 40;
+    chmod(oct($mode),$target)==1 or exit 40;
+  ' "$1" "$2" "$3" "$4"
+}
+
+[ "$#" -eq 2 ] && [ "$1" = evaluate ] || emit_error E_USAGE
+input=$2
+self=${BASH_SOURCE[0]}
+case "$self" in /*) ;; *) self="$(pwd -P)/$self" ;; esac
+[ -f "$self" ] && [ ! -L "$self" ] || emit_error E_RUNTIME
+source_dir=$(CDPATH='' cd -P -- "${self%/*}" 2>/dev/null && pwd -P) || emit_error E_RUNTIME
+[ "$self" = "$source_dir/run.sh" ] || emit_error E_RUNTIME
+repo=$(CDPATH='' cd -P -- "$source_dir/../.." 2>/dev/null && pwd -P) || emit_error E_RUNTIME
+[ "$source_dir" = "$repo/evals/v1" ] || emit_error E_RUNTIME
+
+program_sha=6fb15ba938580b340cfdd6f628de0f72ee0bdb589c0673c47e5444daf16f51bf
+schema_sha=8d1d02d36ac7ada778f05248f9413062b3fc251499914c15d79f003bbd009ade
+registry_sha=f55b697716dc13a6d2c71bde7769493b3f4b091fd7a94d3280c5d417974df3a1
+generation=g-392d20099dfa99872764009b268c8871914b4dbc0da467ec346baa921818ae3e
+platform=$(/usr/bin/uname -s):$(/usr/bin/uname -m)
+case "$platform" in
+  Darwin:*) jq_asset=jq-osx-amd64; jq_sha=5c0a0a3ea600f302ee458b30317425dd9632d1ad8882259fcaf4e9b868b2b1ef ;;
+  Linux:x86_64) jq_asset=jq-linux64; jq_sha=af986793a515d500ab2d35f8d2aecd656e764504b789b66d7e1a0b727a124c44 ;;
+  *) emit_error E_RUNTIME ;;
+esac
+jq_source=''
+for candidate in "${TMPDIR:-/tmp}/ystack-portable-core-jq16/$jq_asset" /usr/bin/jq; do
+  if [ -f "$candidate" ] && [ ! -L "$candidate" ] &&
+     [ "$(sha_file "$candidate")" = "$jq_sha" ]; then jq_source=$candidate; break; fi
+done
+[ -n "$jq_source" ] || emit_error E_RUNTIME
+
+scratch=$(/usr/bin/mktemp -d "${TMPDIR:-/tmp}/ystack-eval-run.XXXXXX") || emit_error E_RUNTIME
+cleanup() { /bin/rm -rf -- "$scratch" >/dev/null 2>&1 || :; }
+signal_exit() { trap - EXIT HUP INT TERM; cleanup; exit 1; }
+trap cleanup EXIT
+trap signal_exit HUP INT TERM
+snapshot_file "$input" "$scratch/input.json" 4194304 0400 || {
+  status=$?; [ "$status" -eq 42 ] && emit_error E_LIMIT; emit_error E_RUNTIME;
+}
+snapshot_file "$source_dir/framework.jq" "$scratch/framework.jq" 1048576 0400 || emit_error E_RUNTIME
+schema="$repo/core/v2/generations/$generation/modules/schema.jq"
+snapshot_file "$schema" "$scratch/schema.jq" 1048576 0400 || emit_error E_RUNTIME
+snapshot_file "$repo/core/v2/generation-registry.json" "$scratch/registry.json" 1048576 0400 ||
+  emit_error E_RUNTIME
+snapshot_file "$jq_source" "$scratch/jq" 33554432 0500 || emit_error E_RUNTIME
+[ "$(sha_file "$scratch/framework.jq")" = "$program_sha" ] || emit_error E_STALE
+[ "$(sha_file "$scratch/schema.jq")" = "$schema_sha" ] || emit_error E_STALE
+[ "$(sha_file "$scratch/registry.json")" = "$registry_sha" ] || emit_error E_STALE
+[ "$(sha_file "$scratch/jq")" = "$jq_sha" ] || emit_error E_RUNTIME
+[ "$("$scratch/jq" -r --arg generation "$generation" \
+    'length==1 and .[0].generation_id==$generation' "$scratch/registry.json")" = true ] ||
+  emit_error E_STALE
+
+if ! "$scratch/jq" -S -c . "$scratch/input.json" > "$scratch/canonical.json" 2>/dev/null; then
+  emit_error E_PARSE
+fi
+/usr/bin/cmp -s "$scratch/input.json" "$scratch/canonical.json" || emit_error E_CANONICAL
+bundle_sha=$(sha_file "$scratch/input.json") || emit_error E_RUNTIME
+if ! "$scratch/jq" -e '
+  . as $bundle |
+  type=="object" and (.body|type=="object") and
+  (.body.suite|type=="object") and
+  all(["cases","trials","grades"][]; . as $key | ($bundle.body[$key]|type)=="array") and
+  (.body.cases|length)<=32 and (.body.trials|length)<=128 and (.body.grades|length)<=256
+' "$scratch/input.json" >/dev/null 2>&1; then emit_error E_SHAPE; fi
+
+: > "$scratch/measured.tsv"
+for section in suite cases trials grades; do
+  if [ "$section" = suite ]; then count=1; else
+    count=$("$scratch/jq" -r --arg section "$section" '.body[$section]|length' "$scratch/input.json") ||
+      emit_error E_SHAPE
+  fi
+  index=0
+  while [ "$index" -lt "$count" ]; do
+    value=$("$scratch/jq" -S -c --arg section "$section" --argjson index "$index" '
+      if $section=="suite" then .body.suite.value else .body[$section][$index].value end
+    ' "$scratch/input.json") || emit_error E_SHAPE
+    /usr/bin/printf '%s\t%s\t%s\n' "$section" "$index" "$(sha_line "$value")" >> "$scratch/measured.tsv"
+    index=$((index + 1))
+  done
+done
+"$scratch/jq" -R -s -c '
+  split("\n") | map(select(length>0) | split("\t") |
+    {section:.[0],index:(.[1]|tonumber),sha256:.[2]})
+' "$scratch/measured.tsv" > "$scratch/measured.json" || emit_error E_RUNTIME
+
+status=0
+/usr/bin/env -i LC_ALL=C PATH=/usr/bin:/bin \
+  "$scratch/jq" -S -c -L "$scratch" --arg program_sha256 "$program_sha" \
+  --arg bundle_sha256 "$bundle_sha" --slurpfile measured_docs "$scratch/measured.json" \
+  -f "$scratch/framework.jq" "$scratch/input.json" > "$scratch/output.json" \
+  2> "$scratch/error" || status=$?
+if [ "$status" -ne 0 ]; then
+  [ ! -s "$scratch/output.json" ] || emit_error E_RUNTIME
+  for token in E_SHAPE E_STALE E_RELATION; do
+    /usr/bin/grep -Fq "$token" "$scratch/error" && emit_error "$token"
+  done
+  emit_error E_RUNTIME
+fi
+[ ! -s "$scratch/error" ] || emit_error E_RUNTIME
+[ "$(sha_file "$scratch/input.json")" = "$bundle_sha" ] || emit_error E_RUNTIME
+[ "$(sha_file "$scratch/framework.jq")" = "$program_sha" ] || emit_error E_RUNTIME
+[ "$(sha_file "$scratch/schema.jq")" = "$schema_sha" ] || emit_error E_RUNTIME
+[ "$(sha_file "$scratch/registry.json")" = "$registry_sha" ] || emit_error E_RUNTIME
+[ "$(sha_file "$scratch/jq")" = "$jq_sha" ] || emit_error E_RUNTIME
+/bin/cat "$scratch/output.json" || emit_error E_RUNTIME
+trap - EXIT HUP INT TERM
+cleanup
diff --git a/scripts/test/eval-framework.test.sh b/scripts/test/eval-framework.test.sh
new file mode 100755
index 00000000..000c0415
--- /dev/null
+++ b/scripts/test/eval-framework.test.sh
@@ -0,0 +1,222 @@
+#!/bin/bash
+# shellcheck disable=SC2016
+set -euo pipefail
+export LC_ALL=C
+umask 077
+
+root=$(CDPATH='' cd -P -- "${BASH_SOURCE[0]%/*}/../.." && pwd -P)
+runner="$root/evals/v1/run.sh"
+tmp=$(/usr/bin/mktemp -d "${TMPDIR:-/tmp}/ystack-eval-framework-test.XXXXXX")
+cleanup() { /bin/rm -rf -- "$tmp"; }
+trap cleanup EXIT
+fail() { /usr/bin/printf 'FAIL: %s\n' "$1" >&2; exit 1; }
+passes=0
+pass() { passes=$((passes + 1)); /usr/bin/printf 'ok %s - %s\n' "$passes" "$1"; }
+sha_file() { /usr/bin/shasum -a 256 "$1" | /usr/bin/awk '{print $1}'; }
+
+platform=$(/usr/bin/uname -s):$(/usr/bin/uname -m)
+case "$platform" in
+  Darwin:*) jq_asset=jq-osx-amd64; jq_sha=5c0a0a3ea600f302ee458b30317425dd9632d1ad8882259fcaf4e9b868b2b1ef ;;
+  Linux:x86_64) jq_asset=jq-linux64; jq_sha=af986793a515d500ab2d35f8d2aecd656e764504b789b66d7e1a0b727a124c44 ;;
+  *) fail "unsupported host $platform" ;;
+esac
+jq_cache_dir="${TMPDIR:-/tmp}/ystack-portable-core-jq16"
+/bin/mkdir -p "$jq_cache_dir"
+jq_bin="$jq_cache_dir/$jq_asset"
+if [ ! -f "$jq_bin" ] || [ "$(sha_file "$jq_bin")" != "$jq_sha" ]; then
+  download=$(/usr/bin/mktemp "$jq_cache_dir/.jq-1.6.XXXXXX")
+  /usr/bin/curl --proto '=https' --tlsv1.2 -fsSL \
+    "https://github.com/jqlang/jq/releases/download/jq-1.6/$jq_asset" -o "$download"
+  [ "$(sha_file "$download")" = "$jq_sha" ] || fail 'jq release digest'
+  /bin/chmod 0555 "$download"
+  /bin/mv "$download" "$jq_bin"
+fi
+[ "$($jq_bin --version)" = jq-1.6 ] || fail 'jq identity'
+
+ref() {
+  "$jq_bin" -S -c -n --arg id "$1" --arg media "$2" --arg sha "$3" \
+    '{content_id:$id,media_type:$media,sha256:$sha}'
+}
+wrap() {
+  local source=$1 target=$2 id digest
+  id=$("$jq_bin" -r .id "$source")
+  digest=$(sha_file "$source")
+  "$jq_bin" -S -c -n --arg id "$id" --arg sha "$digest" --slurpfile value "$source" '
+    {ref:{content_id:$id,media_type:"application/vnd.ystack.eval-record+json",sha256:$sha},
+     value:$value[0]}
+  ' > "$target"
+}
+grader() {
+  "$jq_bin" -S -c -n --arg id "$1" --arg kind "$2" --arg digit "$3" '
+    def ref($id;$sha):
+      {content_id:$id,media_type:"application/vnd.ystack.eval-grader+json",sha256:$sha};
+    {grader_id:$id,grader_kind:$kind,
+     implementation_ref:ref(($id+".implementation");($digit*64)),
+     instructions_ref:ref(($id+".instructions");($digit*64))}
+  '
+}
+rehash() {
+  local source=$1 section=$2 index=$3 target=$4 value digest
+  value=$("$jq_bin" -S -c --arg section "$section" --argjson index "$index" '
+    if $section=="suite" then .body.suite.value else .body[$section][$index].value end
+  ' "$source")
+  digest=$(builtin printf '%s\n' "$value" | /usr/bin/shasum -a 256 | /usr/bin/awk '{print $1}')
+  "$jq_bin" -S -c --arg section "$section" --argjson index "$index" --arg sha "$digest" '
+    if $section=="suite" then .body.suite.ref.sha256=$sha
+    else .body[$section][$index].ref.sha256=$sha end
+  ' "$source" > "$target"
+}
+expect_error() {
+  local name=$1 token=$2 input=$3 status=0
+  "$runner" evaluate "$input" > "$tmp/$name.out" 2> "$tmp/$name.err" || status=$?
+  [ "$status" -ne 0 ] && [ ! -s "$tmp/$name.out" ] &&
+    [ "$(/bin/cat "$tmp/$name.err")" = "$token" ] || fail "$name"
+  pass "$name"
+}
+
+framework_sha=$(sha_file "$root/evals/v1/framework.jq")
+framework_ref=$(ref eval-framework.v1 application/vnd.ystack.eval-framework+jq "$framework_sha")
+scope_ref=$(ref eval-scope.definition application/vnd.ystack.eval-scope+json "$(printf 'a%.0s' {1..64})")
+"$jq_bin" -S -c -n --argjson framework "$framework_ref" --argjson definition "$scope_ref" '
+  {schema_version:1,kind:"eval_suite",id:"suite.default-adapters",body:{
+    suite_version:"v1",framework_ref:$framework,
+    scope:{scope_id:"scope.default-adapters",scope_version:"v1",
+      definition_ref:$definition,scope_sha256:$definition.sha256},
+    case_ids:["case.deterministic","case.model"]}}
+' > "$tmp/suite.value"
+wrap "$tmp/suite.value" "$tmp/suite.wrap"
+suite_ref=$("$jq_bin" -c .ref "$tmp/suite.wrap")
+
+grader_det=$(grader grader.det deterministic b)
+grader_human=$(grader grader.human human c)
+grader_model=$(grader grader.model model d)
+input_det=$(ref input.det application/json "$(printf '1%.0s' {1..64})")
+expected_det=$(ref expected.det application/json "$(printf '2%.0s' {1..64})")
+input_model=$(ref input.model application/json "$(printf '3%.0s' {1..64})")
+expected_model=$(ref expected.model application/json "$(printf '4%.0s' {1..64})")
+"$jq_bin" -S -c -n --argjson suite "$suite_ref" --argjson input "$input_det" \
+  --argjson expected "$expected_det" --argjson det "$grader_det" --argjson human "$grader_human" '
+  {schema_version:1,kind:"eval_case",id:"case.deterministic",body:{case_version:"v1",
+   suite_ref:$suite,execution_kind:"deterministic",input_ref:$input,expected_ref:$expected,
+   trial_count:1,trial_ids:["trial.det.1"],graders:[$det,$human]}}
+' > "$tmp/case.det.value"
+"$jq_bin" -S -c -n --argjson suite "$suite_ref" --argjson input "$input_model" \
+  --argjson expected "$expected_model" --argjson det "$grader_det" --argjson human "$grader_human" \
+  --argjson model "$grader_model" '
+  {schema_version:1,kind:"eval_case",id:"case.model",body:{case_version:"v1",
+   suite_ref:$suite,execution_kind:"model",input_ref:$input,expected_ref:$expected,
+   trial_count:2,trial_ids:["trial.model.1","trial.model.2"],graders:[$det,$human,$model]}}
+' > "$tmp/case.model.value"
+wrap "$tmp/case.det.value" "$tmp/case.det.wrap"
+wrap "$tmp/case.model.value" "$tmp/case.model.wrap"
+"$jq_bin" -S -c -s . "$tmp/case.det.wrap" "$tmp/case.model.wrap" > "$tmp/cases.json"
+case_det_ref=$("$jq_bin" -c .ref "$tmp/case.det.wrap")
+case_model_ref=$("$jq_bin" -c .ref "$tmp/case.model.wrap")
+
+make_trial() {
+  local id=$1 case_ref=$2 index=$3 digit=$4 output
+  output=$(ref "$id.output" application/json "$(printf "$digit%.0s" {1..64})")
+  "$jq_bin" -S -c -n --arg id "$id" --argjson case_ref "$case_ref" --argjson index "$index" \
+    --argjson output "$output" '{schema_version:1,kind:"eval_trial",id:$id,body:{
+      trial_version:"v1",case_ref:$case_ref,trial_index:$index,status:"completed",
+      output_ref:{state:"present",value:$output},reason:{state:"absent"}}}'
+}
+make_trial trial.det.1 "$case_det_ref" 1 5 > "$tmp/trial.det.value"
+make_trial trial.model.1 "$case_model_ref" 1 6 > "$tmp/trial.model1.value"
+make_trial trial.model.2 "$case_model_ref" 2 7 > "$tmp/trial.model2.value"
+for name in det model1 model2; do wrap "$tmp/trial.$name.value" "$tmp/trial.$name.wrap"; done
+"$jq_bin" -S -c -s . "$tmp/trial.det.wrap" "$tmp/trial.model1.wrap" "$tmp/trial.model2.wrap" > "$tmp/trials.json"
+
+make_grade() {
+  local id=$1 trial=$2 grader_json=$3 status=$4 digit=$5
+  local grader_id grader_kind grader_ref evidence evidence_state reason
+  grader_id=$("$jq_bin" -r .grader_id <<< "$grader_json")
+  grader_kind=$("$jq_bin" -r .grader_kind <<< "$grader_json")
+  grader_ref=$("$jq_bin" -c .implementation_ref <<< "$grader_json")
+  evidence=$(ref "$id.evidence" application/json "$(printf "$digit%.0s" {1..64})")
+  if [ "$status" = inconclusive ]; then
+    evidence_state='{"state":"absent"}'; reason='{"state":"present","value":"grader.no-consensus"}'
+  else
+    evidence_state=$("$jq_bin" -c -n --argjson value "$evidence" '{state:"present",value:$value}')
+    reason='{"state":"absent"}'
+  fi
+  "$jq_bin" -S -c -n --arg id "$id" --argjson trial "$trial" --arg grader_id "$grader_id" \
+    --arg grader_kind "$grader_kind" --argjson grader_ref "$grader_ref" --arg status "$status" \
+    --argjson evidence "$evidence_state" --argjson reason "$reason" '
+    {schema_version:1,kind:"eval_grade",id:$id,body:{grade_version:"v1",trial_ref:$trial,
+     grader_id:$grader_id,grader_kind:$grader_kind,grader_ref:$grader_ref,status:$status,
+     evidence_ref:$evidence,reason:$reason}}'
+}
+trial_det_ref=$("$jq_bin" -c .ref "$tmp/trial.det.wrap")
+trial_model1_ref=$("$jq_bin" -c .ref "$tmp/trial.model1.wrap")
+trial_model2_ref=$("$jq_bin" -c .ref "$tmp/trial.model2.wrap")
+grade_specs=(
+  'det.det' 'det.human' 'model1.det' 'model1.human' 'model1.model'
+  'model2.det' 'model2.human' 'model2.model'
+)
+for spec in "${grade_specs[@]}"; do
+  trial_name=${spec%%.*}; grader_name=${spec##*.}; status=passed; digit=8
+  [ "$spec" = model2.human ] && status=inconclusive
+  case "$trial_name" in det) trial_ref=$trial_det_ref ;; model1) trial_ref=$trial_model1_ref ;; model2) trial_ref=$trial_model2_ref ;; esac
+  case "$grader_name" in det) grader_json=$grader_det ;; human) grader_json=$grader_human ;; model) grader_json=$grader_model ;; esac
+  make_grade "grade.$spec" "$trial_ref" "$grader_json" "$status" "$digit" > "$tmp/grade.$spec.value"
+  wrap "$tmp/grade.$spec.value" "$tmp/grade.$spec.wrap"
+done
+"$jq_bin" -S -c -s . "$tmp"/grade.*.wrap | "$jq_bin" -S -c 'sort_by([.value.body.trial_ref.content_id,.value.body.grader_id])' > "$tmp/grades.json"
+"$jq_bin" -S -c -n --slurpfile suite "$tmp/suite.wrap" --slurpfile cases "$tmp/cases.json" \
+  --slurpfile trials "$tmp/trials.json" --slurpfile grades "$tmp/grades.json" '
+  {schema_version:1,kind:"eval_bundle",id:"run.default-adapters",
+   body:{suite:$suite[0],cases:$cases[0],trials:$trials[0],grades:$grades[0]}}
+' > "$tmp/bundle.json"
+
+"$runner" evaluate "$tmp/bundle.json" > "$tmp/report.json"
+"$jq_bin" -e '
+  .kind=="eval_report" and .body.mode=="evaluation-only" and
+  .body.status=="inconclusive" and
+  [.body.cases[].status]==["passed","inconclusive"] and
+  [.body.cases[1].trials[].trial_index]==[1,2]
+' "$tmp/report.json" >/dev/null || fail 'valid multi-trial report'
+pass valid-multi-trial-report
+"$runner" evaluate "$tmp/bundle.json" > "$tmp/repeat.json"
+/usr/bin/cmp -s "$tmp/report.json" "$tmp/repeat.json" || fail 'deterministic report'
+pass deterministic-report
+
+"$jq_bin" -S -c '.body.cases[0].value.body.expected_ref.sha256=("f"*64)' \
+  "$tmp/bundle.json" > "$tmp/tamper.json"
+expect_error tampered-record E_STALE "$tmp/tamper.json"
+"$jq_bin" -S -c '.body.cases[0].value.body.suite_ref.sha256=("e"*64)' \
+  "$tmp/bundle.json" > "$tmp/stale-unhashed.json"
+rehash "$tmp/stale-unhashed.json" cases 0 "$tmp/stale.json"
+expect_error stale-record-link E_RELATION "$tmp/stale.json"
+"$jq_bin" -S -c '.body.cases|=reverse' "$tmp/bundle.json" > "$tmp/order.json"
+expect_error record-order E_RELATION "$tmp/order.json"
+"$jq_bin" -S -c '.body.trials += [.body.trials[0]]' "$tmp/bundle.json" > "$tmp/duplicate.json"
+expect_error duplicate-trial E_RELATION "$tmp/duplicate.json"
+"$jq_bin" -S -c '.body.grades |= map(select(.value.id!="grade.model2.model"))' \
+  "$tmp/bundle.json" > "$tmp/missing-grade.json"
+expect_error missing-grade E_RELATION "$tmp/missing-grade.json"
+"$jq_bin" -S -c '.body.suite.value.body.extra_command="printf unsafe"' \
+  "$tmp/bundle.json" > "$tmp/command.json"
+expect_error arbitrary-command-field E_SHAPE "$tmp/command.json"
+
+"$jq_bin" -S -c '
+  .body.grades |= map(if .value.id=="grade.model2.human" then
+    .value.body.status="unavailable" | .value.body.evidence_ref={state:"absent"} |
+    .value.body.reason={state:"present",value:"grader.unavailable"}
+    else . end)
+' "$tmp/bundle.json" > "$tmp/unavailable-unhashed.json"
+grade_index=$("$jq_bin" -r '.body.grades|map(.value.id)|index("grade.model2.human")' "$tmp/unavailable-unhashed.json")
+rehash "$tmp/unavailable-unhashed.json" grades "$grade_index" "$tmp/unavailable.json"
+"$runner" evaluate "$tmp/unavailable.json" > "$tmp/unavailable-report.json"
+[ "$("$jq_bin" -r .body.status "$tmp/unavailable-report.json")" = unavailable ] || fail 'unavailable result'
+pass explicit-unavailable
+
+"$jq_bin" . "$tmp/bundle.json" > "$tmp/noncanonical.json"
+expect_error canonical-bytes E_CANONICAL "$tmp/noncanonical.json"
+"$jq_bin" -e '
+  ([..|objects|keys[]|select(.=="authority" or .=="permissions" or .=="credential" or
+    .=="network" or .=="command" or .=="activation" or .=="qualification")] | length)==0
+' "$tmp/report.json" >/dev/null || fail 'inactive data-only output'
+pass inactive-data-only-output
+
+/usr/bin/printf 'PASS: %s eval framework checks\n' "$passes"

From 31d05f88260a21e21df9abb9bb357a60ab8c8330 Mon Sep 17 00:00:00 2001
From: ci <ci@example.com>
Date: Wed, 2 Sep 2026 18:13:30 -0400
Subject: [PATCH 2/6] Harden eval record relationships

---
 README.md                           |  6 ++-
 RESTORE.md                          | 10 +++--
 evals/v1/framework.jq               | 24 +++++++++--
 evals/v1/run.sh                     | 10 ++++-
 scripts/test/eval-framework.test.sh | 65 +++++++++++++++++++++++++++--
 5 files changed, 99 insertions(+), 16 deletions(-)

diff --git a/README.md b/README.md
index 8868b47d..5ed25ca0 100644
--- a/README.md
+++ b/README.md
@@ -251,12 +251,14 @@ grant authority or qualification, or activate a profile.
 its cases, trials, and grades, then emits one deterministic report. Every record
 is content-bound. The suite pins its scope and framework version. Model cases
 require multiple trials, while deterministic, model, and human graders remain
-immutable data references rather than commands the runner can invoke.
+immutable data references rather than commands the runner can invoke. Declared
+trial and attempt identities are exact, and trial/grade timestamps must agree.
 
 The runner snapshots a fixed jq program and the selected public-core schema into
 a private directory. It accepts only canonical JSON, rejects stale links,
 tampering, duplicates, and non-canonical order, and keeps `unavailable` distinct
-from `inconclusive`. It has no adapter or arbitrary-command seam, uses no network
+from `inconclusive`. Failed grades cannot be hidden by another trial state. It
+has no adapter or arbitrary-command seam, uses no network
 or credential, grants no authority, and makes no activation or qualification
 claim.
 
diff --git a/RESTORE.md b/RESTORE.md
index 1212d8ac..5691714f 100644
--- a/RESTORE.md
+++ b/RESTORE.md
@@ -487,11 +487,13 @@ bash scripts/test/eval-framework.test.sh
 ```
 
 This checks canonical suite, case, trial, and grade records; immutable framework
-and scope references; deterministic and stochastic multi-trial results; explicit
+and scope references; exact trial/attempt identities; coherent trial and grade
+timestamps; deterministic and stochastic multi-trial results; explicit
 `unavailable` and `inconclusive` states; and stale, tampered, duplicate, missing,
-or out-of-order input rejection. The runner invokes no grader, model, adapter, or
-arbitrary command. It uses no network or credential and grants no authority,
-activation, qualification, or external effect.
+or out-of-order input rejection. Failed grades cannot be hidden by an unavailable
+trial. The runner invokes no grader, model, adapter, or arbitrary command. It uses
+no network or credential and grants no authority, activation, qualification, or
+external effect.
 
 ---
 
diff --git a/evals/v1/framework.jq b/evals/v1/framework.jq
index 8ddb3e9a..63715e04 100644
--- a/evals/v1/framework.jq
+++ b/evals/v1/framework.jq
@@ -50,7 +50,7 @@ def case_ok:
   (.body |
    schema::exact_fields(
      ["case_version","suite_ref","execution_kind","input_ref","expected_ref",
-      "trial_count","trial_ids","graders"];[]) and
+      "trial_count","trial_ids","attempt_ids","graders"];[]) and
    .case_version == "v1" and
    (.suite_ref | schema::content_ref_ok) and
    (.execution_kind == "deterministic" or .execution_kind == "model") and
@@ -59,17 +59,23 @@ def case_ok:
    (.trial_count | schema::int_ok) and .trial_count >= 1 and .trial_count <= 16 and
    (if .execution_kind == "model" then .trial_count >= 2 else true end) and
    (.trial_ids | schema::bounded_set(1;16;schema::id_ok;.)) and
+   (.attempt_ids | schema::bounded_set(1;16;schema::id_ok;.)) and
    .trial_count == (.trial_ids | length) and
+   .trial_count == (.attempt_ids | length) and
    (.graders | schema::bounded_set(1;8;grader_ok;.grader_id)));
 
 def trial_ok:
   envelope_ok("eval_trial") and
   (.body |
    schema::exact_fields(
-     ["trial_version","case_ref","trial_index","status","output_ref","reason"];[]) and
+     ["trial_version","case_ref","trial_index","attempt_id","started_at","finished_at",
+      "status","output_ref","reason"];[]) and
    .trial_version == "v1" and
    (.case_ref | schema::content_ref_ok) and
    (.trial_index | schema::int_ok) and .trial_index >= 1 and .trial_index <= 16 and
+   (.attempt_id | schema::id_ok) and
+   (.started_at | schema::time_ok) and (.finished_at | schema::time_ok) and
+   .started_at <= .finished_at and
    (.output_ref | present_ref_ok) and (.reason | present_reason_ok) and
    ((.status == "completed" and .output_ref.state == "present" and
      .reason.state == "absent") or
@@ -81,13 +87,14 @@ def grade_ok:
   (.body |
    schema::exact_fields(
      ["grade_version","trial_ref","grader_id","grader_kind","grader_ref",
-      "status","evidence_ref","reason"];[]) and
+      "graded_at","status","evidence_ref","reason"];[]) and
    .grade_version == "v1" and
    (.trial_ref | schema::content_ref_ok) and
    (.grader_id | schema::id_ok) and
    (.grader_kind == "deterministic" or .grader_kind == "model" or
     .grader_kind == "human") and
    (.grader_ref | schema::content_ref_ok) and
+   (.graded_at | schema::time_ok) and
    (.evidence_ref | present_ref_ok) and (.reason | present_reason_ok) and
    (((.status == "passed" or .status == "failed") and
      .evidence_ref.state == "present" and .reason.state == "absent") or
@@ -146,6 +153,8 @@ def bundle_order_ok:
   ($body.trials | map([.value.body.case_ref.content_id,.value.body.trial_index])) as $trial_keys |
   $trial_keys == ($trial_keys | sort) and
   ($trial_keys | length) == ($trial_keys | unique | length) and
+  ($body.trials | map(.value.body.attempt_id)) as $attempt_ids |
+  ($attempt_ids | length) == ($attempt_ids | unique | length) and
   ($body.grades | map([.value.body.trial_ref.content_id,.value.body.grader_id])) as $grade_keys |
   $grade_keys == ($grade_keys | sort) and
   ($grade_keys | length) == ($grade_keys | unique | length);
@@ -159,6 +168,7 @@ def bundle_relations_ok:
   all($body.cases[]; . as $case |
     [$body.trials[] | select(.value.body.case_ref == $case.ref)] as $trials |
     ($trials | map(.value.id)) == $case.value.body.trial_ids and
+    ($trials | map(.value.body.attempt_id)) == $case.value.body.attempt_ids and
     ($trials | map(.value.body.trial_index)) ==
       [range(1;($case.value.body.trial_count + 1))]) and
   all($body.trials[]; . as $trial |
@@ -167,6 +177,10 @@ def bundle_relations_ok:
     ($cases[0].value.body.graders) as $graders |
     [$body.grades[] | select(.value.body.trial_ref == $trial.ref)] as $grades |
     ($grades | map(.value.body.grader_id)) == ($graders | map(.grader_id)) and
+    all($grades[]; .value.body.graded_at >= $trial.value.body.finished_at) and
+    (if $trial.value.body.status == "unavailable"
+     then all($grades[]; .value.body.status == "unavailable")
+     else true end) and
     all($grades[]; . as $grade |
       [$graders[] | select(.grader_id == $grade.value.body.grader_id)] as $matches |
       ($matches | length) == 1 and
@@ -188,7 +202,9 @@ def trial_summary($body;$trial):
   (if $trial.value.body.status == "unavailable" then "unavailable"
    else fold_status($grade_statuses)
    end) as $status |
-  {trial_ref:$trial.ref,trial_index:$trial.value.body.trial_index,status:$status};
+  {trial_ref:$trial.ref,trial_index:$trial.value.body.trial_index,
+   attempt_id:$trial.value.body.attempt_id,started_at:$trial.value.body.started_at,
+   finished_at:$trial.value.body.finished_at,status:$status};
 
 def case_summary($body;$case):
   [$body.trials[] | select(.value.body.case_ref == $case.ref) |
diff --git a/evals/v1/run.sh b/evals/v1/run.sh
index 3f2928e3..e57a58a5 100755
--- a/evals/v1/run.sh
+++ b/evals/v1/run.sh
@@ -50,7 +50,7 @@ source_dir=$(CDPATH='' cd -P -- "${self%/*}" 2>/dev/null && pwd -P) || emit_erro
 repo=$(CDPATH='' cd -P -- "$source_dir/../.." 2>/dev/null && pwd -P) || emit_error E_RUNTIME
 [ "$source_dir" = "$repo/evals/v1" ] || emit_error E_RUNTIME
 
-program_sha=6fb15ba938580b340cfdd6f628de0f72ee0bdb589c0673c47e5444daf16f51bf
+program_sha=1cdb63cd246f0fd2a2de484d62a0be297cb24ad1dffd5fc9a6c57b967d10778d
 schema_sha=8d1d02d36ac7ada778f05248f9413062b3fc251499914c15d79f003bbd009ade
 registry_sha=f55b697716dc13a6d2c71bde7769493b3f4b091fd7a94d3280c5d417974df3a1
 generation=g-392d20099dfa99872764009b268c8871914b4dbc0da467ec346baa921818ae3e
@@ -136,11 +136,17 @@ if [ "$status" -ne 0 ]; then
   emit_error E_RUNTIME
 fi
 [ ! -s "$scratch/error" ] || emit_error E_RUNTIME
+status=0
+snapshot_file "$scratch/output.json" "$scratch/bounded-output.json" 1048576 0400 || status=$?
+[ "$status" -eq 0 ] || { [ "$status" -eq 42 ] && emit_error E_LIMIT; emit_error E_RUNTIME; }
+"$scratch/jq" -S -c . "$scratch/bounded-output.json" > "$scratch/canonical-output.json" 2>/dev/null ||
+  emit_error E_RUNTIME
+/usr/bin/cmp -s "$scratch/bounded-output.json" "$scratch/canonical-output.json" || emit_error E_RUNTIME
 [ "$(sha_file "$scratch/input.json")" = "$bundle_sha" ] || emit_error E_RUNTIME
 [ "$(sha_file "$scratch/framework.jq")" = "$program_sha" ] || emit_error E_RUNTIME
 [ "$(sha_file "$scratch/schema.jq")" = "$schema_sha" ] || emit_error E_RUNTIME
 [ "$(sha_file "$scratch/registry.json")" = "$registry_sha" ] || emit_error E_RUNTIME
 [ "$(sha_file "$scratch/jq")" = "$jq_sha" ] || emit_error E_RUNTIME
-/bin/cat "$scratch/output.json" || emit_error E_RUNTIME
+/bin/cat "$scratch/bounded-output.json" || emit_error E_RUNTIME
 trap - EXIT HUP INT TERM
 cleanup
diff --git a/scripts/test/eval-framework.test.sh b/scripts/test/eval-framework.test.sh
index 000c0415..92456f7a 100755
--- a/scripts/test/eval-framework.test.sh
+++ b/scripts/test/eval-framework.test.sh
@@ -98,14 +98,16 @@ expected_model=$(ref expected.model application/json "$(printf '4%.0s' {1..64})"
   --argjson expected "$expected_det" --argjson det "$grader_det" --argjson human "$grader_human" '
   {schema_version:1,kind:"eval_case",id:"case.deterministic",body:{case_version:"v1",
    suite_ref:$suite,execution_kind:"deterministic",input_ref:$input,expected_ref:$expected,
-   trial_count:1,trial_ids:["trial.det.1"],graders:[$det,$human]}}
+   trial_count:1,trial_ids:["trial.det.1"],attempt_ids:["attempt.trial.det.1"],
+   graders:[$det,$human]}}
 ' > "$tmp/case.det.value"
 "$jq_bin" -S -c -n --argjson suite "$suite_ref" --argjson input "$input_model" \
   --argjson expected "$expected_model" --argjson det "$grader_det" --argjson human "$grader_human" \
   --argjson model "$grader_model" '
   {schema_version:1,kind:"eval_case",id:"case.model",body:{case_version:"v1",
    suite_ref:$suite,execution_kind:"model",input_ref:$input,expected_ref:$expected,
-   trial_count:2,trial_ids:["trial.model.1","trial.model.2"],graders:[$det,$human,$model]}}
+   trial_count:2,trial_ids:["trial.model.1","trial.model.2"],
+   attempt_ids:["attempt.trial.model.1","attempt.trial.model.2"],graders:[$det,$human,$model]}}
 ' > "$tmp/case.model.value"
 wrap "$tmp/case.det.value" "$tmp/case.det.wrap"
 wrap "$tmp/case.model.value" "$tmp/case.model.wrap"
@@ -118,7 +120,8 @@ make_trial() {
   output=$(ref "$id.output" application/json "$(printf "$digit%.0s" {1..64})")
   "$jq_bin" -S -c -n --arg id "$id" --argjson case_ref "$case_ref" --argjson index "$index" \
     --argjson output "$output" '{schema_version:1,kind:"eval_trial",id:$id,body:{
-      trial_version:"v1",case_ref:$case_ref,trial_index:$index,status:"completed",
+      trial_version:"v1",case_ref:$case_ref,trial_index:$index,attempt_id:("attempt."+$id),
+      started_at:"2026-09-01T00:00:00Z",finished_at:"2026-09-01T00:00:10Z",status:"completed",
       output_ref:{state:"present",value:$output},reason:{state:"absent"}}}'
 }
 make_trial trial.det.1 "$case_det_ref" 1 5 > "$tmp/trial.det.value"
@@ -144,7 +147,8 @@ make_grade() {
     --arg grader_kind "$grader_kind" --argjson grader_ref "$grader_ref" --arg status "$status" \
     --argjson evidence "$evidence_state" --argjson reason "$reason" '
     {schema_version:1,kind:"eval_grade",id:$id,body:{grade_version:"v1",trial_ref:$trial,
-     grader_id:$grader_id,grader_kind:$grader_kind,grader_ref:$grader_ref,status:$status,
+     grader_id:$grader_id,grader_kind:$grader_kind,grader_ref:$grader_ref,
+     graded_at:"2026-09-01T00:00:20Z",status:$status,
      evidence_ref:$evidence,reason:$reason}}'
 }
 trial_det_ref=$("$jq_bin" -c .ref "$tmp/trial.det.wrap")
@@ -179,6 +183,8 @@ done
 pass valid-multi-trial-report
 "$runner" evaluate "$tmp/bundle.json" > "$tmp/repeat.json"
 /usr/bin/cmp -s "$tmp/report.json" "$tmp/repeat.json" || fail 'deterministic report'
+[ "$(/usr/bin/wc -c < "$tmp/report.json" | /usr/bin/tr -d ' ')" -le 1048576 ] ||
+  fail 'bounded report'
 pass deterministic-report
 
 "$jq_bin" -S -c '.body.cases[0].value.body.expected_ref.sha256=("f"*64)' \
@@ -188,6 +194,18 @@ expect_error tampered-record E_STALE "$tmp/tamper.json"
   "$tmp/bundle.json" > "$tmp/stale-unhashed.json"
 rehash "$tmp/stale-unhashed.json" cases 0 "$tmp/stale.json"
 expect_error stale-record-link E_RELATION "$tmp/stale.json"
+"$jq_bin" -S -c '.body.trials[0].value.body.started_at="2026-09-01T00:00:11Z"' \
+  "$tmp/bundle.json" > "$tmp/trial-time-unhashed.json"
+rehash "$tmp/trial-time-unhashed.json" trials 0 "$tmp/trial-time.json"
+expect_error inverted-trial-time E_SHAPE "$tmp/trial-time.json"
+"$jq_bin" -S -c '.body.grades[0].value.body.graded_at="2026-09-01T00:00:09Z"' \
+  "$tmp/bundle.json" > "$tmp/grade-time-unhashed.json"
+rehash "$tmp/grade-time-unhashed.json" grades 0 "$tmp/grade-time.json"
+expect_error early-grade-time E_RELATION "$tmp/grade-time.json"
+"$jq_bin" -S -c '.body.trials[1].value.body.attempt_id=.body.trials[0].value.body.attempt_id' \
+  "$tmp/bundle.json" > "$tmp/attempt-unhashed.json"
+rehash "$tmp/attempt-unhashed.json" trials 1 "$tmp/attempt.json"
+expect_error duplicate-attempt E_RELATION "$tmp/attempt.json"
 "$jq_bin" -S -c '.body.cases|=reverse' "$tmp/bundle.json" > "$tmp/order.json"
 expect_error record-order E_RELATION "$tmp/order.json"
 "$jq_bin" -S -c '.body.trials += [.body.trials[0]]' "$tmp/bundle.json" > "$tmp/duplicate.json"
@@ -198,6 +216,45 @@ expect_error missing-grade E_RELATION "$tmp/missing-grade.json"
 "$jq_bin" -S -c '.body.suite.value.body.extra_command="printf unsafe"' \
   "$tmp/bundle.json" > "$tmp/command.json"
 expect_error arbitrary-command-field E_SHAPE "$tmp/command.json"
+"$jq_bin" -S -c '.body.cases[0].value.body.graders[0].implementation_ref.content_id=("x"*129)' \
+  "$tmp/bundle.json" > "$tmp/grader-metadata.json"
+expect_error bounded-grader-metadata E_SHAPE "$tmp/grader-metadata.json"
+
+"$jq_bin" -S -c '
+  .body.trials[2].value.body.status="unavailable" |
+  .body.trials[2].value.body.output_ref={state:"absent"} |
+  .body.trials[2].value.body.reason={state:"present",value:"trial.output-unavailable"}
+' "$tmp/bundle.json" > "$tmp/hidden-trial-unhashed.json"
+rehash "$tmp/hidden-trial-unhashed.json" trials 2 "$tmp/hidden-trial.json"
+hidden_trial_ref=$("$jq_bin" -c '.body.trials[2].ref' "$tmp/hidden-trial.json")
+"$jq_bin" -S -c --argjson trial "$hidden_trial_ref" '
+  .body.grades |= map(if .value.body.trial_ref.content_id=="trial.model.2" then
+    .value.body.trial_ref=$trial |
+    if .value.id=="grade.model2.model" then .value.body.status="failed"
+    else .value.body.status="unavailable" | .value.body.evidence_ref={state:"absent"} |
+      .value.body.reason={state:"present",value:"grader.unavailable"} end
+    else . end)
+' "$tmp/hidden-trial.json" > "$tmp/hidden-grades-0.json"
+hidden_current="$tmp/hidden-grades-0.json"
+hidden_count=0
+for hidden_id in grade.model2.det grade.model2.human grade.model2.model; do
+  hidden_index=$("$jq_bin" -r --arg id "$hidden_id" '.body.grades|map(.value.id)|index($id)' "$hidden_current")
+  hidden_next="$tmp/hidden-grades-$((hidden_count + 1)).json"
+  rehash "$hidden_current" grades "$hidden_index" "$hidden_next"
+  hidden_current=$hidden_next
+  hidden_count=$((hidden_count + 1))
+done
+expect_error unavailable-trial-hides-failure E_RELATION "$hidden_current"
+
+"$jq_bin" -S -c '
+  .body.grades |= map(if .value.id=="grade.model2.model" then
+    .value.body.status="failed" else . end)
+' "$tmp/bundle.json" > "$tmp/failed-unhashed.json"
+failed_index=$("$jq_bin" -r '.body.grades|map(.value.id)|index("grade.model2.model")' "$tmp/failed-unhashed.json")
+rehash "$tmp/failed-unhashed.json" grades "$failed_index" "$tmp/failed.json"
+"$runner" evaluate "$tmp/failed.json" > "$tmp/failed-report.json"
+[ "$("$jq_bin" -r .body.status "$tmp/failed-report.json")" = failed ] || fail 'failed grade precedence'
+pass failed-grade-precedence
 
 "$jq_bin" -S -c '
   .body.grades |= map(if .value.id=="grade.model2.human" then

From 7e6d84fcb6e81bae084994f7b6e481a6f5a5476b Mon Sep 17 00:00:00 2001
From: ci <ci@example.com>
Date: Wed, 2 Sep 2026 18:40:18 -0400
Subject: [PATCH 3/6] Authorize eval schema import path

---
 evals/v1/run.sh                           | 15 ++++++++-------
 scripts/test/portable-core-schema.test.sh |  2 +-
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/evals/v1/run.sh b/evals/v1/run.sh
index e57a58a5..d7468ca9 100755
--- a/evals/v1/run.sh
+++ b/evals/v1/run.sh
@@ -53,7 +53,6 @@ repo=$(CDPATH='' cd -P -- "$source_dir/../.." 2>/dev/null && pwd -P) || emit_err
 program_sha=1cdb63cd246f0fd2a2de484d62a0be297cb24ad1dffd5fc9a6c57b967d10778d
 schema_sha=8d1d02d36ac7ada778f05248f9413062b3fc251499914c15d79f003bbd009ade
 registry_sha=f55b697716dc13a6d2c71bde7769493b3f4b091fd7a94d3280c5d417974df3a1
-generation=g-392d20099dfa99872764009b268c8871914b4dbc0da467ec346baa921818ae3e
 platform=$(/usr/bin/uname -s):$(/usr/bin/uname -m)
 case "$platform" in
   Darwin:*) jq_asset=jq-osx-amd64; jq_sha=5c0a0a3ea600f302ee458b30317425dd9632d1ad8882259fcaf4e9b868b2b1ef ;;
@@ -76,18 +75,20 @@ snapshot_file "$input" "$scratch/input.json" 4194304 0400 || {
   status=$?; [ "$status" -eq 42 ] && emit_error E_LIMIT; emit_error E_RUNTIME;
 }
 snapshot_file "$source_dir/framework.jq" "$scratch/framework.jq" 1048576 0400 || emit_error E_RUNTIME
-schema="$repo/core/v2/generations/$generation/modules/schema.jq"
-snapshot_file "$schema" "$scratch/schema.jq" 1048576 0400 || emit_error E_RUNTIME
 snapshot_file "$repo/core/v2/generation-registry.json" "$scratch/registry.json" 1048576 0400 ||
   emit_error E_RUNTIME
 snapshot_file "$jq_source" "$scratch/jq" 33554432 0500 || emit_error E_RUNTIME
 [ "$(sha_file "$scratch/framework.jq")" = "$program_sha" ] || emit_error E_STALE
-[ "$(sha_file "$scratch/schema.jq")" = "$schema_sha" ] || emit_error E_STALE
 [ "$(sha_file "$scratch/registry.json")" = "$registry_sha" ] || emit_error E_STALE
 [ "$(sha_file "$scratch/jq")" = "$jq_sha" ] || emit_error E_RUNTIME
-[ "$("$scratch/jq" -r --arg generation "$generation" \
-    'length==1 and .[0].generation_id==$generation' "$scratch/registry.json")" = true ] ||
-  emit_error E_STALE
+generation=$("$scratch/jq" -er '
+  if length==1 and .[0].semantic_identity=="core.contracts.v2" and
+     (.[0].generation_id | type=="string" and test("\\Ag-[0-9a-f]{64}\\z"))
+  then .[0].generation_id else error("E_STALE") end
+' "$scratch/registry.json" 2>/dev/null) || emit_error E_STALE
+schema="$repo/core/v2/generations/$generation/modules/schema.jq"
+snapshot_file "$schema" "$scratch/schema.jq" 1048576 0400 || emit_error E_STALE
+[ "$(sha_file "$scratch/schema.jq")" = "$schema_sha" ] || emit_error E_STALE
 
 if ! "$scratch/jq" -S -c . "$scratch/input.json" > "$scratch/canonical.json" 2>/dev/null; then
   emit_error E_PARSE
diff --git a/scripts/test/portable-core-schema.test.sh b/scripts/test/portable-core-schema.test.sh
index 414d6926..1142d9ca 100755
--- a/scripts/test/portable-core-schema.test.sh
+++ b/scripts/test/portable-core-schema.test.sh
@@ -776,7 +776,7 @@ schema_import_path_ok() {
   local import_path="$1"
   local test_path
   case "$import_path" in
-    orchestrator/v1/reconciliation-plan.jq|orchestrator/v1/state-scanner.jq) ;;
+    evals/v1/framework.jq|orchestrator/v1/reconciliation-plan.jq|orchestrator/v1/state-scanner.jq) ;;
     scripts/test/default-github-forge-adapter.test.sh) ;;
     scripts/test/portable-core-*)
       test_path="${import_path#scripts/test/}"

From b4ce778823e0bf150d0a12a45316f91267c681c1 Mon Sep 17 00:00:00 2001
From: ci <ci@example.com>
Date: Fri, 4 Sep 2026 17:41:49 -0400
Subject: [PATCH 4/6] Clarify eval record evaluator scope

---
 README.md                                 | 19 ++++++++++++-------
 RESTORE.md                                | 15 ++++++++-------
 ci/required-files.txt                     |  2 +-
 scripts/test/eval-framework.test.sh       |  2 +-
 scripts/test/portable-core-schema.test.sh |  8 ++++----
 5 files changed, 26 insertions(+), 20 deletions(-)

diff --git a/README.md b/README.md
index 4989bee3..2b647cd5 100644
--- a/README.md
+++ b/README.md
@@ -267,14 +267,15 @@ main. The payload is offline and unqualified. It does not call GitHub or a CLI,
 use a credential, rerun or cancel work, dispatch a workflow, change a repository,
 grant authority or qualification, or activate a profile.
 
-## Inactive hermetic eval framework
+## Inactive hermetic eval-record evaluator
 
-`evals/v1/run.sh evaluate BUNDLE.json` validates a bounded canonical eval suite,
-its cases, trials, and grades, then emits one deterministic report. Every record
-is content-bound. The suite pins its scope and framework version. Model cases
-require multiple trials, while deterministic, model, and human graders remain
-immutable data references rather than commands the runner can invoke. Declared
-trial and attempt identities are exact, and trial/grade timestamps must agree.
+`evals/v1/run.sh evaluate BUNDLE.json` validates already-recorded, bounded
+canonical eval suites, cases, trials, and grades, then emits one deterministic
+report. It does not execute trials or invoke graders. Every record is
+content-bound. The suite pins its scope and framework version. Model cases require
+multiple supplied trials, while deterministic, model, and human graders remain
+immutable data references. Declared trial and attempt identities are exact, and
+trial/grade timestamps must agree.
 
 The runner snapshots a fixed jq program and the selected public-core schema into
 a private directory. It accepts only canonical JSON, rejects stale links,
@@ -283,6 +284,10 @@ from `inconclusive`. Failed grades cannot be hidden by another trial state. It
 has no adapter or arbitrary-command seam, uses no network
 or credential, grants no authority, and makes no activation or qualification
 claim.
+
+This is a bounded first slice for eval records, not a runnable eval system or
+qualification evidence. A later unit must provide hermetic built-in trial and
+grader execution before it can make either claim.
 ## Inactive local Git candidate materializer
 
 `adapters/local-git-materializer/v1/` implements the existing portable-core v2
diff --git a/RESTORE.md b/RESTORE.md
index c2d1665a..4529ce11 100644
--- a/RESTORE.md
+++ b/RESTORE.md
@@ -508,21 +508,22 @@ default-set wiring. The pure jq payload is offline and unqualified. It does not
 call GitHub, use a credential, rerun, cancel, or dispatch work, change a
 repository, grant authority or qualification, or activate a profile.
 
-Restore the three paths in the manifest's inactive hermetic eval framework block,
+Restore the three paths in the manifest's inactive hermetic eval-record evaluator block,
 then run:
 
 ```sh
 bash scripts/test/eval-framework.test.sh
 ```
 
-This checks canonical suite, case, trial, and grade records; immutable framework
-and scope references; exact trial/attempt identities; coherent trial and grade
-timestamps; deterministic and stochastic multi-trial results; explicit
+This checks supplied canonical suite, case, trial, and grade records; immutable
+framework and scope references; exact trial/attempt identities; coherent trial and
+grade timestamps; deterministic and stochastic multi-trial results; explicit
 `unavailable` and `inconclusive` states; and stale, tampered, duplicate, missing,
 or out-of-order input rejection. Failed grades cannot be hidden by an unavailable
-trial. The runner invokes no grader, model, adapter, or arbitrary command. It uses
-no network or credential and grants no authority, activation, qualification, or
-external effect.
+trial. The evaluator does not run trials or invoke a grader, model, adapter, or
+arbitrary command. It uses no network or credential and grants no authority,
+activation, qualification, or external effect. Hermetic built-in trial and grader
+execution is later work.
 Restore the four paths in the manifest's inactive local Git materializer block,
 then run:
 
diff --git a/ci/required-files.txt b/ci/required-files.txt
index 4ae614a7..b3797d9c 100644
--- a/ci/required-files.txt
+++ b/ci/required-files.txt
@@ -252,7 +252,7 @@ scripts/test/default-codex-native-reviewer-adapter.test.sh
 adapters/github-actions-ci/v1/normalize.jq
 scripts/test/default-github-actions-ci-adapter.test.sh
 
-# Inactive hermetic eval framework
+# Inactive hermetic eval-record evaluator
 evals/v1/framework.jq
 evals/v1/run.sh
 scripts/test/eval-framework.test.sh
diff --git a/scripts/test/eval-framework.test.sh b/scripts/test/eval-framework.test.sh
index 92456f7a..81e8282c 100755
--- a/scripts/test/eval-framework.test.sh
+++ b/scripts/test/eval-framework.test.sh
@@ -276,4 +276,4 @@ expect_error canonical-bytes E_CANONICAL "$tmp/noncanonical.json"
 ' "$tmp/report.json" >/dev/null || fail 'inactive data-only output'
 pass inactive-data-only-output
 
-/usr/bin/printf 'PASS: %s eval framework checks\n' "$passes"
+/usr/bin/printf 'PASS: %s eval-record evaluator checks\n' "$passes"
diff --git a/scripts/test/portable-core-schema.test.sh b/scripts/test/portable-core-schema.test.sh
index f46c2986..eef7ed46 100755
--- a/scripts/test/portable-core-schema.test.sh
+++ b/scripts/test/portable-core-schema.test.sh
@@ -826,12 +826,12 @@ schema_import_path_ok() {
   local test_path
   case "$import_path" in
     adapters/local-git-materializer/v1/protocol.jq|\
-     adapters/deterministic-verifier/v1/normalize.jq|\
-     evals/v1/framework.jq|\
-     orchestrator/v1/reconciliation-plan.jq|orchestrator/v1/state-scanner.jq) ;;
+    adapters/deterministic-verifier/v1/normalize.jq|\
+    evals/v1/framework.jq|\
+    orchestrator/v1/reconciliation-plan.jq|orchestrator/v1/state-scanner.jq) ;;
     scripts/test/default-codex-native-reviewer-adapter.test.sh|\
     scripts/test/default-dormant-publisher-adapter.test.sh|\
-     scripts/test/default-deterministic-verifier-adapter.test.sh|\
+    scripts/test/default-deterministic-verifier-adapter.test.sh|\
     scripts/test/default-github-forge-adapter.test.sh) ;;
     scripts/test/portable-core-*)
       test_path="${import_path#scripts/test/}"

From 43a1d564b5c65ea1e889a7de473cad68e7610f9f Mon Sep 17 00:00:00 2001
From: ci <ci@example.com>
Date: Sat, 5 Sep 2026 18:12:01 -0400
Subject: [PATCH 5/6] Bind trial ids by position; bind the runner to its own
 directory

Two review findings on the Codex-authored head.

Trial and attempt id lists were checked as sorted sets, so ids whose
lexical order differs from trial order (trial.9, trial.10) were refused
although the relation checks bind ids by position. They are now checked
as distinct ids in trial order.

The runner compared its unnormalized invocation path with its resolved
directory, so a relative invocation such as ./evals/v1/run.sh failed as
E_RUNTIME before evaluating. The path is normalized to the resolved
directory first, as the other launchers do. Program digest re-pinned.

Regressions: a relative-path invocation yields the identical report;
lexically unsorted, position-ordered ids pass the case shape and a
repeated id does not. scripts/test/eval-framework.test.sh 19/19,
shellcheck 0.11.0 clean.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
---
 evals/v1/framework.jq               |  8 ++++++--
 evals/v1/run.sh                     |  3 ++-
 scripts/test/eval-framework.test.sh | 21 +++++++++++++++++++++
 3 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/evals/v1/framework.jq b/evals/v1/framework.jq
index 63715e04..9f7d21de 100644
--- a/evals/v1/framework.jq
+++ b/evals/v1/framework.jq
@@ -58,8 +58,12 @@ def case_ok:
    (.expected_ref | schema::content_ref_ok) and
    (.trial_count | schema::int_ok) and .trial_count >= 1 and .trial_count <= 16 and
    (if .execution_kind == "model" then .trial_count >= 2 else true end) and
-   (.trial_ids | schema::bounded_set(1;16;schema::id_ok;.)) and
-   (.attempt_ids | schema::bounded_set(1;16;schema::id_ok;.)) and
+   # Trial and attempt ids are listed in trial order, not sorted: each must be
+   # a distinct id, and the relation checks bind position to trial index.
+   (.trial_ids | type == "array" and length >= 1 and length <= 16 and
+    all(.[]; schema::id_ok) and length == (unique | length)) and
+   (.attempt_ids | type == "array" and length >= 1 and length <= 16 and
+    all(.[]; schema::id_ok) and length == (unique | length)) and
    .trial_count == (.trial_ids | length) and
    .trial_count == (.attempt_ids | length) and
    (.graders | schema::bounded_set(1;8;grader_ok;.grader_id)));
diff --git a/evals/v1/run.sh b/evals/v1/run.sh
index 1047b7cf..bd21a4c7 100755
--- a/evals/v1/run.sh
+++ b/evals/v1/run.sh
@@ -46,11 +46,12 @@ self=${BASH_SOURCE[0]}
 case "$self" in /*) ;; *) self="$(pwd -P)/$self" ;; esac
 [ -f "$self" ] && [ ! -L "$self" ] || emit_error E_RUNTIME
 source_dir=$(CDPATH='' cd -P -- "${self%/*}" 2>/dev/null && pwd -P) || emit_error E_RUNTIME
+self="$source_dir/${self##*/}"
 [ "$self" = "$source_dir/run.sh" ] || emit_error E_RUNTIME
 repo=$(CDPATH='' cd -P -- "$source_dir/../.." 2>/dev/null && pwd -P) || emit_error E_RUNTIME
 [ "$source_dir" = "$repo/evals/v1" ] || emit_error E_RUNTIME
 
-program_sha=1cdb63cd246f0fd2a2de484d62a0be297cb24ad1dffd5fc9a6c57b967d10778d
+program_sha=d164a102919f42ef002e93c8b515b58028604819e6c09b87c6e4afadb0bbcce4
 schema_sha=8d1d02d36ac7ada778f05248f9413062b3fc251499914c15d79f003bbd009ade
 registry_sha=3950ce43c3073b97759db23fb7e4ce533cbc1d8a8fe4917db6ee1ee0a8e78f94
 platform=$(/usr/bin/uname -s):$(/usr/bin/uname -m)
diff --git a/scripts/test/eval-framework.test.sh b/scripts/test/eval-framework.test.sh
index 81e8282c..e06aed02 100755
--- a/scripts/test/eval-framework.test.sh
+++ b/scripts/test/eval-framework.test.sh
@@ -183,6 +183,27 @@ done
 pass valid-multi-trial-report
 "$runner" evaluate "$tmp/bundle.json" > "$tmp/repeat.json"
 /usr/bin/cmp -s "$tmp/report.json" "$tmp/repeat.json" || fail 'deterministic report'
+# Invoked by a relative path, the runner still binds itself to its own directory.
+( cd "$root" && ./evals/v1/run.sh evaluate "$tmp/bundle.json" ) > "$tmp/relative.json" ||
+  fail 'relative-path invocation'
+/usr/bin/cmp -s "$tmp/report.json" "$tmp/relative.json" || fail 'relative-path report differs'
+pass relative-path-invocation
+# Trial ids are bound by position, so ids whose lexical order differs from
+# their trial order are still one valid case shape; a repeated id is not.
+generation=$(/usr/bin/sed -n "s/^PORTABLE_CORE_GENERATION='\\(g-[0-9a-f]\\{64\\}\\)'$/\\1/p" \
+  "$root/scripts/core-contract.sh")
+modules="$root/core/v2/generations/$generation/modules"
+[ -d "$modules" ] || fail 'selected generation modules'
+/usr/bin/awk 'NR > 48 && /^def / { exit } { print }' "$root/evals/v1/framework.jq" \
+  > "$tmp/framework-shapes.jq"
+"$jq_bin" -L "$tmp" -L "$modules" -e --slurpfile c "$tmp/case.model.value" -n '
+  import "framework-shapes" as shapes;
+  ($c[0] | .body.trial_ids = ["trial.model.9","trial.model.10"] |
+   .body.attempt_ids = ["attempt.trial.model.9","attempt.trial.model.10"] | shapes::case_ok) and
+  ($c[0] | .body.trial_ids = ["trial.model.1","trial.model.1"] | shapes::case_ok | not) and
+  ($c[0] | .body.attempt_ids = ["attempt.trial.model.1","attempt.trial.model.1"] | shapes::case_ok | not)
+' >/dev/null || fail 'trial ids are not bound by position'
+pass trial-ids-bound-by-position
 [ "$(/usr/bin/wc -c < "$tmp/report.json" | /usr/bin/tr -d ' ')" -le 1048576 ] ||
   fail 'bounded report'
 pass deterministic-report

From d4ae469fa40e5e2ef8605d3c53bd208b1b2e5816 Mon Sep 17 00:00:00 2001
From: ci <ci@example.com>
Date: Sun, 6 Sep 2026 02:06:58 -0400
Subject: [PATCH 6/6] Require exactly one JSON text in the eval bundle input

jq reads JSON streams, so canonical bytes alone admitted a file holding
several bundles whose later filters would run against the stream. The runner
now refuses any input that is not exactly one JSON text, before shape checks.
The test feeds two concatenated bundles and expects E_PARSE.

Proof: eval-framework 20/20, shellcheck clean.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
---
 evals/v1/run.sh                     | 4 ++++
 scripts/test/eval-framework.test.sh | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/evals/v1/run.sh b/evals/v1/run.sh
index bd21a4c7..c1214c0e 100755
--- a/evals/v1/run.sh
+++ b/evals/v1/run.sh
@@ -95,6 +95,10 @@ snapshot_file "$schema" "$scratch/schema.jq" 1048576 0400 || emit_error E_STALE
 if ! "$scratch/jq" -S -c . "$scratch/input.json" > "$scratch/canonical.json" 2>/dev/null; then
   emit_error E_PARSE
 fi
+# jq reads JSON streams, so canonical bytes alone would admit a file holding
+# several bundles. The input must be exactly one JSON text.
+"$scratch/jq" -e -n --slurpfile roots "$scratch/input.json" '($roots | length) == 1' \
+  >/dev/null 2>&1 || emit_error E_PARSE
 /usr/bin/cmp -s "$scratch/input.json" "$scratch/canonical.json" || emit_error E_CANONICAL
 bundle_sha=$(sha_file "$scratch/input.json") || emit_error E_RUNTIME
 if ! "$scratch/jq" -e '
diff --git a/scripts/test/eval-framework.test.sh b/scripts/test/eval-framework.test.sh
index e06aed02..1e3eeb3b 100755
--- a/scripts/test/eval-framework.test.sh
+++ b/scripts/test/eval-framework.test.sh
@@ -291,6 +291,8 @@ pass explicit-unavailable
 
 "$jq_bin" . "$tmp/bundle.json" > "$tmp/noncanonical.json"
 expect_error canonical-bytes E_CANONICAL "$tmp/noncanonical.json"
+/bin/cat "$tmp/bundle.json" "$tmp/bundle.json" > "$tmp/two-bundles.json"
+expect_error multi-root-stream E_PARSE "$tmp/two-bundles.json"
 "$jq_bin" -e '
   ([..|objects|keys[]|select(.=="authority" or .=="permissions" or .=="credential" or
     .=="network" or .=="command" or .=="activation" or .=="qualification")] | length)==0