diff --git a/README.md b/README.md index f403211c..b3bc6cc5 100644 --- a/README.md +++ b/README.md @@ -306,6 +306,27 @@ main. The payload is offline and unqualified. It does not call GitHub or a CLI, use a credential, rerun or cancel work, dispatch a workflow, change a repository, grant authority or qualification, or activate a profile. +## Inactive hermetic eval-record evaluator + +`evals/v1/run.sh evaluate BUNDLE.json` validates already-recorded, bounded +canonical eval suites, cases, trials, and grades, then emits one deterministic +report. It does not execute trials or invoke graders. Every record is +content-bound. The suite pins its scope and framework version. Model cases require +multiple supplied trials, while deterministic, model, and human graders remain +immutable data references. Declared trial and attempt identities are exact, and +trial/grade timestamps must agree. + +The runner snapshots a fixed jq program and the selected public-core schema into +a private directory. It accepts only canonical JSON, rejects stale links, +tampering, duplicates, and non-canonical order, and keeps `unavailable` distinct +from `inconclusive`. Failed grades cannot be hidden by another trial state. It +has no adapter or arbitrary-command seam, uses no network +or credential, grants no authority, and makes no activation or qualification +claim. + +This is a bounded first slice for eval records, not a runnable eval system or +qualification evidence. A later unit must provide hermetic built-in trial and +grader execution before it can make either claim. ## Inactive local Git candidate materializer `adapters/local-git-materializer/v1/` implements the existing portable-core v2 diff --git a/RESTORE.md b/RESTORE.md index 82d8a2b7..2d27a42b 100644 --- a/RESTORE.md +++ b/RESTORE.md @@ -537,6 +537,22 @@ default-set wiring. The pure jq payload is offline and unqualified. It does not call GitHub, use a credential, rerun, cancel, or dispatch work, change a repository, grant authority or qualification, or activate a profile. +Restore the three paths in the manifest's inactive hermetic eval-record evaluator block, +then run: + +```sh +bash scripts/test/eval-framework.test.sh +``` + +This checks supplied canonical suite, case, trial, and grade records; immutable +framework and scope references; exact trial/attempt identities; coherent trial and +grade timestamps; deterministic and stochastic multi-trial results; explicit +`unavailable` and `inconclusive` states; and stale, tampered, duplicate, missing, +or out-of-order input rejection. Failed grades cannot be hidden by an unavailable +trial. The evaluator does not run trials or invoke a grader, model, adapter, or +arbitrary command. It uses no network or credential and grants no authority, +activation, qualification, or external effect. Hermetic built-in trial and grader +execution is later work. Restore the four paths in the manifest's inactive local Git materializer block, then run: diff --git a/ci/required-files.txt b/ci/required-files.txt index 8cec8f84..93a061ea 100644 --- a/ci/required-files.txt +++ b/ci/required-files.txt @@ -256,6 +256,10 @@ scripts/test/default-codex-native-reviewer-adapter.test.sh adapters/github-actions-ci/v1/normalize.jq scripts/test/default-github-actions-ci-adapter.test.sh +# Inactive hermetic eval-record evaluator +evals/v1/framework.jq +evals/v1/run.sh +scripts/test/eval-framework.test.sh # Inactive provider-neutral local Git candidate materializer adapters/local-git-materializer/v1/materialize.sh adapters/local-git-materializer/v1/object-closure.c diff --git a/evals/v1/framework.jq b/evals/v1/framework.jq new file mode 100644 index 00000000..9f7d21de --- /dev/null +++ b/evals/v1/framework.jq @@ -0,0 +1,242 @@ +import "schema" as schema; + +def record_media_type: "application/vnd.ystack.eval-record+json"; +def bundle_media_type: "application/vnd.ystack.eval-bundle+json"; + +def present_ref_ok: + (schema::exact_fields(["state"];[]) and .state == "absent") or + (schema::exact_fields(["state","value"];[]) and .state == "present" and + (.value | schema::content_ref_ok)); + +def present_reason_ok: + (schema::exact_fields(["state"];[]) and .state == "absent") or + (schema::exact_fields(["state","value"];[]) and .state == "present" and + (.value | schema::id_ok)); + +def envelope_ok($kind): + schema::exact_fields(["schema_version","kind","id","body"];[]) and + .schema_version == 1 and .kind == $kind and + (.id | schema::id_ok) and (.body | type == "object"); + +def grader_ok: + schema::exact_fields( + ["grader_id","grader_kind","implementation_ref","instructions_ref"];[]) and + (.grader_id | schema::id_ok) and + (.grader_kind == "deterministic" or .grader_kind == "model" or + .grader_kind == "human") and + (.implementation_ref | schema::content_ref_ok) and + (.instructions_ref | schema::content_ref_ok); + +def scope_ok: + schema::exact_fields( + ["scope_id","scope_version","definition_ref","scope_sha256"];[]) and + (.scope_id | schema::id_ok) and (.scope_version | schema::version_ok) and + (.definition_ref | schema::content_ref_ok) and + (.scope_sha256 | schema::sha256_ok) and + .scope_sha256 == .definition_ref.sha256; + +def suite_ok: + envelope_ok("eval_suite") and + (.body | + schema::exact_fields( + ["suite_version","framework_ref","scope","case_ids"];[]) and + .suite_version == "v1" and + (.framework_ref | schema::content_ref_ok) and + (.scope | scope_ok) and + (.case_ids | schema::bounded_set(1;32;schema::id_ok;.))); + +def case_ok: + envelope_ok("eval_case") and + (.body | + schema::exact_fields( + ["case_version","suite_ref","execution_kind","input_ref","expected_ref", + "trial_count","trial_ids","attempt_ids","graders"];[]) and + .case_version == "v1" and + (.suite_ref | schema::content_ref_ok) and + (.execution_kind == "deterministic" or .execution_kind == "model") and + (.input_ref | schema::content_ref_ok) and + (.expected_ref | schema::content_ref_ok) and + (.trial_count | schema::int_ok) and .trial_count >= 1 and .trial_count <= 16 and + (if .execution_kind == "model" then .trial_count >= 2 else true end) and + # Trial and attempt ids are listed in trial order, not sorted: each must be + # a distinct id, and the relation checks bind position to trial index. + (.trial_ids | type == "array" and length >= 1 and length <= 16 and + all(.[]; schema::id_ok) and length == (unique | length)) and + (.attempt_ids | type == "array" and length >= 1 and length <= 16 and + all(.[]; schema::id_ok) and length == (unique | length)) and + .trial_count == (.trial_ids | length) and + .trial_count == (.attempt_ids | length) and + (.graders | schema::bounded_set(1;8;grader_ok;.grader_id))); + +def trial_ok: + envelope_ok("eval_trial") and + (.body | + schema::exact_fields( + ["trial_version","case_ref","trial_index","attempt_id","started_at","finished_at", + "status","output_ref","reason"];[]) and + .trial_version == "v1" and + (.case_ref | schema::content_ref_ok) and + (.trial_index | schema::int_ok) and .trial_index >= 1 and .trial_index <= 16 and + (.attempt_id | schema::id_ok) and + (.started_at | schema::time_ok) and (.finished_at | schema::time_ok) and + .started_at <= .finished_at and + (.output_ref | present_ref_ok) and (.reason | present_reason_ok) and + ((.status == "completed" and .output_ref.state == "present" and + .reason.state == "absent") or + (.status == "unavailable" and .output_ref.state == "absent" and + .reason.state == "present"))); + +def grade_ok: + envelope_ok("eval_grade") and + (.body | + schema::exact_fields( + ["grade_version","trial_ref","grader_id","grader_kind","grader_ref", + "graded_at","status","evidence_ref","reason"];[]) and + .grade_version == "v1" and + (.trial_ref | schema::content_ref_ok) and + (.grader_id | schema::id_ok) and + (.grader_kind == "deterministic" or .grader_kind == "model" or + .grader_kind == "human") and + (.grader_ref | schema::content_ref_ok) and + (.graded_at | schema::time_ok) and + (.evidence_ref | present_ref_ok) and (.reason | present_reason_ok) and + (((.status == "passed" or .status == "failed") and + .evidence_ref.state == "present" and .reason.state == "absent") or + (.status == "inconclusive" and .reason.state == "present") or + (.status == "unavailable" and .evidence_ref.state == "absent" and + .reason.state == "present"))); + +def measured_sha($section;$index): + [$measured_docs[0][] | select(.section == $section and .index == $index)] | + if length == 1 then .[0].sha256 else null end; + +def wrapper_shape_ok(value_ok): + schema::exact_fields(["ref","value"];[]) and + (.ref | schema::content_ref_ok) and .ref.media_type == record_media_type and + (.value | value_ok) and .ref.content_id == .value.id; + +def wrapper_hash_ok($section;$index): + .ref.sha256 == measured_sha($section;$index); + +def bundle_shape_ok: + . as $bundle | + schema::parsed_limits_ok and + schema::exact_fields(["schema_version","kind","id","body"];[]) and + .schema_version == 1 and .kind == "eval_bundle" and (.id | schema::id_ok) and + (.body | schema::exact_fields(["suite","cases","trials","grades"];[])) and + ($bundle.body.suite | wrapper_shape_ok(suite_ok)) and + ($bundle.body.cases | type == "array" and length >= 1 and length <= 32) and + all($bundle.body.cases[]; wrapper_shape_ok(case_ok)) and + ($bundle.body.trials | type == "array" and length >= 1 and length <= 128) and + all($bundle.body.trials[]; wrapper_shape_ok(trial_ok)) and + ($bundle.body.grades | type == "array" and length >= 1 and length <= 256) and + all($bundle.body.grades[]; wrapper_shape_ok(grade_ok)); + +def bundle_hashes_ok($program_sha256): + . as $bundle | + ($bundle.body.suite | wrapper_hash_ok("suite";0)) and + all(range(0;($bundle.body.cases | length)); . as $index | + $bundle.body.cases[$index] | wrapper_hash_ok("cases";$index)) and + all(range(0;($bundle.body.trials | length)); . as $index | + $bundle.body.trials[$index] | wrapper_hash_ok("trials";$index)) and + all(range(0;($bundle.body.grades | length)); . as $index | + $bundle.body.grades[$index] | wrapper_hash_ok("grades";$index)) and + $bundle.body.suite.value.body.framework_ref == { + content_id:"eval-framework.v1", + media_type:"application/vnd.ystack.eval-framework+jq", + sha256:$program_sha256 + }; + +def bundle_order_ok: + .body as $body | + ([$body.suite] + $body.cases + $body.trials + $body.grades | + map(.value.id)) as $all_ids | + ($all_ids | length) == ($all_ids | unique | length) and + ($body.cases | map(.value.id)) as $case_ids | + $case_ids == ($case_ids | sort) and + ($body.trials | map([.value.body.case_ref.content_id,.value.body.trial_index])) as $trial_keys | + $trial_keys == ($trial_keys | sort) and + ($trial_keys | length) == ($trial_keys | unique | length) and + ($body.trials | map(.value.body.attempt_id)) as $attempt_ids | + ($attempt_ids | length) == ($attempt_ids | unique | length) and + ($body.grades | map([.value.body.trial_ref.content_id,.value.body.grader_id])) as $grade_keys | + $grade_keys == ($grade_keys | sort) and + ($grade_keys | length) == ($grade_keys | unique | length); + +def bundle_relations_ok: + .body as $body | + $body.suite.ref as $suite_ref | + ($body.cases | map(.value.id)) as $case_ids | + $body.suite.value.body.case_ids == $case_ids and + all($body.cases[]; .value.body.suite_ref == $suite_ref) and + all($body.cases[]; . as $case | + [$body.trials[] | select(.value.body.case_ref == $case.ref)] as $trials | + ($trials | map(.value.id)) == $case.value.body.trial_ids and + ($trials | map(.value.body.attempt_id)) == $case.value.body.attempt_ids and + ($trials | map(.value.body.trial_index)) == + [range(1;($case.value.body.trial_count + 1))]) and + all($body.trials[]; . as $trial | + [$body.cases[] | select(.ref == $trial.value.body.case_ref)] as $cases | + ($cases | length) == 1 and + ($cases[0].value.body.graders) as $graders | + [$body.grades[] | select(.value.body.trial_ref == $trial.ref)] as $grades | + ($grades | map(.value.body.grader_id)) == ($graders | map(.grader_id)) and + all($grades[]; .value.body.graded_at >= $trial.value.body.finished_at) and + (if $trial.value.body.status == "unavailable" + then all($grades[]; .value.body.status == "unavailable") + else true end) and + all($grades[]; . as $grade | + [$graders[] | select(.grader_id == $grade.value.body.grader_id)] as $matches | + ($matches | length) == 1 and + $grade.value.body.grader_kind == $matches[0].grader_kind and + $grade.value.body.grader_ref == $matches[0].implementation_ref)) and + all($body.grades[]; . as $grade | + any($body.trials[]; .ref == $grade.value.body.trial_ref)); + +def fold_status($statuses): + if any($statuses[]; . == "failed") then "failed" + elif any($statuses[]; . == "unavailable") then "unavailable" + elif any($statuses[]; . == "inconclusive") then "inconclusive" + else "passed" + end; + +def trial_summary($body;$trial): + [$body.grades[] | select(.value.body.trial_ref == $trial.ref) | + .value.body.status] as $grade_statuses | + (if $trial.value.body.status == "unavailable" then "unavailable" + else fold_status($grade_statuses) + end) as $status | + {trial_ref:$trial.ref,trial_index:$trial.value.body.trial_index, + attempt_id:$trial.value.body.attempt_id,started_at:$trial.value.body.started_at, + finished_at:$trial.value.body.finished_at,status:$status}; + +def case_summary($body;$case): + [$body.trials[] | select(.value.body.case_ref == $case.ref) | + trial_summary($body;.)] as $trials | + {case_ref:$case.ref,execution_kind:$case.value.body.execution_kind, + status:fold_status($trials | map(.status)),trials:$trials}; + +def build_report($bundle_sha256): + . as $bundle | + [$bundle.body.cases[] | case_summary($bundle.body;.)] as $cases | + { + schema_version:1, + kind:"eval_report", + id:$bundle.id, + body:{ + mode:"evaluation-only", + bundle_ref:{content_id:$bundle.id,media_type:bundle_media_type,sha256:$bundle_sha256}, + framework_ref:$bundle.body.suite.value.body.framework_ref, + suite_ref:$bundle.body.suite.ref, + status:fold_status($cases | map(.status)), + cases:$cases + } + }; + +. as $input | +if ($input | bundle_shape_ok | not) then error("E_SHAPE") +elif ($input | bundle_hashes_ok($program_sha256) | not) then error("E_STALE") +elif ($input | bundle_order_ok | not) then error("E_RELATION") +elif ($input | bundle_relations_ok | not) then error("E_RELATION") +else $input | build_report($bundle_sha256) +end diff --git a/evals/v1/run.sh b/evals/v1/run.sh new file mode 100755 index 00000000..c1214c0e --- /dev/null +++ b/evals/v1/run.sh @@ -0,0 +1,159 @@ +#!/bin/bash +# shellcheck disable=SC2016 +set -euo pipefail +export LC_ALL=C +umask 077 + +emit_error() { + local token=${1:-E_RUNTIME} + case "$token" in + E_USAGE|E_RUNTIME|E_LIMIT|E_PARSE|E_CANONICAL|E_SHAPE|E_RELATION|E_STALE) ;; + *) token=E_RUNTIME ;; + esac + /usr/bin/printf '%s\n' "$token" >&2 + exit 1 +} + +sha_file() { /usr/bin/shasum -a 256 "$1" | /usr/bin/awk '{print $1}'; } +sha_line() { + builtin printf '%s\n' "$1" | /usr/bin/shasum -a 256 | /usr/bin/awk '{print $1}' +} +snapshot_file() { + /usr/bin/perl -MFcntl=:DEFAULT,:mode -e ' + my ($source,$target,$limit,$mode)=@ARGV; + sysopen(my $in,$source,O_RDONLY|O_NOFOLLOW) or exit 40; + my @stat=stat($in); @stat && S_ISREG($stat[2]) or exit 40; + $stat[7] <= $limit or exit 42; + sysopen(my $out,$target,O_WRONLY|O_CREAT|O_EXCL|O_NOFOLLOW,oct($mode)) or exit 40; + my $total=0; + while (1) { + my $read=sysread($in,my $buffer,65536); defined($read) or exit 40; + last if $read==0; $total += $read; $total <= $limit or exit 42; + my $offset=0; + while ($offset < $read) { + my $written=syswrite($out,$buffer,$read-$offset,$offset); + defined($written) && $written>0 or exit 40; $offset += $written; + } + } + close($in) or exit 40; close($out) or exit 40; + chmod(oct($mode),$target)==1 or exit 40; + ' "$1" "$2" "$3" "$4" +} + +[ "$#" -eq 2 ] && [ "$1" = evaluate ] || emit_error E_USAGE +input=$2 +self=${BASH_SOURCE[0]} +case "$self" in /*) ;; *) self="$(pwd -P)/$self" ;; esac +[ -f "$self" ] && [ ! -L "$self" ] || emit_error E_RUNTIME +source_dir=$(CDPATH='' cd -P -- "${self%/*}" 2>/dev/null && pwd -P) || emit_error E_RUNTIME +self="$source_dir/${self##*/}" +[ "$self" = "$source_dir/run.sh" ] || emit_error E_RUNTIME +repo=$(CDPATH='' cd -P -- "$source_dir/../.." 2>/dev/null && pwd -P) || emit_error E_RUNTIME +[ "$source_dir" = "$repo/evals/v1" ] || emit_error E_RUNTIME + +program_sha=d164a102919f42ef002e93c8b515b58028604819e6c09b87c6e4afadb0bbcce4 +schema_sha=8d1d02d36ac7ada778f05248f9413062b3fc251499914c15d79f003bbd009ade +registry_sha=3950ce43c3073b97759db23fb7e4ce533cbc1d8a8fe4917db6ee1ee0a8e78f94 +platform=$(/usr/bin/uname -s):$(/usr/bin/uname -m) +case "$platform" in + Darwin:*) jq_asset=jq-osx-amd64; jq_sha=5c0a0a3ea600f302ee458b30317425dd9632d1ad8882259fcaf4e9b868b2b1ef ;; + Linux:x86_64) jq_asset=jq-linux64; jq_sha=af986793a515d500ab2d35f8d2aecd656e764504b789b66d7e1a0b727a124c44 ;; + *) emit_error E_RUNTIME ;; +esac +jq_source='' +for candidate in "${TMPDIR:-/tmp}/ystack-portable-core-jq16/$jq_asset" /usr/bin/jq; do + if [ -f "$candidate" ] && [ ! -L "$candidate" ] && + [ "$(sha_file "$candidate")" = "$jq_sha" ]; then jq_source=$candidate; break; fi +done +[ -n "$jq_source" ] || emit_error E_RUNTIME + +scratch=$(/usr/bin/mktemp -d "${TMPDIR:-/tmp}/ystack-eval-run.XXXXXX") || emit_error E_RUNTIME +cleanup() { /bin/rm -rf -- "$scratch" >/dev/null 2>&1 || :; } +signal_exit() { trap - EXIT HUP INT TERM; cleanup; exit 1; } +trap cleanup EXIT +trap signal_exit HUP INT TERM +snapshot_file "$input" "$scratch/input.json" 4194304 0400 || { + status=$?; [ "$status" -eq 42 ] && emit_error E_LIMIT; emit_error E_RUNTIME; +} +snapshot_file "$source_dir/framework.jq" "$scratch/framework.jq" 1048576 0400 || emit_error E_RUNTIME +snapshot_file "$repo/core/v2/generation-registry.json" "$scratch/registry.json" 1048576 0400 || + emit_error E_RUNTIME +snapshot_file "$jq_source" "$scratch/jq" 33554432 0500 || emit_error E_RUNTIME +[ "$(sha_file "$scratch/framework.jq")" = "$program_sha" ] || emit_error E_STALE +[ "$(sha_file "$scratch/registry.json")" = "$registry_sha" ] || emit_error E_STALE +[ "$(sha_file "$scratch/jq")" = "$jq_sha" ] || emit_error E_RUNTIME +generation=$("$scratch/jq" -er ' + if type=="array" and length>=1 and + all(.[]; .semantic_identity=="core.contracts.v2" and + (.generation_id | type=="string" and test("\\Ag-[0-9a-f]{64}\\z"))) + then .[-1].generation_id else error("E_STALE") end +' "$scratch/registry.json" 2>/dev/null) || emit_error E_STALE +schema="$repo/core/v2/generations/$generation/modules/schema.jq" +snapshot_file "$schema" "$scratch/schema.jq" 1048576 0400 || emit_error E_STALE +[ "$(sha_file "$scratch/schema.jq")" = "$schema_sha" ] || emit_error E_STALE + +if ! "$scratch/jq" -S -c . "$scratch/input.json" > "$scratch/canonical.json" 2>/dev/null; then + emit_error E_PARSE +fi +# jq reads JSON streams, so canonical bytes alone would admit a file holding +# several bundles. The input must be exactly one JSON text. +"$scratch/jq" -e -n --slurpfile roots "$scratch/input.json" '($roots | length) == 1' \ + >/dev/null 2>&1 || emit_error E_PARSE +/usr/bin/cmp -s "$scratch/input.json" "$scratch/canonical.json" || emit_error E_CANONICAL +bundle_sha=$(sha_file "$scratch/input.json") || emit_error E_RUNTIME +if ! "$scratch/jq" -e ' + . as $bundle | + type=="object" and (.body|type=="object") and + (.body.suite|type=="object") and + all(["cases","trials","grades"][]; . as $key | ($bundle.body[$key]|type)=="array") and + (.body.cases|length)<=32 and (.body.trials|length)<=128 and (.body.grades|length)<=256 +' "$scratch/input.json" >/dev/null 2>&1; then emit_error E_SHAPE; fi + +: > "$scratch/measured.tsv" +for section in suite cases trials grades; do + if [ "$section" = suite ]; then count=1; else + count=$("$scratch/jq" -r --arg section "$section" '.body[$section]|length' "$scratch/input.json") || + emit_error E_SHAPE + fi + index=0 + while [ "$index" -lt "$count" ]; do + value=$("$scratch/jq" -S -c --arg section "$section" --argjson index "$index" ' + if $section=="suite" then .body.suite.value else .body[$section][$index].value end + ' "$scratch/input.json") || emit_error E_SHAPE + /usr/bin/printf '%s\t%s\t%s\n' "$section" "$index" "$(sha_line "$value")" >> "$scratch/measured.tsv" + index=$((index + 1)) + done +done +"$scratch/jq" -R -s -c ' + split("\n") | map(select(length>0) | split("\t") | + {section:.[0],index:(.[1]|tonumber),sha256:.[2]}) +' "$scratch/measured.tsv" > "$scratch/measured.json" || emit_error E_RUNTIME + +status=0 +/usr/bin/env -i LC_ALL=C PATH=/usr/bin:/bin \ + "$scratch/jq" -S -c -L "$scratch" --arg program_sha256 "$program_sha" \ + --arg bundle_sha256 "$bundle_sha" --slurpfile measured_docs "$scratch/measured.json" \ + -f "$scratch/framework.jq" "$scratch/input.json" > "$scratch/output.json" \ + 2> "$scratch/error" || status=$? +if [ "$status" -ne 0 ]; then + [ ! -s "$scratch/output.json" ] || emit_error E_RUNTIME + for token in E_SHAPE E_STALE E_RELATION; do + /usr/bin/grep -Fq "$token" "$scratch/error" && emit_error "$token" + done + emit_error E_RUNTIME +fi +[ ! -s "$scratch/error" ] || emit_error E_RUNTIME +status=0 +snapshot_file "$scratch/output.json" "$scratch/bounded-output.json" 1048576 0400 || status=$? +[ "$status" -eq 0 ] || { [ "$status" -eq 42 ] && emit_error E_LIMIT; emit_error E_RUNTIME; } +"$scratch/jq" -S -c . "$scratch/bounded-output.json" > "$scratch/canonical-output.json" 2>/dev/null || + emit_error E_RUNTIME +/usr/bin/cmp -s "$scratch/bounded-output.json" "$scratch/canonical-output.json" || emit_error E_RUNTIME +[ "$(sha_file "$scratch/input.json")" = "$bundle_sha" ] || emit_error E_RUNTIME +[ "$(sha_file "$scratch/framework.jq")" = "$program_sha" ] || emit_error E_RUNTIME +[ "$(sha_file "$scratch/schema.jq")" = "$schema_sha" ] || emit_error E_RUNTIME +[ "$(sha_file "$scratch/registry.json")" = "$registry_sha" ] || emit_error E_RUNTIME +[ "$(sha_file "$scratch/jq")" = "$jq_sha" ] || emit_error E_RUNTIME +/bin/cat "$scratch/bounded-output.json" || emit_error E_RUNTIME +trap - EXIT HUP INT TERM +cleanup diff --git a/scripts/test/eval-framework.test.sh b/scripts/test/eval-framework.test.sh new file mode 100755 index 00000000..1e3eeb3b --- /dev/null +++ b/scripts/test/eval-framework.test.sh @@ -0,0 +1,302 @@ +#!/bin/bash +# shellcheck disable=SC2016 +set -euo pipefail +export LC_ALL=C +umask 077 + +root=$(CDPATH='' cd -P -- "${BASH_SOURCE[0]%/*}/../.." && pwd -P) +runner="$root/evals/v1/run.sh" +tmp=$(/usr/bin/mktemp -d "${TMPDIR:-/tmp}/ystack-eval-framework-test.XXXXXX") +cleanup() { /bin/rm -rf -- "$tmp"; } +trap cleanup EXIT +fail() { /usr/bin/printf 'FAIL: %s\n' "$1" >&2; exit 1; } +passes=0 +pass() { passes=$((passes + 1)); /usr/bin/printf 'ok %s - %s\n' "$passes" "$1"; } +sha_file() { /usr/bin/shasum -a 256 "$1" | /usr/bin/awk '{print $1}'; } + +platform=$(/usr/bin/uname -s):$(/usr/bin/uname -m) +case "$platform" in + Darwin:*) jq_asset=jq-osx-amd64; jq_sha=5c0a0a3ea600f302ee458b30317425dd9632d1ad8882259fcaf4e9b868b2b1ef ;; + Linux:x86_64) jq_asset=jq-linux64; jq_sha=af986793a515d500ab2d35f8d2aecd656e764504b789b66d7e1a0b727a124c44 ;; + *) fail "unsupported host $platform" ;; +esac +jq_cache_dir="${TMPDIR:-/tmp}/ystack-portable-core-jq16" +/bin/mkdir -p "$jq_cache_dir" +jq_bin="$jq_cache_dir/$jq_asset" +if [ ! -f "$jq_bin" ] || [ "$(sha_file "$jq_bin")" != "$jq_sha" ]; then + download=$(/usr/bin/mktemp "$jq_cache_dir/.jq-1.6.XXXXXX") + /usr/bin/curl --proto '=https' --tlsv1.2 -fsSL \ + "https://github.com/jqlang/jq/releases/download/jq-1.6/$jq_asset" -o "$download" + [ "$(sha_file "$download")" = "$jq_sha" ] || fail 'jq release digest' + /bin/chmod 0555 "$download" + /bin/mv "$download" "$jq_bin" +fi +[ "$($jq_bin --version)" = jq-1.6 ] || fail 'jq identity' + +ref() { + "$jq_bin" -S -c -n --arg id "$1" --arg media "$2" --arg sha "$3" \ + '{content_id:$id,media_type:$media,sha256:$sha}' +} +wrap() { + local source=$1 target=$2 id digest + id=$("$jq_bin" -r .id "$source") + digest=$(sha_file "$source") + "$jq_bin" -S -c -n --arg id "$id" --arg sha "$digest" --slurpfile value "$source" ' + {ref:{content_id:$id,media_type:"application/vnd.ystack.eval-record+json",sha256:$sha}, + value:$value[0]} + ' > "$target" +} +grader() { + "$jq_bin" -S -c -n --arg id "$1" --arg kind "$2" --arg digit "$3" ' + def ref($id;$sha): + {content_id:$id,media_type:"application/vnd.ystack.eval-grader+json",sha256:$sha}; + {grader_id:$id,grader_kind:$kind, + implementation_ref:ref(($id+".implementation");($digit*64)), + instructions_ref:ref(($id+".instructions");($digit*64))} + ' +} +rehash() { + local source=$1 section=$2 index=$3 target=$4 value digest + value=$("$jq_bin" -S -c --arg section "$section" --argjson index "$index" ' + if $section=="suite" then .body.suite.value else .body[$section][$index].value end + ' "$source") + digest=$(builtin printf '%s\n' "$value" | /usr/bin/shasum -a 256 | /usr/bin/awk '{print $1}') + "$jq_bin" -S -c --arg section "$section" --argjson index "$index" --arg sha "$digest" ' + if $section=="suite" then .body.suite.ref.sha256=$sha + else .body[$section][$index].ref.sha256=$sha end + ' "$source" > "$target" +} +expect_error() { + local name=$1 token=$2 input=$3 status=0 + "$runner" evaluate "$input" > "$tmp/$name.out" 2> "$tmp/$name.err" || status=$? + [ "$status" -ne 0 ] && [ ! -s "$tmp/$name.out" ] && + [ "$(/bin/cat "$tmp/$name.err")" = "$token" ] || fail "$name" + pass "$name" +} + +framework_sha=$(sha_file "$root/evals/v1/framework.jq") +framework_ref=$(ref eval-framework.v1 application/vnd.ystack.eval-framework+jq "$framework_sha") +scope_ref=$(ref eval-scope.definition application/vnd.ystack.eval-scope+json "$(printf 'a%.0s' {1..64})") +"$jq_bin" -S -c -n --argjson framework "$framework_ref" --argjson definition "$scope_ref" ' + {schema_version:1,kind:"eval_suite",id:"suite.default-adapters",body:{ + suite_version:"v1",framework_ref:$framework, + scope:{scope_id:"scope.default-adapters",scope_version:"v1", + definition_ref:$definition,scope_sha256:$definition.sha256}, + case_ids:["case.deterministic","case.model"]}} +' > "$tmp/suite.value" +wrap "$tmp/suite.value" "$tmp/suite.wrap" +suite_ref=$("$jq_bin" -c .ref "$tmp/suite.wrap") + +grader_det=$(grader grader.det deterministic b) +grader_human=$(grader grader.human human c) +grader_model=$(grader grader.model model d) +input_det=$(ref input.det application/json "$(printf '1%.0s' {1..64})") +expected_det=$(ref expected.det application/json "$(printf '2%.0s' {1..64})") +input_model=$(ref input.model application/json "$(printf '3%.0s' {1..64})") +expected_model=$(ref expected.model application/json "$(printf '4%.0s' {1..64})") +"$jq_bin" -S -c -n --argjson suite "$suite_ref" --argjson input "$input_det" \ + --argjson expected "$expected_det" --argjson det "$grader_det" --argjson human "$grader_human" ' + {schema_version:1,kind:"eval_case",id:"case.deterministic",body:{case_version:"v1", + suite_ref:$suite,execution_kind:"deterministic",input_ref:$input,expected_ref:$expected, + trial_count:1,trial_ids:["trial.det.1"],attempt_ids:["attempt.trial.det.1"], + graders:[$det,$human]}} +' > "$tmp/case.det.value" +"$jq_bin" -S -c -n --argjson suite "$suite_ref" --argjson input "$input_model" \ + --argjson expected "$expected_model" --argjson det "$grader_det" --argjson human "$grader_human" \ + --argjson model "$grader_model" ' + {schema_version:1,kind:"eval_case",id:"case.model",body:{case_version:"v1", + suite_ref:$suite,execution_kind:"model",input_ref:$input,expected_ref:$expected, + trial_count:2,trial_ids:["trial.model.1","trial.model.2"], + attempt_ids:["attempt.trial.model.1","attempt.trial.model.2"],graders:[$det,$human,$model]}} +' > "$tmp/case.model.value" +wrap "$tmp/case.det.value" "$tmp/case.det.wrap" +wrap "$tmp/case.model.value" "$tmp/case.model.wrap" +"$jq_bin" -S -c -s . "$tmp/case.det.wrap" "$tmp/case.model.wrap" > "$tmp/cases.json" +case_det_ref=$("$jq_bin" -c .ref "$tmp/case.det.wrap") +case_model_ref=$("$jq_bin" -c .ref "$tmp/case.model.wrap") + +make_trial() { + local id=$1 case_ref=$2 index=$3 digit=$4 output + output=$(ref "$id.output" application/json "$(printf "$digit%.0s" {1..64})") + "$jq_bin" -S -c -n --arg id "$id" --argjson case_ref "$case_ref" --argjson index "$index" \ + --argjson output "$output" '{schema_version:1,kind:"eval_trial",id:$id,body:{ + trial_version:"v1",case_ref:$case_ref,trial_index:$index,attempt_id:("attempt."+$id), + started_at:"2026-09-01T00:00:00Z",finished_at:"2026-09-01T00:00:10Z",status:"completed", + output_ref:{state:"present",value:$output},reason:{state:"absent"}}}' +} +make_trial trial.det.1 "$case_det_ref" 1 5 > "$tmp/trial.det.value" +make_trial trial.model.1 "$case_model_ref" 1 6 > "$tmp/trial.model1.value" +make_trial trial.model.2 "$case_model_ref" 2 7 > "$tmp/trial.model2.value" +for name in det model1 model2; do wrap "$tmp/trial.$name.value" "$tmp/trial.$name.wrap"; done +"$jq_bin" -S -c -s . "$tmp/trial.det.wrap" "$tmp/trial.model1.wrap" "$tmp/trial.model2.wrap" > "$tmp/trials.json" + +make_grade() { + local id=$1 trial=$2 grader_json=$3 status=$4 digit=$5 + local grader_id grader_kind grader_ref evidence evidence_state reason + grader_id=$("$jq_bin" -r .grader_id <<< "$grader_json") + grader_kind=$("$jq_bin" -r .grader_kind <<< "$grader_json") + grader_ref=$("$jq_bin" -c .implementation_ref <<< "$grader_json") + evidence=$(ref "$id.evidence" application/json "$(printf "$digit%.0s" {1..64})") + if [ "$status" = inconclusive ]; then + evidence_state='{"state":"absent"}'; reason='{"state":"present","value":"grader.no-consensus"}' + else + evidence_state=$("$jq_bin" -c -n --argjson value "$evidence" '{state:"present",value:$value}') + reason='{"state":"absent"}' + fi + "$jq_bin" -S -c -n --arg id "$id" --argjson trial "$trial" --arg grader_id "$grader_id" \ + --arg grader_kind "$grader_kind" --argjson grader_ref "$grader_ref" --arg status "$status" \ + --argjson evidence "$evidence_state" --argjson reason "$reason" ' + {schema_version:1,kind:"eval_grade",id:$id,body:{grade_version:"v1",trial_ref:$trial, + grader_id:$grader_id,grader_kind:$grader_kind,grader_ref:$grader_ref, + graded_at:"2026-09-01T00:00:20Z",status:$status, + evidence_ref:$evidence,reason:$reason}}' +} +trial_det_ref=$("$jq_bin" -c .ref "$tmp/trial.det.wrap") +trial_model1_ref=$("$jq_bin" -c .ref "$tmp/trial.model1.wrap") +trial_model2_ref=$("$jq_bin" -c .ref "$tmp/trial.model2.wrap") +grade_specs=( + 'det.det' 'det.human' 'model1.det' 'model1.human' 'model1.model' + 'model2.det' 'model2.human' 'model2.model' +) +for spec in "${grade_specs[@]}"; do + trial_name=${spec%%.*}; grader_name=${spec##*.}; status=passed; digit=8 + [ "$spec" = model2.human ] && status=inconclusive + case "$trial_name" in det) trial_ref=$trial_det_ref ;; model1) trial_ref=$trial_model1_ref ;; model2) trial_ref=$trial_model2_ref ;; esac + case "$grader_name" in det) grader_json=$grader_det ;; human) grader_json=$grader_human ;; model) grader_json=$grader_model ;; esac + make_grade "grade.$spec" "$trial_ref" "$grader_json" "$status" "$digit" > "$tmp/grade.$spec.value" + wrap "$tmp/grade.$spec.value" "$tmp/grade.$spec.wrap" +done +"$jq_bin" -S -c -s . "$tmp"/grade.*.wrap | "$jq_bin" -S -c 'sort_by([.value.body.trial_ref.content_id,.value.body.grader_id])' > "$tmp/grades.json" +"$jq_bin" -S -c -n --slurpfile suite "$tmp/suite.wrap" --slurpfile cases "$tmp/cases.json" \ + --slurpfile trials "$tmp/trials.json" --slurpfile grades "$tmp/grades.json" ' + {schema_version:1,kind:"eval_bundle",id:"run.default-adapters", + body:{suite:$suite[0],cases:$cases[0],trials:$trials[0],grades:$grades[0]}} +' > "$tmp/bundle.json" + +"$runner" evaluate "$tmp/bundle.json" > "$tmp/report.json" +"$jq_bin" -e ' + .kind=="eval_report" and .body.mode=="evaluation-only" and + .body.status=="inconclusive" and + [.body.cases[].status]==["passed","inconclusive"] and + [.body.cases[1].trials[].trial_index]==[1,2] +' "$tmp/report.json" >/dev/null || fail 'valid multi-trial report' +pass valid-multi-trial-report +"$runner" evaluate "$tmp/bundle.json" > "$tmp/repeat.json" +/usr/bin/cmp -s "$tmp/report.json" "$tmp/repeat.json" || fail 'deterministic report' +# Invoked by a relative path, the runner still binds itself to its own directory. +( cd "$root" && ./evals/v1/run.sh evaluate "$tmp/bundle.json" ) > "$tmp/relative.json" || + fail 'relative-path invocation' +/usr/bin/cmp -s "$tmp/report.json" "$tmp/relative.json" || fail 'relative-path report differs' +pass relative-path-invocation +# Trial ids are bound by position, so ids whose lexical order differs from +# their trial order are still one valid case shape; a repeated id is not. +generation=$(/usr/bin/sed -n "s/^PORTABLE_CORE_GENERATION='\\(g-[0-9a-f]\\{64\\}\\)'$/\\1/p" \ + "$root/scripts/core-contract.sh") +modules="$root/core/v2/generations/$generation/modules" +[ -d "$modules" ] || fail 'selected generation modules' +/usr/bin/awk 'NR > 48 && /^def / { exit } { print }' "$root/evals/v1/framework.jq" \ + > "$tmp/framework-shapes.jq" +"$jq_bin" -L "$tmp" -L "$modules" -e --slurpfile c "$tmp/case.model.value" -n ' + import "framework-shapes" as shapes; + ($c[0] | .body.trial_ids = ["trial.model.9","trial.model.10"] | + .body.attempt_ids = ["attempt.trial.model.9","attempt.trial.model.10"] | shapes::case_ok) and + ($c[0] | .body.trial_ids = ["trial.model.1","trial.model.1"] | shapes::case_ok | not) and + ($c[0] | .body.attempt_ids = ["attempt.trial.model.1","attempt.trial.model.1"] | shapes::case_ok | not) +' >/dev/null || fail 'trial ids are not bound by position' +pass trial-ids-bound-by-position +[ "$(/usr/bin/wc -c < "$tmp/report.json" | /usr/bin/tr -d ' ')" -le 1048576 ] || + fail 'bounded report' +pass deterministic-report + +"$jq_bin" -S -c '.body.cases[0].value.body.expected_ref.sha256=("f"*64)' \ + "$tmp/bundle.json" > "$tmp/tamper.json" +expect_error tampered-record E_STALE "$tmp/tamper.json" +"$jq_bin" -S -c '.body.cases[0].value.body.suite_ref.sha256=("e"*64)' \ + "$tmp/bundle.json" > "$tmp/stale-unhashed.json" +rehash "$tmp/stale-unhashed.json" cases 0 "$tmp/stale.json" +expect_error stale-record-link E_RELATION "$tmp/stale.json" +"$jq_bin" -S -c '.body.trials[0].value.body.started_at="2026-09-01T00:00:11Z"' \ + "$tmp/bundle.json" > "$tmp/trial-time-unhashed.json" +rehash "$tmp/trial-time-unhashed.json" trials 0 "$tmp/trial-time.json" +expect_error inverted-trial-time E_SHAPE "$tmp/trial-time.json" +"$jq_bin" -S -c '.body.grades[0].value.body.graded_at="2026-09-01T00:00:09Z"' \ + "$tmp/bundle.json" > "$tmp/grade-time-unhashed.json" +rehash "$tmp/grade-time-unhashed.json" grades 0 "$tmp/grade-time.json" +expect_error early-grade-time E_RELATION "$tmp/grade-time.json" +"$jq_bin" -S -c '.body.trials[1].value.body.attempt_id=.body.trials[0].value.body.attempt_id' \ + "$tmp/bundle.json" > "$tmp/attempt-unhashed.json" +rehash "$tmp/attempt-unhashed.json" trials 1 "$tmp/attempt.json" +expect_error duplicate-attempt E_RELATION "$tmp/attempt.json" +"$jq_bin" -S -c '.body.cases|=reverse' "$tmp/bundle.json" > "$tmp/order.json" +expect_error record-order E_RELATION "$tmp/order.json" +"$jq_bin" -S -c '.body.trials += [.body.trials[0]]' "$tmp/bundle.json" > "$tmp/duplicate.json" +expect_error duplicate-trial E_RELATION "$tmp/duplicate.json" +"$jq_bin" -S -c '.body.grades |= map(select(.value.id!="grade.model2.model"))' \ + "$tmp/bundle.json" > "$tmp/missing-grade.json" +expect_error missing-grade E_RELATION "$tmp/missing-grade.json" +"$jq_bin" -S -c '.body.suite.value.body.extra_command="printf unsafe"' \ + "$tmp/bundle.json" > "$tmp/command.json" +expect_error arbitrary-command-field E_SHAPE "$tmp/command.json" +"$jq_bin" -S -c '.body.cases[0].value.body.graders[0].implementation_ref.content_id=("x"*129)' \ + "$tmp/bundle.json" > "$tmp/grader-metadata.json" +expect_error bounded-grader-metadata E_SHAPE "$tmp/grader-metadata.json" + +"$jq_bin" -S -c ' + .body.trials[2].value.body.status="unavailable" | + .body.trials[2].value.body.output_ref={state:"absent"} | + .body.trials[2].value.body.reason={state:"present",value:"trial.output-unavailable"} +' "$tmp/bundle.json" > "$tmp/hidden-trial-unhashed.json" +rehash "$tmp/hidden-trial-unhashed.json" trials 2 "$tmp/hidden-trial.json" +hidden_trial_ref=$("$jq_bin" -c '.body.trials[2].ref' "$tmp/hidden-trial.json") +"$jq_bin" -S -c --argjson trial "$hidden_trial_ref" ' + .body.grades |= map(if .value.body.trial_ref.content_id=="trial.model.2" then + .value.body.trial_ref=$trial | + if .value.id=="grade.model2.model" then .value.body.status="failed" + else .value.body.status="unavailable" | .value.body.evidence_ref={state:"absent"} | + .value.body.reason={state:"present",value:"grader.unavailable"} end + else . end) +' "$tmp/hidden-trial.json" > "$tmp/hidden-grades-0.json" +hidden_current="$tmp/hidden-grades-0.json" +hidden_count=0 +for hidden_id in grade.model2.det grade.model2.human grade.model2.model; do + hidden_index=$("$jq_bin" -r --arg id "$hidden_id" '.body.grades|map(.value.id)|index($id)' "$hidden_current") + hidden_next="$tmp/hidden-grades-$((hidden_count + 1)).json" + rehash "$hidden_current" grades "$hidden_index" "$hidden_next" + hidden_current=$hidden_next + hidden_count=$((hidden_count + 1)) +done +expect_error unavailable-trial-hides-failure E_RELATION "$hidden_current" + +"$jq_bin" -S -c ' + .body.grades |= map(if .value.id=="grade.model2.model" then + .value.body.status="failed" else . end) +' "$tmp/bundle.json" > "$tmp/failed-unhashed.json" +failed_index=$("$jq_bin" -r '.body.grades|map(.value.id)|index("grade.model2.model")' "$tmp/failed-unhashed.json") +rehash "$tmp/failed-unhashed.json" grades "$failed_index" "$tmp/failed.json" +"$runner" evaluate "$tmp/failed.json" > "$tmp/failed-report.json" +[ "$("$jq_bin" -r .body.status "$tmp/failed-report.json")" = failed ] || fail 'failed grade precedence' +pass failed-grade-precedence + +"$jq_bin" -S -c ' + .body.grades |= map(if .value.id=="grade.model2.human" then + .value.body.status="unavailable" | .value.body.evidence_ref={state:"absent"} | + .value.body.reason={state:"present",value:"grader.unavailable"} + else . end) +' "$tmp/bundle.json" > "$tmp/unavailable-unhashed.json" +grade_index=$("$jq_bin" -r '.body.grades|map(.value.id)|index("grade.model2.human")' "$tmp/unavailable-unhashed.json") +rehash "$tmp/unavailable-unhashed.json" grades "$grade_index" "$tmp/unavailable.json" +"$runner" evaluate "$tmp/unavailable.json" > "$tmp/unavailable-report.json" +[ "$("$jq_bin" -r .body.status "$tmp/unavailable-report.json")" = unavailable ] || fail 'unavailable result' +pass explicit-unavailable + +"$jq_bin" . "$tmp/bundle.json" > "$tmp/noncanonical.json" +expect_error canonical-bytes E_CANONICAL "$tmp/noncanonical.json" +/bin/cat "$tmp/bundle.json" "$tmp/bundle.json" > "$tmp/two-bundles.json" +expect_error multi-root-stream E_PARSE "$tmp/two-bundles.json" +"$jq_bin" -e ' + ([..|objects|keys[]|select(.=="authority" or .=="permissions" or .=="credential" or + .=="network" or .=="command" or .=="activation" or .=="qualification")] | length)==0 +' "$tmp/report.json" >/dev/null || fail 'inactive data-only output' +pass inactive-data-only-output + +/usr/bin/printf 'PASS: %s eval-record evaluator checks\n' "$passes" diff --git a/scripts/test/portable-core-schema.test.sh b/scripts/test/portable-core-schema.test.sh index 04c85e44..107020fb 100755 --- a/scripts/test/portable-core-schema.test.sh +++ b/scripts/test/portable-core-schema.test.sh @@ -828,6 +828,7 @@ schema_import_path_ok() { case "$import_path" in adapters/local-git-materializer/v1/protocol.jq|\ adapters/deterministic-verifier/v1/normalize.jq|\ + evals/v1/framework.jq|\ evals/v1/evals.jq|\ orchestrator/v1/reconciliation-plan.jq|orchestrator/v1/state-scanner.jq) ;; scripts/test/default-codex-native-reviewer-adapter.test.sh|\