From 1e83914c6ec7957acfad3683b87f9736f1a6ec42 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 24 Jul 2026 19:31:30 -0600 Subject: [PATCH] fix(bench): read current Pier candidate contract --- bench/CHANGELOG.md | 6 + bench/package.json | 2 +- bench/pier_agents/candidate_contract.py | 262 +++++++++++-- bench/pier_agents/tangle_candidate.py | 16 +- bench/pier_agents/tangle_candidate_test.py | 410 ++++++++++++++++++--- bench/scripts/verify-pier-agent.mts | 10 +- 6 files changed, 617 insertions(+), 89 deletions(-) diff --git a/bench/CHANGELOG.md b/bench/CHANGELOG.md index 831a7cc0..7542d569 100644 --- a/bench/CHANGELOG.md +++ b/bench/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 0.4.1 + +- Read Runtime 0.105 candidate plans from their signed run cell, benchmark records, and profile activation. +- Allow only the fixed public executable path signed by the Runtime plan. +- Restore the real Pier failure/success proof against the current receipt layout. + ## 0.4.0 - Add the resumable SWE improvement loop backed by the official GEPA engine. diff --git a/bench/package.json b/bench/package.json index 0bb82726..2931d24a 100644 --- a/bench/package.json +++ b/bench/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-bench", - "version": "0.4.0", + "version": "0.4.1", "type": "module", "description": "The unified benchmark suite for agent-runtime agents: 31 adapters (commit0, enterpriseops-gym, ragbench, crag, nomiracl, open-rag-bench, t2-ragbench, tau3-banking, bfcl, finresearchbench, …) behind one resolveAdapter registry, each with a real judge or fail-loud unsupported scorer. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.", "repository": { diff --git a/bench/pier_agents/candidate_contract.py b/bench/pier_agents/candidate_contract.py index bc773401..1724e62b 100644 --- a/bench/pier_agents/candidate_contract.py +++ b/bench/pier_agents/candidate_contract.py @@ -34,8 +34,13 @@ _PLAN_KIND = "agent-candidate-execution-plan-material" _RECEIPT_KIND = "agent-candidate-materialization" _PROFILE_PLAN_KIND = "agent-profile-workspace-plan" +_PROFILE_ACTIVATION_KIND = "agent-candidate-profile-activation" _EXECUTION_EVIDENCE_KIND = "agent-candidate-execution-plan" +_RUN_CELL_KIND = "agent-candidate-run-cell" +_BENCHMARK_SUITE_KIND = "agent-candidate-benchmark-suite" +_BENCHMARK_TASK_KIND = "agent-candidate-benchmark-task" _STDIN_TASK_PATH = "/tangle/input/stdin.txt" +_MAX_SAFE_INTEGER = 9_007_199_254_740_991 class CandidateContractError(ValueError): @@ -130,6 +135,30 @@ def _integer(value: Any, label: str, *, minimum: int = 0) -> int: return value +def _safe_integer(value: Any, label: str) -> int: + if ( + isinstance(value, bool) + or not isinstance(value, int) + or value < -_MAX_SAFE_INTEGER + or value > _MAX_SAFE_INTEGER + ): + raise CandidateContractError(f"{label} must be a safe integer") + return value + + +def _canonical_bytes(value: Any, label: str) -> bytes: + try: + return json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + except (TypeError, ValueError, UnicodeEncodeError) as exc: + raise CandidateContractError(f"{label} is not canonical JSON") from exc + + def _number(value: Any, label: str, *, minimum: float = 0) -> float: if ( isinstance(value, bool) @@ -246,6 +275,21 @@ def _embedded_bytes(value: Any, label: str) -> bytes: return raw +def _embedded_contract( + value: Any, label: str, *, kind: str +) -> tuple[dict[str, Any], bytes, str]: + evidence = _object(value, label) + digest = _digest(evidence.get("digest"), f"{label}.digest") + raw = _embedded_bytes(evidence.get("material"), f"{label}.material") + if sha256_bytes(raw) != digest: + raise CandidateContractError(f"{label} material does not match its digest") + try: + material = json.loads(raw) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise CandidateContractError(f"{label} material is not UTF-8 JSON") from exc + return _contract_object(material, f"{label}.material", kind=kind), raw, digest + + def _workspace_files(value: Any, label: str) -> tuple[WorkspaceFile, ...]: snapshot = _contract_object( value, label, kind="agent-candidate-workspace-snapshot" @@ -318,11 +362,10 @@ def _profile_files(value: Any) -> tuple[ProfileFile, ...]: return tuple(files) -def _instruction(value: Any) -> InstructionEvidence: - obj = _object(value, "task.instruction") - if obj.get("encoding") != "utf8": - raise CandidateContractError("task.instruction.encoding must equal utf8") - delivery = _object(obj.get("delivery"), "task.instruction.delivery") +def _instruction(value: Any, delivery_value: Any) -> InstructionEvidence: + text = _string(value, "benchmark task instruction") + raw = text.encode("utf-8") + delivery = _object(delivery_value, "plan.instructionDelivery") kind = delivery.get("kind") if kind == "argv-append" or kind == "stdin-utf8": if set(delivery) != {"kind"}: @@ -342,10 +385,8 @@ def _instruction(value: Any) -> InstructionEvidence: else: raise CandidateContractError("unsupported task instruction delivery") return InstructionEvidence( - sha256=_digest(obj.get("sha256"), "task.instruction.sha256"), - byte_length=_integer( - obj.get("byteLength"), "task.instruction.byteLength", minimum=1 - ), + sha256=sha256_bytes(raw), + byte_length=len(raw), delivery_kind=kind, delivery_env=env, delivery_path=path, @@ -400,12 +441,87 @@ def load_prepared_candidate_contract( "receipt execution-plan artifact differs from the executed bytes" ) - bundle_digest = _digest(plan.get("bundleDigest"), "plan.bundleDigest") + run_cell = _contract_object( + plan.get("runCell"), "plan.runCell", kind=_RUN_CELL_KIND + ) + _digest(run_cell.get("experimentDigest"), "plan.runCell.experimentDigest") + run_cell_digest = _digest(run_cell.get("digest"), "plan.runCell.digest") + if run_cell.get("arm") not in {"baseline", "candidate"}: + raise CandidateContractError("plan.runCell.arm is unsupported") + run_cell_seed = _safe_integer(run_cell.get("seed"), "plan.runCell.seed") + if sha256_bytes( + _canonical_bytes( + {key: value for key, value in run_cell.items() if key != "digest"}, + "plan.runCell", + ) + ) != run_cell_digest: + raise CandidateContractError("plan.runCell digest does not match") + bundle_digest = _digest( + run_cell.get("bundleDigest"), "plan.runCell.bundleDigest" + ) if receipt.get("bundleDigest") != bundle_digest: raise CandidateContractError("receipt and plan bundle digests differ") execution_id = _string(plan.get("executionId"), "plan.executionId") - task = _object(plan.get("task"), "plan.task") + benchmark = _object(receipt.get("benchmark"), "receipt.benchmark") + suite, _, suite_digest = _embedded_contract( + benchmark.get("suite"), + "receipt.benchmark.suite", + kind=_BENCHMARK_SUITE_KIND, + ) + task, _, task_digest = _embedded_contract( + benchmark.get("task"), + "receipt.benchmark.task", + kind=_BENCHMARK_TASK_KIND, + ) + if ( + run_cell.get("suiteDigest") != suite_digest + or run_cell.get("taskDigest") != task_digest + ): + raise CandidateContractError( + "run cell does not bind the receipt benchmark suite and task" + ) + task_index = _integer(run_cell.get("taskIndex"), "plan.runCell.taskIndex") + repetition = _integer(run_cell.get("repetition"), "plan.runCell.repetition") + if suite.get("digestAlgorithm") != "rfc8785-sha256": + raise CandidateContractError( + "benchmark suite digestAlgorithm must equal rfc8785-sha256" + ) + reps = _integer(suite.get("reps"), "benchmark suite reps", minimum=1) + task_digests = suite.get("taskDigests") + seeds = suite.get("seeds") + if ( + not isinstance(task_digests, list) + or not task_digests + or len(set(task_digests)) != len(task_digests) + ): + raise CandidateContractError( + "benchmark suite taskDigests must be a non-empty unique array" + ) + for index, value in enumerate(task_digests): + _digest(value, f"benchmark suite taskDigests[{index}]") + if ( + not isinstance(seeds, list) + or len(seeds) != len(task_digests) * reps + ): + raise CandidateContractError( + "benchmark suite must provide one seed per task repetition" + ) + for index, value in enumerate(seeds): + _safe_integer(value, f"benchmark suite seeds[{index}]") + if ( + task_index >= len(task_digests) + or task_digests[task_index] != task_digest + or repetition >= reps + or seeds[task_index * reps + repetition] != run_cell_seed + ): + raise CandidateContractError( + "run cell coordinates do not match the receipt benchmark suite" + ) + if task.get("digestAlgorithm") != "rfc8785-sha256": + raise CandidateContractError( + "benchmark task digestAlgorithm must equal rfc8785-sha256" + ) outcome = _object(task.get("outcome"), "plan.task.outcome") if outcome.get("kind") != "workspace": raise CandidateContractError("Pier requires a workspace task outcome") @@ -418,7 +534,9 @@ def load_prepared_candidate_contract( ) if len(base_commit) != len(base_tree): raise CandidateContractError("task Git objects use different hash formats") - instruction = _instruction(task.get("instruction")) + instruction = _instruction( + task.get("instruction"), plan.get("instructionDelivery") + ) task_files = _workspace_files(task.get("workspace"), "plan.task.workspace") if not task_files: raise CandidateContractError("task workspace cannot be empty") @@ -480,16 +598,40 @@ def load_prepared_candidate_contract( if receipt.get("candidateWorkspace") != candidate_snapshot: raise CandidateContractError("receipt and plan candidate workspaces differ") + profile_activation = _contract_object( + receipt.get("profileActivation"), + "receipt.profileActivation", + kind=_PROFILE_ACTIVATION_KIND, + ) + profile_activation_digest = _digest( + profile_activation.get("digest"), + "receipt.profileActivation.digest", + ) + if sha256_bytes( + _canonical_bytes( + { + key: value + for key, value in profile_activation.items() + if key != "digest" + }, + "receipt.profileActivation", + ) + ) != profile_activation_digest: + raise CandidateContractError( + "profile activation digest does not match" + ) profile_evidence = _contract_object( - receipt.get("profilePlan"), - "receipt.profilePlan", + profile_activation.get("profilePlan"), + "receipt.profileActivation.profilePlan", kind=_PROFILE_PLAN_KIND, ) profile_digest = _digest( - profile_evidence.get("digest"), "receipt.profilePlan.digest" + profile_evidence.get("digest"), + "receipt.profileActivation.profilePlan.digest", ) profile_raw = _embedded_bytes( - profile_evidence.get("artifact"), "receipt.profilePlan.artifact" + profile_evidence.get("artifact"), + "receipt.profileActivation.profilePlan.artifact", ) if sha256_bytes(profile_raw) != profile_digest: raise CandidateContractError("profile artifact does not match its digest") @@ -498,11 +640,50 @@ def load_prepared_candidate_contract( except (UnicodeDecodeError, json.JSONDecodeError) as exc: raise CandidateContractError("profile artifact is not UTF-8 JSON") from exc profile_material = _contract_object( - profile_evidence.get("material"), "receipt.profilePlan.material" + profile_evidence.get("material"), + "receipt.profileActivation.profilePlan.material", ) if profile_artifact_material != profile_material: raise CandidateContractError("profile artifact and material differ") + if profile_material.get("harness") != plan.get("harness"): + raise CandidateContractError( + "profile plan harness differs from the execution plan" + ) profile_files = _profile_files(profile_material) + activation_files = profile_activation.get("files") + if not isinstance(activation_files, list): + raise CandidateContractError( + "receipt.profileActivation.files must be an array" + ) + activated: list[ProfileFile] = [] + for index, value in enumerate(activation_files): + item = _object(value, f"receipt.profileActivation.files[{index}]") + path = _safe_relative( + item.get("path"), f"receipt.profileActivation.files[{index}].path" + ) + mode = _integer( + item.get("mode"), f"receipt.profileActivation.files[{index}].mode" + ) + content = item.get("content") + if ( + mode > 0o777 + or not isinstance(content, str) + or "\0" in content + ): + raise CandidateContractError( + f"receipt.profileActivation.files[{index}] is invalid" + ) + activated.append( + ProfileFile( + path=path, + mode=mode, + sha256=sha256_bytes(content.encode("utf-8")), + ) + ) + if activated != list(profile_files): + raise CandidateContractError( + "profile activation files differ from the signed profile plan" + ) profile_application = _object(plan.get("profile"), "plan.profile") if profile_application.get("planDigest") != profile_digest: raise CandidateContractError("plan does not bind the profile digest") @@ -563,6 +744,10 @@ def load_prepared_candidate_contract( raise CandidateContractError("launch cwd names an unavailable workspace") cwd_path = _safe_relative(cwd.get("path"), "plan.launch.cwd.path", allow_dot=True) limits = _object(plan.get("limits"), "plan.limits") + if limits != task.get("limits"): + raise CandidateContractError( + "execution plan limits differ from the benchmark task" + ) timeout_ms = _integer(limits.get("timeoutMs"), "plan.limits.timeoutMs", minimum=1) _integer(limits.get("maxSteps"), "plan.limits.maxSteps", minimum=1) max_model_calls = _integer( @@ -571,11 +756,27 @@ def load_prepared_candidate_contract( _integer(limits.get("maxInputTokens"), "plan.limits.maxInputTokens") _integer(limits.get("maxOutputTokens"), "plan.limits.maxOutputTokens") _number(limits.get("maxCostUsd"), "plan.limits.maxCostUsd") - attempt = _object(plan.get("attempt"), "plan.attempt") - _integer(attempt.get("number"), "plan.attempt.number", minimum=1) - _integer(attempt.get("maxAttempts"), "plan.attempt.maxAttempts", minimum=1) - if attempt.get("retryPolicy") != "none": - raise CandidateContractError("Pier execution requires runtime-owned retries") + attempt = _object(task.get("attempt"), "benchmark task attempt") + attempt_number = _integer( + run_cell.get("attempt"), "plan.runCell.attempt", minimum=1 + ) + max_attempts = _integer( + attempt.get("maxAttempts"), "benchmark task attempt.maxAttempts", minimum=1 + ) + if attempt_number > max_attempts: + raise CandidateContractError( + "run cell attempt exceeds the benchmark task maximum" + ) + retry_policy = attempt.get("retryPolicy") + if retry_policy not in { + "none", + "pre-model-infrastructure-only", + }: + raise CandidateContractError("benchmark task retry policy is unsupported") + if retry_policy == "none" and max_attempts != 1: + raise CandidateContractError( + "a no-retry benchmark task must allow exactly one attempt" + ) container = _object(plan.get("container"), "plan.container") if receipt.get("container") != container: raise CandidateContractError("receipt and plan container identities differ") @@ -586,11 +787,21 @@ def load_prepared_candidate_contract( raise CandidateContractError( "container image must be an unpinned OCI reference without credentials" ) - if container.get("source") not in { + container_source = container.get("source") + if container_source not in { "pinned-container", "evaluator-task-container", }: raise CandidateContractError("plan container source is unsupported") + task_container = task.get("evaluatorTaskContainer") + if container_source == "evaluator-task-container" and task_container != container: + raise CandidateContractError( + "execution plan container differs from the benchmark task container" + ) + if container_source == "pinned-container" and task_container is not None: + raise CandidateContractError( + "pinned execution cannot override a benchmark task container" + ) platform = _object(container.get("platform"), "plan.container.platform") if platform.get("os") != "linux" or platform.get("architecture") not in { "amd64", @@ -605,7 +816,10 @@ def load_prepared_candidate_contract( _string(resolved_model.get("provider"), "plan.model.resolved.provider") _string(resolved_model.get("model"), "plan.model.resolved.model") _string(resolved_model.get("snapshot"), "plan.model.resolved.snapshot") - if receipt.get("resolvedModel") != resolved_model: + if ( + receipt.get("resolvedModel") != resolved_model + or task.get("model") != resolved_model + ): raise CandidateContractError("receipt and plan resolved models differ") model_access = _object(model.get("access"), "plan.model.access") if set(model_access) != {"kind", "grantDigest", "network"}: diff --git a/bench/pier_agents/tangle_candidate.py b/bench/pier_agents/tangle_candidate.py index ae27d2cb..951394cd 100644 --- a/bench/pier_agents/tangle_candidate.py +++ b/bench/pier_agents/tangle_candidate.py @@ -55,6 +55,7 @@ _CANDIDATE_TMP = "/tangle/tmp" _CONTROL_ROOT = "/tangle/control" _FIXED_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +_FIXED_PATH_ENTRIES = frozenset(_FIXED_PATH.split(":")) _RESERVED_PROCESS_ENV = { "GIT_CONFIG_COUNT", "GIT_CONFIG_KEY_0", @@ -241,12 +242,25 @@ def _validate_protected_env( ) -> dict[str, str]: protected: dict[str, str] = {} public_names = set(self._contract.env) - reserved_public = sorted(public_names & _RESERVED_PROCESS_ENV) + reserved_public = sorted( + (public_names & _RESERVED_PROCESS_ENV) - {"PATH"} + ) if reserved_public: raise PierCandidateError( "signed public env collides with evaluator Git safety fields: " + ", ".join(reserved_public) ) + public_path = self._contract.env.get("PATH") + if public_path is not None: + entries = public_path.split(":") + if ( + not entries + or len(entries) != len(set(entries)) + or any(entry not in _FIXED_PATH_ENTRIES for entry in entries) + ): + raise PierCandidateError( + "signed public PATH contains an empty, duplicate, or untrusted directory" + ) expected_trace = _trace_env(self._contract, trace_run_id) for name, expected in expected_trace.items(): if values.get(name) != expected: diff --git a/bench/pier_agents/tangle_candidate_test.py b/bench/pier_agents/tangle_candidate_test.py index cc77be39..9307e5c0 100644 --- a/bench/pier_agents/tangle_candidate_test.py +++ b/bench/pier_agents/tangle_candidate_test.py @@ -248,6 +248,7 @@ def _fixture(root: Path): candidate_snapshot = _workspace([("runner.py", 0o755, runner)]) profile_material = { "harness": "codex", + "sourceProfileDigest": f"sha256:{'a' * 64}", "files": [ { "relPath": "AGENTS.md", @@ -261,32 +262,122 @@ def _fixture(root: Path): } profile_raw = _canonical(profile_material) profile_digest = _sha(profile_raw) + profile_evidence = { + "kind": "agent-profile-workspace-plan", + "digest": profile_digest, + "material": profile_material, + "artifact": _embedded(profile_raw), + } + profile_activation_material = { + "kind": "agent-candidate-profile-activation", + "profilePlan": profile_evidence, + "files": [ + { + "path": "AGENTS.md", + "mode": 0o600, + "content": profile.decode(), + } + ], + } + profile_activation = { + **profile_activation_material, + "digest": _sha(_canonical(profile_activation_material)), + } bundle_digest = f"sha256:{'b' * 64}" - plan = { - "kind": "agent-candidate-execution-plan-material", - "bundleDigest": bundle_digest, - "executionId": "pier-fixture-execution", - "attempt": {"number": 1, "maxAttempts": 1, "retryPolicy": "none"}, - "task": { - "benchmark": "pier-fixture", - "benchmarkVersion": "1", - "taskId": "fixture-1", + limits = { + "timeoutMs": 60_000, + "maxSteps": 8, + "maxModelCalls": 0, + "maxInputTokens": 0, + "maxOutputTokens": 0, + "maxCostUsd": 0, + } + container = { + "source": "evaluator-task-container", + "image": "ghcr.io/tangle-network/fixture:latest", + "indexDigest": f"sha256:{'2' * 64}", + "manifestDigest": f"sha256:{'3' * 64}", + "platform": {"os": "linux", "architecture": "amd64"}, + } + resolved_model = { + "requested": "openai/gpt-5.4", + "provider": "openai", + "model": "gpt-5.4", + "snapshot": "fixture", + "reasoningEffort": "xhigh", + } + task = { + "kind": "agent-candidate-benchmark-task", + "digestAlgorithm": "rfc8785-sha256", + "scenario": { + "id": "fixture-1", + "kind": "coding", + "scenarioDigest": f"sha256:{'5' * 64}", + }, + "benchmark": { + "name": "pier-fixture", + "version": "1", "splitDigest": f"sha256:{'1' * 64}", - "instruction": { - "encoding": "utf8", - "sha256": _sha(instruction), - "byteLength": len(instruction), - "delivery": {"kind": "argv-append"}, - }, - "repository": { - "identity": "fixture/repository", - "rootIdentity": "fixture/repository", - "baseCommit": base_commit, - "baseTree": base_tree, + }, + "instruction": instruction.decode(), + "repository": { + "identity": "fixture/repository", + "rootIdentity": "fixture/repository", + "baseCommit": base_commit, + "baseTree": base_tree, + }, + "outcome": {"kind": "workspace"}, + "workspace": task_snapshot, + "evaluatorTaskContainer": container, + "model": resolved_model, + "limits": limits, + "attempt": {"maxAttempts": 1, "retryPolicy": "none"}, + "grader": { + "name": "fixture-grader", + "version": "1.0.0", + "format": "tangle-grader", + "artifact": { + "sha256": f"sha256:{'6' * 64}", + "byteLength": 1, + "locator": { + "kind": "s3", + "bucket": "fixture", + "key": "grader", + }, }, - "outcome": {"kind": "workspace"}, - "workspace": task_snapshot, }, + } + task_raw = _canonical(task) + task_digest = _sha(task_raw) + suite = { + "kind": "agent-candidate-benchmark-suite", + "digestAlgorithm": "rfc8785-sha256", + "taskDigests": [task_digest], + "reps": 1, + "seeds": [42], + } + suite_raw = _canonical(suite) + suite_digest = _sha(suite_raw) + run_cell_material = { + "kind": "agent-candidate-run-cell", + "experimentDigest": f"sha256:{'7' * 64}", + "bundleDigest": bundle_digest, + "suiteDigest": suite_digest, + "taskDigest": task_digest, + "arm": "candidate", + "taskIndex": 0, + "repetition": 0, + "seed": 42, + "attempt": 1, + } + run_cell = { + **run_cell_material, + "digest": _sha(_canonical(run_cell_material)), + } + plan = { + "kind": "agent-candidate-execution-plan-material", + "runCell": run_cell, + "executionId": "pier-fixture-execution", "workspaces": { "taskRoot": str(task_root), "candidateRoot": str(candidate_root), @@ -300,22 +391,11 @@ def _fixture(root: Path): }, "harness": "codex", "harnessVersion": "fixture", - "container": { - "source": "evaluator-task-container", - "image": "ghcr.io/tangle-network/fixture:latest", - "indexDigest": f"sha256:{'2' * 64}", - "manifestDigest": f"sha256:{'3' * 64}", - "platform": {"os": "linux", "architecture": "amd64"}, - }, + "instructionDelivery": {"kind": "argv-append"}, + "container": container, "model": { "policy": "single", - "resolved": { - "requested": "openai/gpt-5.4", - "provider": "openai", - "model": "gpt-5.4", - "snapshot": "fixture", - "reasoningEffort": "xhigh", - }, + "resolved": resolved_model, "access": { "kind": "evaluator-mediated", "grantDigest": f"sha256:{'4' * 64}", @@ -326,18 +406,16 @@ def _fixture(root: Path): "launch": { "executable": "python3", "args": [{"kind": "public", "value": str(candidate_root / "runner.py")}], - "env": {}, + "env": { + "PATH": { + "kind": "public", + "value": "/usr/local/bin:/usr/bin:/bin", + } + }, "cwd": {"workspace": "task", "path": "."}, }, "memory": {"mode": "disabled"}, - "limits": { - "timeoutMs": 60_000, - "maxSteps": 8, - "maxModelCalls": 0, - "maxInputTokens": 0, - "maxOutputTokens": 0, - "maxCostUsd": 0, - }, + "limits": limits, "network": {"mode": "disabled"}, } plan_raw = _canonical(plan) @@ -348,17 +426,21 @@ def _fixture(root: Path): "material": plan, "artifact": _embedded(plan_raw), } - profile_evidence = { - "kind": "agent-profile-workspace-plan", - "digest": profile_digest, - "material": profile_material, - "artifact": _embedded(profile_raw), - } receipt = { "kind": "agent-candidate-materialization", "digestAlgorithm": "rfc8785-sha256", "bundleDigest": bundle_digest, - "profilePlan": profile_evidence, + "benchmark": { + "suite": { + "digest": suite_digest, + "material": _embedded(suite_raw), + }, + "task": { + "digest": task_digest, + "material": _embedded(task_raw), + }, + }, + "profileActivation": profile_activation, "executionPlan": execution_evidence, "candidateWorkspace": candidate_snapshot, "codeKind": "no-op", @@ -390,13 +472,20 @@ def _fixture(root: Path): "logs": logs, "base_commit": base_commit, "plan": plan, + "task": task, + "suite": suite, } -def _rewrite_signed_plan(fixture): +def _write_receipt(fixture, receipt): + receipt_raw = _canonical(receipt) + fixture["receipt_path"].write_bytes(receipt_raw) + fixture["receipt_digest"] = _sha(receipt_raw) + + +def _rebind_plan(fixture, receipt): plan_raw = _canonical(fixture["plan"]) plan_digest = _sha(plan_raw) - receipt = json.loads(fixture["receipt_path"].read_text()) receipt["executionPlan"] = { "kind": "agent-candidate-execution-plan", "digest": plan_digest, @@ -405,10 +494,43 @@ def _rewrite_signed_plan(fixture): } receipt["container"] = fixture["plan"]["container"] receipt["resolvedModel"] = fixture["plan"]["model"]["resolved"] - receipt_raw = _canonical(receipt) fixture["plan_path"].write_bytes(plan_raw) - fixture["receipt_path"].write_bytes(receipt_raw) - fixture["receipt_digest"] = _sha(receipt_raw) + _write_receipt(fixture, receipt) + + +def _rewrite_signed_plan(fixture, *, sync_task_container=True): + task = fixture["task"] + task["limits"] = fixture["plan"]["limits"] + task["model"] = fixture["plan"]["model"]["resolved"] + if sync_task_container: + if fixture["plan"]["container"]["source"] == "evaluator-task-container": + task["evaluatorTaskContainer"] = fixture["plan"]["container"] + else: + task.pop("evaluatorTaskContainer", None) + task_raw = _canonical(task) + task_digest = _sha(task_raw) + suite = fixture["suite"] + suite["taskDigests"] = [task_digest] + suite_raw = _canonical(suite) + suite_digest = _sha(suite_raw) + run_cell = fixture["plan"]["runCell"] + run_cell["taskDigest"] = task_digest + run_cell["suiteDigest"] = suite_digest + run_cell["digest"] = _sha( + _canonical({key: value for key, value in run_cell.items() if key != "digest"}) + ) + receipt = json.loads(fixture["receipt_path"].read_text()) + receipt["benchmark"] = { + "suite": { + "digest": suite_digest, + "material": _embedded(suite_raw), + }, + "task": { + "digest": task_digest, + "material": _embedded(task_raw), + }, + } + _rebind_plan(fixture, receipt) def _agent(fixture): @@ -422,7 +544,9 @@ def _agent(fixture): plan_digest = _sha(fixture["plan_path"].read_bytes()) trace_env = { "TANGLE_CANDIDATE_EXECUTION_ID": fixture["plan"]["executionId"], - "TANGLE_CANDIDATE_BUNDLE_DIGEST": fixture["plan"]["bundleDigest"], + "TANGLE_CANDIDATE_BUNDLE_DIGEST": fixture["plan"]["runCell"][ + "bundleDigest" + ], "TANGLE_CANDIDATE_EXECUTION_PLAN_DIGEST": plan_digest, "TANGLE_CANDIDATE_MATERIALIZATION_RECEIPT_DIGEST": fixture["receipt_digest"], "TANGLE_TRACE_RUN_ID": fixture["plan"]["executionId"], @@ -752,7 +876,7 @@ def test_loads_exact_runtime_artifacts_and_rejects_plan_substitution(self): def test_rejects_extra_utf8_file_delivery_fields(self): with tempfile.TemporaryDirectory() as directory: fixture = _fixture(Path(directory)) - fixture["plan"]["task"]["instruction"]["delivery"] = { + fixture["plan"]["instructionDelivery"] = { "kind": "utf8-file", "env": "TANGLE_CANDIDATE_TASK_PATH", "path": "/tangle/input/task.txt", @@ -816,6 +940,155 @@ def test_accepts_only_frozen_public_model_gateway_domains(self): finally: agent._snapshots.cleanup() + def test_accepts_runtime_owned_pre_model_retry_policy(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + fixture["task"]["attempt"][ + "retryPolicy" + ] = "pre-model-infrastructure-only" + _rewrite_signed_plan(fixture) + loaded = contract_module.load_prepared_candidate_contract( + fixture["plan_path"], + fixture["receipt_path"], + fixture["receipt_digest"], + ) + self.assertEqual(loaded.execution_id, "pier-fixture-execution") + + def test_accepts_candidate_pinned_container_without_task_container(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + fixture["plan"]["container"]["source"] = "pinned-container" + _rewrite_signed_plan(fixture) + loaded = contract_module.load_prepared_candidate_contract( + fixture["plan_path"], + fixture["receipt_path"], + fixture["receipt_digest"], + ) + self.assertEqual(loaded.container["source"], "pinned-container") + + def test_rejects_missing_run_cell_digest(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + receipt = json.loads(fixture["receipt_path"].read_text()) + fixture["plan"]["runCell"].pop("digest") + _rebind_plan(fixture, receipt) + with self.assertRaisesRegex( + contract_module.CandidateContractError, "runCell.digest" + ): + contract_module.load_prepared_candidate_contract( + fixture["plan_path"], + fixture["receipt_path"], + fixture["receipt_digest"], + ) + + def test_rejects_invalid_suite_identity_and_seed(self): + for label, suite_value, cell_value, message in ( + ("algorithm", "not-rfc8785", 42, "digestAlgorithm"), + ("seed", "rfc8785-sha256", "42", "safe integer"), + ): + with ( + self.subTest(case=label), + tempfile.TemporaryDirectory() as directory, + ): + fixture = _fixture(Path(directory)) + fixture["suite"]["digestAlgorithm"] = suite_value + fixture["suite"]["seeds"] = [cell_value] + fixture["plan"]["runCell"]["seed"] = cell_value + _rewrite_signed_plan(fixture) + with self.assertRaisesRegex( + contract_module.CandidateContractError, message + ): + contract_module.load_prepared_candidate_contract( + fixture["plan_path"], + fixture["receipt_path"], + fixture["receipt_digest"], + ) + + def test_rejects_no_retry_task_with_multiple_attempts(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + fixture["task"]["attempt"] = { + "maxAttempts": 2, + "retryPolicy": "none", + } + fixture["plan"]["runCell"]["attempt"] = 2 + _rewrite_signed_plan(fixture) + with self.assertRaisesRegex( + contract_module.CandidateContractError, "no-retry" + ): + contract_module.load_prepared_candidate_contract( + fixture["plan_path"], + fixture["receipt_path"], + fixture["receipt_digest"], + ) + + def test_rejects_invalid_profile_activation_digest(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + receipt = json.loads(fixture["receipt_path"].read_text()) + receipt["profileActivation"]["digest"] = f"sha256:{'f' * 64}" + _write_receipt(fixture, receipt) + with self.assertRaisesRegex( + contract_module.CandidateContractError, + "profile activation digest", + ): + contract_module.load_prepared_candidate_contract( + fixture["plan_path"], + fixture["receipt_path"], + fixture["receipt_digest"], + ) + + def test_rejects_pinned_container_for_evaluator_container_task(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + evaluator_container = fixture["plan"]["container"] + fixture["plan"]["container"] = { + **evaluator_container, + "source": "pinned-container", + } + fixture["task"]["evaluatorTaskContainer"] = evaluator_container + _rewrite_signed_plan(fixture, sync_task_container=False) + with self.assertRaisesRegex( + contract_module.CandidateContractError, + "pinned execution", + ): + contract_module.load_prepared_candidate_contract( + fixture["plan_path"], + fixture["receipt_path"], + fixture["receipt_digest"], + ) + + def test_rejects_profile_plan_for_another_harness(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + receipt = json.loads(fixture["receipt_path"].read_text()) + activation = receipt["profileActivation"] + profile = activation["profilePlan"] + profile["material"]["harness"] = "claude-code" + profile_raw = _canonical(profile["material"]) + profile["digest"] = _sha(profile_raw) + profile["artifact"] = _embedded(profile_raw) + activation["digest"] = _sha( + _canonical( + { + key: value + for key, value in activation.items() + if key != "digest" + } + ) + ) + fixture["plan"]["profile"]["planDigest"] = profile["digest"] + _rebind_plan(fixture, receipt) + with self.assertRaisesRegex( + contract_module.CandidateContractError, + "profile plan harness", + ): + contract_module.load_prepared_candidate_contract( + fixture["plan_path"], + fixture["receipt_path"], + fixture["receipt_digest"], + ) + def test_rejects_wildcard_or_unsorted_model_gateway_domains(self): for domains in ( ["*.tangle.tools"], @@ -856,6 +1129,25 @@ def test_rejects_model_gateway_when_call_budget_is_zero(self): fixture["receipt_digest"], ) + def test_rejects_untrusted_signed_public_path(self): + for value in ( + "/app", + ".", + "/usr/local/bin::/usr/bin", + "/usr/local/bin:/opt/bin", + ): + with ( + self.subTest(value=value), + tempfile.TemporaryDirectory() as directory, + ): + fixture = _fixture(Path(directory)) + fixture["plan"]["launch"]["env"]["PATH"]["value"] = value + _rewrite_signed_plan(fixture) + with self.assertRaisesRegex( + candidate.PierCandidateError, "signed public PATH" + ): + _agent(fixture) + def test_rejects_unsigned_candidate_workspace_files(self): with tempfile.TemporaryDirectory() as directory: fixture = _fixture(Path(directory)) diff --git a/bench/scripts/verify-pier-agent.mts b/bench/scripts/verify-pier-agent.mts index 8e40cd60..3f162f46 100644 --- a/bench/scripts/verify-pier-agent.mts +++ b/bench/scripts/verify-pier-agent.mts @@ -435,7 +435,7 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner= workspace: taskWorkspace, evaluatorTaskContainer: container, limits: { - timeoutMs: 60_000, + timeoutMs: 180_000, maxSteps: 8, maxModelCalls: 0, maxInputTokens: 0, @@ -671,12 +671,14 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner= ) } assertTreeOmits(scratch, 'zero-model-proof') - const usage = finalized.receipt.value.usage + const usage = finalized.receipt.value.modelSettlement.material.usage if ( usage.modelCalls !== 0 || usage.inputTokens !== 0 || usage.outputTokens !== 0 || - usage.costUsd !== 0 || + usage.cachedInputTokens !== 0 || + usage.reasoningTokens !== 0 || + usage.costUsdNanos !== 0 || finalized.receipt.value.trace.modelCallCount !== 0 ) { throw new Error(`runtime minted nonzero usage for a zero-model run: ${JSON.stringify(usage)}`) @@ -691,7 +693,7 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner= modelCalls: usage.modelCalls, inputTokens: usage.inputTokens, outputTokens: usage.outputTokens, - costUsd: usage.costUsd, + costUsd: usage.costUsdNanos / 1_000_000_000, pierContextUsage: null, profileExcludedByVerifier: true, container: {