From fadb01ed9c81a49d22d07dacdb90e0c727b390f9 Mon Sep 17 00:00:00 2001 From: Lumen Date: Thu, 20 Aug 2026 03:46:29 +0000 Subject: [PATCH 1/2] feat(evidence): record engine-probed adapter CLI version per turn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements issue #5 proposal 2. Accepted-turn evidence gains an optional cli_version record: the adapter's --version argv, exit status, verbatim output (truncated to 500 chars), output SHA-256, and probe timestamp. Probing is engine-run (never adapter self-report), bounded (15s), and informational: a failed probe is recorded with an 'error' key instead of failing the turn — acceptance still rests on the identity evidence alone. - version_probe_argv hook on Adapter; implemented for hermes-cli (settings.command_name), fable-session (settings.command_name), and command (worker argv[0]) - runtime-evidence.schema.json gains the optional cli_version property (additionalProperties stays false; evidence_version unchanged: the field is additive and optional) - fakes learn --version: fake-hermes 1.1.0, fake-fable-session 0.3.0b1, fake-worker 1.0.0 - tests: probe recorded for all three adapters, two stubbed versions yield distinct evidence (canary), failed probe recorded not fatal --- README.md | 2 + examples/fakes/bin/fake-fable-session | 3 + examples/fakes/bin/fake-hermes | 3 + examples/fakes/bin/fake-worker | 3 + schemas/runtime-evidence.schema.json | 83 +++++++++-- src/multi_agent_dialogue/adapters/base.py | 51 ++++++- .../adapters/claude_fable.py | 8 + src/multi_agent_dialogue/adapters/command.py | 6 + src/multi_agent_dialogue/adapters/hermes.py | 6 + tests/test_real_contracts.py | 138 ++++++++++++++++++ 10 files changed, 289 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 38d4830..b13c6ba 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,8 @@ madp owner-decide DIR --decision DECISION.md - **Hermes Agent** — provider, model, session, API usage, and final active message derived from the actor profile's `state.db`. - **Generic command** — requires a separate identity-verifier command; worker self-report is rejected. +Every accepted turn also records the engine-probed adapter CLI version (`--version` output, verbatim plus SHA-256) as `cli_version` in the turn's evidence record, since real adapters depend on the exact installed CLI versions. + Real adapters depend on the exact installed CLI versions. Run the fake demo first, then perform a harmless live smoke and bounded canary on your own machine before relying on a real transport. ## Why trust it? diff --git a/examples/fakes/bin/fake-fable-session b/examples/fakes/bin/fake-fable-session index 4694be0..fa4caa7 100755 --- a/examples/fakes/bin/fake-fable-session +++ b/examples/fakes/bin/fake-fable-session @@ -441,6 +441,9 @@ def cmd_audit(argv: list[str]) -> int: def main() -> int: argv = sys.argv[1:] + if argv == ["--version"]: + print("fake-fable-session 0.3.0b1 (fixture)") + return 0 if not argv: print("usage: fake-fable-session {run,audit,watch} ...", file=sys.stderr) return 2 diff --git a/examples/fakes/bin/fake-hermes b/examples/fakes/bin/fake-hermes index 7166d9b..238f557 100755 --- a/examples/fakes/bin/fake-hermes +++ b/examples/fakes/bin/fake-hermes @@ -114,6 +114,9 @@ def parse_chat_args(argv: list[str]) -> dict: def main() -> int: argv = sys.argv[1:] + if argv == ["--version"]: + print("fake-hermes 1.1.0 (fixture)") + return 0 if not argv or argv[0] != "chat": print( "fake-hermes: error: this fixture only implements the real " diff --git a/examples/fakes/bin/fake-worker b/examples/fakes/bin/fake-worker index d8a7705..00b9b3b 100755 --- a/examples/fakes/bin/fake-worker +++ b/examples/fakes/bin/fake-worker @@ -21,6 +21,9 @@ import _fakelib # noqa: E402 if __name__ == "__main__": parser = argparse.ArgumentParser(prog="fake-worker") + parser.add_argument( + "--version", action="version", version="fake-worker 1.0.0 (fixture)" + ) parser.add_argument("--task", required=True) parser.add_argument("--turn-output", required=True) parser.add_argument("--round", required=True) diff --git a/schemas/runtime-evidence.schema.json b/schemas/runtime-evidence.schema.json index 36eb006..c2916df 100644 --- a/schemas/runtime-evidence.schema.json +++ b/schemas/runtime-evidence.schema.json @@ -22,13 +22,28 @@ ], "additionalProperties": false, "properties": { - "evidence_version": {"const": 1}, - "actor_id": {"type": "string", "minLength": 1}, - "round_id": {"type": "string", "minLength": 1}, - "adapter": {"type": "string", "minLength": 1}, + "evidence_version": { + "const": 1 + }, + "actor_id": { + "type": "string", + "minLength": 1 + }, + "round_id": { + "type": "string", + "minLength": 1 + }, + "adapter": { + "type": "string", + "minLength": 1 + }, "transport": { "type": "string", - "enum": ["command", "fable-session", "hermes-cli"] + "enum": [ + "command", + "fable-session", + "hermes-cli" + ] }, "provider": { "type": "string", @@ -47,12 +62,25 @@ }, "outcome": { "type": "string", - "enum": ["success", "failure", "timeout", "cancelled"], + "enum": [ + "success", + "failure", + "timeout", + "cancelled" + ], "description": "Terminal outcome; anything other than success blocks completion." }, - "exit_status": {"type": "integer"}, - "artifact_path": {"type": "string", "minLength": 1}, - "artifact_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "exit_status": { + "type": "integer" + }, + "artifact_path": { + "type": "string", + "minLength": 1 + }, + "artifact_sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, "captured_at": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z$", @@ -60,7 +88,9 @@ }, "proof": { "type": "object", - "required": ["kind"], + "required": [ + "kind" + ], "minProperties": 1, "description": "Adapter-derived proof references from EXTERNAL records — never worker-authored. kind names the record family; the remaining keys point at the backing records (manifest/audit/stream for fable-session, state.db facts for hermes-cli, the verifier report for command).", "properties": { @@ -70,6 +100,21 @@ "description": "External record family, e.g. fable-session, hermes-state-db, external-command-verifier." } } + }, + "cli_version": { + "type": "object", + "description": "Engine-probed adapter CLI version record: argv, exit_status, verbatim output (truncated), output_sha256, probed_at; an 'error' key appears instead of output fields when the probe itself failed. Informational provenance only — never part of turn acceptance.", + "required": [ + "argv" + ], + "properties": { + "argv": { + "type": "array", + "items": { + "type": "string" + } + } + } } }, "examples": [ @@ -89,9 +134,21 @@ "captured_at": "2026-07-16T00:00:00Z", "proof": { "kind": "external-command-verifier", - "worker_argv": ["fake-worker", "--task", "task.md"], - "verifier_argv": ["fake-verifier", "--turn", "turn.md"], - "report": {"fake": true, "provider": "provider-a", "model": "model-a"} + "worker_argv": [ + "fake-worker", + "--task", + "task.md" + ], + "verifier_argv": [ + "fake-verifier", + "--turn", + "turn.md" + ], + "report": { + "fake": true, + "provider": "provider-a", + "model": "model-a" + } } } ] diff --git a/src/multi_agent_dialogue/adapters/base.py b/src/multi_agent_dialogue/adapters/base.py index aa1463e..e72c0cf 100644 --- a/src/multi_agent_dialogue/adapters/base.py +++ b/src/multi_agent_dialogue/adapters/base.py @@ -22,6 +22,7 @@ from datetime import datetime, timezone from pathlib import Path +from .. import artifacts from ..config import Actor, TurnSpec EVIDENCE_VERSION = 1 @@ -161,6 +162,12 @@ def command_failed(result: subprocess.CompletedProcess, what: str) -> AdapterErr return AdapterError(f"{what} exited {result.returncode}: {detail}") +# Bounded probe for the adapter CLI's own version string. The README +# warns that real adapters depend on the exact installed CLI versions, +# so accepted-turn evidence records the probed version verbatim. +VERSION_PROBE_TIMEOUT_SECONDS = 15 + + class Adapter(ABC): name: str = "abstract" transport: str = "" @@ -186,10 +193,48 @@ def output_contract(self, context: PrepareContext) -> tuple[str, ...]: f"{context.turn_file}", ) + def version_probe_argv(self, context: PrepareContext) -> list[str] | None: + """Argv that prints the adapter CLI's version, or None if unknown. + + Engine-probed, never adapter self-report: the output of THIS + command is what lands in evidence. + """ + return None + + def cli_version_evidence(self, context: PrepareContext) -> dict | None: + """Probe the adapter CLI version for the evidence record. + + Informational only: a failed probe is recorded, never fatal — + the turn's acceptance still rests on the identity evidence. + """ + argv = self.version_probe_argv(context) + if argv is None: + return None + where = f"actor {context.actor.actor_id!r} ({self.name})" + try: + result = run_command( + argv, env={}, cwd=context.work_dir, + timeout=VERSION_PROBE_TIMEOUT_SECONDS, + what=f"{where}: version probe", + ) + except AdapterError as exc: + return {"argv": list(argv), "error": str(exc)} + output = (result.stdout or result.stderr or "").strip()[:500] + record = { + "argv": list(argv), + "exit_status": result.returncode, + "output": output, + "output_sha256": artifacts.sha256_bytes(output.encode("utf-8")), + "probed_at": utc_now(), + } + if result.returncode != 0: + record["error"] = f"version probe exited {result.returncode}" + return record + def base_evidence(self, context: PrepareContext, *, provider: str, model: str, session_id: str, exit_status: int, artifact_sha256: str, proof: dict) -> dict: - return { + record = { "evidence_version": EVIDENCE_VERSION, "actor_id": context.actor.actor_id, "round_id": context.turn.round_id, @@ -205,6 +250,10 @@ def base_evidence(self, context: PrepareContext, *, provider: str, model: str, "captured_at": utc_now(), "proof": proof, } + cli_version = self.cli_version_evidence(context) + if cli_version is not None: + record["cli_version"] = cli_version + return record def get_adapter(transport: str) -> Adapter: diff --git a/src/multi_agent_dialogue/adapters/claude_fable.py b/src/multi_agent_dialogue/adapters/claude_fable.py index 3ee659e..757d173 100644 --- a/src/multi_agent_dialogue/adapters/claude_fable.py +++ b/src/multi_agent_dialogue/adapters/claude_fable.py @@ -62,6 +62,14 @@ class ClaudeFableAdapter(Adapter): name = "claude-fable" transport = "fable-session" + def version_probe_argv(self, context: PrepareContext) -> list[str]: + settings = context.actor.settings + where = f"actor {context.actor.actor_id!r} (claude-fable)" + command_name = require_str_setting( + settings, "command_name", where, "fable-session" + ) + return [command_name, "--version"] + # -- packet ------------------------------------------------------------ def _settings(self, context: PrepareContext) -> dict: diff --git a/src/multi_agent_dialogue/adapters/command.py b/src/multi_agent_dialogue/adapters/command.py index 2dae14c..4d7dc80 100644 --- a/src/multi_agent_dialogue/adapters/command.py +++ b/src/multi_agent_dialogue/adapters/command.py @@ -52,6 +52,12 @@ class CommandAdapter(Adapter): name = "command" transport = "command" + def version_probe_argv(self, context: PrepareContext) -> list[str]: + settings = context.actor.settings + where = f"actor {context.actor.actor_id!r} (command)" + argv_setting = _argv_setting(settings, "argv", where, required=True) + return [argv_setting[0], "--version"] + def prepare(self, context: PrepareContext) -> CommandPacket: settings = context.actor.settings where = f"actor {context.actor.actor_id!r} (command)" diff --git a/src/multi_agent_dialogue/adapters/hermes.py b/src/multi_agent_dialogue/adapters/hermes.py index 22b1d68..4d72fe4 100644 --- a/src/multi_agent_dialogue/adapters/hermes.py +++ b/src/multi_agent_dialogue/adapters/hermes.py @@ -97,6 +97,12 @@ def _source(self, context: PrepareContext) -> str: f"{secrets.token_hex(4)}" ) + def version_probe_argv(self, context: PrepareContext) -> list[str]: + settings = context.actor.settings + where = f"actor {context.actor.actor_id!r} (hermes)" + command_name = require_str_setting(settings, "command_name", where) + return [command_name, "--version"] + def _argv(self, command_name: str, prompt: str, source: str) -> tuple[str, ...]: return ( command_name, diff --git a/tests/test_real_contracts.py b/tests/test_real_contracts.py index e8919a2..0bd7e8c 100644 --- a/tests/test_real_contracts.py +++ b/tests/test_real_contracts.py @@ -36,7 +36,9 @@ from __future__ import annotations +import hashlib import json +import os import re import sqlite3 import tempfile @@ -923,5 +925,141 @@ def test_verifier_identity_mismatch_fails_closed(self) -> None: self.assertEqual(dialogue.state()["turn_index"], 0) +class CliVersionEvidenceTests(HermesTestCase): + """Accepted-turn evidence records the engine-probed adapter CLI + version (verbatim output + hash) — the README ties adapter behavior + to the exact installed CLI versions, so the record must name them.""" + + def _version_stub(self, name: str, version_line: str, + fail: bool = False) -> str: + stub = self.base / name + if fail: + body = ( + "#!/bin/sh\n" + 'if [ "$1" = "--version" ]; then exit 1; fi\n' + f'exec "{HERMES}" "$@"\n' + ) + else: + body = ( + "#!/bin/sh\n" + 'if [ "$1" = "--version" ]; then\n' + f' echo "{version_line}"\n' + " exit 0\n" + "fi\n" + f'exec "{HERMES}" "$@"\n' + ) + stub.write_text(body, encoding="utf-8") + os.chmod(stub, 0o755) + return str(stub) + + def _dialogue_with_command(self, command_name: str, + directory: str) -> engine.Dialogue: + raw = self.definition_raw() + raw["actors"][0]["settings"]["command_name"] = command_name + definition = config.parse_definition(raw) + return engine.init_dialogue(definition, self.base / directory) + + def test_hermes_turn_records_probed_cli_version(self) -> None: + dialogue = self.make_dialogue() + runner.launch(dialogue, "hermes-north") + cli = self.evidence_for(dialogue, 0)["cli_version"] + self.assertEqual(cli["argv"], [HERMES, "--version"]) + self.assertEqual(cli["exit_status"], 0) + self.assertEqual(cli["output"], "fake-hermes 1.1.0 (fixture)") + self.assertEqual( + cli["output_sha256"], + hashlib.sha256(cli["output"].encode("utf-8")).hexdigest(), + ) + self.assertNotIn("error", cli) + + def test_two_cli_versions_produce_distinct_evidence(self) -> None: + # The canary shape: same turn under two stubbed CLI versions + # yields different version records; both turns still validate. + stub_a = self._version_stub("hermes-a", "fake-hermes 1.0.0 (canary-a)") + stub_b = self._version_stub("hermes-b", "fake-hermes 2.0.0 (canary-b)") + dialogue_a = self._dialogue_with_command(stub_a, "dialogue-a") + dialogue_b = self._dialogue_with_command(stub_b, "dialogue-b") + runner.launch(dialogue_a, "hermes-north") + runner.launch(dialogue_b, "hermes-north") + out_a = self.evidence_for(dialogue_a, 0)["cli_version"]["output"] + out_b = self.evidence_for(dialogue_b, 0)["cli_version"]["output"] + self.assertEqual(out_a, "fake-hermes 1.0.0 (canary-a)") + self.assertEqual(out_b, "fake-hermes 2.0.0 (canary-b)") + self.assertNotEqual(out_a, out_b) + self.assertEqual(dialogue_a.state()["turn_index"], 1) + self.assertEqual(dialogue_b.state()["turn_index"], 1) + + def test_failed_probe_is_recorded_not_fatal(self) -> None: + stub = self._version_stub("hermes-broken-version", "", fail=True) + dialogue = self._dialogue_with_command(stub, "dialogue-broken") + runner.launch(dialogue, "hermes-north") + cli = self.evidence_for(dialogue, 0)["cli_version"] + self.assertIn("error", cli) + self.assertEqual(cli["exit_status"], 1) + self.assertEqual(dialogue.state()["turn_index"], 1) + + +class FableCliVersionTests(FableTestCase): + def test_fable_turn_records_probed_cli_version(self) -> None: + dialogue = self.make_dialogue() + runner.launch(dialogue, "fable-a") + record = dialogue.state()["completed_turns"][0] + evidence_record = json.loads( + (dialogue.directory / record["evidence_file"]).read_text( + encoding="utf-8" + ) + ) + cli = evidence_record["cli_version"] + self.assertEqual(cli["argv"], [FABLE, "--version"]) + self.assertEqual(cli["exit_status"], 0) + self.assertEqual(cli["output"], "fake-fable-session 0.3.0b1 (fixture)") + + +class CommandCliVersionTests(CommandIdentityTests): + def _settings(self, argv0: str) -> dict: + argv = [argv0, *self.worker_argv()[1:]] + return { + "argv": argv, + "identity_verifier_argv": self.verifier_argv(), + "env": { + "FAKE_PROVIDER": "fake-provider-a", + "FAKE_MODEL": "fake-model-a", + "FAKE_SPAWN_MARKER": str(self.marker), + }, + } + + def _read_evidence(self, dialogue: engine.Dialogue) -> dict: + record = dialogue.state()["completed_turns"][0] + return json.loads( + (dialogue.directory / record["evidence_file"]).read_text( + encoding="utf-8" + ) + ) + + def test_worker_version_recorded(self) -> None: + dialogue = self.make_dialogue(self._settings(WORKER)) + runner.launch(dialogue, "worker-a") + cli = self._read_evidence(dialogue)["cli_version"] + self.assertEqual(cli["argv"], [WORKER, "--version"]) + self.assertEqual(cli["exit_status"], 0) + self.assertEqual(cli["output"], "fake-worker 1.0.0 (fixture)") + + def test_worker_without_version_flag_is_recorded_not_fatal(self) -> None: + wrapper = self.base / "worker-no-version" + wrapper.write_text( + "#!/bin/sh\n" + 'if [ "$1" = "--version" ]; then exit 1; fi\n' + f'exec "{WORKER}" "$@"\n', + encoding="utf-8", + ) + os.chmod(wrapper, 0o755) + dialogue = self.make_dialogue(self._settings(str(wrapper))) + runner.launch(dialogue, "worker-a") + cli = self._read_evidence(dialogue)["cli_version"] + self.assertIn("error", cli) + self.assertEqual(cli["exit_status"], 1) + self.assertEqual(dialogue.state()["turn_index"], 1) + + if __name__ == "__main__": unittest.main() From 0d2cbfe76efb980e54ac641c4050f2bd2643f3b5 Mon Sep 17 00:00:00 2001 From: Vesper Date: Thu, 20 Aug 2026 05:06:43 +0000 Subject: [PATCH 2/2] fix(evidence): harden cli_version probe per review - version_probe_argv hook call moved inside the try: a subclass AdapterError degrades to a recorded error, never fails the turn. - output_sha256 now attests the FULL probe output, computed before the 500-char storage truncation; output_truncated flags the prefix case. - Probe runs under the actor substituted settings env so PATH- or env-dependent CLIs resolve the binary the turn actually used. - Schema: cli_version gains exit_status/output/output_sha256(64-hex)/ output_truncated/probed_at/error with additionalProperties=false. - Tests: CommandFixtureBase extracted (no verbatim re-run of the identity matrix); new guards for truncation hashing, hook-error degradation, and settings-env propagation. --- README.md | 2 +- schemas/runtime-evidence.schema.json | 25 +++++++-- src/multi_agent_dialogue/adapters/base.py | 34 +++++++++--- tests/test_real_contracts.py | 64 ++++++++++++++++++++++- 4 files changed, 110 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index b13c6ba..09d546a 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ madp owner-decide DIR --decision DECISION.md - **Hermes Agent** — provider, model, session, API usage, and final active message derived from the actor profile's `state.db`. - **Generic command** — requires a separate identity-verifier command; worker self-report is rejected. -Every accepted turn also records the engine-probed adapter CLI version (`--version` output, verbatim plus SHA-256) as `cli_version` in the turn's evidence record, since real adapters depend on the exact installed CLI versions. +Every accepted turn also records the engine-probed adapter CLI version (`--version` output, stored verbatim up to 500 chars with SHA-256 over the full untruncated output) as `cli_version` in the turn's evidence record, since real adapters depend on the exact installed CLI versions. The probe runs under the actor's own settings env and is informational only: a failed probe is recorded, never fatal to the turn. Real adapters depend on the exact installed CLI versions. Run the fake demo first, then perform a harmless live smoke and bounded canary on your own machine before relying on a real transport. diff --git a/schemas/runtime-evidence.schema.json b/schemas/runtime-evidence.schema.json index c2916df..c82b19b 100644 --- a/schemas/runtime-evidence.schema.json +++ b/schemas/runtime-evidence.schema.json @@ -103,16 +103,33 @@ }, "cli_version": { "type": "object", - "description": "Engine-probed adapter CLI version record: argv, exit_status, verbatim output (truncated), output_sha256, probed_at; an 'error' key appears instead of output fields when the probe itself failed. Informational provenance only — never part of turn acceptance.", - "required": [ - "argv" - ], + "description": "Engine-probed adapter CLI version record. argv/exit_status/output/probed_at describe the probe; output is truncated at 500 chars while output_sha256 always attests the FULL untruncated output (output_truncated=true marks that case). An 'error' key appears when the probe itself failed; if even the probe argv could not be built, only 'error' is present. Informational provenance only — never part of turn acceptance.", + "additionalProperties": false, "properties": { "argv": { "type": "array", "items": { "type": "string" } + }, + "exit_status": { + "type": "integer" + }, + "output": { + "type": "string" + }, + "output_sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "output_truncated": { + "type": "boolean" + }, + "probed_at": { + "type": "string" + }, + "error": { + "type": "string" } } } diff --git a/src/multi_agent_dialogue/adapters/base.py b/src/multi_agent_dialogue/adapters/base.py index e72c0cf..a7a7d3b 100644 --- a/src/multi_agent_dialogue/adapters/base.py +++ b/src/multi_agent_dialogue/adapters/base.py @@ -207,26 +207,44 @@ def cli_version_evidence(self, context: PrepareContext) -> dict | None: Informational only: a failed probe is recorded, never fatal — the turn's acceptance still rests on the identity evidence. """ - argv = self.version_probe_argv(context) - if argv is None: - return None where = f"actor {context.actor.actor_id!r} ({self.name})" + argv: list[str] | None = None try: + # Inside the try: a subclass hook raising AdapterError (bad + # settings, malformed env) must degrade to a recorded error, + # never fail the accepted turn. + argv = self.version_probe_argv(context) + if argv is None: + return None + # Probe under the actor's own settings env (same substitution + # as the turn packet) so PATH- or env-dependent CLIs resolve + # the binary the turn actually used. + probe_env = substitute_env( + context.actor.settings.get("env"), context.placeholders(), where + ) result = run_command( - argv, env={}, cwd=context.work_dir, + argv, env=probe_env, cwd=context.work_dir, timeout=VERSION_PROBE_TIMEOUT_SECONDS, what=f"{where}: version probe", ) except AdapterError as exc: - return {"argv": list(argv), "error": str(exc)} - output = (result.stdout or result.stderr or "").strip()[:500] + record: dict = {"error": str(exc)} + if argv is not None: + record["argv"] = list(argv) + return record + raw_output = (result.stdout or result.stderr or "").strip() record = { "argv": list(argv), "exit_status": result.returncode, - "output": output, - "output_sha256": artifacts.sha256_bytes(output.encode("utf-8")), + "output": raw_output[:500], + # The hash attests the FULL probe output, computed before the + # 500-char storage truncation; output_truncated flags when the + # stored output is a prefix of what the hash covers. + "output_sha256": artifacts.sha256_bytes(raw_output.encode("utf-8")), "probed_at": utc_now(), } + if len(raw_output) > 500: + record["output_truncated"] = True if result.returncode != 0: record["error"] = f"version probe exited {result.returncode}" return record diff --git a/tests/test_real_contracts.py b/tests/test_real_contracts.py index 0bd7e8c..4aa075b 100644 --- a/tests/test_real_contracts.py +++ b/tests/test_real_contracts.py @@ -44,6 +44,7 @@ import tempfile import unittest from pathlib import Path +from unittest import mock import support @@ -791,7 +792,10 @@ def test_start_outside_window_is_rejected_even_with_null_terminal(self) -> None: self.assert_refused("outside this") -class CommandIdentityTests(unittest.TestCase): +class CommandFixtureBase(unittest.TestCase): + """Shared command-adapter dialogue fixture (worker/verifier fakes, + definition builder, dialogue factory); no test methods itself.""" + def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory() self.addCleanup(self._tmp.cleanup) @@ -858,6 +862,10 @@ def make_dialogue(self, settings_a: dict) -> engine.Dialogue: definition = config.parse_definition(self.definition_raw(settings_a)) return engine.init_dialogue(definition, self.base / "dialogue") + +class CommandIdentityTests(CommandFixtureBase): + """Identity acceptance/refusal for the command transport.""" + def test_command_without_external_verifier_fails_closed(self) -> None: # Self-reported output is not identity proof: with no external # identity verifier configured, identity-sensitive completion is @@ -998,6 +1006,58 @@ def test_failed_probe_is_recorded_not_fatal(self) -> None: self.assertEqual(cli["exit_status"], 1) self.assertEqual(dialogue.state()["turn_index"], 1) + def test_long_output_hash_covers_full_untruncated_output(self) -> None: + # output_sha256 attests what the CLI actually printed; the stored + # output is a 500-char prefix flagged by output_truncated. + version_line = "fake-hermes " + ("9" * 600) + " (long)" + stub = self._version_stub("hermes-long-version", version_line) + dialogue = self._dialogue_with_command(stub, "dialogue-long") + runner.launch(dialogue, "hermes-north") + cli = self.evidence_for(dialogue, 0)["cli_version"] + self.assertEqual(cli["output"], version_line[:500]) + self.assertTrue(cli["output_truncated"]) + self.assertEqual( + cli["output_sha256"], + hashlib.sha256(version_line.encode("utf-8")).hexdigest(), + ) + + def test_probe_argv_hook_error_is_recorded_not_fatal(self) -> None: + # A subclass hook raising AdapterError (bad settings/env) must + # degrade to a recorded error, never fail the accepted turn. + dialogue = self.make_dialogue() + with mock.patch.object( + hermes_adapter.HermesAdapter, + "version_probe_argv", + side_effect=adapters.AdapterError("settings broke"), + ): + runner.launch(dialogue, "hermes-north") + cli = self.evidence_for(dialogue, 0)["cli_version"] + self.assertIn("settings broke", cli["error"]) + self.assertNotIn("argv", cli) + self.assertEqual(dialogue.state()["turn_index"], 1) + + def test_probe_runs_under_actor_settings_env(self) -> None: + # PATH- or env-dependent CLIs must probe the binary the turn used: + # the probe inherits the actor's substituted settings env. + stub = self.base / "hermes-env-version" + stub.write_text( + "#!/bin/sh\n" + 'if [ "$1" = "--version" ]; then\n' + ' echo "fake-hermes ${FAKE_VERSION_TAG:-unset} (env)"\n' + " exit 0\n" + "fi\n" + f'exec "{HERMES}" "$@"\n', + encoding="utf-8", + ) + os.chmod(stub, 0o755) + raw = self.definition_raw({"FAKE_VERSION_TAG": "from-settings-env"}) + raw["actors"][0]["settings"]["command_name"] = str(stub) + definition = config.parse_definition(raw) + dialogue = engine.init_dialogue(definition, self.base / "dialogue-env") + runner.launch(dialogue, "hermes-north") + cli = self.evidence_for(dialogue, 0)["cli_version"] + self.assertEqual(cli["output"], "fake-hermes from-settings-env (env)") + class FableCliVersionTests(FableTestCase): def test_fable_turn_records_probed_cli_version(self) -> None: @@ -1015,7 +1075,7 @@ def test_fable_turn_records_probed_cli_version(self) -> None: self.assertEqual(cli["output"], "fake-fable-session 0.3.0b1 (fixture)") -class CommandCliVersionTests(CommandIdentityTests): +class CommandCliVersionTests(CommandFixtureBase): def _settings(self, argv0: str) -> dict: argv = [argv0, *self.worker_argv()[1:]] return {