diff --git a/docs/technical-reference.md b/docs/technical-reference.md index b6ad64d..1ba4eea 100644 --- a/docs/technical-reference.md +++ b/docs/technical-reference.md @@ -252,11 +252,22 @@ usage, and records that basis in the proof (`terminal_basis`) instead of claiming a DB terminal state. If the terminal fields *are* present they must be consistent (`ended_at` inside the invocation window; never an `end_reason` without an `ended_at`) and -clean (`end_reason` NULL, empty, or exactly `completed`); ambiguous +clean (`end_reason` NULL, empty, `completed`, or — Hermes v0.20+ +one-shot finalization — `cli_close`); ambiguous sources, missing final messages, zero API calls, mixed models or providers, out-of-window starts, nonzero exits, and dirty or inconsistent terminal fields all fail closed. +Model/provider identity and the recorded `api_call_count` come from +`session_model_usage` **main-conversation rows only** (`task` NULL or +empty): auxiliary Hermes calls (title generation, vision, compression, +...) share the table under a non-empty `task` and say nothing about +which model reasoned the turn, so they are excluded from both identity +and the count by design. Schemas predating the `task` column keep the +all-rows behavior (every row is a main-conversation call there). The +column check matches SQLite semantics case-insensitively, so a schema +declaring `"Task"` still triggers the filter. + **Turn briefings**: every actor receives one `## Goal` / `## Checks` / `## Boundaries` / `## Report` task file (the exact sections `fable-session` requires; one shape serves every transport). From the diff --git a/src/multi_agent_dialogue/adapters/hermes.py b/src/multi_agent_dialogue/adapters/hermes.py index 22b1d68..dc5cf1b 100644 --- a/src/multi_agent_dialogue/adapters/hermes.py +++ b/src/multi_agent_dialogue/adapters/hermes.py @@ -26,7 +26,8 @@ terminal fields ARE present they must be consistent (``ended_at`` inside the invocation window, never an ``end_reason`` without an ``ended_at``) and clean (``end_reason`` NULL, -empty, or exactly ``completed``); anything else fails closed. +empty, ``completed``, or ``cli_close`` — the normal one-shot exit reason +written by Hermes v0.20+); anything else fails closed. Two Hermes actors are only independent if they use two different ``hermes_home`` directories; the setting is mandatory and never derived @@ -59,10 +60,14 @@ # invocation window (same host, same clock; generous margin). WINDOW_SLACK_SECONDS = 5.0 -# The only persisted end_reason that counts as a clean completion. +# Persisted end_reason values that count as a clean completion. # Both terminal fields may be NULL; an unset reason is handled -# separately from this value. -CLEAN_END_REASON = "completed" +# separately from these values. "cli_close" is how Hermes v0.20+ +# finalizes one-shot (-q/-Q) sessions on normal CLI exit +# (_flush_one_shot_session_store in cli.py); it carries the same +# meaning as "completed" for the one-shot contract and is still gated +# by ended_at consistency plus the process-exit terminal basis below. +CLEAN_END_REASONS = frozenset({"completed", "cli_close"}) # Proof labels for what actually established the turn's completion. TERMINAL_BASIS_DB = "state-db-ended" @@ -265,7 +270,7 @@ def _observe_session(db_path: Path, source: str, started_before: float, f"{end_reason!r} without a usable ended_at; terminal " "fields are inconsistent and completion is refused" ) - if end_reason != CLEAN_END_REASON: + if end_reason not in CLEAN_END_REASONS: raise AdapterError( f"{where}: session {session_id} ended with reason " f"{end_reason!r}, not a clean completion" @@ -293,9 +298,33 @@ def _observe_session(db_path: Path, source: str, started_before: float, ) api_calls = 0 try: + # Only main-conversation calls (task ''/NULL) prove the + # actor's model identity. Auxiliary Hermes calls (title + # generation, vision, compression, ...) are recorded in the + # same table with a non-empty task and a different model; + # they say nothing about which model reasoned the turn. + # Older Hermes state.db schemas have no task column; there + # every usage row is a main-conversation call. + # Column names are matched case-insensitively (SQLite + # resolves identifiers the same way) so an oddly-cased + # "Task" column still triggers the filter instead of + # silently reverting to all-rows counting. Consequence for + # consumers: on task-column schemas, api_call_count covers + # main-conversation calls only; auxiliary calls are + # excluded from the count by design. + cols = { + str(row[1]).lower() + for row in con.execute( + "PRAGMA table_info(session_model_usage)" + ).fetchall() + } + task_filter = ( + " AND (task IS NULL OR task = '')" if "task" in cols else "" + ) usage_rows = con.execute( "SELECT model, billing_provider, api_call_count " - "FROM session_model_usage WHERE session_id = ?", + "FROM session_model_usage WHERE session_id = ?" + + task_filter, (session_id,), ).fetchall() except sqlite3.Error as exc: diff --git a/tests/test_real_contracts.py b/tests/test_real_contracts.py index e8919a2..020b0ab 100644 --- a/tests/test_real_contracts.py +++ b/tests/test_real_contracts.py @@ -738,6 +738,15 @@ def test_ended_with_completed_reason_is_accepted(self) -> None: self.assertEqual(observed["ended_at"], self.before + 2.0) self.assertEqual(observed["end_reason"], "completed") + def test_ended_with_cli_close_reason_is_accepted(self) -> None: + # Hermes v0.20+ finalizes one-shot (-q/-Q) sessions with + # end_reason "cli_close" on normal CLI exit; it must count as a + # clean completion exactly like "completed". + self.build_db(ended_at=self.before + 2.0, end_reason="cli_close") + observed = self.observe() + self.assertEqual(observed["ended_at"], self.before + 2.0) + self.assertEqual(observed["end_reason"], "cli_close") + def test_ended_with_unset_reason_is_accepted(self) -> None: for reason in (None, ""): with self.subTest(reason=reason): @@ -789,6 +798,82 @@ def test_start_outside_window_is_rejected_even_with_null_terminal(self) -> None: self.assert_refused("outside this") +class HermesUsageTaskFilterTests(HermesTerminalFieldMatrixTests): + """Auxiliary usage rows (title generation, vision, compression, ...) + share session_model_usage with the main turn under a non-empty task; + they must not pollute the actor's model/provider identity.""" + + SCHEMA = """ + CREATE TABLE sessions ( + id TEXT PRIMARY KEY, source TEXT NOT NULL, model TEXT, + model_config TEXT, started_at REAL NOT NULL, ended_at REAL, + end_reason TEXT, billing_provider TEXT, profile_name TEXT + ); + CREATE TABLE messages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + session_id TEXT NOT NULL, role TEXT NOT NULL, content TEXT, + timestamp REAL NOT NULL, active INTEGER NOT NULL DEFAULT 1, + compacted INTEGER NOT NULL DEFAULT 0 + ); + CREATE TABLE session_model_usage ( + session_id TEXT NOT NULL, model TEXT NOT NULL, + billing_provider TEXT NOT NULL DEFAULT '', + api_call_count INTEGER NOT NULL DEFAULT 0, + task TEXT DEFAULT '' + ); + """ + + def add_usage_row(self, model: str, provider: str, calls: int, + task: str) -> None: + con = sqlite3.connect(self.db) + try: + con.execute( + "INSERT INTO session_model_usage (session_id, model, " + "billing_provider, api_call_count, task) " + "VALUES (?, ?, ?, ?, ?)", + ("sess-1", model, provider, calls, task), + ) + con.commit() + finally: + con.close() + + def test_auxiliary_task_rows_do_not_pollute_identity(self) -> None: + self.build_db() + self.add_usage_row("kimi-k2.7-code", "kimi", 1, "title_generation") + observed = self.observe() + self.assertEqual(observed["models"], ["hermes-4-405b"]) + self.assertEqual(observed["api_call_count"], 3) + + def test_null_task_row_counts_as_main_conversation(self) -> None: + self.build_db() + con = sqlite3.connect(self.db) + try: + con.execute( + "INSERT INTO session_model_usage (session_id, model, " + "billing_provider, api_call_count, task) " + "VALUES ('sess-1', 'hermes-4-405b', 'nousresearch', 1, NULL)" + ) + con.commit() + finally: + con.close() + self.assertEqual(self.observe()["api_call_count"], 4) + + def test_two_main_task_models_are_still_rejected(self) -> None: + self.build_db() + self.add_usage_row("other-model", "other-provider", 1, "") + self.assert_refused("not exactly one model") + + +class HermesUsageOddlyCasedTaskColumnTests(HermesUsageTaskFilterTests): + """SQLite resolves identifiers case-insensitively; a schema declaring + the column as ``Task`` must still trigger the main-conversation + filter instead of silently reverting to all-rows counting.""" + + SCHEMA = HermesUsageTaskFilterTests.SCHEMA.replace( + "task TEXT DEFAULT ''", '"Task" TEXT DEFAULT \'\'' + ) + + class CommandIdentityTests(unittest.TestCase): def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory()