Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion docs/technical-reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -252,11 +252,22 @@ usage, and records that basis in the proof (`terminal_basis`) instead
of claiming a DB terminal state. If the terminal
fields *are* present they must be consistent (`ended_at` inside the
invocation window; never an `end_reason` without an `ended_at`) and
clean (`end_reason` NULL, empty, or exactly `completed`); ambiguous
clean (`end_reason` NULL, empty, `completed`, or — Hermes v0.20+
one-shot finalization — `cli_close`); ambiguous
sources, missing final messages, zero API calls, mixed models or
providers, out-of-window starts, nonzero exits, and dirty or
inconsistent terminal fields all fail closed.

Model/provider identity and the recorded `api_call_count` come from
`session_model_usage` **main-conversation rows only** (`task` NULL or
empty): auxiliary Hermes calls (title generation, vision, compression,
...) share the table under a non-empty `task` and say nothing about
which model reasoned the turn, so they are excluded from both identity
and the count by design. Schemas predating the `task` column keep the
all-rows behavior (every row is a main-conversation call there). The
column check matches SQLite semantics case-insensitively, so a schema
declaring `"Task"` still triggers the filter.

**Turn briefings**: every actor receives one `## Goal` / `## Checks` /
`## Boundaries` / `## Report` task file (the exact sections
`fable-session` requires; one shape serves every transport). From the
Expand Down
41 changes: 35 additions & 6 deletions src/multi_agent_dialogue/adapters/hermes.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@
terminal fields ARE present they must be
consistent (``ended_at`` inside the invocation window, never an
``end_reason`` without an ``ended_at``) and clean (``end_reason`` NULL,
empty, or exactly ``completed``); anything else fails closed.
empty, ``completed``, or ``cli_close`` — the normal one-shot exit reason
written by Hermes v0.20+); anything else fails closed.

Two Hermes actors are only independent if they use two different
``hermes_home`` directories; the setting is mandatory and never derived
Expand Down Expand Up @@ -59,10 +60,14 @@
# invocation window (same host, same clock; generous margin).
WINDOW_SLACK_SECONDS = 5.0

# The only persisted end_reason that counts as a clean completion.
# Persisted end_reason values that count as a clean completion.
# Both terminal fields may be NULL; an unset reason is handled
# separately from this value.
CLEAN_END_REASON = "completed"
# separately from these values. "cli_close" is how Hermes v0.20+
# finalizes one-shot (-q/-Q) sessions on normal CLI exit
# (_flush_one_shot_session_store in cli.py); it carries the same
# meaning as "completed" for the one-shot contract and is still gated
# by ended_at consistency plus the process-exit terminal basis below.
CLEAN_END_REASONS = frozenset({"completed", "cli_close"})

# Proof labels for what actually established the turn's completion.
TERMINAL_BASIS_DB = "state-db-ended"
Expand Down Expand Up @@ -265,7 +270,7 @@ def _observe_session(db_path: Path, source: str, started_before: float,
f"{end_reason!r} without a usable ended_at; terminal "
"fields are inconsistent and completion is refused"
)
if end_reason != CLEAN_END_REASON:
if end_reason not in CLEAN_END_REASONS:
raise AdapterError(
f"{where}: session {session_id} ended with reason "
f"{end_reason!r}, not a clean completion"
Expand Down Expand Up @@ -293,9 +298,33 @@ def _observe_session(db_path: Path, source: str, started_before: float,
)
api_calls = 0
try:
# Only main-conversation calls (task ''/NULL) prove the
# actor's model identity. Auxiliary Hermes calls (title
# generation, vision, compression, ...) are recorded in the
# same table with a non-empty task and a different model;
# they say nothing about which model reasoned the turn.
# Older Hermes state.db schemas have no task column; there
# every usage row is a main-conversation call.
# Column names are matched case-insensitively (SQLite
# resolves identifiers the same way) so an oddly-cased
# "Task" column still triggers the filter instead of
# silently reverting to all-rows counting. Consequence for
# consumers: on task-column schemas, api_call_count covers
# main-conversation calls only; auxiliary calls are
# excluded from the count by design.
cols = {
str(row[1]).lower()
for row in con.execute(
"PRAGMA table_info(session_model_usage)"
).fetchall()
}
task_filter = (
" AND (task IS NULL OR task = '')" if "task" in cols else ""
Comment thread
askclaw-vesper marked this conversation as resolved.
)
usage_rows = con.execute(
"SELECT model, billing_provider, api_call_count "
"FROM session_model_usage WHERE session_id = ?",
"FROM session_model_usage WHERE session_id = ?"
+ task_filter,
Comment thread
askclaw-vesper marked this conversation as resolved.
(session_id,),
).fetchall()
except sqlite3.Error as exc:
Expand Down
85 changes: 85 additions & 0 deletions tests/test_real_contracts.py
Original file line number Diff line number Diff line change
Expand Up @@ -738,6 +738,15 @@ def test_ended_with_completed_reason_is_accepted(self) -> None:
self.assertEqual(observed["ended_at"], self.before + 2.0)
self.assertEqual(observed["end_reason"], "completed")

def test_ended_with_cli_close_reason_is_accepted(self) -> None:
# Hermes v0.20+ finalizes one-shot (-q/-Q) sessions with
# end_reason "cli_close" on normal CLI exit; it must count as a
# clean completion exactly like "completed".
self.build_db(ended_at=self.before + 2.0, end_reason="cli_close")
observed = self.observe()
self.assertEqual(observed["ended_at"], self.before + 2.0)
self.assertEqual(observed["end_reason"], "cli_close")

def test_ended_with_unset_reason_is_accepted(self) -> None:
for reason in (None, ""):
with self.subTest(reason=reason):
Expand Down Expand Up @@ -789,6 +798,82 @@ def test_start_outside_window_is_rejected_even_with_null_terminal(self) -> None:
self.assert_refused("outside this")


class HermesUsageTaskFilterTests(HermesTerminalFieldMatrixTests):
"""Auxiliary usage rows (title generation, vision, compression, ...)
share session_model_usage with the main turn under a non-empty task;
they must not pollute the actor's model/provider identity."""

SCHEMA = """
CREATE TABLE sessions (
id TEXT PRIMARY KEY, source TEXT NOT NULL, model TEXT,
model_config TEXT, started_at REAL NOT NULL, ended_at REAL,
end_reason TEXT, billing_provider TEXT, profile_name TEXT
);
CREATE TABLE messages (
id INTEGER PRIMARY KEY AUTOINCREMENT,
session_id TEXT NOT NULL, role TEXT NOT NULL, content TEXT,
timestamp REAL NOT NULL, active INTEGER NOT NULL DEFAULT 1,
compacted INTEGER NOT NULL DEFAULT 0
);
CREATE TABLE session_model_usage (
session_id TEXT NOT NULL, model TEXT NOT NULL,
billing_provider TEXT NOT NULL DEFAULT '',
api_call_count INTEGER NOT NULL DEFAULT 0,
task TEXT DEFAULT ''
);
"""

def add_usage_row(self, model: str, provider: str, calls: int,
task: str) -> None:
con = sqlite3.connect(self.db)
try:
con.execute(
"INSERT INTO session_model_usage (session_id, model, "
"billing_provider, api_call_count, task) "
"VALUES (?, ?, ?, ?, ?)",
("sess-1", model, provider, calls, task),
)
con.commit()
finally:
con.close()

def test_auxiliary_task_rows_do_not_pollute_identity(self) -> None:
self.build_db()
self.add_usage_row("kimi-k2.7-code", "kimi", 1, "title_generation")
observed = self.observe()
self.assertEqual(observed["models"], ["hermes-4-405b"])
self.assertEqual(observed["api_call_count"], 3)

def test_null_task_row_counts_as_main_conversation(self) -> None:
self.build_db()
con = sqlite3.connect(self.db)
try:
con.execute(
"INSERT INTO session_model_usage (session_id, model, "
"billing_provider, api_call_count, task) "
"VALUES ('sess-1', 'hermes-4-405b', 'nousresearch', 1, NULL)"
)
con.commit()
finally:
con.close()
self.assertEqual(self.observe()["api_call_count"], 4)

def test_two_main_task_models_are_still_rejected(self) -> None:
self.build_db()
self.add_usage_row("other-model", "other-provider", 1, "")
self.assert_refused("not exactly one model")


class HermesUsageOddlyCasedTaskColumnTests(HermesUsageTaskFilterTests):
"""SQLite resolves identifiers case-insensitively; a schema declaring
the column as ``Task`` must still trigger the main-conversation
filter instead of silently reverting to all-rows counting."""

SCHEMA = HermesUsageTaskFilterTests.SCHEMA.replace(
"task TEXT DEFAULT ''", '"Task" TEXT DEFAULT \'\''
)


class CommandIdentityTests(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
Expand Down