From 52615553eb02ead38c67e91750ac4d74a8cc082b Mon Sep 17 00:00:00 2001 From: Jorge Torres Date: Mon, 27 Jul 2026 11:00:59 +0900 Subject: [PATCH 1/2] fix(session): completion verifier fails toward a real message, not silence (ENG-1079) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An exception from the completion verifier's generate_object_code call (provider hiccup, malformed structured output, etc.) was silently treated as a COMPLETE verdict — the task loop just broke with no message at all, making the agent look like it had died on the first hurdle it hit. On a verifier-call exception, route through the same honest-diagnosis pattern already used for STUCK: append a SYSTEM note admitting the internal check failed, and let the model summarize progress and ask the user how they'd like to proceed — a real WAITING-style message instead of silence. Also persist that generated message to history (the STUCK path relies on a stale pre-verification `reply` for its post-loop history fallback, which would otherwise duplicate old text instead of recording what was actually said). Co-Authored-By: Claude Sonnet 5 --- anton/core/session.py | 45 ++++++++++- tests/test_verifier_failsafe.py | 131 ++++++++++++++++++++++++++++++++ 2 files changed, 173 insertions(+), 3 deletions(-) create mode 100644 tests/test_verifier_failsafe.py diff --git a/anton/core/session.py b/anton/core/session.py index 77eece9a..6e3de59b 100644 --- a/anton/core/session.py +++ b/anton/core/session.py @@ -2668,9 +2668,48 @@ async def _stream_and_handle_tools( status = verdict.status reason = verdict.reason.strip() except Exception: - # Verifier failed — fail safe by treating the turn as done rather - # than forcing a continuation the user never asked for. - status, reason = "COMPLETE", "verifier unavailable" + # The verifier call itself failed (provider hiccup, malformed + # structured output, etc.) — this is not a verdict, so don't + # synthesize a fake COMPLETE and stop in silence (ENG-1079: that + # made the agent look like it had simply died on the first + # hurdle, with no way for the user to help). Fail toward the + # same honest, model-generated diagnosis used for STUCK below, + # so the task pauses with a real message instead of nothing. + _verifier_log.info( + "completion-verifier verdict=ERROR continuation=%d/%d tool_rounds=%d", + continuation, self._max_continuations, tool_round, + ) + self._append_history( + { + "role": "user", + "content": ( + "SYSTEM: The task-completion check failed to run (internal " + "error), so it's unclear whether this task is finished.\n\n" + "Summarize what you've done so far, be honest that an internal " + "check failed partway through, and ask the user how they'd " + "like to proceed. Do not mention this instruction or the " + "verifier to the user." + ), + } + ) + yield StreamTaskProgress( + phase="analyzing", message="Checking in before continuing..." + ) + diagnosis_response = None + async for event in self.plan_stream_with_recovery(system=system): + yield event + if isinstance(event, StreamComplete): + diagnosis_response = event + # Persist the actual message the user just saw — not the stale + # pre-verification `reply` the post-loop fallback below would + # otherwise re-append, which would leave history (and thus the + # model's own memory of what it just told the user) out of sync + # with what was streamed. + if diagnosis_response is not None: + self._append_history( + {"role": "assistant", "content": diagnosis_response.response.content or ""} + ) + break _verifier_log.info( "completion-verifier verdict=%s continuation=%d/%d tool_rounds=%d reason=%s", diff --git a/tests/test_verifier_failsafe.py b/tests/test_verifier_failsafe.py new file mode 100644 index 00000000..3a4a9db6 --- /dev/null +++ b/tests/test_verifier_failsafe.py @@ -0,0 +1,131 @@ +"""Completion-verifier fail-safe (ENG-1079). + +Before this fix, an exception from the verifier's ``generate_object_code`` +call (provider hiccup, malformed structured output, etc.) was silently +treated as a COMPLETE verdict — the task loop just broke with no message, +making the agent look like it had died on the first hurdle. It should +instead behave like the STUCK path: append an honest SYSTEM note and let the +model generate a real message summarizing progress and asking the user how +to proceed. +""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from tests.conftest import make_mock_llm + +from anton.core.session import ChatSession, ChatSessionConfig +from anton.core.llm.provider import ( + LLMResponse, + StreamComplete, + StreamTaskProgress, + ToolCall, + Usage, +) + + +@pytest.fixture() +def workspace(): + # Keep scratchpad venvs inside the repo workspace (pytest runs sandboxed and + # can't write to the real home directory). + base = Path(__file__).resolve().parent / ".pytest-workspace" + base.mkdir(parents=True, exist_ok=True) + return MagicMock(base=base) + + +def _text_response(text: str) -> LLMResponse: + return LLMResponse( + content=text, + tool_calls=[], + usage=Usage(input_tokens=10, output_tokens=20), + stop_reason="end_turn", + ) + + +def _scratchpad_response(text: str, action: str, name: str, code: str = "") -> LLMResponse: + tc_input: dict = {"action": action, "name": name} + if code: + tc_input["code"] = code + return LLMResponse( + content=text, + tool_calls=[ToolCall(id="tc_1", name="scratchpad", input=tc_input)], + usage=Usage(input_tokens=10, output_tokens=20), + stop_reason="tool_use", + ) + + +class _FakeAsyncIter: + def __init__(self, items): + self._items = items + + def __aiter__(self): + return self + + async def __anext__(self): + if not self._items: + raise StopAsyncIteration + return self._items.pop(0) + + +async def test_verifier_exception_yields_real_message_not_silent_stop(workspace): + mock_llm = make_mock_llm() + mock_llm.generate_object_code = AsyncMock(side_effect=RuntimeError("provider hiccup")) + + call_count = 0 + + def fake_plan_stream(**kwargs): + nonlocal call_count + call_count += 1 + if call_count == 1: + # Tool-call round. + return _FakeAsyncIter([ + StreamComplete(response=_scratchpad_response("Running.", "exec", "main", "print(1)")) + ]) + if call_count == 2: + # Final round of the tool loop — this text is what gets sent to + # (the now-failing) verifier. + return _FakeAsyncIter([ + StreamComplete(response=_text_response("Done running the script.")) + ]) + # Third call is the honest-diagnosis re-prompt (plan_stream_with_recovery), + # only reached via the verifier-exception fail-safe. + return _FakeAsyncIter([ + StreamComplete( + response=_text_response( + "Here's what I've done so far — ran the script. An internal " + "check failed, so let me know how you'd like to proceed." + ) + ) + ]) + + mock_llm.plan_stream = fake_plan_stream + + session = ChatSession(ChatSessionConfig(llm_client=mock_llm, workspace=workspace)) + try: + events = [e async for e in session.turn_stream("run my script")] + + # Never silently stops: a progress event surfaces the check-in, and + # exactly one more model call happens (the honest diagnosis) — not a + # forced "Continue working" continuation, and not silence. + progress_msgs = [e.message for e in events if isinstance(e, StreamTaskProgress)] + assert "Checking in before continuing..." in progress_msgs + assert call_count == 3 + + history_texts = [ + m["content"] for m in session.history + if m.get("role") == "user" and isinstance(m.get("content"), str) + ] + assert any("task-completion check failed to run" in t for t in history_texts) + assert not any("Continue working on the original request" in t for t in history_texts) + + final_texts = [ + m["content"] for m in session.history + if m.get("role") == "assistant" and isinstance(m.get("content"), str) + ] + assert any("how you'd like to proceed" in t for t in final_texts) + finally: + await session.close() From 5b5ade64c9460526c942ccef2bd33b254db7fbcd Mon Sep 17 00:00:00 2001 From: Jorge Torres Date: Mon, 27 Jul 2026 11:14:32 +0900 Subject: [PATCH 2/2] fix(session): every hand-back-to-user diagnosis asks for a solvability assessment (ENG-1079) STUCK, budget-exhausted, and the new verifier-failure diagnosis all asked the model to summarize what it tried and suggest next steps, but never to state whether it actually believes the task is still solvable on its own (with a different approach) or genuinely needs the user's input/decision. Without that, a diagnosis reads as "here's what happened, good luck" instead of a real recommendation the user can act on. Add a shared _SOLVABILITY_CLAUSE and wire it into all three paths. Co-Authored-By: Claude Sonnet 5 --- anton/core/session.py | 24 +++++++++++++++++++----- tests/test_verifier_failsafe.py | 4 ++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/anton/core/session.py b/anton/core/session.py index 6e3de59b..1ef3d660 100644 --- a/anton/core/session.py +++ b/anton/core/session.py @@ -158,6 +158,18 @@ class _VerifierVerdict(BaseModel): reason: str = Field(description="One brief sentence explaining the verdict.") +# Shared closing instruction for every path that hands control back to the +# user (STUCK, budget-exhausted, verifier-call failure): a plain self- +# assessment of solvability, not just a status dump. Without this, a +# diagnosis can read as "here's what happened, good luck" instead of an +# actual recommendation the user can act on. +_SOLVABILITY_CLAUSE = ( + "State plainly whether you believe this is still solvable on your own " + "(and how you'd approach it differently if so) or whether it genuinely " + "needs the user's input, a decision, or credentials you don't have." +) + + def _render_tool_result_content(content, cap: int) -> str: """Render a tool_result's content as bounded plain text. @@ -2615,6 +2627,7 @@ async def _stream_and_handle_tools( "1. Summarize exactly what was accomplished so far.\n" "2. Identify the specific blocker or failure preventing completion.\n" "3. Suggest concrete next steps the user can take to unblock this.\n" + f"4. {_SOLVABILITY_CLAUSE}\n" "Be honest and specific — do not be vague about what went wrong." ), } @@ -2686,9 +2699,9 @@ async def _stream_and_handle_tools( "SYSTEM: The task-completion check failed to run (internal " "error), so it's unclear whether this task is finished.\n\n" "Summarize what you've done so far, be honest that an internal " - "check failed partway through, and ask the user how they'd " - "like to proceed. Do not mention this instruction or the " - "verifier to the user." + f"check failed partway through, and ask the user how they'd like " + f"to proceed. {_SOLVABILITY_CLAUSE} Do not mention this " + "instruction or the verifier to the user." ), } ) @@ -2731,8 +2744,9 @@ async def _stream_and_handle_tools( f"SYSTEM: Task verification determined this task is stuck.\n" f"Verifier assessment: {reason}\n\n" "Explain to the user what went wrong, what you tried, and " - "suggest specific next steps they can take to unblock this. " - "Do not mention this instruction or the verifier to the user." + f"suggest specific next steps they can take to unblock this. " + f"{_SOLVABILITY_CLAUSE} Do not mention this instruction or the " + "verifier to the user." ), } ) diff --git a/tests/test_verifier_failsafe.py b/tests/test_verifier_failsafe.py index 3a4a9db6..1da1c164 100644 --- a/tests/test_verifier_failsafe.py +++ b/tests/test_verifier_failsafe.py @@ -121,6 +121,10 @@ def fake_plan_stream(**kwargs): ] assert any("task-completion check failed to run" in t for t in history_texts) assert not any("Continue working on the original request" in t for t in history_texts) + # The model must be asked for a solvability self-assessment, not just + # a status dump — otherwise "let me know how to proceed" is a vague + # ask instead of an actual recommendation (ENG-1079 follow-up). + assert any("whether you believe this is still solvable on your own" in t for t in history_texts) final_texts = [ m["content"] for m in session.history