diff --git a/anton/core/session.py b/anton/core/session.py index 77eece9a..1ef3d660 100644 --- a/anton/core/session.py +++ b/anton/core/session.py @@ -158,6 +158,18 @@ class _VerifierVerdict(BaseModel): reason: str = Field(description="One brief sentence explaining the verdict.") +# Shared closing instruction for every path that hands control back to the +# user (STUCK, budget-exhausted, verifier-call failure): a plain self- +# assessment of solvability, not just a status dump. Without this, a +# diagnosis can read as "here's what happened, good luck" instead of an +# actual recommendation the user can act on. +_SOLVABILITY_CLAUSE = ( + "State plainly whether you believe this is still solvable on your own " + "(and how you'd approach it differently if so) or whether it genuinely " + "needs the user's input, a decision, or credentials you don't have." +) + + def _render_tool_result_content(content, cap: int) -> str: """Render a tool_result's content as bounded plain text. @@ -2615,6 +2627,7 @@ async def _stream_and_handle_tools( "1. Summarize exactly what was accomplished so far.\n" "2. Identify the specific blocker or failure preventing completion.\n" "3. Suggest concrete next steps the user can take to unblock this.\n" + f"4. {_SOLVABILITY_CLAUSE}\n" "Be honest and specific — do not be vague about what went wrong." ), } @@ -2668,9 +2681,48 @@ async def _stream_and_handle_tools( status = verdict.status reason = verdict.reason.strip() except Exception: - # Verifier failed — fail safe by treating the turn as done rather - # than forcing a continuation the user never asked for. - status, reason = "COMPLETE", "verifier unavailable" + # The verifier call itself failed (provider hiccup, malformed + # structured output, etc.) — this is not a verdict, so don't + # synthesize a fake COMPLETE and stop in silence (ENG-1079: that + # made the agent look like it had simply died on the first + # hurdle, with no way for the user to help). Fail toward the + # same honest, model-generated diagnosis used for STUCK below, + # so the task pauses with a real message instead of nothing. + _verifier_log.info( + "completion-verifier verdict=ERROR continuation=%d/%d tool_rounds=%d", + continuation, self._max_continuations, tool_round, + ) + self._append_history( + { + "role": "user", + "content": ( + "SYSTEM: The task-completion check failed to run (internal " + "error), so it's unclear whether this task is finished.\n\n" + "Summarize what you've done so far, be honest that an internal " + f"check failed partway through, and ask the user how they'd like " + f"to proceed. {_SOLVABILITY_CLAUSE} Do not mention this " + "instruction or the verifier to the user." + ), + } + ) + yield StreamTaskProgress( + phase="analyzing", message="Checking in before continuing..." + ) + diagnosis_response = None + async for event in self.plan_stream_with_recovery(system=system): + yield event + if isinstance(event, StreamComplete): + diagnosis_response = event + # Persist the actual message the user just saw — not the stale + # pre-verification `reply` the post-loop fallback below would + # otherwise re-append, which would leave history (and thus the + # model's own memory of what it just told the user) out of sync + # with what was streamed. + if diagnosis_response is not None: + self._append_history( + {"role": "assistant", "content": diagnosis_response.response.content or ""} + ) + break _verifier_log.info( "completion-verifier verdict=%s continuation=%d/%d tool_rounds=%d reason=%s", @@ -2692,8 +2744,9 @@ async def _stream_and_handle_tools( f"SYSTEM: Task verification determined this task is stuck.\n" f"Verifier assessment: {reason}\n\n" "Explain to the user what went wrong, what you tried, and " - "suggest specific next steps they can take to unblock this. " - "Do not mention this instruction or the verifier to the user." + f"suggest specific next steps they can take to unblock this. " + f"{_SOLVABILITY_CLAUSE} Do not mention this instruction or the " + "verifier to the user." ), } ) diff --git a/tests/test_verifier_failsafe.py b/tests/test_verifier_failsafe.py new file mode 100644 index 00000000..1da1c164 --- /dev/null +++ b/tests/test_verifier_failsafe.py @@ -0,0 +1,135 @@ +"""Completion-verifier fail-safe (ENG-1079). + +Before this fix, an exception from the verifier's ``generate_object_code`` +call (provider hiccup, malformed structured output, etc.) was silently +treated as a COMPLETE verdict — the task loop just broke with no message, +making the agent look like it had died on the first hurdle. It should +instead behave like the STUCK path: append an honest SYSTEM note and let the +model generate a real message summarizing progress and asking the user how +to proceed. +""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from tests.conftest import make_mock_llm + +from anton.core.session import ChatSession, ChatSessionConfig +from anton.core.llm.provider import ( + LLMResponse, + StreamComplete, + StreamTaskProgress, + ToolCall, + Usage, +) + + +@pytest.fixture() +def workspace(): + # Keep scratchpad venvs inside the repo workspace (pytest runs sandboxed and + # can't write to the real home directory). + base = Path(__file__).resolve().parent / ".pytest-workspace" + base.mkdir(parents=True, exist_ok=True) + return MagicMock(base=base) + + +def _text_response(text: str) -> LLMResponse: + return LLMResponse( + content=text, + tool_calls=[], + usage=Usage(input_tokens=10, output_tokens=20), + stop_reason="end_turn", + ) + + +def _scratchpad_response(text: str, action: str, name: str, code: str = "") -> LLMResponse: + tc_input: dict = {"action": action, "name": name} + if code: + tc_input["code"] = code + return LLMResponse( + content=text, + tool_calls=[ToolCall(id="tc_1", name="scratchpad", input=tc_input)], + usage=Usage(input_tokens=10, output_tokens=20), + stop_reason="tool_use", + ) + + +class _FakeAsyncIter: + def __init__(self, items): + self._items = items + + def __aiter__(self): + return self + + async def __anext__(self): + if not self._items: + raise StopAsyncIteration + return self._items.pop(0) + + +async def test_verifier_exception_yields_real_message_not_silent_stop(workspace): + mock_llm = make_mock_llm() + mock_llm.generate_object_code = AsyncMock(side_effect=RuntimeError("provider hiccup")) + + call_count = 0 + + def fake_plan_stream(**kwargs): + nonlocal call_count + call_count += 1 + if call_count == 1: + # Tool-call round. + return _FakeAsyncIter([ + StreamComplete(response=_scratchpad_response("Running.", "exec", "main", "print(1)")) + ]) + if call_count == 2: + # Final round of the tool loop — this text is what gets sent to + # (the now-failing) verifier. + return _FakeAsyncIter([ + StreamComplete(response=_text_response("Done running the script.")) + ]) + # Third call is the honest-diagnosis re-prompt (plan_stream_with_recovery), + # only reached via the verifier-exception fail-safe. + return _FakeAsyncIter([ + StreamComplete( + response=_text_response( + "Here's what I've done so far — ran the script. An internal " + "check failed, so let me know how you'd like to proceed." + ) + ) + ]) + + mock_llm.plan_stream = fake_plan_stream + + session = ChatSession(ChatSessionConfig(llm_client=mock_llm, workspace=workspace)) + try: + events = [e async for e in session.turn_stream("run my script")] + + # Never silently stops: a progress event surfaces the check-in, and + # exactly one more model call happens (the honest diagnosis) — not a + # forced "Continue working" continuation, and not silence. + progress_msgs = [e.message for e in events if isinstance(e, StreamTaskProgress)] + assert "Checking in before continuing..." in progress_msgs + assert call_count == 3 + + history_texts = [ + m["content"] for m in session.history + if m.get("role") == "user" and isinstance(m.get("content"), str) + ] + assert any("task-completion check failed to run" in t for t in history_texts) + assert not any("Continue working on the original request" in t for t in history_texts) + # The model must be asked for a solvability self-assessment, not just + # a status dump — otherwise "let me know how to proceed" is a vague + # ask instead of an actual recommendation (ENG-1079 follow-up). + assert any("whether you believe this is still solvable on your own" in t for t in history_texts) + + final_texts = [ + m["content"] for m in session.history + if m.get("role") == "assistant" and isinstance(m.get("content"), str) + ] + assert any("how you'd like to proceed" in t for t in final_texts) + finally: + await session.close()