fix(session): completion verifier fails toward a real message, not silence (ENG-1079)

anton/core/session.py

@@ -158,6 +158,18 @@ class _VerifierVerdict(BaseModel):
     reason: str = Field(description="One brief sentence explaining the verdict.")
 
 
+# Shared closing instruction for every path that hands control back to the
+# user (STUCK, budget-exhausted, verifier-call failure): a plain self-
+# assessment of solvability, not just a status dump. Without this, a
+# diagnosis can read as "here's what happened, good luck" instead of an
+# actual recommendation the user can act on.
+_SOLVABILITY_CLAUSE = (
+    "State plainly whether you believe this is still solvable on your own "
+    "(and how you'd approach it differently if so) or whether it genuinely "
+    "needs the user's input, a decision, or credentials you don't have."
+)
+
+
 def _render_tool_result_content(content, cap: int) -> str:
     """Render a tool_result's content as bounded plain text.
 
@@ -2615,6 +2627,7 @@ async def _stream_and_handle_tools(
                             "1. Summarize exactly what was accomplished so far.\n"
                             "2. Identify the specific blocker or failure preventing completion.\n"
                             "3. Suggest concrete next steps the user can take to unblock this.\n"
+                            f"4. {_SOLVABILITY_CLAUSE}\n"
                             "Be honest and specific — do not be vague about what went wrong."
                         ),
                     }
@@ -2668,9 +2681,48 @@ async def _stream_and_handle_tools(
                 status = verdict.status
                 reason = verdict.reason.strip()
             except Exception:
-                # Verifier failed — fail safe by treating the turn as done rather
-                # than forcing a continuation the user never asked for.
-                status, reason = "COMPLETE", "verifier unavailable"
+                # The verifier call itself failed (provider hiccup, malformed
+                # structured output, etc.) — this is not a verdict, so don't
+                # synthesize a fake COMPLETE and stop in silence (ENG-1079: that
+                # made the agent look like it had simply died on the first
+                # hurdle, with no way for the user to help). Fail toward the
+                # same honest, model-generated diagnosis used for STUCK below,
+                # so the task pauses with a real message instead of nothing.
+                _verifier_log.info(
+                    "completion-verifier verdict=ERROR continuation=%d/%d tool_rounds=%d",
+                    continuation, self._max_continuations, tool_round,
+                )
+                self._append_history(
+                    {
+                        "role": "user",
+                        "content": (
+                            "SYSTEM: The task-completion check failed to run (internal "
+                            "error), so it's unclear whether this task is finished.\n\n"
+                            "Summarize what you've done so far, be honest that an internal "
+                            f"check failed partway through, and ask the user how they'd like "
+                            f"to proceed. {_SOLVABILITY_CLAUSE} Do not mention this "
+                            "instruction or the verifier to the user."
+                        ),
+                    }
+                )
+                yield StreamTaskProgress(
+                    phase="analyzing", message="Checking in before continuing..."
+                )
+                diagnosis_response = None
+                async for event in self.plan_stream_with_recovery(system=system):
+                    yield event
+                    if isinstance(event, StreamComplete):
+                        diagnosis_response = event
+                # Persist the actual message the user just saw — not the stale
+                # pre-verification `reply` the post-loop fallback below would
+                # otherwise re-append, which would leave history (and thus the
+                # model's own memory of what it just told the user) out of sync
+                # with what was streamed.
+                if diagnosis_response is not None:
+                    self._append_history(
+                        {"role": "assistant", "content": diagnosis_response.response.content or ""}
+                    )
+                break
 
             _verifier_log.info(
                 "completion-verifier verdict=%s continuation=%d/%d tool_rounds=%d reason=%s",
@@ -2692,8 +2744,9 @@ async def _stream_and_handle_tools(
                             f"SYSTEM: Task verification determined this task is stuck.\n"
                             f"Verifier assessment: {reason}\n\n"
                             "Explain to the user what went wrong, what you tried, and "
-                            "suggest specific next steps they can take to unblock this. "
-                            "Do not mention this instruction or the verifier to the user."
+                            f"suggest specific next steps they can take to unblock this. "
+                            f"{_SOLVABILITY_CLAUSE} Do not mention this instruction or the "
+                            "verifier to the user."
                         ),
                     }
                 )

tests/test_verifier_failsafe.py

@@ -0,0 +1,135 @@
+"""Completion-verifier fail-safe (ENG-1079).
+
+Before this fix, an exception from the verifier's ``generate_object_code``
+call (provider hiccup, malformed structured output, etc.) was silently
+treated as a COMPLETE verdict — the task loop just broke with no message,
+making the agent look like it had died on the first hurdle. It should
+instead behave like the STUCK path: append an honest SYSTEM note and let the
+model generate a real message summarizing progress and asking the user how
+to proceed.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from tests.conftest import make_mock_llm
+
+from anton.core.session import ChatSession, ChatSessionConfig
+from anton.core.llm.provider import (
+    LLMResponse,
+    StreamComplete,
+    StreamTaskProgress,
+    ToolCall,
+    Usage,
+)
+
+
+@pytest.fixture()
+def workspace():
+    # Keep scratchpad venvs inside the repo workspace (pytest runs sandboxed and
+    # can't write to the real home directory).
+    base = Path(__file__).resolve().parent / ".pytest-workspace"
+    base.mkdir(parents=True, exist_ok=True)
+    return MagicMock(base=base)
+
+
+def _text_response(text: str) -> LLMResponse:
+    return LLMResponse(
+        content=text,
+        tool_calls=[],
+        usage=Usage(input_tokens=10, output_tokens=20),
+        stop_reason="end_turn",
+    )
+
+
+def _scratchpad_response(text: str, action: str, name: str, code: str = "") -> LLMResponse:
+    tc_input: dict = {"action": action, "name": name}
+    if code:
+        tc_input["code"] = code
+    return LLMResponse(
+        content=text,
+        tool_calls=[ToolCall(id="tc_1", name="scratchpad", input=tc_input)],
+        usage=Usage(input_tokens=10, output_tokens=20),
+        stop_reason="tool_use",
+    )
+
+
+class _FakeAsyncIter:
+    def __init__(self, items):
+        self._items = items
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self):
+        if not self._items:
+            raise StopAsyncIteration
+        return self._items.pop(0)
+
+
+async def test_verifier_exception_yields_real_message_not_silent_stop(workspace):
+    mock_llm = make_mock_llm()
+    mock_llm.generate_object_code = AsyncMock(side_effect=RuntimeError("provider hiccup"))
+
+    call_count = 0
+
+    def fake_plan_stream(**kwargs):
+        nonlocal call_count
+        call_count += 1
+        if call_count == 1:
+            # Tool-call round.
+            return _FakeAsyncIter([
+                StreamComplete(response=_scratchpad_response("Running.", "exec", "main", "print(1)"))
+            ])
+        if call_count == 2:
+            # Final round of the tool loop — this text is what gets sent to
+            # (the now-failing) verifier.
+            return _FakeAsyncIter([
+                StreamComplete(response=_text_response("Done running the script."))
+            ])
+        # Third call is the honest-diagnosis re-prompt (plan_stream_with_recovery),
+        # only reached via the verifier-exception fail-safe.
+        return _FakeAsyncIter([
+            StreamComplete(
+                response=_text_response(
+                    "Here's what I've done so far — ran the script. An internal "
+                    "check failed, so let me know how you'd like to proceed."
+                )
+            )
+        ])
+
+    mock_llm.plan_stream = fake_plan_stream
+
+    session = ChatSession(ChatSessionConfig(llm_client=mock_llm, workspace=workspace))
+    try:
+        events = [e async for e in session.turn_stream("run my script")]
+
+        # Never silently stops: a progress event surfaces the check-in, and
+        # exactly one more model call happens (the honest diagnosis) — not a
+        # forced "Continue working" continuation, and not silence.
+        progress_msgs = [e.message for e in events if isinstance(e, StreamTaskProgress)]
+        assert "Checking in before continuing..." in progress_msgs
+        assert call_count == 3
+
+        history_texts = [
+            m["content"] for m in session.history
+            if m.get("role") == "user" and isinstance(m.get("content"), str)
+        ]
+        assert any("task-completion check failed to run" in t for t in history_texts)
+        assert not any("Continue working on the original request" in t for t in history_texts)
+        # The model must be asked for a solvability self-assessment, not just
+        # a status dump — otherwise "let me know how to proceed" is a vague
+        # ask instead of an actual recommendation (ENG-1079 follow-up).
+        assert any("whether you believe this is still solvable on your own" in t for t in history_texts)
+
+        final_texts = [
+            m["content"] for m in session.history
+            if m.get("role") == "assistant" and isinstance(m.get("content"), str)
+        ]
+        assert any("how you'd like to proceed" in t for t in final_texts)
+    finally:
+        await session.close()