fix(evaluation): reject empty intake answers and protocol fallback (#5434)

Signed-off-by: huangruiteng <huangrt01@163.com>

Author: huangruiteng

docs/architecture/rfcs/app-conversation-and-async-inbox-v0.md

@@ -544,9 +544,14 @@ skips rather than presenting an unqualified profile as passing. The suite uses
 the production prompt/parser and fixed public contexts. Its frozen outcomes
 cover new work, existing owners, ambiguity, stopped/ungranted recipients, scope
 correction, current-Goal follow-ups, quotes, negation and ordinary questions in
-Chinese/English. Output conflicts, omitted envelopes and truncated generations
-fail rather than being counted as successful intent recognition. Report model,
-prompt/case hashes, request settings, token usage and repeat count. This layer
+Chinese/English. An empty answer fails for either provider. The API lane also
+checks raw envelope conflicts, omitted tags and truncated generations; the
+Codex lane consumes adapter protocol warnings, including missing tags, rather
+than certifying its readable fallback. Codex raw envelope integrity beyond the
+adapter's warnings remains unqualified. Both receive the same fixture context
+and normalized request. Report model, template and per-case effective prompt
+hashes, case hash, request settings, available token usage and repeat count.
+Offline provider fixtures exercise the CLI and report without paid calls. This layer
 qualifies model interpretation of supplied evidence, not live discovery, actual
 dispatch, stop enforcement or full GQ01/GQ02 completion. Packaged browser and
 real collaboration transport tests qualify those separate boundaries.

examples/evaluations/chat-intake.py

@@ -23,6 +23,9 @@
 def score(case, response):
     observed = "handoff" if response.get("context_handoff") else "draft" if response.get("goal_draft") else "answer"
     errors = []
+    message = response.get("message")
+    if not isinstance(message, str) or not message.strip():
+        errors.append("missing_answer")
     if observed != case["expected"]:
         errors.append(f"expected_{case['expected']}_got_{observed}")
     if response.get("proposals") or response.get("protected_action"):
@@ -70,19 +73,36 @@ def main():
 
     def run(case):
         started = time.monotonic()
-        prompt = _turn_prompt(case["request"], context_summary=json.dumps(case["context"], ensure_ascii=False))
+        # Match the real Codex adapter's input normalization for both providers.
+        message = " ".join(case["request"].split())
+        context = json.dumps(case["context"], ensure_ascii=False)
+        prompt = _turn_prompt(message, context_summary=context)
+        prompt_sha256 = hashlib.sha256(prompt.encode()).hexdigest()
         if args.provider == "codex":
+            protocol_warning = False
+
+            def observe(event, _payload):
+                nonlocal protocol_warning
+                # Keep only the fact; provider payloads can contain secrets.
+                if event == "protocol.warning":
+                    protocol_warning = True
+
             try:
                 with tempfile.TemporaryDirectory(prefix="loopx-public-intake-") as work:
                     with CodexChatAgentSession.start(
                         codex_bin="codex", work_dir=Path(work), goal_id="intake-evaluation",
                         objective=json.dumps(case["context"], ensure_ascii=False), model=args.model,
                         reasoning_effort="high", hard_timeout_sec=180,
                     ) as session:
-                        row = score(case, session.send(case["request"]))
+                        session.context_summary = context
+                        row = score(case, session.send(message, on_event=observe))
+                        if protocol_warning:
+                            row["passed"] = False
+                            row["errors"].append("protocol_warning")
             except Exception as error:
                 row = {"id": case["id"], "passed": False, "errors": [type(error).__name__]}
             row["seconds"] = round(time.monotonic() - started, 2)
+            row["prompt_sha256"] = prompt_sha256
             print(json.dumps(row), flush=True)
             return row
         body = {"model": args.model, "messages": [{"role": "user", "content": prompt}],
@@ -111,6 +131,7 @@ def run(case):
             # Provider errors can contain secrets, URLs or echoed prompts. Store only type.
             row = {"id": case["id"], "passed": False, "errors": [type(error).__name__]}
         row["seconds"] = round(time.monotonic() - started, 2)
+        row["prompt_sha256"] = prompt_sha256
         print(json.dumps({k: row[k] for k in ("id", "passed", "errors")}), flush=True)
         return row
 
@@ -120,7 +141,7 @@ def run(case):
               "prompt_sha256": hashlib.sha256(_turn_prompt("").encode()).hexdigest(),
               "request_settings": {"reasoning_effort": "high", "runtime_profile": "restricted"} if args.provider == "codex" else {"temperature": 0, "max_tokens": 8192, "thinking": "provider_default"},
               "passed": sum(row["passed"] for row in rows), "total": len(rows), "results": rows,
-              "boundary": "Fixed public context with production prompt/parser; synthetic authoritative observations test intake, not real fact lookup. Scoring checks effects, recipients and evidence pointers; review factual conclusions separately. Codex uses the real restricted Chat adapter; operator-api also checks raw envelope integrity. No dynamic discovery, dispatch or work completion qualification."}
+              "boundary": "Fixed public context with production prompt/parser; synthetic authoritative observations test intake, not real fact lookup. Top-level prompt_sha256 identifies the template; each result hashes its effective prompt including context and normalized request. Scoring requires a nonempty answer and checks effects, recipients and evidence pointers; review factual conclusions separately. Codex uses the real restricted Chat adapter and fails on its protocol warnings; operator-api also checks raw envelope integrity. No dynamic discovery, dispatch or work completion qualification."}
     args.output.parent.mkdir(parents=True, exist_ok=True)
     args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
     return 0 if report["passed"] == report["total"] else 1

tests/test_chat_intake_evaluation.py

@@ -1,6 +1,10 @@
 """Offline oracles for the opt-in model evaluation; no credentials or paid calls."""
 import importlib.util
+import json
 from pathlib import Path
+import sys
+
+import pytest
 
 ROOT = Path(__file__).resolve().parents[1]
 spec = importlib.util.spec_from_file_location("chat_intake_eval", ROOT / "examples/evaluations/chat-intake.py")
@@ -25,8 +29,8 @@ def test_existing_work_requires_the_right_recipient_without_a_second_gate():
 def test_complete_new_request_does_not_need_another_question():
     case = {"id": "new", "expected": "draft", "ready": True}
     draft = {"completion_criteria": "Cited report", "question": ""}
-    assert evaluation.score(case, {"goal_draft": draft})["passed"]
-    assert not evaluation.score(case, {"goal_draft": {**draft, "question": "Ready to begin?"}})["passed"]
+    assert evaluation.score(case, {"message": "Here is the draft.", "goal_draft": draft})["passed"]
+    assert not evaluation.score(case, {"message": "Here is the draft.", "goal_draft": {**draft, "question": "Ready to begin?"}})["passed"]
 
 
 def test_ordinary_question_cannot_silently_become_work():
@@ -61,3 +65,103 @@ def test_shared_intake_guidance_precedes_handoff_without_widening_runtime():
     for profile in ["restricted", "trusted_owner"]:
         assert CONVERSATION_INTENT_RESOLUTION_INSTRUCTION in manager_agent_objective(profile)
     assert CONVERSATION_INTENT_RESOLUTION_INSTRUCTION not in _turn_prompt("Execute.", execution_mode=True)
+
+
+@pytest.mark.parametrize("message", [None, "", " \n\t", 7, ["Answer"]])
+def test_an_ordinary_answer_needs_visible_text(message):
+    row = evaluation.score({"id": "ordinary", "expected": "answer"}, {"message": message})
+    assert not row["passed"] and "missing_answer" in row["errors"]
+
+
+@pytest.mark.parametrize("provider", ["codex", "operator-api"])
+@pytest.mark.parametrize("message", ["A sourced explanation.", ""])
+def test_release_cli_scores_provider_output_and_hashes_effective_prompt(
+    tmp_path, monkeypatch, provider, message
+):
+    """Exercise the real CLI/report flow; only provider inference is simulated."""
+    from contextlib import contextmanager
+    import hashlib
+    from io import BytesIO
+
+    suite = {"contexts": {"ordinary": {"scope": "public", "goals": []}},
+             "cases": [{"id": "ordinary", "context_ref": "ordinary", "expected": "answer",
+                        "request": "Explain\n  what a Goal means."}]}
+    cases = tmp_path / "cases.json"
+    cases.write_text(json.dumps(suite))
+    output = tmp_path / "report.json"
+    monkeypatch.setattr(sys, "argv", ["chat-intake", "--live", "--provider", provider,
+                                      "--model", "fixture-model", "--cases", str(cases), "--output", str(output)])
+    monkeypatch.setattr(evaluation, "operator_provider_environ", lambda _: {"DEEPSEEK_API_KEY": "synthetic-test-only"})
+    observed = []
+    response = {"message": message, "proposals": [], "protected_action": None, "gate": None}
+
+    class CodexFixture:
+        context_summary = "wrong startup context"
+
+        def send(self, request, *, on_event):
+            observed.append(evaluation._turn_prompt(request, context_summary=self.context_summary))
+            on_event("answer.final", {"response": response})
+            return response
+
+    @contextmanager
+    def start(**kwargs):
+        assert kwargs["model"] == "fixture-model"
+        yield CodexFixture()
+
+    def urlopen(request, **_):
+        observed.append(json.loads(request.data)["messages"][0]["content"])
+        envelope = evaluation.CHAT_REVIEW_OPEN_TAG + json.dumps(response) + evaluation.CHAT_REVIEW_CLOSE_TAG
+        return BytesIO(json.dumps({"choices": [{"finish_reason": "stop", "message": {"content": envelope}}]}).encode())
+
+    monkeypatch.setattr(evaluation.CodexChatAgentSession, "start", start)
+    monkeypatch.setattr(evaluation.urllib.request, "urlopen", urlopen)
+    assert evaluation.main() == (0 if message else 1)
+    report = json.loads(output.read_text())
+    assert report["total"] == len(observed) == 1
+    expected = evaluation._turn_prompt("Explain what a Goal means.", context_summary=json.dumps(suite["contexts"]["ordinary"], ensure_ascii=False))
+    assert observed == [expected], "Both providers must receive the same case context and request"
+    assert report["results"][0]["prompt_sha256"] == hashlib.sha256(expected.encode()).hexdigest()
+    assert report["passed"] == bool(message)
+    assert "synthetic-test-only" not in output.read_text()
+
+
+def test_codex_protocol_warning_fails_even_with_a_readable_fallback(tmp_path, monkeypatch, capsys):
+    from contextlib import contextmanager
+
+    cases = tmp_path / "cases.json"
+    cases.write_text(json.dumps({"contexts": {"ordinary": {}}, "cases": [
+        {"id": "ordinary", "context_ref": "ordinary", "expected": "answer", "request": "Explain Goals."}]}))
+    output = tmp_path / "report.json"
+    monkeypatch.setattr(sys, "argv", ["chat-intake", "--live", "--provider", "codex", "--model", "fixture-model",
+                                      "--cases", str(cases), "--output", str(output)])
+    monkeypatch.setattr(evaluation, "operator_provider_environ", lambda _: {})
+
+    class WarningFixture:
+        def send(self, request, *, on_event):
+            on_event("protocol.warning", {"error_code": "missing_review_envelope", "detail": "do-not-persist-provider-payload"})
+            return {"message": "A readable fallback that is not protocol-qualified."}
+
+    @contextmanager
+    def start(**_):
+        yield WarningFixture()
+
+    monkeypatch.setattr(evaluation.CodexChatAgentSession, "start", start)
+    assert evaluation.main() == 1
+    row = json.loads(output.read_text())["results"][0]
+    assert row["errors"] == ["protocol_warning"] and not row["passed"]
+    assert "do-not-persist-provider-payload" not in output.read_text() + capsys.readouterr().out
+
+
+def test_release_cli_without_live_never_reads_credentials_or_calls_provider(tmp_path, monkeypatch):
+    monkeypatch.setattr(sys, "argv", ["chat-intake", "--model", "fixture-model", "--output", str(tmp_path / "report.json")])
+
+    def forbidden(*_, **__):
+        pytest.fail("No provider or credential access without explicit live opt-in")
+
+    monkeypatch.setattr(evaluation, "operator_provider_environ", forbidden)
+    monkeypatch.setattr(evaluation.CodexChatAgentSession, "start", forbidden)
+    monkeypatch.setattr(evaluation.urllib.request, "urlopen", forbidden)
+    with pytest.raises(SystemExit) as error:
+        evaluation.main()
+    assert error.value.code == 2
+    assert not (tmp_path / "report.json").exists()