From cd87b865f1ddc91f5c7a9d0e93cadf44ac7d86e0 Mon Sep 17 00:00:00 2001 From: huangruiteng <14976749+huangruiteng@users.noreply.github.com> Date: Fri, 25 Sep 2026 09:13:30 +0800 Subject: [PATCH 1/3] fix(runtime): allow bounded typed responses under host load Signed-off-by: huangruiteng <14976749+huangruiteng@users.noreply.github.com> --- loopx/control_plane/effect_runtime.py | 5 ++-- .../test_effect_runtime_integration.py | 26 +++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/loopx/control_plane/effect_runtime.py b/loopx/control_plane/effect_runtime.py index f0e75891eb..d4c7be6188 100644 --- a/loopx/control_plane/effect_runtime.py +++ b/loopx/control_plane/effect_runtime.py @@ -37,6 +37,7 @@ STARTUP_LOCK_TIMEOUT_SECONDS = 15.0 STARTUP_READY_TIMEOUT_SECONDS = 15.0 STARTUP_POLL_SECONDS = 0.025 +DEFAULT_REQUEST_TIMEOUT_SECONDS = 10.0 _NODE_VERSION_RE = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)(?:[-+].*)?$") _RUNTIME_SOURCE_SUFFIXES = frozenset({".json", ".ts"}) _RuntimeSourceSnapshot = tuple[tuple[str, int, int, int], ...] @@ -802,7 +803,7 @@ def effect_runtime_request( method: str, params: Mapping[str, Any], *, - timeout: float = 5.0, + timeout: float = DEFAULT_REQUEST_TIMEOUT_SECONDS, retry_safe: bool = True, ) -> dict[str, Any]: """Call the managed TS runtime, retrying only idempotent typed effects.""" @@ -870,7 +871,7 @@ def effect_runtime_result( method: str, params: Mapping[str, Any], *, - timeout: float = 5.0, + timeout: float = DEFAULT_REQUEST_TIMEOUT_SECONDS, retry_safe: bool = True, ) -> Any: return effect_runtime_request( diff --git a/tests/control_plane/test_effect_runtime_integration.py b/tests/control_plane/test_effect_runtime_integration.py index 9d96d7fb36..a0a83b30ab 100644 --- a/tests/control_plane/test_effect_runtime_integration.py +++ b/tests/control_plane/test_effect_runtime_integration.py @@ -100,6 +100,32 @@ def probe(pid: object) -> bool: assert calls == [1234] +def test_default_runtime_request_budget_covers_typed_projection_calls( + tmp_path: Path, + monkeypatch, +) -> None: + info = {"token": "fixture"} + observed: list[tuple[str, float]] = [] + monkeypatch.setattr(effect_runtime, "_runtime_fingerprint_for_request", lambda: "fixture") + monkeypatch.setattr(effect_runtime, "_runtime_info_path", lambda _: tmp_path / "runtime.json") + monkeypatch.setattr(effect_runtime, "_read_info", lambda *_args, **_kwargs: info) + + def respond(_info: object, **kwargs: object) -> dict[str, object]: + observed.append((str(kwargs["method"]), float(kwargs["timeout"]))) + return {"result": {"ok": True}} + + monkeypatch.setattr(effect_runtime, "_request_with_info", respond) + assert effect_runtime.effect_runtime_result("todo.succession.project", {}) == {"ok": True} + assert effect_runtime.effect_runtime_request("scheduler.monitor_target.select", {}) == { + "result": {"ok": True} + } + assert observed == [ + ("todo.succession.project", effect_runtime.DEFAULT_REQUEST_TIMEOUT_SECONDS), + ("scheduler.monitor_target.select", effect_runtime.DEFAULT_REQUEST_TIMEOUT_SECONDS), + ] + assert effect_runtime.DEFAULT_REQUEST_TIMEOUT_SECONDS == 10.0 + + def test_managed_runtime_is_reused_and_restart_safe_for_typed_write( tmp_path: Path, monkeypatch, From 028136a20b98cf78cb6bff27ee1540c3082ceece Mon Sep 17 00:00:00 2001 From: huangruiteng <14976749+huangruiteng@users.noreply.github.com> Date: Fri, 25 Sep 2026 09:13:58 +0800 Subject: [PATCH 2/3] fix(qualification): complete bounded vision closeout and retain safe diagnostics Signed-off-by: huangruiteng <14976749+huangruiteng@users.noreply.github.com> --- ...actual_default_model_behavior_portfolio.py | 67 ++++++++++++++----- .../replan_vision_closeout_behavior.py | 2 +- ...actual_default_model_behavior_portfolio.py | 53 +++++++++++++++ .../test_required_vision_closeout_behavior.py | 4 +- 4 files changed, 106 insertions(+), 20 deletions(-) diff --git a/loopx/control_plane/testing/actual_default_model_behavior_portfolio.py b/loopx/control_plane/testing/actual_default_model_behavior_portfolio.py index 0406c35df6..b3cb91cbae 100644 --- a/loopx/control_plane/testing/actual_default_model_behavior_portfolio.py +++ b/loopx/control_plane/testing/actual_default_model_behavior_portfolio.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import re from collections.abc import Callable, Mapping, Sequence from copy import deepcopy from dataclasses import dataclass @@ -75,6 +76,7 @@ } ) _TURN_ACTOR_KINDS = frozenset({"turn", *_TOOL_ACTOR_KINDS}) +_DIAGNOSTIC_CODE = re.compile(r"[a-z][a-z0-9_]{0,63}\Z") @dataclass(frozen=True) @@ -1169,6 +1171,35 @@ def _receipt_alignment( return not mismatches, sorted(set(mismatches)) +def _tool_repeat_diagnostic(receipt: Mapping[str, Any], repeat: int) -> dict[str, Any]: + """Retain bounded failure context without commands or provider content.""" + def count(value: Any) -> int | None: + return value if type(value) is int and 0 <= value <= 1_000 else None + + def code(value: Any) -> str | None: + if value is None: + return None + return value if isinstance(value, str) and _DIAGNOSTIC_CODE.fullmatch(value) else "unclassified" + + steps = receipt.get("tool_call_receipts") + entries = steps if isinstance(steps, list) else [] + errors: dict[str, int] = {} + for entry in entries[:64]: + if not isinstance(entry, Mapping) or not entry.get("error_code"): + continue + error = code(entry["error_code"]) or "unclassified" + errors[error] = errors.get(error, 0) + 1 + return { + "repeat": repeat, + "actor_passed": receipt.get("qualification_passed") is True, + "failure_code": code(receipt.get("failure_code")), + "tool_call_count": count(receipt.get("tool_call_count")), + "tool_call_limit": count(receipt.get("tool_call_limit")), + "tool_error_counts": dict(sorted(errors.items())), + "tool_errors_truncated": len(entries) > 64, + } + + def _scenario_result( spec: _ScenarioSpec, packet: Mapping[str, Any], @@ -1187,6 +1218,7 @@ def _scenario_result( observed_routes: list[str] = [] observed_action_kind_sequences: list[list[str]] = [] failure_codes: list[str] = [] + repeat_diagnostics: list[dict[str, Any]] = [] actor_error = False observations: list[dict[str, Any]] = [] for repeat_index in range(ACTUAL_DEFAULT_MODEL_BEHAVIOR_REPEAT_ATTEMPTS): @@ -1251,6 +1283,8 @@ def _scenario_result( break aligned, mismatches = _receipt_alignment(spec, receipt, expected) receipt_digests.append(_digest(dict(receipt))) + if spec.actor_kind in _TOOL_ACTOR_KINDS: + repeat_diagnostics.append(_tool_repeat_diagnostic(receipt, repeat_index + 1)) observations.append( {field: receipt.get(field) for field in _HARD_INVARIANT_FIELDS} ) @@ -1272,23 +1306,22 @@ def _scenario_result( not failure_codes and repeats_completed == ACTUAL_DEFAULT_MODEL_BEHAVIOR_REPEAT_ATTEMPTS ) - return ( - { - "scenario_id": spec.scenario_id, - "actor_kind": spec.actor_kind, - "phase": spec.phase, - "expected_route": spec.expected_route, - "status": "passed" if passed else "failed", - "repeats_required": ACTUAL_DEFAULT_MODEL_BEHAVIOR_REPEAT_ATTEMPTS, - "repeats_completed": repeats_completed, - "observed_routes": observed_routes, - "observed_action_kind_sequences": observed_action_kind_sequences, - "failure_codes": sorted(set(failure_codes)), - "receipt_digests": receipt_digests, - }, - actor_error, - observations, - ) + result = { + "scenario_id": spec.scenario_id, + "actor_kind": spec.actor_kind, + "phase": spec.phase, + "expected_route": spec.expected_route, + "status": "passed" if passed else "failed", + "repeats_required": ACTUAL_DEFAULT_MODEL_BEHAVIOR_REPEAT_ATTEMPTS, + "repeats_completed": repeats_completed, + "observed_routes": observed_routes, + "observed_action_kind_sequences": observed_action_kind_sequences, + "failure_codes": sorted(set(failure_codes)), + "receipt_digests": receipt_digests, + } + if spec.actor_kind in _TOOL_ACTOR_KINDS: + result["repeat_diagnostics"] = repeat_diagnostics + return result, actor_error, observations def _contrast_result( diff --git a/loopx/control_plane/testing/replan_vision_closeout_behavior.py b/loopx/control_plane/testing/replan_vision_closeout_behavior.py index a7e24a3a54..a07d1c17e4 100644 --- a/loopx/control_plane/testing/replan_vision_closeout_behavior.py +++ b/loopx/control_plane/testing/replan_vision_closeout_behavior.py @@ -19,7 +19,7 @@ # Full closeout includes evidence discovery, authoring, refresh and settlement; # its resource bound is independent of the narrower single-action qualifier. -REQUIRED_VISION_CLOSEOUT_MAX_CALLS = 32 +REQUIRED_VISION_CLOSEOUT_MAX_CALLS = 40 class VisionHostAdmissionRejected(ValueError): diff --git a/tests/control_plane/test_actual_default_model_behavior_portfolio.py b/tests/control_plane/test_actual_default_model_behavior_portfolio.py index f58b8b2f14..1b74a90cf7 100644 --- a/tests/control_plane/test_actual_default_model_behavior_portfolio.py +++ b/tests/control_plane/test_actual_default_model_behavior_portfolio.py @@ -366,6 +366,59 @@ def test_required_vision_rejects_narrow_or_incomplete_actor_receipts(overrides: assert failures +def test_tool_portfolio_keeps_bounded_repeat_diagnostics_without_raw_commands() -> None: + from loopx.control_plane.testing.actual_default_model_behavior_portfolio import ( + _SCENARIOS, _scenario_result, + ) + + spec = next(item for item in _SCENARIOS if item.scenario_id == "turn_required_vision_replan") + complete = _replan_semantic_action_actor("fixture") + expected = {key: complete[key] for key in ( + "qualification_scope", "trigger_kinds", "required_semantic_outcomes", "vision_closeout", + )} + + def replan_actor(run_id: str) -> dict[str, Any]: + failed = run_id.endswith(":r1") + return { + **complete, + "qualification_passed": not failed, + "failure_code": "tool_call_budget_exhausted" if failed else None, + "semantic_action_accepted": not failed, + "selected_semantic_outcomes": [] if failed else ["fresh_vision_path_outcome"], + "vision_closeout": None if failed else complete["vision_closeout"], + "tool_call_count": 40 if failed else 16, + "tool_call_limit": 40, + "tool_call_receipts": [ + {"error_code": "shell_nonzero", "raw_command": "never-publish-this-command"}, + {"error_code": "private/path", "raw_command": "never-publish-this-command"}, + ], + } + + def unused(_: Any) -> dict[str, Any]: + return {} # Only the replan actor is invoked for this scenario. + scenario, actor_error, _ = _scenario_result( + spec, {}, expected=expected, qualification_id="diagnostic-test", + turn_actor=unused, onboarding_actor=unused, + selected_todo_actor=unused, replan_semantic_action_actor=replan_actor, + scoped_gate_successor_actor=unused, capability_monitor_repair_actor=unused, + terminal_settlement_actor=unused, + ) + assert actor_error is False + assert scenario["status"] == "failed" + assert scenario["repeat_diagnostics"] == [ + {"repeat": 1, "actor_passed": False, "failure_code": "tool_call_budget_exhausted", + "tool_call_count": 40, "tool_call_limit": 40, + "tool_error_counts": {"shell_nonzero": 1, "unclassified": 1}, + "tool_errors_truncated": False}, + {"repeat": 2, "actor_passed": True, "failure_code": None, + "tool_call_count": 16, "tool_call_limit": 40, + "tool_error_counts": {"shell_nonzero": 1, "unclassified": 1}, + "tool_errors_truncated": False}, + ] + assert "never-publish-this-command" not in json.dumps(scenario) + assert "private/path" not in json.dumps(scenario) + + def _capability_monitor_repair_actor(_: str) -> dict[str, Any]: return _passing_tool_receipt( "capability_monitor_repair_tool_behavior_receipt_v1", diff --git a/tests/control_plane/test_required_vision_closeout_behavior.py b/tests/control_plane/test_required_vision_closeout_behavior.py index 718a2ef327..debcf2a5a7 100644 --- a/tests/control_plane/test_required_vision_closeout_behavior.py +++ b/tests/control_plane/test_required_vision_closeout_behavior.py @@ -179,7 +179,7 @@ def refresh(request: Mapping[str, Any]) -> ScriptedExecToolAction: assert not (result.get("vision_closeout") or {}).get("settled") -@pytest.mark.parametrize("extra_reads,passed", [(27, True), (28, False)]) +@pytest.mark.parametrize("extra_reads,passed", [(35, True), (36, False)]) def test_budget_boundary_still_requires_the_final_spend(tmp_path: Path, extra_reads: int, passed: bool) -> None: fixture = _build_fixture(tmp_path / "oracle", required_vision=True) result = _qualify(tmp_path, [ @@ -189,7 +189,7 @@ def test_budget_boundary_still_requires_the_final_spend(tmp_path: Path, extra_re vision_patch_action, projected_refresh, projected_spend, ]) assert result["qualification_passed"] is passed - assert result["tool_call_count"] == result["tool_call_limit"] == 32 + assert result["tool_call_count"] == result["tool_call_limit"] == 40 assert result["vision_closeout"]["settled"] is passed From 785cf0615ee81c3646c60c954a5faa0e45ff5e3a Mon Sep 17 00:00:00 2001 From: huangruiteng <14976749+huangruiteng@users.noreply.github.com> Date: Fri, 25 Sep 2026 09:17:43 +0800 Subject: [PATCH 3/3] docs(qualification): align live portfolio and vision limits Signed-off-by: huangruiteng <14976749+huangruiteng@users.noreply.github.com> --- docs/development/testing-and-quality.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/development/testing-and-quality.md b/docs/development/testing-and-quality.md index 097e412427..4c5be58bf2 100644 --- a/docs/development/testing-and-quality.md +++ b/docs/development/testing-and-quality.md @@ -830,7 +830,7 @@ and qualification scope; a successful ordinary `typed_progress_repeat` refresh cannot qualify this journey. The narrow semantic-action gate remains useful but does not prove full closeout. Run this focused journey with `uv run --extra test python scripts/qualify-doubao-replan-semantic-action-live.py --required-vision --qualification-id `. -The complete required-vision journey has a 32-call bound; the narrow +The complete required-vision journey has a 40-call bound; the narrow single-semantic-action qualifier retains seven. The increased budget covers evidence discovery, JSON authoring, refresh, settlement and bounded recovery, including multiple field-validation corrections before a final spend; @@ -894,14 +894,14 @@ projection; every repeat must pass and hard actor errors are not retried. The remaining live turn actor cases consume the default CLI hot-path `quota should-run` projection used by Codex App automation and return runtime-facing decisions rather than echoing a global testing-only semantic -contract. The suite has 38 bounded scenario attempts. Five scenarios +contract. The suite has 42 bounded scenario attempts. Five scenarios exercise real tool loops; their per-scenario provider-call ceilings are owned by the corresponding typed behavior harnesses instead of being duplicated here. Exact scheduler, vision, writeback, and warning fields stay in deterministic action-signature coverage; pair mode keeps TurnEnvelope semantic extraction for explicit packet differentials or outcome claims. -常规 live suite 是 `actual_default_model_behavior_portfolio_v0`：19 个 one-arm +常规 live suite 是 `actual_default_model_behavior_portfolio_v0`：21 个 one-arm 场景，每个重复 2 次。9 个 core-contract 场景覆盖正常接入、agent 身份与 goal 选择、selected todo、peer 身份路由、same-agent 续接、最终 human gate、 健康继续和 projection repair；1 个 effect-settlement 场景覆盖 terminal closeout； @@ -924,6 +924,7 @@ actor 硬错误不自动重试。selected-Todo 场景从正式 thin heartbeat 缺失 vision 的 hermetic 状态，执行真实 quota，并要求模型读取 host 投影的 frontier 与 工作源，再通过真实写路径提交 typed semantic action；其他 turn 场景仍直接读取 Codex App automation 使用的默认 CLI hot-path `quota should-run` projection 并返回运行时决策， +完整 required-vision 闭环最多允许 40 次工具调用，窄范围单动作验收仍是 7 次；耗尽预算但未完成最终结算仍判失败。 scoped-gate successor 场景也从 hermetic Goal 与正式 heartbeat 开始：真实 quota 必须 同时投影非阻塞 user notice 和 ready deferred successor，模型随后既要呈现提醒，也要 实际执行被选中的 successor；capability re-entry 场景则要求模型先执行原 blocked Todo @@ -932,7 +933,7 @@ scoped-gate successor 场景也从 hermetic Goal 与正式 heartbeat 开始： 其他 turn 场景仍属于 packet interpretation。scheduler、vision、writeback 与 warning 的精确字段继续由 action-signature 确定性覆盖；pair 中的 TurnEnvelope 只用于 -明确的 packet 差分或结果提升声明。全套是 38 个有界 scenario attempt；5 个真实工具 +明确的 packet 差分或结果提升声明。全套是 42 个有界 scenario attempt；5 个真实工具 场景的 provider 调用上限由各自 typed behavior harness 持有，本文不再复制易漂移的总数。 For onboarding packets, the suite uses the shipped guided packet builder and