Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions docs/development/testing-and-quality.md
Original file line number Diff line number Diff line change
Expand Up @@ -830,7 +830,7 @@ and qualification scope; a successful ordinary `typed_progress_repeat` refresh
cannot qualify this journey. The narrow semantic-action gate remains useful
but does not prove full closeout. Run this focused journey with
`uv run --extra test python scripts/qualify-doubao-replan-semantic-action-live.py --required-vision --qualification-id <public-safe-run-id>`.
The complete required-vision journey has a 32-call bound; the narrow
The complete required-vision journey has a 40-call bound; the narrow
single-semantic-action qualifier retains seven. The increased budget covers
evidence discovery, JSON authoring, refresh, settlement and bounded recovery,
including multiple field-validation corrections before a final spend;
Expand Down Expand Up @@ -894,14 +894,14 @@ projection; every repeat must pass and hard actor errors are not retried. The
remaining live turn actor cases consume the default CLI hot-path
`quota should-run` projection used by Codex App automation and return
runtime-facing decisions rather than echoing a global testing-only semantic
contract. The suite has 38 bounded scenario attempts. Five scenarios
contract. The suite has 42 bounded scenario attempts. Five scenarios
exercise real tool loops; their per-scenario provider-call ceilings are owned by
the corresponding typed behavior harnesses instead of being duplicated here.
Exact scheduler, vision, writeback, and warning fields stay in deterministic
action-signature coverage; pair mode keeps TurnEnvelope semantic extraction for
explicit packet differentials or outcome claims.

常规 live suite 是 `actual_default_model_behavior_portfolio_v0`:19 个 one-arm
常规 live suite 是 `actual_default_model_behavior_portfolio_v0`:21 个 one-arm
场景,每个重复 2 次。9 个 core-contract 场景覆盖正常接入、agent 身份与
goal 选择、selected todo、peer 身份路由、same-agent 续接、最终 human gate、
健康继续和 projection repair;1 个 effect-settlement 场景覆盖 terminal closeout;
Expand All @@ -924,6 +924,7 @@ actor 硬错误不自动重试。selected-Todo 场景从正式 thin heartbeat
缺失 vision 的 hermetic 状态,执行真实 quota,并要求模型读取 host 投影的 frontier 与
工作源,再通过真实写路径提交 typed semantic action;其他 turn 场景仍直接读取 Codex App
automation 使用的默认 CLI hot-path `quota should-run` projection 并返回运行时决策,
完整 required-vision 闭环最多允许 40 次工具调用,窄范围单动作验收仍是 7 次;耗尽预算但未完成最终结算仍判失败。
scoped-gate successor 场景也从 hermetic Goal 与正式 heartbeat 开始:真实 quota 必须
同时投影非阻塞 user notice 和 ready deferred successor,模型随后既要呈现提醒,也要
实际执行被选中的 successor;capability re-entry 场景则要求模型先执行原 blocked Todo
Expand All @@ -932,7 +933,7 @@ scoped-gate successor 场景也从 hermetic Goal 与正式 heartbeat 开始:
其他 turn 场景仍属于
packet interpretation。scheduler、vision、writeback 与
warning 的精确字段继续由 action-signature 确定性覆盖;pair 中的 TurnEnvelope 只用于
明确的 packet 差分或结果提升声明。全套是 38 个有界 scenario attempt;5 个真实工具
明确的 packet 差分或结果提升声明。全套是 42 个有界 scenario attempt;5 个真实工具
场景的 provider 调用上限由各自 typed behavior harness 持有,本文不再复制易漂移的总数。

For onboarding packets, the suite uses the shipped guided packet builder and
Expand Down
5 changes: 3 additions & 2 deletions loopx/control_plane/effect_runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
STARTUP_LOCK_TIMEOUT_SECONDS = 15.0
STARTUP_READY_TIMEOUT_SECONDS = 15.0
STARTUP_POLL_SECONDS = 0.025
DEFAULT_REQUEST_TIMEOUT_SECONDS = 10.0
_NODE_VERSION_RE = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)(?:[-+].*)?$")
_RUNTIME_SOURCE_SUFFIXES = frozenset({".json", ".ts"})
_RuntimeSourceSnapshot = tuple[tuple[str, int, int, int], ...]
Expand Down Expand Up @@ -802,7 +803,7 @@ def effect_runtime_request(
method: str,
params: Mapping[str, Any],
*,
timeout: float = 5.0,
timeout: float = DEFAULT_REQUEST_TIMEOUT_SECONDS,
retry_safe: bool = True,
) -> dict[str, Any]:
"""Call the managed TS runtime, retrying only idempotent typed effects."""
Expand Down Expand Up @@ -870,7 +871,7 @@ def effect_runtime_result(
method: str,
params: Mapping[str, Any],
*,
timeout: float = 5.0,
timeout: float = DEFAULT_REQUEST_TIMEOUT_SECONDS,
retry_safe: bool = True,
) -> Any:
return effect_runtime_request(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from __future__ import annotations

import json
import re
from collections.abc import Callable, Mapping, Sequence
from copy import deepcopy
from dataclasses import dataclass
Expand Down Expand Up @@ -75,6 +76,7 @@
}
)
_TURN_ACTOR_KINDS = frozenset({"turn", *_TOOL_ACTOR_KINDS})
_DIAGNOSTIC_CODE = re.compile(r"[a-z][a-z0-9_]{0,63}\Z")


@dataclass(frozen=True)
Expand Down Expand Up @@ -1169,6 +1171,35 @@ def _receipt_alignment(
return not mismatches, sorted(set(mismatches))


def _tool_repeat_diagnostic(receipt: Mapping[str, Any], repeat: int) -> dict[str, Any]:
"""Retain bounded failure context without commands or provider content."""
def count(value: Any) -> int | None:
return value if type(value) is int and 0 <= value <= 1_000 else None

def code(value: Any) -> str | None:
if value is None:
return None
return value if isinstance(value, str) and _DIAGNOSTIC_CODE.fullmatch(value) else "unclassified"

steps = receipt.get("tool_call_receipts")
entries = steps if isinstance(steps, list) else []
errors: dict[str, int] = {}
for entry in entries[:64]:
if not isinstance(entry, Mapping) or not entry.get("error_code"):
continue
error = code(entry["error_code"]) or "unclassified"
errors[error] = errors.get(error, 0) + 1
return {
"repeat": repeat,
"actor_passed": receipt.get("qualification_passed") is True,
"failure_code": code(receipt.get("failure_code")),
"tool_call_count": count(receipt.get("tool_call_count")),
"tool_call_limit": count(receipt.get("tool_call_limit")),
"tool_error_counts": dict(sorted(errors.items())),
"tool_errors_truncated": len(entries) > 64,
}


def _scenario_result(
spec: _ScenarioSpec,
packet: Mapping[str, Any],
Expand All @@ -1187,6 +1218,7 @@ def _scenario_result(
observed_routes: list[str] = []
observed_action_kind_sequences: list[list[str]] = []
failure_codes: list[str] = []
repeat_diagnostics: list[dict[str, Any]] = []
actor_error = False
observations: list[dict[str, Any]] = []
for repeat_index in range(ACTUAL_DEFAULT_MODEL_BEHAVIOR_REPEAT_ATTEMPTS):
Expand Down Expand Up @@ -1251,6 +1283,8 @@ def _scenario_result(
break
aligned, mismatches = _receipt_alignment(spec, receipt, expected)
receipt_digests.append(_digest(dict(receipt)))
if spec.actor_kind in _TOOL_ACTOR_KINDS:
repeat_diagnostics.append(_tool_repeat_diagnostic(receipt, repeat_index + 1))
observations.append(
{field: receipt.get(field) for field in _HARD_INVARIANT_FIELDS}
)
Expand All @@ -1272,23 +1306,22 @@ def _scenario_result(
not failure_codes
and repeats_completed == ACTUAL_DEFAULT_MODEL_BEHAVIOR_REPEAT_ATTEMPTS
)
return (
{
"scenario_id": spec.scenario_id,
"actor_kind": spec.actor_kind,
"phase": spec.phase,
"expected_route": spec.expected_route,
"status": "passed" if passed else "failed",
"repeats_required": ACTUAL_DEFAULT_MODEL_BEHAVIOR_REPEAT_ATTEMPTS,
"repeats_completed": repeats_completed,
"observed_routes": observed_routes,
"observed_action_kind_sequences": observed_action_kind_sequences,
"failure_codes": sorted(set(failure_codes)),
"receipt_digests": receipt_digests,
},
actor_error,
observations,
)
result = {
"scenario_id": spec.scenario_id,
"actor_kind": spec.actor_kind,
"phase": spec.phase,
"expected_route": spec.expected_route,
"status": "passed" if passed else "failed",
"repeats_required": ACTUAL_DEFAULT_MODEL_BEHAVIOR_REPEAT_ATTEMPTS,
"repeats_completed": repeats_completed,
"observed_routes": observed_routes,
"observed_action_kind_sequences": observed_action_kind_sequences,
"failure_codes": sorted(set(failure_codes)),
"receipt_digests": receipt_digests,
}
if spec.actor_kind in _TOOL_ACTOR_KINDS:
result["repeat_diagnostics"] = repeat_diagnostics
return result, actor_error, observations


def _contrast_result(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

# Full closeout includes evidence discovery, authoring, refresh and settlement;
# its resource bound is independent of the narrower single-action qualifier.
REQUIRED_VISION_CLOSEOUT_MAX_CALLS = 32
REQUIRED_VISION_CLOSEOUT_MAX_CALLS = 40


class VisionHostAdmissionRejected(ValueError):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -366,6 +366,59 @@ def test_required_vision_rejects_narrow_or_incomplete_actor_receipts(overrides:
assert failures


def test_tool_portfolio_keeps_bounded_repeat_diagnostics_without_raw_commands() -> None:
from loopx.control_plane.testing.actual_default_model_behavior_portfolio import (
_SCENARIOS, _scenario_result,
)

spec = next(item for item in _SCENARIOS if item.scenario_id == "turn_required_vision_replan")
complete = _replan_semantic_action_actor("fixture")
expected = {key: complete[key] for key in (
"qualification_scope", "trigger_kinds", "required_semantic_outcomes", "vision_closeout",
)}

def replan_actor(run_id: str) -> dict[str, Any]:
failed = run_id.endswith(":r1")
return {
**complete,
"qualification_passed": not failed,
"failure_code": "tool_call_budget_exhausted" if failed else None,
"semantic_action_accepted": not failed,
"selected_semantic_outcomes": [] if failed else ["fresh_vision_path_outcome"],
"vision_closeout": None if failed else complete["vision_closeout"],
"tool_call_count": 40 if failed else 16,
"tool_call_limit": 40,
"tool_call_receipts": [
{"error_code": "shell_nonzero", "raw_command": "never-publish-this-command"},
{"error_code": "private/path", "raw_command": "never-publish-this-command"},
],
}

def unused(_: Any) -> dict[str, Any]:
return {} # Only the replan actor is invoked for this scenario.
scenario, actor_error, _ = _scenario_result(
spec, {}, expected=expected, qualification_id="diagnostic-test",
turn_actor=unused, onboarding_actor=unused,
selected_todo_actor=unused, replan_semantic_action_actor=replan_actor,
scoped_gate_successor_actor=unused, capability_monitor_repair_actor=unused,
terminal_settlement_actor=unused,
)
assert actor_error is False
assert scenario["status"] == "failed"
assert scenario["repeat_diagnostics"] == [
{"repeat": 1, "actor_passed": False, "failure_code": "tool_call_budget_exhausted",
"tool_call_count": 40, "tool_call_limit": 40,
"tool_error_counts": {"shell_nonzero": 1, "unclassified": 1},
"tool_errors_truncated": False},
{"repeat": 2, "actor_passed": True, "failure_code": None,
"tool_call_count": 16, "tool_call_limit": 40,
"tool_error_counts": {"shell_nonzero": 1, "unclassified": 1},
"tool_errors_truncated": False},
]
assert "never-publish-this-command" not in json.dumps(scenario)
assert "private/path" not in json.dumps(scenario)


def _capability_monitor_repair_actor(_: str) -> dict[str, Any]:
return _passing_tool_receipt(
"capability_monitor_repair_tool_behavior_receipt_v1",
Expand Down
26 changes: 26 additions & 0 deletions tests/control_plane/test_effect_runtime_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,32 @@ def probe(pid: object) -> bool:
assert calls == [1234]


def test_default_runtime_request_budget_covers_typed_projection_calls(
tmp_path: Path,
monkeypatch,
) -> None:
info = {"token": "fixture"}
observed: list[tuple[str, float]] = []
monkeypatch.setattr(effect_runtime, "_runtime_fingerprint_for_request", lambda: "fixture")
monkeypatch.setattr(effect_runtime, "_runtime_info_path", lambda _: tmp_path / "runtime.json")
monkeypatch.setattr(effect_runtime, "_read_info", lambda *_args, **_kwargs: info)

def respond(_info: object, **kwargs: object) -> dict[str, object]:
observed.append((str(kwargs["method"]), float(kwargs["timeout"])))
return {"result": {"ok": True}}

monkeypatch.setattr(effect_runtime, "_request_with_info", respond)
assert effect_runtime.effect_runtime_result("todo.succession.project", {}) == {"ok": True}
assert effect_runtime.effect_runtime_request("scheduler.monitor_target.select", {}) == {
"result": {"ok": True}
}
assert observed == [
("todo.succession.project", effect_runtime.DEFAULT_REQUEST_TIMEOUT_SECONDS),
("scheduler.monitor_target.select", effect_runtime.DEFAULT_REQUEST_TIMEOUT_SECONDS),
]
assert effect_runtime.DEFAULT_REQUEST_TIMEOUT_SECONDS == 10.0


def test_managed_runtime_is_reused_and_restart_safe_for_typed_write(
tmp_path: Path,
monkeypatch,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ def refresh(request: Mapping[str, Any]) -> ScriptedExecToolAction:
assert not (result.get("vision_closeout") or {}).get("settled")


@pytest.mark.parametrize("extra_reads,passed", [(27, True), (28, False)])
@pytest.mark.parametrize("extra_reads,passed", [(35, True), (36, False)])
def test_budget_boundary_still_requires_the_final_spend(tmp_path: Path, extra_reads: int, passed: bool) -> None:
fixture = _build_fixture(tmp_path / "oracle", required_vision=True)
result = _qualify(tmp_path, [
Expand All @@ -189,7 +189,7 @@ def test_budget_boundary_still_requires_the_final_spend(tmp_path: Path, extra_re
vision_patch_action, projected_refresh, projected_spend,
])
assert result["qualification_passed"] is passed
assert result["tool_call_count"] == result["tool_call_limit"] == 32
assert result["tool_call_count"] == result["tool_call_limit"] == 40
assert result["vision_closeout"]["settled"] is passed


Expand Down
Loading