From 4f2085bff5fbd4350ba0ec94e790e4cbe0ab9488 Mon Sep 17 00:00:00 2001 From: wchwawa Date: Wed, 2 Sep 2026 07:42:43 +1000 Subject: [PATCH 01/12] feat(authority): add local post-commit shadow Signed-off-by: wchwawa --- loopx/cli_commands/registry_admin.py | 6 + .../cli_commands/registry_admin_configure.py | 16 + loopx/configuration_catalog.py | 51 ++ loopx/configure_goal.py | 56 ++ .../coordination/local_authority_shadow.ts | 313 +++++++++++ .../local_authority_shadow_adapter.py | 358 ++++++++++++ .../control_plane/effect_runtime_handlers.ts | 2 + loopx/control_plane/todos/handoff_mode.py | 12 + .../work_items/task_lease_acquire_adapter.py | 65 +++ loopx/state_migration.py | 134 ++++- loopx/todo_followups.py | 18 +- loopx/todos.py | 78 ++- pyproject.toml | 1 + .../test_local_authority_shadow_config.py | 159 ++++++ .../test_local_authority_shadow_runtime.py | 508 ++++++++++++++++++ .../test_state_migration_authority_shadow.py | 262 +++++++++ .../local_authority_shadow.test.ts | 197 +++++++ tsconfig.control-plane.json | 2 + 18 files changed, 2227 insertions(+), 11 deletions(-) create mode 100644 loopx/control_plane/coordination/local_authority_shadow.ts create mode 100644 loopx/control_plane/coordination/local_authority_shadow_adapter.py create mode 100644 tests/control_plane/test_local_authority_shadow_config.py create mode 100644 tests/control_plane/test_local_authority_shadow_runtime.py create mode 100644 tests/control_plane/test_state_migration_authority_shadow.py create mode 100644 tests/control_plane_ts/local_authority_shadow.test.ts diff --git a/loopx/cli_commands/registry_admin.py b/loopx/cli_commands/registry_admin.py index 1a8d96490c..2419449eb3 100644 --- a/loopx/cli_commands/registry_admin.py +++ b/loopx/cli_commands/registry_admin.py @@ -494,6 +494,12 @@ def handle_registry_admin_command( write_scope=args.write_scope, replace_write_scope=bool(args.replace_write_scope), clear_write_scope=bool(args.clear_write_scope), + local_authority_shadow_file=bool( + args.local_authority_shadow_file + ), + clear_local_authority_shadow=bool( + args.clear_local_authority_shadow + ), waiting_on=args.waiting_on, clear_waiting_on=bool(args.clear_waiting_on), boundary_authority_scopes=args.boundary_authority_scope, diff --git a/loopx/cli_commands/registry_admin_configure.py b/loopx/cli_commands/registry_admin_configure.py index c9b904920d..61e4482b65 100644 --- a/loopx/cli_commands/registry_admin_configure.py +++ b/loopx/cli_commands/registry_admin_configure.py @@ -251,6 +251,22 @@ def register_configure_goal_command(subparsers: argparse._SubParsersAction) -> N action="store_true", help="Clear coordination.write_scope.", ) + configure_goal_parser.add_argument( + "--local-authority-shadow-file", + action="store_true", + help=( + "Enable the default-off, one-way post-commit FileAuthorityStore " + "qualification shadow. Legacy local writers remain authoritative." + ), + ) + configure_goal_parser.add_argument( + "--clear-local-authority-shadow", + action="store_true", + help=( + "Disable the local authority shadow. This does not delete retained " + "qualification evidence." + ), + ) configure_goal_parser.add_argument( "--waiting-on", choices=["codex", "user_or_controller", "controller", "external_evidence"], diff --git a/loopx/configuration_catalog.py b/loopx/configuration_catalog.py index 11de69072f..36375a494b 100644 --- a/loopx/configuration_catalog.py +++ b/loopx/configuration_catalog.py @@ -77,6 +77,11 @@ def build_goal_configuration_catalog( if isinstance(feature_summary.get("peer_task_coordination"), Mapping) else {} ) + local_authority_shadow = ( + feature_summary.get("local_authority_shadow") + if isinstance(feature_summary.get("local_authority_shadow"), Mapping) + else {} + ) graph_enable_args = ("--explore-graph-enabled",) harness_enable_args = ( "--explore-harness-enabled", @@ -100,6 +105,52 @@ def build_goal_configuration_catalog( ), }, "features": [ + { + "feature_id": "local_authority_shadow", + "display_name": "Local authority parity shadow", + "availability": "qualification_opt_in", + "default": {"enabled": False}, + "current": { + "enabled": local_authority_shadow.get("enabled") is True, + "mode": local_authority_shadow.get("mode"), + "status": local_authority_shadow.get("status", "disabled"), + }, + "consider_when": ( + "A Goal needs post-commit parity evidence before any local " + "authority-source promotion." + ), + "effect": ( + "Observes committed Todo and task-lease state through the " + "FileAuthorityStore contract." + ), + "does_not": [ + "read the candidate for lifecycle decisions", + "write candidate state back into Markdown or task-lease files", + "promote shared authority or fence legacy writers", + ], + "commands": { + "preview_enable": _configure_command( + goal_id, "--local-authority-shadow-file" + ), + "apply_enable": _configure_command( + goal_id, "--local-authority-shadow-file", execute=True + ), + "preview_disable": _configure_command( + goal_id, "--clear-local-authority-shadow" + ), + "apply_disable": _configure_command( + goal_id, "--clear-local-authority-shadow", execute=True + ), + "verify": [inspect_command], + }, + "documentation": { + "path": "docs/architecture/rfcs/shared-goal-authority-state-provider-v0.md", + "url": ( + "https://github.com/huangruiteng/loopx/blob/main/" + "docs/architecture/rfcs/shared-goal-authority-state-provider-v0.md" + ), + }, + }, { "feature_id": "multi_subagent", "display_name": "Adaptive child capacity", diff --git a/loopx/configure_goal.py b/loopx/configure_goal.py index d73987af3c..c9eb0084bd 100644 --- a/loopx/configure_goal.py +++ b/loopx/configure_goal.py @@ -70,6 +70,9 @@ MULTI_SUBAGENT_FEATURE_CHOICES = ("off", "enabled") DEFAULT_MULTI_SUBAGENT_MAX_CHILDREN = 2 AGENT_MODEL_CHOICES = tuple(model.value for model in AgentRuntimeModel) +LOCAL_AUTHORITY_SHADOW_CONFIG_SCHEMA_VERSION = ( + "loopx_local_authority_shadow_config_v0" +) def _control_plane(goal: dict[str, Any]) -> dict[str, Any]: @@ -233,6 +236,29 @@ def _clean_write_scope(values: list[str] | None) -> list[str] | None: return scopes +def _local_authority_shadow_summary(goal: Mapping[str, Any]) -> dict[str, Any]: + coordination = ( + goal.get("coordination") + if isinstance(goal.get("coordination"), Mapping) + else {} + ) + raw = coordination.get("authority_shadow") + if raw is None: + return {"enabled": False, "mode": None, "status": "disabled"} + valid = bool( + isinstance(raw, Mapping) + and set(raw) == {"schema_version", "mode"} + and raw.get("schema_version") + == LOCAL_AUTHORITY_SHADOW_CONFIG_SCHEMA_VERSION + and raw.get("mode") == "file_one_way" + ) + return { + "enabled": valid, + "mode": raw.get("mode") if isinstance(raw, Mapping) else None, + "status": "enabled" if valid else "invalid", + } + + def _settings_summary(goal: dict[str, Any]) -> dict[str, Any]: quota = goal_quota_config(goal) control_plane = compact_control_plane_policy(goal.get("control_plane")) @@ -260,6 +286,7 @@ def _settings_summary(goal: dict[str, Any]) -> dict[str, Any]: "orchestration": orchestration, "waiting_on": goal.get("waiting_on"), "write_scope": _clean_write_scope(coordination.get("write_scope") or []) or [], + "local_authority_shadow": _local_authority_shadow_summary(goal), "checkpointed_boundary_authority": checkpointed_boundary_authority_summary( coordination ), @@ -461,6 +488,8 @@ def configure_goal( write_scope: list[str] | None = None, replace_write_scope: bool = False, clear_write_scope: bool = False, + local_authority_shadow_file: bool = False, + clear_local_authority_shadow: bool = False, waiting_on: str | None = None, clear_waiting_on: bool = False, boundary_authority_scopes: list[str] | None = None, @@ -535,6 +564,11 @@ def configure_goal( raise ValueError( "--clear-write-scope cannot be combined with --replace-write-scope" ) + if local_authority_shadow_file and clear_local_authority_shadow: + raise ValueError( + "--local-authority-shadow-file cannot be combined with " + "--clear-local-authority-shadow" + ) if clear_waiting_on and waiting_on: raise ValueError("--clear-waiting-on cannot be combined with --waiting-on") adding_boundary_authority = any( @@ -1241,6 +1275,24 @@ def configure_goal( coordination["checkpointed_boundary_authority"] = [*entries, entry] goal["coordination"] = coordination + if local_authority_shadow_file or clear_local_authority_shadow: + coordination = ( + goal.get("coordination") + if isinstance(goal.get("coordination"), dict) + else {} + ) + if clear_local_authority_shadow: + coordination.pop("authority_shadow", None) + else: + coordination["authority_shadow"] = { + "schema_version": LOCAL_AUTHORITY_SHADOW_CONFIG_SCHEMA_VERSION, + "mode": "file_one_way", + } + if coordination: + goal["coordination"] = coordination + else: + goal.pop("coordination", None) + after = _settings_summary(goal) changed_fields = _changed_fields(before, after) if goal != before_goal and not changed_fields: @@ -1279,6 +1331,10 @@ def configure_goal( "peer_task_coordination": deepcopy( after.get("peer_task_coordination") or {"enabled": False} ), + "local_authority_shadow": deepcopy( + after.get("local_authority_shadow") + or {"enabled": False, "mode": None, "status": "disabled"} + ), "lark_event_inbox": _lark_event_inbox_config_summary(goal), "lark_kanban_heartbeat_sync": _lark_kanban_heartbeat_config_summary(goal), "reward_memory": reward_memory_goal_policy_summary(goal), diff --git a/loopx/control_plane/coordination/local_authority_shadow.ts b/loopx/control_plane/coordination/local_authority_shadow.ts new file mode 100644 index 0000000000..802d1666f5 --- /dev/null +++ b/loopx/control_plane/coordination/local_authority_shadow.ts @@ -0,0 +1,313 @@ +import { join } from "node:path"; + +import type { JsonObject } from "../effect_program.ts"; +import { EffectRuntimeRequestError } from "../effect_runtime_errors.ts"; +import { requireJsonObject, requireNonEmptyString } from "../runtime_decode.ts"; +import type { + AuthorityStore, + AuthorityStoreLoadResult, + AuthorityStoreReceiptResult, +} from "./authority_store.ts"; +import { FileAuthorityStore } from "./file_authority_store.ts"; + +export const LOCAL_AUTHORITY_SHADOW_REQUEST_SCHEMA = + "loopx_local_authority_shadow_request_v0"; +export const LOCAL_AUTHORITY_SHADOW_PROJECTION_SCHEMA = + "loopx_local_authority_shadow_projection_v0"; +export const LOCAL_AUTHORITY_SHADOW_EVIDENCE_SCHEMA = + "loopx_local_authority_shadow_evidence_v0"; +export const LOCAL_AUTHORITY_SHADOW_RECEIPT_SCHEMA = + "loopx_local_authority_shadow_receipt_v0"; + +const REQUEST_FIELDS = new Set([ + "schema_version", + "mode", + "runtime_root", + "goal_id", + "operation_id", + "source_operation", + "source_digest", + "source_projection", +]); +export type LocalAuthorityShadowOutcome = + | "advanced" + | "replayed" + | "ambiguous_reconciled" + | "ambiguous_unproved" + | "unavailable" + | "failed" + | "protocol_mismatch" + | "conflict_retry_required"; + +export interface LocalAuthorityShadowEvidence extends JsonObject { + schema_version: typeof LOCAL_AUTHORITY_SHADOW_EVIDENCE_SCHEMA; + outcome: LocalAuthorityShadowOutcome; + reason_code: string | null; + goal_id: string; + operation_id: string; + source_digest: string; + primary_authority: "legacy_local"; + candidate_provider: "file"; + candidate_read_for_decision: false; + provider_to_local_writes: false; + primary_writeback_preserved: true; + store_identity: string | null; + provider_revision: string | null; + cursor: string | null; +} + +interface LocalAuthorityShadowRequest { + mode: "file_one_way"; + runtime_root: string; + goal_id: string; + operation_id: string; + source_operation: string; + source_digest: string; + source_projection: JsonObject; +} + +export interface LocalAuthorityShadowDependencies { + openStore?: (directory: string, goalId: string) => AuthorityStore; +} + +function decodeRequest(value: unknown): LocalAuthorityShadowRequest { + const request = requireJsonObject(value, "local authority shadow request"); + const unexpected = Object.keys(request).filter((field) => !REQUEST_FIELDS.has(field)); + if (unexpected.length > 0) { + throw new EffectRuntimeRequestError( + `Local authority shadow request has unsupported fields: ${unexpected.sort().join(", ")}`, + ); + } + if (request.schema_version !== LOCAL_AUTHORITY_SHADOW_REQUEST_SCHEMA) { + throw new EffectRuntimeRequestError("Local authority shadow request schema mismatch"); + } + if (request.mode !== "file_one_way") { + throw new EffectRuntimeRequestError("Local authority shadow mode must be file_one_way"); + } + const goalId = requireNonEmptyString(request.goal_id, "goal_id"); + if (goalId === "." || goalId === ".." || goalId.includes("/") || goalId.includes("\\")) { + throw new EffectRuntimeRequestError( + "Local authority shadow goal id must be a single path segment", + ); + } + const projection = requireJsonObject(request.source_projection, "source_projection"); + if ( + projection.schema_version !== LOCAL_AUTHORITY_SHADOW_PROJECTION_SCHEMA || + projection.goal_id !== goalId + ) { + throw new EffectRuntimeRequestError( + "Local authority shadow projection schema or goal identity mismatch", + ); + } + const sourceDigest = requireNonEmptyString(request.source_digest, "source_digest"); + if (!/^sha256:[a-f0-9]{64}$/u.test(sourceDigest)) { + throw new EffectRuntimeRequestError("source_digest must be sha256:<64 lowercase hex>"); + } + return { + mode: "file_one_way", + runtime_root: requireNonEmptyString(request.runtime_root, "runtime_root"), + goal_id: goalId, + operation_id: requireNonEmptyString(request.operation_id, "operation_id"), + source_operation: requireNonEmptyString( + request.source_operation, + "source_operation", + ), + source_digest: sourceDigest, + source_projection: structuredClone(projection), + }; +} + +function evidence( + request: LocalAuthorityShadowRequest, + outcome: LocalAuthorityShadowOutcome, + options: { + reasonCode?: string | null; + storeIdentity?: string | null; + providerRevision?: string | null; + cursor?: string | null; + } = {}, +): LocalAuthorityShadowEvidence { + return { + schema_version: LOCAL_AUTHORITY_SHADOW_EVIDENCE_SCHEMA, + outcome, + reason_code: options.reasonCode ?? null, + goal_id: request.goal_id, + operation_id: request.operation_id, + source_digest: request.source_digest, + primary_authority: "legacy_local", + candidate_provider: "file", + candidate_read_for_decision: false, + provider_to_local_writes: false, + primary_writeback_preserved: true, + store_identity: options.storeIdentity ?? null, + provider_revision: options.providerRevision ?? null, + cursor: options.cursor ?? null, + }; +} + +function readFailureEvidence( + request: LocalAuthorityShadowRequest, + result: Extract, + storeIdentity: string | null, +): LocalAuthorityShadowEvidence { + return evidence(request, result.status, { + reasonCode: result.reason_code, + storeIdentity, + }); +} + +function receiptMatches( + request: LocalAuthorityShadowRequest, + result: Extract, +): boolean { + return result.receipts.some((raw) => { + const receipt = raw as Record; + return receipt.schema_version === LOCAL_AUTHORITY_SHADOW_RECEIPT_SCHEMA && + receipt.operation_id === request.operation_id && + receipt.source_digest === request.source_digest && + receipt.primary_authority === "legacy_local" && + receipt.provider_to_local_writes === false; + }); +} + +async function reconcileReceipt( + store: AuthorityStore, + request: LocalAuthorityShadowRequest, + storeIdentity: string, + reconciledOutcome: "replayed" | "ambiguous_reconciled", +): Promise { + const result = await store.readReceipt(request.operation_id); + if (result.status === "found" && receiptMatches(request, result)) { + return evidence(request, reconciledOutcome, { + storeIdentity, + providerRevision: result.provider_revision, + cursor: result.cursor, + }); + } + if (result.status === "unavailable") { + return evidence(request, "unavailable", { + reasonCode: result.reason_code, + storeIdentity, + }); + } + if (result.status === "failed") { + return evidence(request, "failed", { + reasonCode: result.reason_code, + storeIdentity, + }); + } + return evidence( + request, + reconciledOutcome === "ambiguous_reconciled" + ? "ambiguous_unproved" + : "protocol_mismatch", + { + reasonCode: result.status === "missing" + ? "operation_receipt_missing" + : "operation_receipt_mismatch", + storeIdentity, + }, + ); +} + +/** + * Record a post-commit observation in a candidate AuthorityStore. + * + * The legacy local writers remain the only decision authority. This function + * receives a completed source projection and has no route back to those files. + */ +export async function recordLocalAuthorityShadow( + value: unknown, + dependencies: LocalAuthorityShadowDependencies = {}, +): Promise { + const request = decodeRequest(value); + let store: AuthorityStore; + try { + const providerDirectory = join( + request.runtime_root, + "authority-shadow", + "file", + request.goal_id, + ); + store = (dependencies.openStore ?? ((directory, goalId) => + new FileAuthorityStore(directory, goalId)))( + providerDirectory, + request.goal_id, + ); + } catch { + return evidence(request, "unavailable", { + reasonCode: "provider_construction_failed", + }); + } + + try { + const identity = await store.storeIdentity(); + if (identity.status !== "available") { + return evidence(request, identity.status, { reasonCode: identity.reason_code }); + } + const storeIdentity = identity.store_identity; + const receipt = { + schema_version: LOCAL_AUTHORITY_SHADOW_RECEIPT_SCHEMA, + operation_id: request.operation_id, + source_digest: request.source_digest, + source_operation: request.source_operation, + primary_authority: "legacy_local", + candidate_read_for_decision: false, + provider_to_local_writes: false, + }; + const event = { + schema_version: "loopx_local_authority_shadow_event_v0", + kind: "source_observed", + operation_id: request.operation_id, + source_operation: request.source_operation, + source_digest: request.source_digest, + }; + + const loaded = await store.loadAuthority(); + if (loaded.status === "unavailable" || loaded.status === "failed") { + return readFailureEvidence(request, loaded, storeIdentity); + } + const committed = await store.commitAuthority({ + expected_provider_revision: + loaded.status === "loaded" ? loaded.provider_revision : null, + operation_id: request.operation_id, + events: [event], + next_projection: request.source_projection, + receipts: [receipt], + }); + if (committed.status === "applied") { + return evidence(request, "advanced", { + storeIdentity, + providerRevision: committed.provider_revision, + cursor: committed.cursor, + }); + } + if (committed.status === "ambiguous") { + return await reconcileReceipt( + store, + request, + storeIdentity, + "ambiguous_reconciled", + ); + } + if (committed.status === "failed") { + return evidence(request, "failed", { + reasonCode: committed.reason_code, + storeIdentity, + }); + } + if (committed.conflict_kind === "operation_id_exists") { + return await reconcileReceipt(store, request, storeIdentity, "replayed"); + } + return evidence(request, "conflict_retry_required", { + reasonCode: "provider_revision_mismatch", + storeIdentity, + providerRevision: committed.current_provider_revision, + cursor: committed.current_cursor, + }); + } catch { + return evidence(request, "unavailable", { + reasonCode: "provider_call_failed", + }); + } +} diff --git a/loopx/control_plane/coordination/local_authority_shadow_adapter.py b/loopx/control_plane/coordination/local_authority_shadow_adapter.py new file mode 100644 index 0000000000..c977cc7bf5 --- /dev/null +++ b/loopx/control_plane/coordination/local_authority_shadow_adapter.py @@ -0,0 +1,358 @@ +"""Post-commit bridge from legacy local authority into a file shadow. + +The adapter deliberately owns no lifecycle decision. It is entered only after +the existing Markdown or task-lease writer has succeeded, projects public-safe +facts, and asks the TypeScript authority-store boundary to retain an +observation. Missing configuration is a zero-effect fast path. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from ...file_lock import LockAcquireTimeoutError, exclusive_file_lock +from ...history import load_registry +from ...paths import resolve_runtime_root +from ...registry import find_registry_goal +from ..effect_runtime import effect_runtime_result + + +LOCAL_AUTHORITY_SHADOW_CONFIG_SCHEMA = "loopx_local_authority_shadow_config_v0" +LOCAL_AUTHORITY_SHADOW_REQUEST_SCHEMA = "loopx_local_authority_shadow_request_v0" +LOCAL_AUTHORITY_SHADOW_PROJECTION_SCHEMA = "loopx_local_authority_shadow_projection_v0" +LOCAL_AUTHORITY_SHADOW_EVIDENCE_SCHEMA = "loopx_local_authority_shadow_evidence_v0" +_CONFIG_FIELDS = {"schema_version", "mode"} +_PROJECTION_ATTEMPTS = 3 +_CONFLICT_RETRY_ATTEMPTS = 3 +_EVIDENCE_OUTCOMES = { + "advanced", + "replayed", + "ambiguous_reconciled", + "ambiguous_unproved", + "unavailable", + "failed", + "protocol_mismatch", + "conflict_retry_required", +} +_TODO_FIELDS = ( + "todo_id", + "role", + "status", + "claimed_by", + "bound_agent", + "goal_bound", + "blocks_agent", + "excluded_agents", + "global_gate", + "task_class", + "action_kind", + "required_write_scopes", + "required_capabilities", + "continuation_policy", + "successor_todo_ids", + "no_followup", + "completion_continuation", +) +_LEASE_FIELDS = ( + "todo_id", + "owner", + "idempotency_key", + "write_scopes", + "version", + "lease_epoch", + "acquired_at", + "updated_at", + "expires_at", + "released_at", + "status", +) + + +def _base_evidence( + *, + goal_id: str, + outcome: str, + reason_code: str, +) -> dict[str, Any]: + return { + "schema_version": LOCAL_AUTHORITY_SHADOW_EVIDENCE_SCHEMA, + "outcome": outcome, + "reason_code": reason_code, + "goal_id": goal_id, + "operation_id": None, + "source_digest": None, + "primary_authority": "legacy_local", + "candidate_provider": "file", + "candidate_read_for_decision": False, + "provider_to_local_writes": False, + "primary_writeback_preserved": True, + "store_identity": None, + "provider_revision": None, + "cursor": None, + } + + +def _shadow_config(registry: dict[str, Any], goal_id: str) -> dict[str, str] | None: + goal = find_registry_goal(registry, goal_id) + coordination = goal.get("coordination") if isinstance(goal, dict) else None + if not isinstance(coordination, dict) or "authority_shadow" not in coordination: + return None + raw = coordination.get("authority_shadow") + if ( + not isinstance(raw, dict) + or set(raw) != _CONFIG_FIELDS + or raw.get("schema_version") != LOCAL_AUTHORITY_SHADOW_CONFIG_SCHEMA + or raw.get("mode") != "file_one_way" + ): + raise ValueError("authority_shadow must be a closed file_one_way config") + return {"mode": "file_one_way"} + + +def _canonical(value: object) -> bytes: + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +def _compact_todo(raw: object) -> dict[str, Any] | None: + if not isinstance(raw, dict): + return None + todo_id = str(raw.get("todo_id") or "").strip() + if not todo_id: + return None + compact = {field: raw[field] for field in _TODO_FIELDS if field in raw} + compact["todo_id"] = todo_id + if "status" not in compact and isinstance(raw.get("done"), bool): + compact["status"] = "done" if raw["done"] else "open" + return json.loads(_canonical(compact)) + + +def _compact_lease(path: Path, *, goal_id: str) -> dict[str, Any]: + raw = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(raw, dict): + raise ValueError("task lease must contain an object") + if raw.get("goal_id") != goal_id or raw.get("todo_id") != path.stem: + raise ValueError("task lease identity does not match its shadow source") + return json.loads( + _canonical({field: raw[field] for field in _LEASE_FIELDS if field in raw}) + ) + + +def _source_projection( + *, + registry_path: Path, + runtime_root: Path, + goal_id: str, +) -> dict[str, Any]: + from ...control_plane.todos.handoff_mode import goal_handoff_mode_for_goal + from ...todos import list_goal_todos + + todo_payload = list_goal_todos( + registry_path=registry_path, + goal_id=goal_id, + runtime_root_arg=str(runtime_root), + ) + todos = [ + compact + for raw in todo_payload.get("todos") or [] + if (compact := _compact_todo(raw)) is not None + ] + todos.sort(key=lambda item: str(item["todo_id"])) + lease_dir = runtime_root / "goals" / goal_id / "task-leases" + leases = ( + [ + _compact_lease(path, goal_id=goal_id) + for path in sorted(lease_dir.glob("*.json")) + ] + if lease_dir.exists() + else [] + ) + projection = { + "schema_version": LOCAL_AUTHORITY_SHADOW_PROJECTION_SCHEMA, + "goal_id": goal_id, + "handoff_mode": goal_handoff_mode_for_goal( + registry_path=registry_path, + goal_id=goal_id, + ), + "todos": todos, + "leases": leases, + } + return json.loads(_canonical(projection)) + + +def _stable_projection( + *, + registry_path: Path, + runtime_root: Path, + goal_id: str, +) -> dict[str, Any]: + previous = _source_projection( + registry_path=registry_path, + runtime_root=runtime_root, + goal_id=goal_id, + ) + for _attempt in range(_PROJECTION_ATTEMPTS): + current = _source_projection( + registry_path=registry_path, + runtime_root=runtime_root, + goal_id=goal_id, + ) + if current == previous: + return current + previous = current + raise RuntimeError("local authority sources did not stabilize for shadowing") + + +def _valid_evidence( + result: object, + *, + goal_id: str, + operation_id: str, + source_digest: str, +) -> bool: + if not isinstance(result, dict): + return False + return ( + result.get("schema_version") == LOCAL_AUTHORITY_SHADOW_EVIDENCE_SCHEMA + and result.get("outcome") in _EVIDENCE_OUTCOMES + and result.get("goal_id") == goal_id + and result.get("operation_id") == operation_id + and result.get("source_digest") == source_digest + and result.get("primary_authority") == "legacy_local" + and result.get("candidate_provider") == "file" + and result.get("candidate_read_for_decision") is False + and result.get("provider_to_local_writes") is False + and result.get("primary_writeback_preserved") is True + and ( + result.get("reason_code") is None + or isinstance(result.get("reason_code"), str) + ) + ) + + +def observe_local_authority_commit( + *, + registry_path: Path, + runtime_root: Path | None, + goal_id: str, + source_operation: str, +) -> dict[str, Any] | None: + """Record one local post-commit observation without changing its verdict.""" + + if not goal_id or goal_id in {".", ".."} or "/" in goal_id or "\\" in goal_id: + return _base_evidence( + goal_id=goal_id, + outcome="failed", + reason_code="invalid_shadow_goal_id", + ) + try: + registry = load_registry(registry_path) + config = _shadow_config(registry, goal_id) + except Exception: + return _base_evidence( + goal_id=goal_id, + outcome="failed", + reason_code="invalid_shadow_config", + ) + if config is None: + return None + + try: + if runtime_root is None: + runtime_root = resolve_runtime_root( + registry, + None, + registry_path=registry_path, + ) + # Candidate-provider bytes live outside the legacy per-goal runtime + # tree. State migration may copy that tree, but it must never copy a + # store identity or revision and accidentally create a second lineage. + shadow_root = runtime_root / "authority-shadow" / "file" / goal_id + with exclusive_file_lock( + shadow_root / "observation", + timeout_seconds=1.0, + operation="local_authority_shadow_observe", + ): + result: dict[str, Any] | None = None + for _attempt in range(_CONFLICT_RETRY_ATTEMPTS): + projection = _stable_projection( + registry_path=registry_path, + runtime_root=runtime_root, + goal_id=goal_id, + ) + source_digest = ( + "sha256:" + hashlib.sha256(_canonical(projection)).hexdigest() + ) + operation_id = ( + "local-shadow:" + + hashlib.sha256( + _canonical( + { + "goal_id": goal_id, + "source_operation": source_operation, + "source_digest": source_digest, + } + ) + ).hexdigest() + ) + raw_result = effect_runtime_result( + "coordination.local_authority_shadow.record", + { + "schema_version": LOCAL_AUTHORITY_SHADOW_REQUEST_SCHEMA, + "mode": config["mode"], + "runtime_root": str(runtime_root), + "goal_id": goal_id, + "operation_id": operation_id, + "source_operation": source_operation, + "source_digest": source_digest, + "source_projection": projection, + }, + timeout=15.0, + ) + if not _valid_evidence( + raw_result, + goal_id=goal_id, + operation_id=operation_id, + source_digest=source_digest, + ): + return _base_evidence( + goal_id=goal_id, + outcome="failed", + reason_code="shadow_evidence_invalid", + ) + result = dict(raw_result) + if result["outcome"] != "conflict_retry_required": + return result + if result is not None: + return result + except LockAcquireTimeoutError: + return _base_evidence( + goal_id=goal_id, + outcome="unavailable", + reason_code="shadow_observation_lock_timeout", + ) + except Exception: + return _base_evidence( + goal_id=goal_id, + outcome="failed", + reason_code="shadow_observation_failed", + ) + return _base_evidence( + goal_id=goal_id, + outcome="failed", + reason_code="shadow_observation_failed", + ) + + +__all__ = [ + "LOCAL_AUTHORITY_SHADOW_CONFIG_SCHEMA", + "LOCAL_AUTHORITY_SHADOW_EVIDENCE_SCHEMA", + "observe_local_authority_commit", +] diff --git a/loopx/control_plane/effect_runtime_handlers.ts b/loopx/control_plane/effect_runtime_handlers.ts index a782d199bf..4b1493c8df 100644 --- a/loopx/control_plane/effect_runtime_handlers.ts +++ b/loopx/control_plane/effect_runtime_handlers.ts @@ -96,6 +96,7 @@ import { executeTaskLeaseAcquire, } from "./work_items/task_lease_acquire.ts"; import { executeTaskLeaseLifecycle } from "./work_items/task_lease_lifecycle.ts"; +import { recordLocalAuthorityShadow } from "./coordination/local_authority_shadow.ts"; import { evaluateTaskLeaseLifecycleDecision } from "./work_items/task_lease_lifecycle_decision.ts"; import { projectTodoPlanningInventory, @@ -355,6 +356,7 @@ export function createEffectRuntimeHandlers( ["task_lease.lifecycle.native", executeTaskLeaseLifecycle], ["task_lease.write_scopes.overlap", evaluateTaskLeaseWriteScopesOverlap], ["quota.monitor_poll.commit", evaluateQuotaMonitorPollCommit], + ["coordination.local_authority_shadow.record", recordLocalAuthorityShadow], [ "effect.program_from_ordered_steps", (params) => effectProgramFromOrderedSteps( diff --git a/loopx/control_plane/todos/handoff_mode.py b/loopx/control_plane/todos/handoff_mode.py index f0129b188f..7ad2e83e73 100644 --- a/loopx/control_plane/todos/handoff_mode.py +++ b/loopx/control_plane/todos/handoff_mode.py @@ -509,4 +509,16 @@ def set_goal_handoff_mode( new_text = "\n".join(lines) + ("\n" if original.endswith("\n") else "") resolved_state_file.write_text(new_text, encoding="utf-8") payload["changed"] = True + from ..coordination.local_authority_shadow_adapter import ( + observe_local_authority_commit, + ) + + evidence = observe_local_authority_commit( + registry_path=registry_path, + runtime_root=runtime_root_from_registry(registry_path, None), + goal_id=goal_id, + source_operation=f"handoff_mode_set:{previous}:{requested}", + ) + if evidence is not None: + payload["authority_shadow"] = evidence return payload diff --git a/loopx/control_plane/work_items/task_lease_acquire_adapter.py b/loopx/control_plane/work_items/task_lease_acquire_adapter.py index e96e335958..da6cc83c29 100644 --- a/loopx/control_plane/work_items/task_lease_acquire_adapter.py +++ b/loopx/control_plane/work_items/task_lease_acquire_adapter.py @@ -24,6 +24,44 @@ TASK_LEASE_AUTHORITY_SNAPSHOT_ATTEMPTS = 3 +def _attach_local_authority_shadow( + result: dict[str, Any], + *, + registry_path: Path | None, + runtime_root: Path, + goal_id: str, + todo_id: str, + operation: str, +) -> dict[str, Any]: + """Observe a committed public lease mutation without changing its verdict.""" + + if registry_path is None: + return result + lease = result.get("lease") if isinstance(result.get("lease"), dict) else {} + source_operation = ":".join( + ( + f"task_lease_{operation}", + str(todo_id), + str(lease.get("version") or "none"), + str(lease.get("lease_epoch") or "none"), + str(lease.get("updated_at") or lease.get("released_at") or "unknown"), + ) + ) + from ..coordination.local_authority_shadow_adapter import ( + observe_local_authority_commit, + ) + + evidence = observe_local_authority_commit( + registry_path=registry_path, + runtime_root=runtime_root, + goal_id=str(goal_id), + source_operation=source_operation, + ) + if evidence is not None: + result["authority_shadow"] = evidence + return result + + def _authority_source_receipt(source_id: str, path: Path) -> dict[str, Any]: resolved = path.expanduser().resolve(strict=False) try: @@ -338,6 +376,15 @@ def execute_native_task_lease_acquire( authority.get("handoff_mode") or HANDOFF_MODE_LEGACY ) result.pop("settlement", None) + if result.get("ok") is True and result.get("acquired") is True: + result = _attach_local_authority_shadow( + result, + registry_path=registry_path, + runtime_root=runtime_root, + goal_id=str(goal_id), + todo_id=str(todo_id), + operation="acquire", + ) return result raise RuntimeError("native task-lease acquire exhausted source-CAS retries") @@ -592,6 +639,24 @@ def execute_native_task_lease_lifecycle( result["handoff_mode"] = authority["handoff_mode"] if _legacy_provider_projection: result.pop("settlement", None) + committed_mutation = ( + normalized_operation == "renew" and result.get("renewed") is True + ) or ( + normalized_operation == "transfer" + and result.get("transferred") is True + ) or ( + normalized_operation == "release" + and result.get("released") is True + ) + if committed_mutation and result.get("idempotent") is not True: + result = _attach_local_authority_shadow( + result, + registry_path=registry_path, + runtime_root=runtime_root, + goal_id=str(goal_id), + todo_id=str(todo_id), + operation=normalized_operation, + ) # lock_token is an internal bridge value. Callers that need a held # fence read it from the nested native payload before redacting it. return result diff --git a/loopx/state_migration.py b/loopx/state_migration.py index 59cf867058..5e9330b549 100644 --- a/loopx/state_migration.py +++ b/loopx/state_migration.py @@ -2,19 +2,29 @@ import copy import json -import os import shutil from datetime import datetime, timezone from pathlib import Path from typing import Any from .global_registry import write_json -from .paths import DEFAULT_RUNTIME_ROOT from .registry import registry_goals LEGACY_RUNTIME_ROOT = Path.home() / ".codex" / "goal-harness" LEGACY_GLOBAL_REGISTRY = LEGACY_RUNTIME_ROOT / "registry.global.json" +AUTHORITY_SHADOW_CONFIG_SCHEMA = "loopx_local_authority_shadow_config_v0" +MIGRATION_SHADOW_SEED_EVIDENCE_SCHEMA = "loopx_state_migration_shadow_seed_evidence_v0" +_SHADOW_EVIDENCE_OUTCOMES = { + "advanced", + "replayed", + "ambiguous_reconciled", + "ambiguous_unproved", + "unavailable", + "failed", + "protocol_mismatch", + "conflict_retry_required", +} def now_local() -> str: @@ -131,7 +141,7 @@ def copy_active_state_files( for source_goal, target_goal in pairs: source_path = resolve_goal_state(source_goal.get("repo"), source_goal.get("state_file")) target_path = resolve_goal_state(target_goal.get("repo"), target_goal.get("state_file")) - row = { + row: dict[str, Any] = { "goal_id": target_goal.get("id"), "source": str(source_path) if source_path else None, "target": str(target_path) if target_path else None, @@ -171,7 +181,7 @@ def copy_runtime_goal_dirs( new_goal_id = goal_id_map.get(old_goal_id, old_goal_id) source_dir = legacy_runtime_root.expanduser() / "goals" / old_goal_id target_dir = target_runtime_root.expanduser() / "goals" / new_goal_id - row = { + row: dict[str, Any] = { "source": str(source_dir), "target": str(target_dir), "goal_id": new_goal_id, @@ -206,6 +216,104 @@ def copy_runtime_goal_dirs( return results +def _uses_file_authority_shadow(goal: dict[str, Any]) -> bool: + coordination = goal.get("coordination") + if not isinstance(coordination, dict): + return False + config = coordination.get("authority_shadow") + return ( + isinstance(config, dict) + and config.get("schema_version") == AUTHORITY_SHADOW_CONFIG_SCHEMA + and config.get("mode") == "file_one_way" + ) + + +def _shadow_seed_evidence( + *, + goal_id: str, + attempted: bool, + outcome: str, + reason_code: str | None = None, +) -> dict[str, Any]: + return { + "schema_version": MIGRATION_SHADOW_SEED_EVIDENCE_SCHEMA, + "goal_id": goal_id, + "attempted": attempted, + "outcome": outcome, + "reason_code": reason_code, + } + + +def _shadow_seed_result(*, goal_id: str, result: object) -> dict[str, Any]: + outcome = result.get("outcome") if isinstance(result, dict) else None + if outcome not in _SHADOW_EVIDENCE_OUTCOMES: + return _shadow_seed_evidence( + goal_id=goal_id, + attempted=True, + outcome="failed", + reason_code="post_migration_shadow_seed_invalid_evidence", + ) + + reason_code = ( + None + if outcome in {"advanced", "replayed", "ambiguous_reconciled"} + else f"post_migration_shadow_seed_{outcome}" + ) + return _shadow_seed_evidence( + goal_id=goal_id, + attempted=True, + outcome=str(outcome), + reason_code=reason_code, + ) + + +def seed_migrated_authority_shadows( + *, + goals: list[dict[str, Any]], + target_registry_path: Path, + target_runtime_root: Path, + execute: bool, +) -> list[dict[str, Any]]: + """Plan or seed fresh candidate lineage from migrated local authority.""" + + results: list[dict[str, Any]] = [] + for goal in goals: + if not _uses_file_authority_shadow(goal): + continue + goal_id = str(goal.get("id") or "") + if not execute: + results.append( + _shadow_seed_evidence( + goal_id=goal_id, + attempted=False, + outcome="planned", + ) + ) + continue + try: + from .control_plane.coordination.local_authority_shadow_adapter import ( + observe_local_authority_commit, + ) + + result = observe_local_authority_commit( + registry_path=target_registry_path, + runtime_root=target_runtime_root, + goal_id=goal_id, + source_operation="state_migration_seed", + ) + results.append(_shadow_seed_result(goal_id=goal_id, result=result)) + except Exception: + results.append( + _shadow_seed_evidence( + goal_id=goal_id, + attempted=True, + outcome="failed", + reason_code="post_migration_shadow_seed_failed", + ) + ) + return results + + def migrate_legacy_state( *, legacy_registry_path: Path, @@ -282,6 +390,13 @@ def migrate_legacy_state( if execute: write_json(target_registry_path, target_payload) + authority_shadow_seeds = seed_migrated_authority_shadows( + goals=incoming_goals, + target_registry_path=target_registry_path, + target_runtime_root=target_runtime_root, + execute=execute, + ) + return { "ok": True, "schema_version": "loopx_state_migration_v0", @@ -299,6 +414,7 @@ def migrate_legacy_state( "project_registry_goal_count": len(target_payload.get("goals", [])), "active_state": active_state_results, "runtime_goals": runtime_results, + "authority_shadow_seeds": authority_shadow_seeds, } @@ -338,6 +454,16 @@ def render_state_migration_markdown(payload: dict[str, Any]) -> str: ) if row.get("skipped_reason"): lines.append(f" - skipped_reason: `{row.get('skipped_reason')}`") + authority_shadow_seeds = payload.get("authority_shadow_seeds") or [] + if authority_shadow_seeds: + lines.extend(["", "## Authority Shadow Seeds"]) + for row in authority_shadow_seeds: + lines.append( + f"- `{row.get('goal_id')}` outcome=`{row.get('outcome')}` " + f"attempted=`{row.get('attempted')}`" + ) + if row.get("reason_code"): + lines.append(f" - reason_code: `{row.get('reason_code')}`") global_sync = payload.get("global_sync") if isinstance(global_sync, dict): lines.extend(["", "## Global Sync"]) diff --git a/loopx/todo_followups.py b/loopx/todo_followups.py index 45024052ae..963cba7878 100644 --- a/loopx/todo_followups.py +++ b/loopx/todo_followups.py @@ -149,7 +149,7 @@ def capture_followup_todos( if not dry_run: resolved_state_file.write_text(new_text, encoding="utf-8") - return { + result = { "ok": True, "dry_run": dry_run, "changed": changed, @@ -166,3 +166,19 @@ def capture_followup_todos( "items": items, "updated_at": updated_at if changed else None, } + if changed and not dry_run: + from .control_plane.coordination.local_authority_shadow_adapter import ( + observe_local_authority_commit, + ) + + shadow = observe_local_authority_commit( + registry_path=registry_path, + runtime_root=None, + goal_id=goal_id, + source_operation=( + f"todo_capture_followups:{recorded_count}:{updated_at}" + ), + ) + if shadow is not None: + result["authority_shadow"] = shadow + return result diff --git a/loopx/todos.py b/loopx/todos.py index baac692e9f..ed5ef7b5a8 100644 --- a/loopx/todos.py +++ b/loopx/todos.py @@ -136,6 +136,38 @@ ARCHIVE_COMPLETED_DEFAULT_MAX_ACTIVE_DONE = max(0, MAX_ACTIVE_DONE_TODOS_BEFORE_ARCHIVE - 2) +def _attach_local_authority_shadow( + payload: dict[str, Any], + *, + registry_path: Path, + goal_id: str, + write_class: str, +) -> dict[str, Any]: + """Observe a completed local mutation without changing its result.""" + + changed = any( + payload.get(field) + for field in ("changed", "added", "metadata_updated", "completed", "superseded") + ) + if payload.get("dry_run") or not changed: + return payload + from .control_plane.coordination.local_authority_shadow_adapter import ( + observe_local_authority_commit, + ) + + todo_id = str(payload.get("todo_id") or "none") + updated_at = str(payload.get("updated_at") or "unknown") + evidence = observe_local_authority_commit( + registry_path=registry_path, + runtime_root=None, + goal_id=goal_id, + source_operation=f"{write_class}:{todo_id}:{updated_at}", + ) + if evidence is not None: + payload["authority_shadow"] = evidence + return payload + + def require_registered_todo_excluded_agents( *, registry_path: Path, @@ -1164,12 +1196,18 @@ def add_goal_todo( "updated_at": updated_at if changed else None, **handoff_gate, } - return _attach_todo_write_correctness_dry_run_packet( + payload = _attach_todo_write_correctness_dry_run_packet( payload, goal_id=goal_id, write_class="todo_add", state_text=original, ) + return _attach_local_authority_shadow( + payload, + registry_path=registry_path, + goal_id=goal_id, + write_class="todo_add", + ) def resolve_todo_state( @@ -1622,12 +1660,18 @@ def update_goal_todo( payload["external_wait_transition"] = external_wait_transition if monitor_poll_transition is not None: payload["monitor_poll_transition"] = monitor_poll_transition - return _attach_todo_write_correctness_dry_run_packet( + payload = _attach_todo_write_correctness_dry_run_packet( payload, goal_id=goal_id, write_class=write_class, state_text=original, ) + return _attach_local_authority_shadow( + payload, + registry_path=registry_path, + goal_id=goal_id, + write_class=write_class, + ) def complete_goal_todo( @@ -1858,7 +1902,12 @@ def complete_goal_todo( task_lease_fence, committed=bool(event_result.get("changed")) and not dry_run, ) - return event_result + return _attach_local_authority_shadow( + event_result, + registry_path=registry_path, + goal_id=goal_id, + write_class="todo_complete_event_projection", + ) if not isinstance(completion_state, dict): raise RuntimeError( "TypeScript Todo completion transaction did not authorize a commit" @@ -2016,7 +2065,12 @@ def complete_goal_todo( if effective_decision_outcome: result["decision_outcome"] = effective_decision_outcome result["self_merged"] = effective_self_merged - return result + return _attach_local_authority_shadow( + result, + registry_path=registry_path, + goal_id=goal_id, + write_class="todo_complete", + ) def supersede_goal_todo( *, @@ -2211,7 +2265,7 @@ def supersede_goal_todo( if changed and not dry_run: resolved_state_file.write_text(new_text, encoding="utf-8") release_verified_task_lease_fence(task_lease_fence, committed=changed and not dry_run) - return { + result = { "ok": True, "dry_run": dry_run, "superseded": True, @@ -2225,6 +2279,12 @@ def supersede_goal_todo( "project": str(resolved_project) if resolved_project else None, "updated_at": updated_at if changed else None, } + return _attach_local_authority_shadow( + result, + registry_path=registry_path, + goal_id=goal_id, + write_class="todo_supersede", + ) def archive_completed_todos( @@ -2266,7 +2326,7 @@ def archive_completed_todos( if changed and not dry_run: resolved_state_file.write_text(new_text, encoding="utf-8") - return { + result = { "ok": True, "dry_run": dry_run, "goal_id": goal_id, @@ -2275,3 +2335,9 @@ def archive_completed_todos( "project": str(resolved_project) if resolved_project else None, "updated_at": updated_at if changed else None, } + return _attach_local_authority_shadow( + result, + registry_path=registry_path, + goal_id=goal_id, + write_class="todo_archive_completed", + ) diff --git a/pyproject.toml b/pyproject.toml index 5f19990194..c4e00a5e05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,7 @@ include = ["loopx*"] "loopx" = ["web/chat/*.html", "web/chat/assets/*", "web/chat/manifest.webmanifest", "web/chat/pwa/*"] "loopx.control_plane" = ["*.json", "*.ts"] "loopx.control_plane.agents" = ["*.ts"] +"loopx.control_plane.coordination" = ["*.ts"] "loopx.control_plane.goals" = ["*.ts"] "loopx.control_plane.quota" = ["*.ts"] "loopx.control_plane.scheduler" = ["*.ts"] diff --git a/tests/control_plane/test_local_authority_shadow_config.py b/tests/control_plane/test_local_authority_shadow_config.py new file mode 100644 index 0000000000..af217d8381 --- /dev/null +++ b/tests/control_plane/test_local_authority_shadow_config.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from loopx.cli import main +from loopx.configure_goal import configure_goal + + +GOAL_ID = "local-authority-shadow-config" + + +def _registry(tmp_path: Path) -> Path: + state = tmp_path / "ACTIVE_GOAL_STATE.md" + state.write_text( + "---\n" + f"goal_id: {GOAL_ID}\n" + "handoff_mode: hard_lease\n" + "---\n\n" + "## Agent Todo\n\n", + encoding="utf-8", + ) + registry = tmp_path / "registry.json" + registry.write_text( + json.dumps( + { + "goals": [ + { + "id": GOAL_ID, + "repo": str(tmp_path), + "state_file": state.name, + "coordination": { + "agent_model": "peer_v1", + "registered_agents": ["agent-a", "agent-b"], + }, + } + ] + } + ), + encoding="utf-8", + ) + return registry + + +def test_configure_goal_enables_and_clears_closed_file_shadow_config( + tmp_path: Path, +) -> None: + registry = _registry(tmp_path) + + preview = configure_goal( + registry_path=registry, + goal_id=GOAL_ID, + local_authority_shadow_file=True, + execute=False, + ) + + assert preview["changed_fields"] == ["local_authority_shadow"] + assert preview["before"]["local_authority_shadow"] == { + "enabled": False, + "mode": None, + "status": "disabled", + } + assert preview["after"]["local_authority_shadow"] == { + "enabled": True, + "mode": "file_one_way", + "status": "enabled", + } + + applied = configure_goal( + registry_path=registry, + goal_id=GOAL_ID, + local_authority_shadow_file=True, + execute=True, + ) + assert applied["written"] is True + goal = json.loads(registry.read_text(encoding="utf-8"))["goals"][0] + assert goal["coordination"]["authority_shadow"] == { + "schema_version": "loopx_local_authority_shadow_config_v0", + "mode": "file_one_way", + } + assert goal["coordination"]["agent_model"] == "peer_v1" + + repeated = configure_goal( + registry_path=registry, + goal_id=GOAL_ID, + local_authority_shadow_file=True, + execute=True, + ) + assert repeated["written"] is False + + cleared = configure_goal( + registry_path=registry, + goal_id=GOAL_ID, + clear_local_authority_shadow=True, + execute=True, + ) + assert cleared["changed_fields"] == ["local_authority_shadow"] + goal = json.loads(registry.read_text(encoding="utf-8"))["goals"][0] + assert "authority_shadow" not in goal["coordination"] + assert goal["coordination"]["registered_agents"] == ["agent-a", "agent-b"] + + +def test_configure_goal_rejects_enable_and_clear_in_one_operation( + tmp_path: Path, +) -> None: + with pytest.raises(ValueError, match="cannot be combined"): + configure_goal( + registry_path=_registry(tmp_path), + goal_id=GOAL_ID, + local_authority_shadow_file=True, + clear_local_authority_shadow=True, + ) + + +def test_configure_goal_cli_exposes_default_off_shadow_boundary( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + registry = _registry(tmp_path) + + exit_code = main( + [ + "--registry", + str(registry), + "--runtime-root", + str(tmp_path / "runtime"), + "--format", + "json", + "configure-goal", + "--goal-id", + GOAL_ID, + "--local-authority-shadow-file", + ] + ) + + assert exit_code == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["dry_run"] is True + assert payload["written"] is False + assert payload["after"]["local_authority_shadow"]["enabled"] is True + feature = next( + item + for item in payload["configuration_catalog"]["features"] + if item["feature_id"] == "local_authority_shadow" + ) + assert feature["availability"] == "qualification_opt_in" + assert feature["does_not"] == [ + "read the candidate for lifecycle decisions", + "write candidate state back into Markdown or task-lease files", + "promote shared authority or fence legacy writers", + ] + assert feature["commands"]["apply_disable"].endswith( + "--clear-local-authority-shadow --execute" + ) + assert "authority_shadow" not in json.loads( + registry.read_text(encoding="utf-8") + )["goals"][0]["coordination"] diff --git a/tests/control_plane/test_local_authority_shadow_runtime.py b/tests/control_plane/test_local_authority_shadow_runtime.py new file mode 100644 index 0000000000..c054f54921 --- /dev/null +++ b/tests/control_plane/test_local_authority_shadow_runtime.py @@ -0,0 +1,508 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from loopx.control_plane.coordination.local_authority_shadow_adapter import ( + LOCAL_AUTHORITY_SHADOW_EVIDENCE_SCHEMA, + observe_local_authority_commit, +) +from loopx.control_plane.todos.handoff_mode import set_goal_handoff_mode +from loopx.control_plane.work_items.task_lease import ( + acquire_task_lease, + release_task_lease, + renew_task_lease, + transfer_task_lease, +) +from loopx.event_sourced_state import ( + TODO_ADDED, + AppendOnlyStateEventStore, + make_state_event, +) +from loopx.todo_followups import capture_followup_todos +from loopx.todos import ( + add_goal_todo, + archive_completed_todos, + complete_goal_todo, + supersede_goal_todo, + update_goal_todo, +) + + +GOAL_ID = "goal-shadow" +AGENT_A = "agent-a" +AGENT_B = "agent-b" + + +def _fixture(tmp_path: Path, *, enabled: bool) -> tuple[Path, Path, Path]: + repo = tmp_path / "repo" + repo.mkdir() + state = repo / "ACTIVE_GOAL_STATE.md" + state.write_text( + "---\n" + f"goal_id: {GOAL_ID}\n" + "handoff_mode: hard_lease\n" + "updated_at: 2026-09-02T00:00:00+00:00\n" + "---\n\n" + "## Agent Todo\n\n", + encoding="utf-8", + ) + runtime_root = tmp_path / "runtime" + coordination: dict[str, object] = { + "agent_model": "peer_v1", + "registered_agents": [AGENT_A, AGENT_B], + } + if enabled: + coordination["authority_shadow"] = { + "schema_version": "loopx_local_authority_shadow_config_v0", + "mode": "file_one_way", + } + registry = tmp_path / "registry.json" + registry.write_text( + json.dumps( + { + "common_runtime_root": str(runtime_root), + "goals": [ + { + "id": GOAL_ID, + "domain": "harness_self_improvement", + "status": "active", + "repo": str(repo), + "state_file": state.name, + "adapter": {"kind": "harness_self_improvement"}, + "coordination": coordination, + } + ], + } + ), + encoding="utf-8", + ) + return registry, state, runtime_root + + +def _add(registry: Path) -> dict: + return add_goal_todo( + registry_path=registry, + goal_id=GOAL_ID, + role="agent", + text="Exercise one-way local authority shadowing.", + task_class="advancement_task", + ) + + +def _shadow_document(runtime_root: Path) -> dict: + paths = list( + ( + runtime_root + / "authority-shadow" + / "file" + / GOAL_ID + ).glob("authority-store-*.json") + ) + assert len(paths) == 1 + return json.loads(paths[0].read_text(encoding="utf-8")) + + +def test_default_off_public_writers_never_call_shadow_runtime( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + registry, _state, runtime_root = _fixture(tmp_path, enabled=False) + calls: list[object] = [] + + def forbidden(*args: object, **kwargs: object) -> object: + calls.append((args, kwargs)) + raise AssertionError("default-off path constructed the shadow runtime") + + monkeypatch.setattr( + "loopx.control_plane.coordination.local_authority_shadow_adapter.effect_runtime_result", + forbidden, + ) + + result = _add(registry) + lease_result = acquire_task_lease( + registry_path=registry, + runtime_root=runtime_root, + goal_id=GOAL_ID, + todo_id=str(result["todo_id"]), + owner=AGENT_A, + idempotency_key="default-off", + ttl_seconds=120, + ) + + assert result["ok"] is True + assert lease_result["ok"] is True + assert "authority_shadow" not in result + assert "authority_shadow" not in lease_result + assert calls == [] + assert not (runtime_root / "authority-shadow").exists() + + +def test_enabled_todo_public_facades_emit_post_commit_evidence(tmp_path: Path) -> None: + registry, _state, runtime_root = _fixture(tmp_path, enabled=True) + + added = _add(registry) + todo_id = str(added["todo_id"]) + acquired = acquire_task_lease( + registry_path=registry, + runtime_root=runtime_root, + goal_id=GOAL_ID, + todo_id=todo_id, + owner=AGENT_A, + idempotency_key="todo-terminal-a", + ttl_seconds=120, + ) + updated = update_goal_todo( + registry_path=registry, + goal_id=GOAL_ID, + todo_id=todo_id, + note="A public-safe update.", + agent_id=AGENT_A, + ) + completed = complete_goal_todo( + registry_path=registry, + goal_id=GOAL_ID, + todo_id=todo_id, + role="agent", + no_followup=True, + agent_id=AGENT_A, + task_lease_idempotency_key="todo-terminal-a", + task_lease_expected_version=int(acquired["lease"]["version"]), + ) + archived = archive_completed_todos( + registry_path=registry, + goal_id=GOAL_ID, + role="agent", + max_active_done=0, + dry_run=False, + ) + replacement = _add(registry) + replacement_lease = acquire_task_lease( + registry_path=registry, + runtime_root=runtime_root, + goal_id=GOAL_ID, + todo_id=str(replacement["todo_id"]), + owner=AGENT_A, + idempotency_key="todo-terminal-b", + ttl_seconds=120, + ) + superseded = supersede_goal_todo( + registry_path=registry, + goal_id=GOAL_ID, + todo_id=str(replacement["todo_id"]), + role="agent", + reason="Replace obsolete work.", + next_agent_todo="Carry the bounded work forward.", + agent_id=AGENT_A, + task_lease_idempotency_key="todo-terminal-b", + task_lease_expected_version=int(replacement_lease["lease"]["version"]), + ) + + for result in (added, updated, completed, archived, replacement, superseded): + assert result["ok"] is True + assert result["authority_shadow"]["schema_version"] == ( + LOCAL_AUTHORITY_SHADOW_EVIDENCE_SCHEMA + ) + assert result["authority_shadow"]["primary_writeback_preserved"] is True + assert result["authority_shadow"]["provider_to_local_writes"] is False + + +def test_enabled_task_lease_facades_shadow_only_committed_mutations( + tmp_path: Path, +) -> None: + registry, _state, runtime_root = _fixture(tmp_path, enabled=True) + todo_id = str(_add(registry)["todo_id"]) + + acquired = acquire_task_lease( + registry_path=registry, + runtime_root=runtime_root, + goal_id=GOAL_ID, + todo_id=todo_id, + owner=AGENT_A, + idempotency_key="lease-a", + ttl_seconds=120, + ) + replayed_acquire = acquire_task_lease( + registry_path=registry, + runtime_root=runtime_root, + goal_id=GOAL_ID, + todo_id=todo_id, + owner=AGENT_A, + idempotency_key="lease-a", + ttl_seconds=120, + ) + renewed = renew_task_lease( + registry_path=registry, + runtime_root=runtime_root, + goal_id=GOAL_ID, + todo_id=todo_id, + owner=AGENT_A, + idempotency_key="lease-a", + expected_version=1, + ttl_seconds=120, + ) + transferred = transfer_task_lease( + registry_path=registry, + runtime_root=runtime_root, + goal_id=GOAL_ID, + todo_id=todo_id, + owner=AGENT_A, + idempotency_key="lease-a", + new_owner=AGENT_B, + new_idempotency_key="lease-b", + expected_version=2, + ttl_seconds=120, + ) + released = release_task_lease( + registry_path=registry, + runtime_root=runtime_root, + goal_id=GOAL_ID, + todo_id=todo_id, + owner=AGENT_B, + idempotency_key="lease-b", + expected_version=3, + ) + + for result in (acquired, renewed, transferred, released): + assert result["authority_shadow"]["outcome"] in { + "advanced", + "replayed", + "ambiguous_reconciled", + } + assert replayed_acquire["idempotent"] is True + assert "authority_shadow" not in replayed_acquire + + +def test_handoff_mode_and_direct_followup_writers_refresh_the_same_shadow( + tmp_path: Path, +) -> None: + registry, _state, runtime_root = _fixture(tmp_path, enabled=True) + + mode = set_goal_handoff_mode( + registry_path=registry, + goal_id=GOAL_ID, + mode="legacy", + ) + followups = capture_followup_todos( + registry_path=registry, + goal_id=GOAL_ID, + followups=["Verify the migrated authority projection."], + evidence="validation://local-shadow-followup", + ) + + assert mode["changed"] is True + assert mode["authority_shadow"]["outcome"] == "advanced" + assert followups["changed"] is True + assert followups["authority_shadow"]["outcome"] == "advanced" + head = _shadow_document(runtime_root)["head"] + assert head["handoff_mode"] == "legacy" + assert [todo["todo_id"] for todo in head["todos"]] == [ + followups["items"][0]["todo_id"] + ] + + +def test_event_projected_completion_refreshes_shadow_after_releasing_lease( + tmp_path: Path, +) -> None: + registry, state, runtime_root = _fixture(tmp_path, enabled=True) + todo_id = "todo_event_shadow" + AppendOnlyStateEventStore(state.with_name("events.jsonl")).append( + make_state_event( + event_id="evt-event-shadow-parent", + goal_id=GOAL_ID, + event_type=TODO_ADDED, + refs={"todo_id": todo_id}, + payload={ + "role": "agent", + "title": "Complete the event-projected shadow task.", + "task_class": "advancement_task", + "claimed_by": AGENT_A, + }, + recorded_at="2026-09-02T00:00:00+00:00", + ) + ) + lease_key = "event-shadow-instance" + acquire_task_lease( + registry_path=registry, + runtime_root=runtime_root, + goal_id=GOAL_ID, + todo_id=todo_id, + owner=AGENT_A, + idempotency_key=lease_key, + ttl_seconds=120, + ) + + completed = complete_goal_todo( + registry_path=registry, + goal_id=GOAL_ID, + todo_id=todo_id, + claimed_by=AGENT_A, + agent_id=AGENT_A, + task_lease_idempotency_key=lease_key, + task_lease_expected_version=1, + evidence="validation://event-shadow-completion", + no_followup=True, + ) + + assert completed["source"] == "event_log" + assert completed["changed"] is True + assert completed["authority_shadow"]["outcome"] == "advanced" + head = _shadow_document(runtime_root)["head"] + assert len(head["leases"]) == 1 + assert head["leases"][0]["todo_id"] == todo_id + assert head["leases"][0]["status"] == "released" + projected = next(todo for todo in head["todos"] if todo["todo_id"] == todo_id) + assert projected["status"] == "done" + + +def test_candidate_failure_never_changes_committed_todo_result( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + registry, state, _runtime_root = _fixture(tmp_path, enabled=True) + + def unavailable( + _method: str, + params: dict[str, object], + **_kwargs: object, + ) -> dict[str, object]: + return { + "schema_version": LOCAL_AUTHORITY_SHADOW_EVIDENCE_SCHEMA, + "outcome": "unavailable", + "reason_code": "injected_outage", + "goal_id": params["goal_id"], + "operation_id": params["operation_id"], + "source_digest": params["source_digest"], + "primary_authority": "legacy_local", + "candidate_provider": "file", + "candidate_read_for_decision": False, + "provider_to_local_writes": False, + "primary_writeback_preserved": True, + "store_identity": None, + "provider_revision": None, + "cursor": None, + } + + monkeypatch.setattr( + "loopx.control_plane.coordination.local_authority_shadow_adapter.effect_runtime_result", + unavailable, + ) + + result = _add(registry) + + assert result["ok"] is True + assert result["added"] is True + assert result["authority_shadow"]["outcome"] == "unavailable" + assert str(result["todo_id"]) in state.read_text(encoding="utf-8") + + +def test_invalid_shadow_config_is_typed_but_preserves_primary_write( + tmp_path: Path, +) -> None: + registry, state, _runtime_root = _fixture(tmp_path, enabled=True) + payload = json.loads(registry.read_text(encoding="utf-8")) + payload["goals"][0]["coordination"]["authority_shadow"]["mode"] = "remote" + registry.write_text(json.dumps(payload), encoding="utf-8") + + result = _add(registry) + + assert result["ok"] is True + assert result["authority_shadow"]["outcome"] == "failed" + assert result["authority_shadow"]["reason_code"] == "invalid_shadow_config" + assert str(result["todo_id"]) in state.read_text(encoding="utf-8") + + +def test_observer_is_default_off_without_creating_lock_or_provider_directory( + tmp_path: Path, +) -> None: + registry, _state, runtime_root = _fixture(tmp_path, enabled=False) + + assert ( + observe_local_authority_commit( + registry_path=registry, + runtime_root=runtime_root, + goal_id=GOAL_ID, + source_operation="todo_update", + ) + is None + ) + assert not (runtime_root / "authority-shadow").exists() + + +def test_provider_revision_conflict_resamples_source_under_same_observation_lock( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + registry, _state, runtime_root = _fixture(tmp_path, enabled=True) + projections = iter( + ( + { + "schema_version": "loopx_local_authority_shadow_projection_v0", + "goal_id": GOAL_ID, + "handoff_mode": "hard_lease", + "todos": [{"todo_id": "todo_old", "status": "open"}], + "leases": [], + }, + { + "schema_version": "loopx_local_authority_shadow_projection_v0", + "goal_id": GOAL_ID, + "handoff_mode": "hard_lease", + "todos": [{"todo_id": "todo_fresh", "status": "open"}], + "leases": [], + }, + ) + ) + requests: list[dict[str, object]] = [] + + monkeypatch.setattr( + "loopx.control_plane.coordination.local_authority_shadow_adapter._stable_projection", + lambda **_kwargs: next(projections), + ) + + def conflict_then_advance( + _method: str, + params: dict[str, object], + **_kwargs: object, + ) -> dict[str, object]: + requests.append(dict(params)) + outcome = "conflict_retry_required" if len(requests) == 1 else "advanced" + return { + "schema_version": LOCAL_AUTHORITY_SHADOW_EVIDENCE_SCHEMA, + "outcome": outcome, + "reason_code": ( + "provider_revision_mismatch" if len(requests) == 1 else None + ), + "goal_id": GOAL_ID, + "operation_id": params["operation_id"], + "source_digest": params["source_digest"], + "primary_authority": "legacy_local", + "candidate_provider": "file", + "candidate_read_for_decision": False, + "provider_to_local_writes": False, + "primary_writeback_preserved": True, + "store_identity": "file:test", + "provider_revision": "file:2:test", + "cursor": "2", + } + + monkeypatch.setattr( + "loopx.control_plane.coordination.local_authority_shadow_adapter.effect_runtime_result", + conflict_then_advance, + ) + + result = observe_local_authority_commit( + registry_path=registry, + runtime_root=runtime_root, + goal_id=GOAL_ID, + source_operation="todo_update:todo_a:now", + ) + + assert result is not None + assert result["outcome"] == "advanced" + assert len(requests) == 2 + assert requests[0]["source_digest"] != requests[1]["source_digest"] + assert requests[0]["operation_id"] != requests[1]["operation_id"] + assert all(request["runtime_root"] == str(runtime_root) for request in requests) + assert all("provider_directory" not in request for request in requests) diff --git a/tests/control_plane/test_state_migration_authority_shadow.py b/tests/control_plane/test_state_migration_authority_shadow.py new file mode 100644 index 0000000000..b3cfa84bf2 --- /dev/null +++ b/tests/control_plane/test_state_migration_authority_shadow.py @@ -0,0 +1,262 @@ +from __future__ import annotations + +import json +from collections.abc import Iterator +from pathlib import Path + +import pytest + +from loopx.state_migration import migrate_legacy_state, render_state_migration_markdown + + +OLD_GOAL_ID = "legacy-goal" +NEW_GOAL_ID = "migrated-goal" +OLD_STORE_IDENTITY = "file:11111111111111111111111111111111" + + +def _migration_fixture(tmp_path: Path) -> dict[str, Path]: + legacy_runtime = tmp_path / "legacy-runtime" + target_runtime = tmp_path / "target-runtime" + source_repo = tmp_path / "legacy-repo" + target_repo = tmp_path / "target-repo" + source_repo.mkdir() + target_repo.mkdir() + + source_state = source_repo / "ACTIVE_GOAL_STATE.md" + source_state.write_text( + "---\n" + f"goal_id: {OLD_GOAL_ID}\n" + "handoff_mode: soft_claim\n" + "updated_at: 2026-09-02T00:00:00+10:00\n" + "---\n\n" + "## Agent Todo\n\n" + "- [ ] Preserve the new local authority only.\n", + encoding="utf-8", + ) + + legacy_registry = tmp_path / "legacy-registry.json" + legacy_registry.write_text( + json.dumps( + { + "schema_version": "0.1", + "common_runtime_root": str(legacy_runtime), + "goals": [ + { + "id": OLD_GOAL_ID, + "status": "active", + "repo": str(source_repo), + "state_file": source_state.name, + "coordination": { + "agent_model": "peer_v1", + "registered_agents": ["agent-a", "agent-b"], + "authority_shadow": { + "schema_version": ( + "loopx_local_authority_shadow_config_v0" + ), + "mode": "file_one_way", + }, + }, + } + ], + } + ), + encoding="utf-8", + ) + + source_goal_runtime = legacy_runtime / "goals" / OLD_GOAL_ID + (source_goal_runtime / "task-leases").mkdir(parents=True) + (source_goal_runtime / "task-leases" / "safe-local.json").write_text( + json.dumps( + { + "goal_id": OLD_GOAL_ID, + "todo_id": "safe-local", + "owner": "agent-a", + "version": 1, + "lease_epoch": 1, + "status": "released", + } + ), + encoding="utf-8", + ) + source_shadow_store = ( + legacy_runtime / "authority-shadow" / "file" / OLD_GOAL_ID + ) + source_shadow_store.mkdir(parents=True) + (source_shadow_store / "store-identity").write_text( + OLD_STORE_IDENTITY, + encoding="utf-8", + ) + (source_shadow_store / "authority-store-legacy.json").write_text( + json.dumps( + { + "goal_id": OLD_GOAL_ID, + "store_identity": OLD_STORE_IDENTITY, + "provider_revision": "file:99:legacy-lineage", + "cursor": "99", + "private_provider_byte": "must-never-migrate", + "source_path": str(source_repo), + } + ), + encoding="utf-8", + ) + + return { + "legacy_registry": legacy_registry, + "target_registry": tmp_path / "target-registry.json", + "legacy_runtime": legacy_runtime, + "target_runtime": target_runtime, + "source_repo": source_repo, + "target_repo": target_repo, + } + + +def _migrate(paths: dict[str, Path], *, execute: bool) -> dict[str, object]: + return migrate_legacy_state( + legacy_registry_path=paths["legacy_registry"], + target_registry_path=paths["target_registry"], + legacy_runtime_root=paths["legacy_runtime"], + target_runtime_root=paths["target_runtime"], + goal_ids=[OLD_GOAL_ID], + goal_id_map={OLD_GOAL_ID: NEW_GOAL_ID}, + path_map={str(paths["source_repo"]): str(paths["target_repo"])}, + copy_active_state=True, + copy_runtime=True, + execute=execute, + ) + + +def test_execute_excludes_old_shadow_and_seeds_fresh_target_lineage( + tmp_path: Path, +) -> None: + paths = _migration_fixture(tmp_path) + + result = _migrate(paths, execute=True) + + assert result["ok"] is True + runtime_result = result["runtime_goals"][0] # type: ignore[index] + assert runtime_result["copied"] is True + + target_goal_runtime = paths["target_runtime"] / "goals" / NEW_GOAL_ID + copied_lease = json.loads( + (target_goal_runtime / "task-leases" / "safe-local.json").read_text( + encoding="utf-8" + ) + ) + assert copied_lease["goal_id"] == NEW_GOAL_ID + + target_store_dir = ( + paths["target_runtime"] / "authority-shadow" / "file" / NEW_GOAL_ID + ) + target_identity = (target_store_dir / "store-identity").read_text(encoding="utf-8") + assert target_identity.startswith("file:") + assert target_identity != OLD_STORE_IDENTITY + + store_paths = list(target_store_dir.glob("authority-store-*.json")) + assert len(store_paths) == 1 + store_payload = json.loads(store_paths[0].read_text(encoding="utf-8")) + assert store_payload["goal_id"] == NEW_GOAL_ID + assert store_payload["store_identity"] == target_identity + assert store_payload["cursor"] == "1" + assert len(store_payload["committed"]) == 1 + serialized_store = json.dumps(store_payload, sort_keys=True) + assert OLD_STORE_IDENTITY not in serialized_store + assert "file:99:legacy-lineage" not in serialized_store + assert str(paths["source_repo"]) not in serialized_store + assert "must-never-migrate" not in serialized_store + + seed = result["authority_shadow_seeds"][0] # type: ignore[index] + assert seed["goal_id"] == NEW_GOAL_ID + assert seed["attempted"] is True + assert seed["outcome"] == "advanced" + + +def test_dry_run_reports_exclusion_and_seed_plan_without_writing( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths = _migration_fixture(tmp_path) + sentinel = b'{"schema_version":"existing","goals":[]}\n' + paths["target_registry"].write_bytes(sentinel) + + def forbidden_observer(**_kwargs: object) -> object: + raise AssertionError("dry-run called the shadow observer") + + original_rglob = Path.rglob + + def guarded_rglob(path: Path, pattern: str) -> Iterator[Path]: + if path.name == "authority-shadow": + raise AssertionError("migration entered candidate-provider storage") + return original_rglob(path, pattern) + + monkeypatch.setattr( + "loopx.control_plane.coordination.local_authority_shadow_adapter." + "observe_local_authority_commit", + forbidden_observer, + ) + monkeypatch.setattr(Path, "rglob", guarded_rglob) + + result = _migrate(paths, execute=False) + + assert result["ok"] is True + assert result["dry_run"] is True + assert paths["target_registry"].read_bytes() == sentinel + assert not paths["target_runtime"].exists() + runtime_result = result["runtime_goals"][0] # type: ignore[index] + assert runtime_result["copied_file_count"] == 0 + seed = result["authority_shadow_seeds"][0] # type: ignore[index] + assert seed == { + "schema_version": "loopx_state_migration_shadow_seed_evidence_v0", + "goal_id": NEW_GOAL_ID, + "attempted": False, + "outcome": "planned", + "reason_code": None, + } + rendered = render_state_migration_markdown(result) + assert "Authority Shadow Seeds" in rendered + assert "outcome=`planned`" in rendered + assert "must-never-migrate" not in json.dumps(result, sort_keys=True) + assert "must-never-migrate" not in rendered + + +def test_seed_failure_is_public_safe_evidence_and_does_not_reverse_migration( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths = _migration_fixture(tmp_path) + + def fail_observer(**_kwargs: object) -> object: + raise RuntimeError("credential=private-provider-value") + + monkeypatch.setattr( + "loopx.control_plane.coordination.local_authority_shadow_adapter." + "observe_local_authority_commit", + fail_observer, + ) + + result = _migrate(paths, execute=True) + + assert result["ok"] is True + assert result["wrote_project_registry"] is True + migrated_registry = json.loads(paths["target_registry"].read_text(encoding="utf-8")) + assert migrated_registry["goals"][0]["id"] == NEW_GOAL_ID + assert ( + paths["target_runtime"] + / "goals" + / NEW_GOAL_ID + / "task-leases" + / "safe-local.json" + ).exists() + + seed = result["authority_shadow_seeds"][0] # type: ignore[index] + assert seed["outcome"] == "failed" + assert seed["reason_code"] == "post_migration_shadow_seed_failed" + assert seed["attempted"] is True + assert "credential" not in json.dumps(result, sort_keys=True) + assert "private-provider-value" not in json.dumps(result, sort_keys=True) + assert not ( + paths["target_runtime"] + / "authority-shadow" + / "file" + / NEW_GOAL_ID + / "authority-store-legacy.json" + ).exists() diff --git a/tests/control_plane_ts/local_authority_shadow.test.ts b/tests/control_plane_ts/local_authority_shadow.test.ts new file mode 100644 index 0000000000..d1499ee58b --- /dev/null +++ b/tests/control_plane_ts/local_authority_shadow.test.ts @@ -0,0 +1,197 @@ +import assert from "node:assert/strict"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import type { + AuthorityStore, + AuthorityStoreCommit, + AuthorityStoreCommitResult, + AuthorityStoreIdentityResult, + AuthorityStoreLoadResult, + AuthorityStoreReceiptResult, + AuthorityStoreScanResult, +} from "../../loopx/control_plane/coordination/authority_store.ts"; +import { FileAuthorityStore } from "../../loopx/control_plane/coordination/file_authority_store.ts"; +import { + LOCAL_AUTHORITY_SHADOW_EVIDENCE_SCHEMA, + recordLocalAuthorityShadow, +} from "../../loopx/control_plane/coordination/local_authority_shadow.ts"; + +function request(directory: string, operationId = "local-operation-a") { + return { + schema_version: "loopx_local_authority_shadow_request_v0", + mode: "file_one_way", + runtime_root: directory, + goal_id: "goal-a", + operation_id: operationId, + source_operation: "todo_update", + source_digest: `sha256:${"a".repeat(64)}`, + source_projection: { + schema_version: "loopx_local_authority_shadow_projection_v0", + goal_id: "goal-a", + handoff_mode: "hard_lease", + todos: [{ todo_id: "todo-a", status: "open", claimed_by: "agent-a" }], + leases: [{ todo_id: "todo-a", version: 2, lease_epoch: 1, status: "active" }], + }, + }; +} + +test("one-way file shadow commits an observation without becoming decision authority", async (t) => { + const root = await mkdtemp(join(tmpdir(), "loopx-local-authority-shadow-")); + t.after(() => rm(root, { recursive: true, force: true })); + + const evidence = await recordLocalAuthorityShadow(request(root)); + + assert.equal(evidence.schema_version, LOCAL_AUTHORITY_SHADOW_EVIDENCE_SCHEMA); + assert.equal(evidence.outcome, "advanced"); + assert.equal(evidence.primary_authority, "legacy_local"); + assert.equal(evidence.candidate_read_for_decision, false); + assert.equal(evidence.provider_to_local_writes, false); + assert.equal(evidence.primary_writeback_preserved, true); + const loaded = await new FileAuthorityStore( + join(root, "authority-shadow", "file", "goal-a"), + "goal-a", + ).loadAuthority(); + assert.equal(loaded.status, "loaded"); + if (loaded.status === "loaded") { + assert.deepEqual(loaded.head, request(root).source_projection); + } +}); + +test("same operation replays its typed observation receipt", async (t) => { + const root = await mkdtemp(join(tmpdir(), "loopx-local-authority-shadow-")); + t.after(() => rm(root, { recursive: true, force: true })); + + assert.equal((await recordLocalAuthorityShadow(request(root))).outcome, "advanced"); + const replay = await recordLocalAuthorityShadow(request(root)); + + assert.equal(replay.outcome, "replayed"); + const page = await new FileAuthorityStore( + join(root, "authority-shadow", "file", "goal-a"), + "goal-a", + ).scanCommitted(null, 10); + assert.equal(page.status, "page"); + if (page.status === "page") assert.equal(page.transactions.length, 1); +}); + +class UnavailableStore implements AuthorityStore { + commits = 0; + + async storeIdentity(): Promise { + return { status: "unavailable", reason_code: "injected", reason: "offline" }; + } + async loadAuthority(): Promise { + return { status: "unavailable", reason_code: "injected", reason: "offline" }; + } + async commitAuthority(_commit: AuthorityStoreCommit): Promise { + this.commits += 1; + return { status: "failed", reason_code: "unexpected", reason: "must not commit" }; + } + async readReceipt(_operationId: string): Promise { + return { status: "missing" }; + } + async scanCommitted(_afterCursor: string | null, _limit: number): Promise { + return { status: "page", transactions: [], next_cursor: null, has_more: false }; + } +} + +test("candidate unavailability is typed evidence and attempts no commit", async () => { + const store = new UnavailableStore(); + const evidence = await recordLocalAuthorityShadow(request("/not-used"), { + openStore: () => store, + }); + + assert.equal(evidence.outcome, "unavailable"); + assert.equal(evidence.reason_code, "injected"); + assert.equal(evidence.primary_writeback_preserved, true); + assert.equal(store.commits, 0); +}); + +class RevisionConflictStore extends UnavailableStore { + override async storeIdentity(): Promise { + return { status: "available", store_identity: "file:test" }; + } + override async loadAuthority(): Promise { + return { + status: "loaded", + head: {}, + provider_revision: "provider-revision-a", + cursor: "1", + }; + } + override async commitAuthority( + _commit: AuthorityStoreCommit, + ): Promise { + this.commits += 1; + return { + status: "conflict", + conflict_kind: "provider_revision_mismatch", + current_provider_revision: "provider-revision-b", + current_cursor: "2", + }; + } +} + +test("provider revision conflict requests a fresh source projection without stale retry", async () => { + const store = new RevisionConflictStore(); + + const result = await recordLocalAuthorityShadow(request("/not-used"), { + openStore: () => store, + }); + + assert.equal(result.outcome, "conflict_retry_required"); + assert.equal(result.reason_code, "provider_revision_mismatch"); + assert.equal(store.commits, 1); +}); + +test("goal id cannot escape the fixed shadow directory", async () => { + await assert.rejects( + recordLocalAuthorityShadow({ + ...request("/not-used"), + goal_id: "../other-goal", + source_projection: { + ...request("/not-used").source_projection, + goal_id: "../other-goal", + }, + }), + /single path segment/u, + ); +}); + +test("ambiguous response is success only when the exact receipt is readable", async () => { + const root = await mkdtemp(join(tmpdir(), "loopx-local-authority-shadow-")); + tCleanup(root); + class AfterCommitStore extends FileAuthorityStore { + protected override async replaceDurably(path: string, payload: Uint8Array): Promise { + await super.replaceDurably(path, payload); + if (path === this.path) throw new Error("lost response after durable replace"); + } + } + const recovered = await recordLocalAuthorityShadow(request(root), { + openStore: (directory, goalId) => new AfterCommitStore(directory, goalId), + }); + assert.equal(recovered.outcome, "ambiguous_reconciled"); + + const beforeRoot = await mkdtemp(join(tmpdir(), "loopx-local-authority-shadow-")); + tCleanup(beforeRoot); + class BeforeCommitStore extends FileAuthorityStore { + protected override async replaceDurably(path: string, _payload: Uint8Array): Promise { + if (path === this.path) throw new Error("failed before durable replace"); + return await super.replaceDurably(path, _payload); + } + } + const unproved = await recordLocalAuthorityShadow(request(beforeRoot), { + openStore: (directory, goalId) => new BeforeCommitStore(directory, goalId), + }); + assert.equal(unproved.outcome, "ambiguous_unproved"); +}); + +const cleanupRoots: string[] = []; +function tCleanup(root: string): void { + cleanupRoots.push(root); +} +test.after(async () => { + await Promise.all(cleanupRoots.map((root) => rm(root, { recursive: true, force: true }))); +}); diff --git a/tsconfig.control-plane.json b/tsconfig.control-plane.json index 4bbfd66289..2f54be74a3 100644 --- a/tsconfig.control-plane.json +++ b/tsconfig.control-plane.json @@ -21,6 +21,7 @@ "loopx/control_plane/coordination/authority_store.ts", "loopx/control_plane/coordination/authority_store_codec.ts", "loopx/control_plane/coordination/file_authority_store.ts", + "loopx/control_plane/coordination/local_authority_shadow.ts", "loopx/control_plane/coordination/postgresql_authority_store.ts", "loopx/control_plane/agents/delivery_workspace.ts", "loopx/control_plane/goals/vision_checkpoint.ts", @@ -52,6 +53,7 @@ "tests/control_plane_ts/interaction_contract.test.ts", "tests/control_plane_ts/runtime_decode.test.ts", "tests/control_plane_ts/authority_store.test.ts", + "tests/control_plane_ts/local_authority_shadow.test.ts", "tests/control_plane_ts/authority_store_conformance.ts", "tests/control_plane_ts/postgresql_authority_store.integration.test.ts", "tests/control_plane_ts/delivery_continuity.test.ts", From 279720e96c6651fbf7e193300c46416ca1d9e152 Mon Sep 17 00:00:00 2001 From: wchwawa Date: Wed, 2 Sep 2026 07:42:47 +1000 Subject: [PATCH 02/12] docs(authority): document local shadow boundary Signed-off-by: wchwawa --- ...shared-goal-authority-state-provider-v0.md | 27 +++++++++++++++++++ ...-goal-authority-state-provider-v0.zh-CN.md | 23 ++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.md b/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.md index 1ac7f3ad16..3ae656da0c 100644 --- a/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.md +++ b/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.md @@ -1078,6 +1078,33 @@ The sequence is: cache, offline projection, and diagnostic material. Never keep a long-lived dual-write or dual-master mode. +#### Stage 2C qualification slice: local post-commit shadow + +The first half of Stage 2C is an explicit, default-off product path. Preview +and enable it with: + +```bash +loopx configure-goal --goal-id GOAL --local-authority-shadow-file +loopx configure-goal --goal-id GOAL --local-authority-shadow-file --execute +``` + +Todo, handoff-mode, follow-up, and task-lease facades observe their committed +local result through `FileAuthorityStore`. Candidate bytes live under +`authority-shadow/file/` outside the legacy per-Goal runtime tree, so state +migration never copies a store identity or revision; an executed migration +seeds a new target lineage from the migrated local state. Candidate failure is +reported as evidence but never reverses the completed local write. + +Disable the observer in one command with +`loopx configure-goal --goal-id GOAL --clear-local-authority-shadow --execute`. +This is rollback of observation only: the local Markdown and task-lease files +remain canonical throughout. The slice does not read the candidate for a +decision, fence a legacy writer, qualify a remote provider, or complete the +second Stage 2C promotion. A process crash after the local commit but before +the observer call may miss that individual observation; a later committed +write or migration seed refreshes the full current projection, but no durable +shadow outbox is claimed here. + ### Implementation prerequisite: put local file mode behind the same coordination contract Before wiring a live NoKV or another remote provider, the runtime should first diff --git a/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.zh-CN.md b/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.zh-CN.md index 0bb4d1d851..8e767a087e 100644 --- a/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.zh-CN.md +++ b/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.zh-CN.md @@ -865,6 +865,29 @@ Stage 3/4 qualification 必须保持以下 ownership 与 proof 边界： LoopX service 成为唯一 writer。本地 `.loopx` 退为 cache、offline projection 与 诊断材料。绝不长期维持 dual-write 或 dual-master。 +#### Stage 2C 资格验证切片：本地提交后 shadow + +Stage 2C 的前半段是一个显式开启、默认关闭的产品路径。先预览，再开启： + +```bash +loopx configure-goal --goal-id GOAL --local-authority-shadow-file +loopx configure-goal --goal-id GOAL --local-authority-shadow-file --execute +``` + +Todo、handoff-mode、follow-up 与 task-lease facade 会在本地主写成功后，通过 +`FileAuthorityStore` 观察完整当前投影。候选数据位于 legacy 单 Goal runtime tree +之外的 `authority-shadow/file/`，因此 state migration 不会复制 store identity 或 +revision；真正执行迁移时，会从迁移后的本地主状态为目标端建立一条新 lineage。 +候选失败只形成 evidence，不会推翻已经完成的本地写入。 + +用 +`loopx configure-goal --goal-id GOAL --clear-local-authority-shadow --execute` +即可关闭 observer。这里回退的只是观察路径：Markdown 与 task-lease 文件始终是 +canonical。本切片不会读取候选来决策，不会 fence legacy writer，不会资格化远端 +provider，也没有完成 Stage 2C 后半段的本地 canonical promotion。若进程恰好在本地 +提交后、observer 调用前崩溃，该次 observation 可能丢失；后续成功写入或 migration +seed 会刷新完整当前投影，但这里不宣称已有 durable shadow outbox。 + ### 实施前置条件：先让本地文件模式经过同一协调合同 在接入 live NoKV 或其他远端 provider 之前，runtime 应先把当前 todo/lease 写路径中的 From 80764e877f5614bda3fc4e642ad938aa7dfdd251 Mon Sep 17 00:00:00 2001 From: wchwawa Date: Wed, 2 Sep 2026 08:00:24 +1000 Subject: [PATCH 03/12] refactor(authority): keep shadow wiring within coordination Signed-off-by: wchwawa --- loopx/configure_goal.py | 87 ++++------------- .../coordination/configuration.py | 25 +++++ .../local_authority_shadow_adapter.py | 94 +++++++++++++++++++ loopx/todos.py | 78 ++------------- .../test_coordination_configuration.py | 11 +++ 5 files changed, 157 insertions(+), 138 deletions(-) create mode 100644 loopx/control_plane/coordination/configuration.py create mode 100644 tests/control_plane/test_coordination_configuration.py diff --git a/loopx/configure_goal.py b/loopx/configure_goal.py index c9eb0084bd..f71c2c7b62 100644 --- a/loopx/configure_goal.py +++ b/loopx/configure_goal.py @@ -34,6 +34,8 @@ ) from .control_plane.agents.supervisor import normalize_peer_supervisor from .control_plane.agents.work_mode import normalize_agent_work_modes +from .control_plane.coordination import local_authority_shadow_adapter as shadow +from .control_plane.coordination.configuration import normalize_goal_write_scope from .control_plane.operator_inbox_binding import local_private_config_digest from .control_plane.reward_memory import ( reward_memory_goal_policy, @@ -70,9 +72,6 @@ MULTI_SUBAGENT_FEATURE_CHOICES = ("off", "enabled") DEFAULT_MULTI_SUBAGENT_MAX_CHILDREN = 2 AGENT_MODEL_CHOICES = tuple(model.value for model in AgentRuntimeModel) -LOCAL_AUTHORITY_SHADOW_CONFIG_SCHEMA_VERSION = ( - "loopx_local_authority_shadow_config_v0" -) def _control_plane(goal: dict[str, Any]) -> dict[str, Any]: @@ -224,41 +223,6 @@ def _clean_registered_agents(values: list[str] | None) -> list[str] | None: return agents -def _clean_write_scope(values: list[str] | None) -> list[str] | None: - if values is None: - return None - scopes: list[str] = [] - for value in values: - for part in str(value).split(","): - scope = part.strip() - if scope and scope not in scopes: - scopes.append(scope) - return scopes - - -def _local_authority_shadow_summary(goal: Mapping[str, Any]) -> dict[str, Any]: - coordination = ( - goal.get("coordination") - if isinstance(goal.get("coordination"), Mapping) - else {} - ) - raw = coordination.get("authority_shadow") - if raw is None: - return {"enabled": False, "mode": None, "status": "disabled"} - valid = bool( - isinstance(raw, Mapping) - and set(raw) == {"schema_version", "mode"} - and raw.get("schema_version") - == LOCAL_AUTHORITY_SHADOW_CONFIG_SCHEMA_VERSION - and raw.get("mode") == "file_one_way" - ) - return { - "enabled": valid, - "mode": raw.get("mode") if isinstance(raw, Mapping) else None, - "status": "enabled" if valid else "invalid", - } - - def _settings_summary(goal: dict[str, Any]) -> dict[str, Any]: quota = goal_quota_config(goal) control_plane = compact_control_plane_policy(goal.get("control_plane")) @@ -285,8 +249,9 @@ def _settings_summary(goal: dict[str, Any]) -> dict[str, Any]: "explore_graph": compact_explore_graph_policy(goal.get("explore_graph")), "orchestration": orchestration, "waiting_on": goal.get("waiting_on"), - "write_scope": _clean_write_scope(coordination.get("write_scope") or []) or [], - "local_authority_shadow": _local_authority_shadow_summary(goal), + "write_scope": normalize_goal_write_scope(coordination.get("write_scope") or []) + or [], + "local_authority_shadow": shadow.local_authority_shadow_summary(goal), "checkpointed_boundary_authority": checkpointed_boundary_authority_summary( coordination ), @@ -564,11 +529,9 @@ def configure_goal( raise ValueError( "--clear-write-scope cannot be combined with --replace-write-scope" ) - if local_authority_shadow_file and clear_local_authority_shadow: - raise ValueError( - "--local-authority-shadow-file cannot be combined with " - "--clear-local-authority-shadow" - ) + shadow.validate_local_authority_shadow_change( + local_authority_shadow_file, clear_local_authority_shadow + ) if clear_waiting_on and waiting_on: raise ValueError("--clear-waiting-on cannot be combined with --waiting-on") adding_boundary_authority = any( @@ -668,7 +631,7 @@ def configure_goal( clear_todo_lifecycle_authority ) supervised_agents = _clean_registered_agents(supervised_agents) - write_scope = _clean_write_scope(write_scope) + write_scope = normalize_goal_write_scope(write_scope) issue_fix_reviewer_notification_config = _local_private_config_path( issue_fix_reviewer_notification_config, label="reviewer notification config", @@ -1252,10 +1215,12 @@ def configure_goal( coordination["write_scope"] = write_scope else: existing_write_scope = ( - _clean_write_scope(coordination.get("write_scope") or []) or [] + normalize_goal_write_scope(coordination.get("write_scope") or []) + or [] ) coordination["write_scope"] = ( - _clean_write_scope([*existing_write_scope, *write_scope]) or [] + normalize_goal_write_scope([*existing_write_scope, *write_scope]) + or [] ) if clear_boundary_authority: coordination.pop("checkpointed_boundary_authority", None) @@ -1275,24 +1240,9 @@ def configure_goal( coordination["checkpointed_boundary_authority"] = [*entries, entry] goal["coordination"] = coordination - if local_authority_shadow_file or clear_local_authority_shadow: - coordination = ( - goal.get("coordination") - if isinstance(goal.get("coordination"), dict) - else {} - ) - if clear_local_authority_shadow: - coordination.pop("authority_shadow", None) - else: - coordination["authority_shadow"] = { - "schema_version": LOCAL_AUTHORITY_SHADOW_CONFIG_SCHEMA_VERSION, - "mode": "file_one_way", - } - if coordination: - goal["coordination"] = coordination - else: - goal.pop("coordination", None) - + shadow.apply_local_authority_shadow_change( + goal, local_authority_shadow_file, clear_local_authority_shadow + ) after = _settings_summary(goal) changed_fields = _changed_fields(before, after) if goal != before_goal and not changed_fields: @@ -1331,10 +1281,7 @@ def configure_goal( "peer_task_coordination": deepcopy( after.get("peer_task_coordination") or {"enabled": False} ), - "local_authority_shadow": deepcopy( - after.get("local_authority_shadow") - or {"enabled": False, "mode": None, "status": "disabled"} - ), + "local_authority_shadow": deepcopy(after["local_authority_shadow"]), "lark_event_inbox": _lark_event_inbox_config_summary(goal), "lark_kanban_heartbeat_sync": _lark_kanban_heartbeat_config_summary(goal), "reward_memory": reward_memory_goal_policy_summary(goal), diff --git a/loopx/control_plane/coordination/configuration.py b/loopx/control_plane/coordination/configuration.py new file mode 100644 index 0000000000..5a5e814966 --- /dev/null +++ b/loopx/control_plane/coordination/configuration.py @@ -0,0 +1,25 @@ +"""Normalization rules for goal-level coordination configuration.""" + +from __future__ import annotations + + +def normalize_goal_write_scope(values: list[str] | None) -> list[str] | None: + """Split comma-delimited goal scopes and preserve first-seen order. + + Goal configuration intentionally accepts a wider token vocabulary than a + Todo's ``required_write_scopes`` contract, so its stricter normalizer is + not interchangeable with this rule. + """ + + if values is None: + return None + scopes: list[str] = [] + for value in values: + for part in str(value).split(","): + scope = part.strip() + if scope and scope not in scopes: + scopes.append(scope) + return scopes + + +__all__ = ["normalize_goal_write_scope"] diff --git a/loopx/control_plane/coordination/local_authority_shadow_adapter.py b/loopx/control_plane/coordination/local_authority_shadow_adapter.py index c977cc7bf5..d729f0075f 100644 --- a/loopx/control_plane/coordination/local_authority_shadow_adapter.py +++ b/loopx/control_plane/coordination/local_authority_shadow_adapter.py @@ -10,6 +10,7 @@ import hashlib import json +from collections.abc import Mapping from pathlib import Path from typing import Any @@ -71,6 +72,68 @@ ) +def local_authority_shadow_summary(goal: Mapping[str, Any]) -> dict[str, Any]: + """Project the closed local-shadow configuration for operator readback.""" + + coordination = ( + goal.get("coordination") + if isinstance(goal.get("coordination"), Mapping) + else {} + ) + raw = coordination.get("authority_shadow") + if raw is None: + return {"enabled": False, "mode": None, "status": "disabled"} + valid = bool( + isinstance(raw, Mapping) + and set(raw) == _CONFIG_FIELDS + and raw.get("schema_version") == LOCAL_AUTHORITY_SHADOW_CONFIG_SCHEMA + and raw.get("mode") == "file_one_way" + ) + return { + "enabled": valid, + "mode": raw.get("mode") if isinstance(raw, Mapping) else None, + "status": "enabled" if valid else "invalid", + } + + +def validate_local_authority_shadow_change( + enable_file: bool, + clear: bool, +) -> None: + """Reject contradictory CLI intent before reading or mutating the registry.""" + + if enable_file and clear: + raise ValueError( + "--local-authority-shadow-file cannot be combined with " + "--clear-local-authority-shadow" + ) + + +def apply_local_authority_shadow_change( + goal: dict[str, Any], + enable_file: bool, + clear: bool, +) -> None: + """Apply a validated default-off local-shadow configuration change.""" + + if not enable_file and not clear: + return + coordination = ( + goal.get("coordination") if isinstance(goal.get("coordination"), dict) else {} + ) + if clear: + coordination.pop("authority_shadow", None) + else: + coordination["authority_shadow"] = { + "schema_version": LOCAL_AUTHORITY_SHADOW_CONFIG_SCHEMA, + "mode": "file_one_way", + } + if coordination: + goal["coordination"] = coordination + else: + goal.pop("coordination", None) + + def _base_evidence( *, goal_id: str, @@ -351,8 +414,39 @@ def observe_local_authority_commit( ) +def observe_todo_local_authority_commit( + payload: dict[str, Any], + registry_path: Path, + goal_id: str, + write_class: str, +) -> dict[str, Any]: + """Attach post-commit shadow evidence without changing the Todo verdict.""" + + changed = any( + payload.get(field) + for field in ("changed", "added", "metadata_updated", "completed", "superseded") + ) + if payload.get("dry_run") or not changed: + return payload + todo_id = str(payload.get("todo_id") or "none") + updated_at = str(payload.get("updated_at") or "unknown") + evidence = observe_local_authority_commit( + registry_path=registry_path, + runtime_root=None, + goal_id=goal_id, + source_operation=f"{write_class}:{todo_id}:{updated_at}", + ) + if evidence is not None: + payload["authority_shadow"] = evidence + return payload + + __all__ = [ "LOCAL_AUTHORITY_SHADOW_CONFIG_SCHEMA", "LOCAL_AUTHORITY_SHADOW_EVIDENCE_SCHEMA", + "apply_local_authority_shadow_change", + "local_authority_shadow_summary", "observe_local_authority_commit", + "observe_todo_local_authority_commit", + "validate_local_authority_shadow_change", ] diff --git a/loopx/todos.py b/loopx/todos.py index ed5ef7b5a8..ce0de7db1b 100644 --- a/loopx/todos.py +++ b/loopx/todos.py @@ -126,6 +126,9 @@ enter_todo_ownership_handoff_gate, resolve_todo_completion_handoff, ) +from .control_plane.coordination.local_authority_shadow_adapter import ( + observe_todo_local_authority_commit as _shadow_todo, +) from .control_plane.work_items.task_lease import ( enter_terminal_todo_lease_fence, hold_task_lease_mutation_fence, @@ -136,38 +139,6 @@ ARCHIVE_COMPLETED_DEFAULT_MAX_ACTIVE_DONE = max(0, MAX_ACTIVE_DONE_TODOS_BEFORE_ARCHIVE - 2) -def _attach_local_authority_shadow( - payload: dict[str, Any], - *, - registry_path: Path, - goal_id: str, - write_class: str, -) -> dict[str, Any]: - """Observe a completed local mutation without changing its result.""" - - changed = any( - payload.get(field) - for field in ("changed", "added", "metadata_updated", "completed", "superseded") - ) - if payload.get("dry_run") or not changed: - return payload - from .control_plane.coordination.local_authority_shadow_adapter import ( - observe_local_authority_commit, - ) - - todo_id = str(payload.get("todo_id") or "none") - updated_at = str(payload.get("updated_at") or "unknown") - evidence = observe_local_authority_commit( - registry_path=registry_path, - runtime_root=None, - goal_id=goal_id, - source_operation=f"{write_class}:{todo_id}:{updated_at}", - ) - if evidence is not None: - payload["authority_shadow"] = evidence - return payload - - def require_registered_todo_excluded_agents( *, registry_path: Path, @@ -1202,12 +1173,7 @@ def add_goal_todo( write_class="todo_add", state_text=original, ) - return _attach_local_authority_shadow( - payload, - registry_path=registry_path, - goal_id=goal_id, - write_class="todo_add", - ) + return _shadow_todo(payload, registry_path, goal_id, "todo_add") def resolve_todo_state( @@ -1666,12 +1632,7 @@ def update_goal_todo( write_class=write_class, state_text=original, ) - return _attach_local_authority_shadow( - payload, - registry_path=registry_path, - goal_id=goal_id, - write_class=write_class, - ) + return _shadow_todo(payload, registry_path, goal_id, write_class) def complete_goal_todo( @@ -1902,12 +1863,8 @@ def complete_goal_todo( task_lease_fence, committed=bool(event_result.get("changed")) and not dry_run, ) - return _attach_local_authority_shadow( - event_result, - registry_path=registry_path, - goal_id=goal_id, - write_class="todo_complete_event_projection", - ) + write_class = "todo_complete_event_projection" + return _shadow_todo(event_result, registry_path, goal_id, write_class) if not isinstance(completion_state, dict): raise RuntimeError( "TypeScript Todo completion transaction did not authorize a commit" @@ -2065,12 +2022,7 @@ def complete_goal_todo( if effective_decision_outcome: result["decision_outcome"] = effective_decision_outcome result["self_merged"] = effective_self_merged - return _attach_local_authority_shadow( - result, - registry_path=registry_path, - goal_id=goal_id, - write_class="todo_complete", - ) + return _shadow_todo(result, registry_path, goal_id, "todo_complete") def supersede_goal_todo( *, @@ -2279,12 +2231,7 @@ def supersede_goal_todo( "project": str(resolved_project) if resolved_project else None, "updated_at": updated_at if changed else None, } - return _attach_local_authority_shadow( - result, - registry_path=registry_path, - goal_id=goal_id, - write_class="todo_supersede", - ) + return _shadow_todo(result, registry_path, goal_id, "todo_supersede") def archive_completed_todos( @@ -2335,9 +2282,4 @@ def archive_completed_todos( "project": str(resolved_project) if resolved_project else None, "updated_at": updated_at if changed else None, } - return _attach_local_authority_shadow( - result, - registry_path=registry_path, - goal_id=goal_id, - write_class="todo_archive_completed", - ) + return _shadow_todo(result, registry_path, goal_id, "todo_archive_completed") diff --git a/tests/control_plane/test_coordination_configuration.py b/tests/control_plane/test_coordination_configuration.py new file mode 100644 index 0000000000..10c0ca4789 --- /dev/null +++ b/tests/control_plane/test_coordination_configuration.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +from loopx.control_plane.coordination.configuration import normalize_goal_write_scope + + +def test_goal_write_scope_normalization_preserves_configuration_semantics() -> None: + assert normalize_goal_write_scope(None) is None + assert normalize_goal_write_scope([]) == [] + assert normalize_goal_write_scope( + [" docs/**, tests/** ", "docs/**", "src/**;generated/**", ""] + ) == ["docs/**", "tests/**", "src/**;generated/**"] From 30c3e6b87a8c8cdfd6a2c740bbf7f7a5ead29dac Mon Sep 17 00:00:00 2001 From: wchwawa Date: Wed, 2 Sep 2026 08:05:07 +1000 Subject: [PATCH 04/12] feat(authority): add NoKV candidate store Signed-off-by: wchwawa --- .../coordination/nokv_authority_store.ts | 657 ++++++++++++++++++ .../coordination/nokv_jsonl_helper.py | 610 ++++++++++++++++ .../coordination/nokv_jsonl_transport.ts | 346 +++++++++ .../nokv_authority_store.test.ts | 312 +++++++++ .../nokv_jsonl_transport.test.ts | 174 +++++ tests/fixtures/nokv_fake_sdk/nokv/__init__.py | 80 +++ tests/fixtures/nokv_jsonl_fake_helper.py | 82 +++ tests/test_nokv_jsonl_helper.py | 590 ++++++++++++++++ tsconfig.control-plane.json | 6 + 9 files changed, 2857 insertions(+) create mode 100644 loopx/control_plane/coordination/nokv_authority_store.ts create mode 100644 loopx/control_plane/coordination/nokv_jsonl_helper.py create mode 100644 loopx/control_plane/coordination/nokv_jsonl_transport.ts create mode 100644 tests/control_plane_ts/nokv_authority_store.test.ts create mode 100644 tests/control_plane_ts/nokv_jsonl_transport.test.ts create mode 100644 tests/fixtures/nokv_fake_sdk/nokv/__init__.py create mode 100644 tests/fixtures/nokv_jsonl_fake_helper.py create mode 100644 tests/test_nokv_jsonl_helper.py diff --git a/loopx/control_plane/coordination/nokv_authority_store.ts b/loopx/control_plane/coordination/nokv_authority_store.ts new file mode 100644 index 0000000000..548118e89e --- /dev/null +++ b/loopx/control_plane/coordination/nokv_authority_store.ts @@ -0,0 +1,657 @@ +import { createHash, randomUUID } from "node:crypto"; +import { TextDecoder } from "node:util"; + +import type { JsonObject } from "../effect_program.ts"; +import type { + AuthorityStore, + AuthorityStoreCommit, + AuthorityStoreCommittedTransaction, + AuthorityStoreCommitResult, + AuthorityStoreIdentityResult, + AuthorityStoreLoadResult, + AuthorityStoreReadFailure, + AuthorityStoreReceiptResult, + AuthorityStoreScanResult, +} from "./authority_store.ts"; +import { + AuthorityStoreProtocolError, + canonicalAuthorityBytes, + canonicalAuthorityObject, + canonicalAuthorityObjectList, + hasExactAuthorityKeys, + isAuthorityJsonObject, + normalizeAuthorityStoreCommit, + parseAuthorityCursor, + requireAuthorityStoreId, +} from "./authority_store_codec.ts"; + +const NOKV_AUTHORITY_STORE_SCHEMA = "loopx_nokv_authority_store_v0"; +const DEFAULT_MAX_ENVELOPE_BYTES = 16 * 1024 * 1024; +const HEX_128_PATTERN = /^[0-9a-f]{32}$/; + +export class NoKVTransportUnavailableError extends Error {} +export class NoKVTransportProtocolError extends Error {} + +export type NoKVTransportFailure = { + status: "unavailable" | "failed"; + reason_code: string; + reason: string; +}; + +export type NoKVStoreIdentityResult = + | { status: "available"; store_identity: string } + | NoKVTransportFailure; + +export type NoKVBlobReadResult = + | { status: "loaded"; bytes: Uint8Array; generation: number } + | { status: "missing" } + | NoKVTransportFailure; + +export interface NoKVBlobCasRequest { + workbench: string; + path: string; + expected_generation: number | null; + bytes: Uint8Array; + operation_id: string; + artifact_revision_id: string; +} + +export type NoKVBlobCasResult = + | { status: "applied"; generation: number } + | { status: "conflict"; current_generation: number | null } + | { status: "ambiguous"; reason_code: string; reason: string } + | { status: "failed"; reason_code: string; reason: string }; + +/** Raw byte-storage contract implemented by the Python SDK JSON-lines helper. */ +export interface NoKVBlobTransport { + storeIdentity(workbench: string): Promise; + readBlob(workbench: string, path: string): Promise; + casPublishBlob(request: NoKVBlobCasRequest): Promise; +} + +export interface NoKVAuthorityStoreOptions { + tenant_id: string; + goal_id: string; + workbench: string; + max_envelope_bytes?: number; +} + +interface NoKVAuthorityStoreDocument extends JsonObject { + schema_version: typeof NOKV_AUTHORITY_STORE_SCHEMA; + tenant_id: string; + goal_id: string; + store_identity: string; + storage_generation: number; + provider_revision: string; + cursor: string; + head: JsonObject; + committed: AuthorityStoreCommittedTransaction[]; +} + +type EnvelopeReadResult = + | { + status: "loaded"; + identity: string; + generation: number; + document: NoKVAuthorityStoreDocument; + } + | { status: "missing"; identity: string } + | AuthorityStoreReadFailure; + +function cloneTransaction( + value: AuthorityStoreCommittedTransaction, +): AuthorityStoreCommittedTransaction { + return structuredClone(value); +} + +function transactionWithoutRevision(value: AuthorityStoreCommittedTransaction) { + return { + cursor: value.cursor, + operation_id: value.operation_id, + events: value.events, + projection: value.projection, + receipts: value.receipts, + }; +} + +function providerRevision( + tenantId: string, + goalId: string, + storeIdentity: string, + storageGeneration: number, + previousRevision: string | null, + transaction: ReturnType, +): string { + const digest = createHash("sha256") + .update(canonicalAuthorityBytes({ + provider: "nokv", + tenant_id: tenantId, + goal_id: goalId, + store_identity: storeIdentity, + storage_generation: storageGeneration, + previous_provider_revision: previousRevision, + transaction, + })) + .digest("hex") + .slice(0, 24); + return `nokv:${transaction.cursor}:${digest}`; +} + +function physicalAttemptIdentity( + domain: "operation" | "artifact_revision", + attemptNonce: string, + operationId: string, + expectedGeneration: number | null, + payload: Uint8Array, +): string { + return createHash("sha256") + .update(`loopx.nokv.${domain}.v1\0`, "utf8") + .update(attemptNonce, "utf8") + .update("\0", "utf8") + .update(operationId, "utf8") + .update("\0", "utf8") + .update(expectedGeneration === null ? "create" : String(expectedGeneration), "utf8") + .update("\0", "utf8") + .update(createHash("sha256").update(payload).digest()) + .digest("hex") + .slice(0, 32); +} + +function requireGeneration(value: unknown, name: string): number { + if (!Number.isSafeInteger(value) || (value as number) < 1) { + throw new AuthorityStoreProtocolError(`${name} must be a positive safe integer`); + } + return value as number; +} + +function decodeTransaction(value: unknown): AuthorityStoreCommittedTransaction { + if (!isAuthorityJsonObject(value) || !hasExactAuthorityKeys(value, [ + "cursor", "provider_revision", "operation_id", "events", "projection", "receipts", + ])) { + throw new AuthorityStoreProtocolError("committed transaction is invalid"); + } + return { + cursor: requireAuthorityStoreId(value.cursor, "transaction cursor"), + provider_revision: requireAuthorityStoreId( + value.provider_revision, + "transaction provider revision", + ), + operation_id: requireAuthorityStoreId(value.operation_id, "operation id"), + events: canonicalAuthorityObjectList(value.events, "transaction events"), + projection: canonicalAuthorityObject(value.projection, "transaction projection"), + receipts: canonicalAuthorityObjectList(value.receipts, "transaction receipts"), + }; +} + +function decodeDocument( + value: unknown, + tenantId: string, + goalId: string, + storeIdentity: string, + observedGeneration: number, +): NoKVAuthorityStoreDocument { + if (!isAuthorityJsonObject(value) || !hasExactAuthorityKeys(value, [ + "schema_version", "tenant_id", "goal_id", "store_identity", "storage_generation", + "provider_revision", "cursor", "head", "committed", + ]) || value.schema_version !== NOKV_AUTHORITY_STORE_SCHEMA) { + throw new AuthorityStoreProtocolError("NoKV authority store schema mismatch"); + } + if (value.tenant_id !== tenantId) { + throw new AuthorityStoreProtocolError("NoKV authority store tenant mismatch"); + } + if (value.goal_id !== goalId) { + throw new AuthorityStoreProtocolError("NoKV authority store goal mismatch"); + } + if (value.store_identity !== storeIdentity) { + throw new AuthorityStoreProtocolError("NoKV authority store lineage mismatch"); + } + const storageGeneration = requireGeneration( + value.storage_generation, + "NoKV storage generation", + ); + if (storageGeneration !== observedGeneration) { + throw new AuthorityStoreProtocolError( + "NoKV authority store storage generation does not match read metadata", + ); + } + const revision = requireAuthorityStoreId(value.provider_revision, "provider revision"); + const cursor = requireAuthorityStoreId(value.cursor, "provider cursor"); + const head = canonicalAuthorityObject(value.head, "NoKV authority store head"); + if (!Array.isArray(value.committed)) { + throw new AuthorityStoreProtocolError("NoKV authority store history is invalid"); + } + const committed = value.committed.map(decodeTransaction); + if ( + committed.length === 0 || + storageGeneration !== committed.length || + parseAuthorityCursor(cursor) !== BigInt(committed.length) + ) { + throw new AuthorityStoreProtocolError("NoKV authority store generation lineage is invalid"); + } + let previousRevision: string | null = null; + const operationIds = new Set(); + for (const [index, entry] of committed.entries()) { + const generation = index + 1; + if (parseAuthorityCursor(entry.cursor) !== BigInt(generation)) { + throw new AuthorityStoreProtocolError("NoKV authority store cursor lineage is invalid"); + } + if (operationIds.has(entry.operation_id)) { + throw new AuthorityStoreProtocolError( + "NoKV authority store operation identity is duplicated", + ); + } + operationIds.add(entry.operation_id); + const expectedRevision = providerRevision( + tenantId, + goalId, + storeIdentity, + generation, + previousRevision, + transactionWithoutRevision(entry), + ); + if (entry.provider_revision !== expectedRevision) { + throw new AuthorityStoreProtocolError("NoKV authority store revision lineage is invalid"); + } + previousRevision = entry.provider_revision; + } + const last = committed.at(-1)!; + if ( + last.cursor !== cursor || + last.provider_revision !== revision || + !canonicalAuthorityBytes(last.projection).equals(canonicalAuthorityBytes(head)) + ) { + throw new AuthorityStoreProtocolError("NoKV authority store head lineage is invalid"); + } + return { + schema_version: NOKV_AUTHORITY_STORE_SCHEMA, + tenant_id: tenantId, + goal_id: goalId, + store_identity: storeIdentity, + storage_generation: storageGeneration, + provider_revision: revision, + cursor, + head, + committed, + }; +} + +function readFailure(error: unknown): AuthorityStoreReadFailure { + if ( + error instanceof AuthorityStoreProtocolError || + error instanceof NoKVTransportProtocolError || + error instanceof SyntaxError + ) { + return { + status: "failed", + reason_code: "provider_protocol_violation", + reason: error.message, + }; + } + return { + status: "unavailable", + reason_code: "nokv_transport_unavailable", + reason: error instanceof Error ? error.message : "NoKV transport unavailable", + }; +} + +function validStoreIdentity(value: string, workbench: string): boolean { + const prefix = `nokv:${workbench}:`; + return value.startsWith(prefix) && HEX_128_PATTERN.test(value.slice(prefix.length)); +} + +/** Stage 2A candidate. No runtime constructs this provider by default. */ +export class NoKVAuthorityStore implements AuthorityStore { + readonly transport: NoKVBlobTransport; + readonly tenantId: string; + readonly goalId: string; + readonly workbench: string; + readonly maxEnvelopeBytes: number; + readonly path: string; + + constructor(transport: NoKVBlobTransport, options: NoKVAuthorityStoreOptions) { + this.transport = transport; + this.tenantId = requireAuthorityStoreId(options.tenant_id, "tenant id"); + this.goalId = requireAuthorityStoreId(options.goal_id, "goal id"); + this.workbench = requireAuthorityStoreId(options.workbench, "workbench"); + this.maxEnvelopeBytes = options.max_envelope_bytes ?? DEFAULT_MAX_ENVELOPE_BYTES; + if (!Number.isSafeInteger(this.maxEnvelopeBytes) || this.maxEnvelopeBytes < 1) { + throw new AuthorityStoreProtocolError( + "max envelope bytes must be a positive safe integer", + ); + } + const digest = createHash("sha256") + .update(canonicalAuthorityBytes({ + tenant_id: this.tenantId, + goal_id: this.goalId, + })) + .digest("hex") + .slice(0, 32); + this.path = `metadata/loopx-authority/${digest}.json`; + } + + async storeIdentity(): Promise { + try { + const result = await this.transport.storeIdentity(this.workbench); + if (result.status !== "available") return result; + if (!validStoreIdentity(result.store_identity, this.workbench)) { + return { + status: "failed", + reason_code: "store_identity_invalid", + reason: "NoKV store identity does not match the bound workbench incarnation", + }; + } + return result; + } catch (error) { + return readFailure(error); + } + } + + private async readEnvelope(): Promise { + const identityResult = await this.storeIdentity(); + if (identityResult.status !== "available") return identityResult; + let result: NoKVBlobReadResult; + try { + result = await this.transport.readBlob(this.workbench, this.path); + } catch (error) { + return readFailure(error); + } + if (result.status === "missing") { + return { status: "missing", identity: identityResult.store_identity }; + } + if (result.status !== "loaded") return result; + try { + const generation = requireGeneration(result.generation, "NoKV read generation"); + let raw: string; + try { + raw = new TextDecoder("utf-8", { fatal: true }).decode(result.bytes); + } catch (error) { + throw new AuthorityStoreProtocolError( + `NoKV authority store bytes are not UTF-8: ${ + error instanceof Error ? error.message : "invalid UTF-8" + }`, + ); + } + let value: unknown; + try { + value = JSON.parse(raw); + } catch (error) { + throw new AuthorityStoreProtocolError( + `NoKV authority store JSON is invalid: ${ + error instanceof Error ? error.message : "invalid JSON" + }`, + ); + } + return { + status: "loaded", + identity: identityResult.store_identity, + generation, + document: decodeDocument( + value, + this.tenantId, + this.goalId, + identityResult.store_identity, + generation, + ), + }; + } catch (error) { + return readFailure(error); + } + } + + async loadAuthority(): Promise { + const result = await this.readEnvelope(); + if (result.status === "missing") return { status: "missing" }; + if (result.status !== "loaded") return result; + return { + status: "loaded", + head: structuredClone(result.document.head), + provider_revision: result.document.provider_revision, + cursor: result.document.cursor, + }; + } + + private async settleUnknownCommit( + expectedProviderRevision: string | null, + intended: AuthorityStoreCommittedTransaction, + fallback: Extract, + ): Promise { + const observed = await this.readEnvelope(); + if (observed.status !== "loaded") return fallback; + const sameOperation = observed.document.committed.find( + (entry) => entry.operation_id === intended.operation_id, + ); + if (sameOperation) { + if ( + canonicalAuthorityBytes(sameOperation).equals(canonicalAuthorityBytes(intended)) + ) { + return { + status: "applied", + provider_revision: sameOperation.provider_revision, + cursor: sameOperation.cursor, + }; + } + return { + status: "conflict", + conflict_kind: "operation_id_exists", + current_provider_revision: observed.document.provider_revision, + current_cursor: observed.document.cursor, + }; + } + if (observed.document.provider_revision !== expectedProviderRevision) { + return { + status: "conflict", + conflict_kind: "provider_revision_mismatch", + current_provider_revision: observed.document.provider_revision, + current_cursor: observed.document.cursor, + }; + } + return fallback; + } + + async commitAuthority(commit: AuthorityStoreCommit): Promise { + let normalized: AuthorityStoreCommit; + try { + normalized = normalizeAuthorityStoreCommit(commit); + } catch (error) { + return { + status: "failed", + reason_code: "invalid_commit_request", + reason: error instanceof Error ? error.message : "invalid commit request", + }; + } + const current = await this.readEnvelope(); + if (current.status !== "loaded" && current.status !== "missing") { + return { + status: "failed", + reason_code: current.reason_code, + reason: current.reason, + }; + } + const currentDocument = current.status === "loaded" ? current.document : null; + if ((currentDocument?.provider_revision ?? null) !== normalized.expected_provider_revision) { + return { + status: "conflict", + conflict_kind: "provider_revision_mismatch", + current_provider_revision: currentDocument?.provider_revision ?? null, + current_cursor: currentDocument?.cursor ?? null, + }; + } + if ( + currentDocument?.committed.some( + (entry) => entry.operation_id === normalized.operation_id, + ) + ) { + return { + status: "conflict", + conflict_kind: "operation_id_exists", + current_provider_revision: currentDocument.provider_revision, + current_cursor: currentDocument.cursor, + }; + } + const cursor = (parseAuthorityCursor(currentDocument?.cursor ?? null) + 1n).toString(); + const generation = (current.status === "loaded" ? current.generation : 0) + 1; + const base = { + cursor, + operation_id: normalized.operation_id, + events: normalized.events, + projection: normalized.next_projection, + receipts: normalized.receipts, + }; + const revision = providerRevision( + this.tenantId, + this.goalId, + current.identity, + generation, + currentDocument?.provider_revision ?? null, + base, + ); + const transaction: AuthorityStoreCommittedTransaction = { + ...base, + provider_revision: revision, + }; + const document: NoKVAuthorityStoreDocument = { + schema_version: NOKV_AUTHORITY_STORE_SCHEMA, + tenant_id: this.tenantId, + goal_id: this.goalId, + store_identity: current.identity, + storage_generation: generation, + provider_revision: revision, + cursor, + head: normalized.next_projection, + committed: [...(currentDocument?.committed ?? []), transaction], + }; + const payload = canonicalAuthorityBytes(document); + if (payload.byteLength > this.maxEnvelopeBytes) { + return { + status: "failed", + reason_code: "authority_envelope_too_large", + reason: `authority envelope exceeds ${this.maxEnvelopeBytes} bytes`, + }; + } + const expectedGeneration = current.status === "loaded" ? current.generation : null; + // NoKV publication identities are terminal after a failed or quarantined + // attempt. Keep the LoopX operation id stable in the authority envelope, + // while giving each physical retry a fresh pair of lower-layer ids. A + // response-lost success is still settled only by reading that envelope. + const attemptNonce = randomUUID(); + let result: NoKVBlobCasResult; + try { + result = await this.transport.casPublishBlob({ + workbench: this.workbench, + path: this.path, + expected_generation: expectedGeneration, + bytes: payload, + operation_id: physicalAttemptIdentity( + "operation", + attemptNonce, + normalized.operation_id, + expectedGeneration, + payload, + ), + artifact_revision_id: physicalAttemptIdentity( + "artifact_revision", + attemptNonce, + normalized.operation_id, + expectedGeneration, + payload, + ), + }); + } catch (error) { + result = { + status: "ambiguous", + reason_code: "nokv_transport_lost", + reason: error instanceof Error ? error.message : "NoKV transport outcome unknown", + }; + } + if (result.status === "applied" && result.generation === generation) { + return { status: "applied", provider_revision: revision, cursor }; + } + if (result.status === "failed") return result; + const fallback: Extract = + result.status === "ambiguous" + ? result + : { + status: "ambiguous", + reason_code: result.status === "conflict" + ? "nokv_cas_conflict_unresolved" + : "nokv_publish_response_invalid", + reason: result.status === "conflict" + ? "NoKV CAS conflict requires authority-envelope readback" + : "NoKV publish generation did not match the attempted envelope", + }; + return await this.settleUnknownCommit( + normalized.expected_provider_revision, + transaction, + fallback, + ); + } + + async readReceipt(operationId: string): Promise { + let normalized: string; + try { + normalized = requireAuthorityStoreId(operationId, "operation id"); + } catch (error) { + return { + status: "failed", + reason_code: "invalid_operation_id", + reason: error instanceof Error ? error.message : "invalid operation id", + }; + } + const result = await this.readEnvelope(); + if (result.status === "missing") return { status: "missing" }; + if (result.status !== "loaded") return result; + const transaction = result.document.committed.find( + (entry) => entry.operation_id === normalized, + ); + return transaction + ? { + status: "found", + cursor: transaction.cursor, + provider_revision: transaction.provider_revision, + receipts: structuredClone(transaction.receipts), + } + : { status: "missing" }; + } + + async scanCommitted( + afterCursor: string | null, + limit: number, + ): Promise { + let offset: bigint; + try { + offset = parseAuthorityCursor(afterCursor); + if (!Number.isSafeInteger(limit) || limit < 1) { + throw new AuthorityStoreProtocolError("scan limit must be a positive safe integer"); + } + } catch (error) { + return { + status: "failed", + reason_code: "invalid_scan_request", + reason: error instanceof Error ? error.message : "invalid scan request", + }; + } + const result = await this.readEnvelope(); + if (result.status === "missing") { + return { status: "page", transactions: [], next_cursor: afterCursor, has_more: false }; + } + if (result.status !== "loaded") return result; + const headCursor = parseAuthorityCursor(result.document.cursor); + if (offset > headCursor || offset > BigInt(Number.MAX_SAFE_INTEGER)) { + return { + status: "failed", + reason_code: "scan_cursor_out_of_range", + reason: "scan cursor is ahead of the provider head", + }; + } + const start = Number(offset); + const transactions = result.document.committed + .slice(start, start + limit) + .map(cloneTransaction); + return { + status: "page", + transactions, + next_cursor: transactions.at(-1)?.cursor ?? afterCursor, + has_more: start + transactions.length < result.document.committed.length, + }; + } +} diff --git a/loopx/control_plane/coordination/nokv_jsonl_helper.py b/loopx/control_plane/coordination/nokv_jsonl_helper.py new file mode 100644 index 0000000000..bbcf8094d9 --- /dev/null +++ b/loopx/control_plane/coordination/nokv_jsonl_helper.py @@ -0,0 +1,610 @@ +"""Narrow JSON-lines bridge from LoopX to the NoKV Python SDK. + +The bridge deliberately knows only byte storage. Authority transitions, +receipts, cursor ordering, and ambiguous-outcome reconciliation stay in the +TypeScript ``NoKVAuthorityStore``. The first JSON line configures one SDK +client; every later line invokes exactly one of ``store_identity``, +``read_blob``, or ``cas_publish_blob``. +""" + +from __future__ import annotations + +import base64 +import binascii +import json +import re +import sys +from collections.abc import Callable, Mapping +from typing import Any, TextIO + +_HEX_128 = re.compile(r"^[0-9a-f]{32}$") +_CLIENT_AVAILABILITY_ERRORS = (RuntimeError, OSError) +QUALIFIED_NOKV_SDK_VERSION = "0.11.0" +QUALIFIED_NOKV_API_VERSION = 1 + +_CONFIG_KEYS = frozenset( + { + "root_id", + "routing", + "object_store", + "max_attempts", + "connect_timeout_ms", + "read_timeout_ms", + "write_timeout_ms", + "handshake_timeout_ms", + "workbench_root", + } +) +_ETCD_ROUTING_KEYS = frozenset({"kind", "endpoints", "key_prefix", "lease_ttl_seconds"}) +_STATIC_ROUTING_KEYS = frozenset( + { + "kind", + "endpoint", + "logical_shard_id", + "object_namespace_id", + "placement_generation", + "owner_epoch", + } +) +_MEMORY_OBJECT_STORE_KEYS = frozenset({"kind"}) +_S3_OBJECT_STORE_KEYS = frozenset( + { + "kind", + "bucket", + "region", + "root", + "endpoint", + "access_key_id", + "secret_access_key", + "session_token", + "virtual_host_style", + "skip_signature", + } +) + + +class RequestError(ValueError): + """The JSON-lines caller violated the raw storage protocol.""" + + +class ProviderProtocolError(RuntimeError): + """The NoKV SDK returned a shape that violates its reviewed contract.""" + + +class ClientAdmissionUnavailable(RuntimeError): + """The configured SDK could not admit a live route or object provider.""" + + +def _request_id(value: object) -> str | None: + if isinstance(value, str) and value.strip() == value and value: + return value + return None + + +def _response( + request_id: str | None, status: str, **values: object +) -> dict[str, object]: + return {"request_id": request_id, "status": status, **values} + + +def _failure( + request_id: str | None, + status: str, + reason_code: str, + error: object, +) -> dict[str, object]: + return _response( + request_id, + status, + reason_code=reason_code, + reason=str(error) if str(error) else reason_code, + ) + + +def _opaque_failure( + request_id: str | None, + status: str, + reason_code: str, + reason: str, +) -> dict[str, object]: + return _response( + request_id, + status, + reason_code=reason_code, + reason=reason, + ) + + +def _mapping(value: object, name: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise RequestError(f"{name} must be an object") + return value + + +def _require_exact_keys( + values: Mapping[str, Any], + allowed: frozenset[str], + name: str, +) -> None: + if any(key not in allowed for key in values): + raise RequestError(f"{name} contains unsupported fields") + + +def _required_string(values: Mapping[str, Any], name: str) -> str: + value = values.get(name) + if not isinstance(value, str) or not value or value.strip() != value: + raise RequestError(f"{name} must be a non-empty trimmed string") + return value + + +def _generation(value: object, name: str, *, nullable: bool = False) -> int | None: + if value is None and nullable: + return None + if not isinstance(value, int) or isinstance(value, bool) or value < 1: + suffix = " or null" if nullable else "" + raise RequestError(f"{name} must be a positive integer{suffix}") + return value + + +def _sdk_generation(value: object, name: str) -> int: + try: + generation = _generation(value, name) + except RequestError as error: + raise ProviderProtocolError(str(error)) from error + assert generation is not None + return generation + + +def _decode_bytes(value: object) -> bytes: + if not isinstance(value, str): + raise RequestError("bytes_base64 must be a string") + try: + return base64.b64decode(value, validate=True) + except (binascii.Error, ValueError) as error: + raise RequestError("bytes_base64 is not canonical base64") from error + + +def _identity(client: Any, workbench: str) -> str: + cursor: bytes | None = None + seen_cursors: set[bytes] = set() + identities: set[str] = set() + while True: + page = _mapping( + client.find_workspaces(cursor=cursor, limit=100), + "find_workspaces result", + ) + workspaces = page.get("workspaces") + if not isinstance(workspaces, list): + raise ProviderProtocolError("find_workspaces omitted workspaces") + for item in workspaces: + entry = _mapping(item, "workspace entry") + workspace = _mapping(entry.get("workspace"), "workspace summary") + if workspace.get("workbench") != workbench: + continue + incarnation = workspace.get("workspace_incarnation_id") + if not isinstance(incarnation, str) or not _HEX_128.fullmatch(incarnation): + raise ProviderProtocolError( + "workspace incarnation identity must be 32 lowercase hex" + ) + identities.add(incarnation) + next_cursor = page.get("next_cursor") + if next_cursor is None: + break + if not isinstance(next_cursor, bytes) or not next_cursor: + raise ProviderProtocolError("find_workspaces next_cursor is invalid") + if next_cursor in seen_cursors: + raise ProviderProtocolError("find_workspaces cursor did not advance") + seen_cursors.add(next_cursor) + cursor = next_cursor + if len(identities) != 1: + raise ProviderProtocolError( + "workbench did not resolve to one incarnation identity" + ) + return f"nokv:{workbench}:{next(iter(identities))}" + + +def _store_identity( + client: Any, + request_id: str, + values: Mapping[str, Any], +) -> dict[str, object]: + workbench = _required_string(values, "workbench") + try: + identity = _identity(client, workbench) + except (ProviderProtocolError, TypeError, ValueError) as error: + return _failure( + request_id, + "failed", + "provider_protocol_violation", + error, + ) + except _CLIENT_AVAILABILITY_ERRORS: + return _opaque_failure( + request_id, + "unavailable", + "nokv_identity_unavailable", + "NoKV identity lookup is unavailable", + ) + return _response(request_id, "available", store_identity=identity) + + +def _read_blob( + client: Any, + request_id: str, + values: Mapping[str, Any], +) -> dict[str, object]: + workbench = _required_string(values, "workbench") + path = _required_string(values, "path") + try: + result = _mapping(client.read(workbench, path), "read result") + except FileNotFoundError: + return _response(request_id, "missing") + except _CLIENT_AVAILABILITY_ERRORS: + return _opaque_failure( + request_id, + "unavailable", + "nokv_read_unavailable", + "NoKV blob read is unavailable", + ) + except (TypeError, ValueError) as error: + return _failure(request_id, "failed", "provider_protocol_violation", error) + try: + raw = result.get("bytes") + if not isinstance(raw, (bytes, bytearray, memoryview)): + raise ProviderProtocolError("read result omitted bytes") + metadata = _mapping(result.get("metadata"), "read metadata") + if metadata.get("workbench") != workbench: + raise ProviderProtocolError("read metadata workbench mismatch") + if metadata.get("path") != path: + raise ProviderProtocolError("read metadata path mismatch") + incarnation = metadata.get("workspace_incarnation_id") + if not isinstance(incarnation, str) or not _HEX_128.fullmatch(incarnation): + raise ProviderProtocolError( + "read metadata workspace incarnation identity is invalid" + ) + current_identity = _identity(client, workbench) + if current_identity != f"nokv:{workbench}:{incarnation}": + raise ProviderProtocolError("read metadata workspace incarnation mismatch") + generation = _sdk_generation(metadata.get("generation"), "read generation") + except (ProviderProtocolError, TypeError, ValueError) as error: + return _failure(request_id, "failed", "provider_protocol_violation", error) + except _CLIENT_AVAILABILITY_ERRORS: + return _opaque_failure( + request_id, + "unavailable", + "nokv_read_unavailable", + "NoKV blob read is unavailable", + ) + return _response( + request_id, + "loaded", + bytes_base64=base64.b64encode(bytes(raw)).decode("ascii"), + generation=generation, + ) + + +def _cas_publish_blob( + client: Any, + request_id: str, + values: Mapping[str, Any], +) -> dict[str, object]: + workbench = _required_string(values, "workbench") + path = _required_string(values, "path") + expected_generation = _generation( + values.get("expected_generation"), + "expected_generation", + nullable=True, + ) + payload = _decode_bytes(values.get("bytes_base64")) + operation_id = _required_string(values, "operation_id") + artifact_revision_id = _required_string(values, "artifact_revision_id") + if not _HEX_128.fullmatch(operation_id): + raise RequestError("operation_id must be 32 lowercase hex") + if not _HEX_128.fullmatch(artifact_revision_id): + raise RequestError("artifact_revision_id must be 32 lowercase hex") + try: + raw_result = client.publish_bytes( + workbench, + path, + payload, + content_type="application/json", + expected_generation=expected_generation, + operation_id=operation_id, + artifact_revision_id=artifact_revision_id, + ) + except FileExistsError: + return _response( + request_id, + "conflict", + current_generation=None, + ) + except (RuntimeError, OSError, TypeError, ValueError): + # RuntimeError covers both a rejected generation and a response lost + # after commit. Human error text cannot distinguish them, so only a + # later authority-envelope readback may settle the outcome. + return _opaque_failure( + request_id, + "ambiguous", + "nokv_publish_outcome_unknown", + "NoKV publish outcome is unknown", + ) + try: + result = _mapping(raw_result, "publish result") + except (TypeError, ValueError): + return _opaque_failure( + request_id, + "ambiguous", + "provider_protocol_violation", + "NoKV publish response violated the storage protocol", + ) + try: + generation = _sdk_generation(result.get("generation"), "publish generation") + expected_result_generation = (expected_generation or 0) + 1 + if result.get("workbench") != workbench: + raise ProviderProtocolError("publish result workbench mismatch") + if result.get("path") != path: + raise ProviderProtocolError("publish result path mismatch") + if result.get("operation_id") != operation_id: + raise ProviderProtocolError("publish result operation identity mismatch") + if result.get("artifact_revision_id") != artifact_revision_id: + raise ProviderProtocolError("publish result artifact revision mismatch") + if generation != expected_result_generation: + raise ProviderProtocolError("publish result generation mismatch") + except ProviderProtocolError: + return _opaque_failure( + request_id, + "ambiguous", + "provider_protocol_violation", + "NoKV publish response violated the storage protocol", + ) + return _response(request_id, "applied", generation=generation) + + +def handle_request(client: Any, value: object) -> dict[str, object]: + """Execute one raw storage request without interpreting stored bytes.""" + + request_id = _request_id( + value.get("request_id") if isinstance(value, Mapping) else None + ) + try: + values = _mapping(value, "request") + if request_id is None: + raise RequestError("request_id must be a non-empty trimmed string") + operation = _required_string(values, "operation") + handlers: dict[ + str, + Callable[[Any, str, Mapping[str, Any]], dict[str, object]], + ] = { + "store_identity": _store_identity, + "read_blob": _read_blob, + "cas_publish_blob": _cas_publish_blob, + } + handler = handlers.get(operation) + if handler is None: + raise RequestError(f"unknown operation {operation!r}") + return handler(client, request_id, values) + except RequestError as error: + return _failure(request_id, "failed", "invalid_request", error) + + +def serve(client: Any, incoming: TextIO, outgoing: TextIO) -> None: + """Serve raw storage requests until EOF.""" + + for line in incoming: + try: + request = json.loads(line) + except json.JSONDecodeError as error: + result = _failure(None, "failed", "invalid_json", error) + else: + result = handle_request(client, request) + outgoing.write(json.dumps(result, sort_keys=True, separators=(",", ":")) + "\n") + outgoing.flush() + + +def _string_list(values: Mapping[str, Any], name: str) -> list[str]: + value = values.get(name) + if not isinstance(value, list) or not value: + raise RequestError(f"{name} must be a non-empty array") + result: list[str] = [] + for item in value: + if not isinstance(item, str) or not item: + raise RequestError(f"{name} entries must be non-empty strings") + result.append(item) + return result + + +def build_client(config_value: object) -> Any: + """Construct one eagerly admitted NoKV client from the open handshake.""" + + config = _mapping(config_value, "config") + _require_exact_keys(config, _CONFIG_KEYS, "config") + routing_value = _mapping(config.get("routing"), "routing") + routing_kind = _required_string(routing_value, "kind") + if routing_kind == "etcd": + _require_exact_keys(routing_value, _ETCD_ROUTING_KEYS, "routing") + routing_arguments: tuple[object, ...] = ( + _string_list(routing_value, "endpoints"), + _required_string(routing_value, "key_prefix"), + _generation( + routing_value.get("lease_ttl_seconds", 10), + "lease_ttl_seconds", + ), + ) + elif routing_kind == "static": + _require_exact_keys(routing_value, _STATIC_ROUTING_KEYS, "routing") + routing_arguments = ( + _required_string(routing_value, "endpoint"), + _required_string(routing_value, "logical_shard_id"), + _required_string(routing_value, "object_namespace_id"), + _generation( + routing_value.get("placement_generation"), + "placement_generation", + ), + _generation(routing_value.get("owner_epoch"), "owner_epoch"), + ) + else: + raise RequestError(f"unsupported routing kind {routing_kind!r}") + + object_value = _mapping(config.get("object_store"), "object_store") + object_kind = _required_string(object_value, "kind") + if object_kind == "memory": + _require_exact_keys( + object_value, + _MEMORY_OBJECT_STORE_KEYS, + "object_store", + ) + object_arguments: dict[str, object] | None = None + elif object_kind == "s3": + _require_exact_keys(object_value, _S3_OBJECT_STORE_KEYS, "object_store") + object_arguments = { + "bucket": _required_string(object_value, "bucket"), + "region": object_value.get("region", "us-east-1"), + "root": object_value.get("root", "/"), + "endpoint": object_value.get("endpoint"), + "access_key_id": object_value.get("access_key_id"), + "secret_access_key": object_value.get("secret_access_key"), + "session_token": object_value.get("session_token"), + "virtual_host_style": object_value.get("virtual_host_style", False), + "skip_signature": object_value.get("skip_signature", False), + } + else: + raise RequestError(f"unsupported object store kind {object_kind!r}") + + root_id = _required_string(config, "root_id") + if not _HEX_128.fullmatch(root_id): + raise RequestError("root_id must be 32 lowercase hex") + max_attempts = _generation(config.get("max_attempts", 3), "max_attempts") + connect_timeout_ms = _generation( + config.get("connect_timeout_ms", 5_000), + "connect_timeout_ms", + ) + read_timeout_ms = _generation( + config.get("read_timeout_ms", 30_000), + "read_timeout_ms", + ) + write_timeout_ms = _generation( + config.get("write_timeout_ms", 30_000), + "write_timeout_ms", + ) + handshake_timeout_ms = _generation( + config.get("handshake_timeout_ms", 5_000), + "handshake_timeout_ms", + ) + workbench_root = config.get("workbench_root") + if workbench_root is not None and ( + not isinstance(workbench_root, str) or not workbench_root + ): + raise RequestError("workbench_root must be a non-empty string or null") + + try: + import nokv + except ImportError as error: # pragma: no cover - exercised by live packaging + raise RequestError("the NoKV Python SDK is not installed") from error + if ( + getattr(nokv, "__version__", None) != QUALIFIED_NOKV_SDK_VERSION + or getattr(nokv, "API_VERSION", None) != QUALIFIED_NOKV_API_VERSION + ): + raise RequestError( + "the NoKV Python SDK must be version " + f"{QUALIFIED_NOKV_SDK_VERSION} with API version " + f"{QUALIFIED_NOKV_API_VERSION}" + ) + try: + Client = nokv.Client + ObjectStoreConfig = nokv.ObjectStoreConfig + RoutingConfig = nokv.RoutingConfig + except AttributeError as error: + raise RequestError("the NoKV Python SDK surface is incomplete") from error + + try: + routing = ( + RoutingConfig.etcd(*routing_arguments) + if routing_kind == "etcd" + else RoutingConfig.static(*routing_arguments) + ) + except (TypeError, ValueError) as error: + raise RequestError("NoKV routing configuration is invalid") from error + try: + object_store = ( + ObjectStoreConfig.memory() + if object_arguments is None + else ObjectStoreConfig.s3(**object_arguments) + ) + except (TypeError, ValueError) as error: + raise RequestError("NoKV object-store configuration is invalid") from error + try: + return Client( + root_id=root_id, + routing=routing, + object_store=object_store, + max_attempts=max_attempts, + connect_timeout_ms=connect_timeout_ms, + read_timeout_ms=read_timeout_ms, + write_timeout_ms=write_timeout_ms, + handshake_timeout_ms=handshake_timeout_ms, + workbench_root=workbench_root, + ) + except (RuntimeError, OSError, ValueError) as error: + raise ClientAdmissionUnavailable from error + except TypeError as error: + raise RequestError("NoKV client configuration is invalid") from error + + +def main() -> int: + first = sys.stdin.readline() + try: + value = json.loads(first) + values = _mapping(value, "open request") + request_id = _request_id(values.get("request_id")) + if request_id is None or values.get("operation") != "open": + raise RequestError("first request must be an open handshake") + client = build_client(values.get("config")) + except json.JSONDecodeError as error: + result = _failure(None, "failed", "invalid_json", error) + except RequestError as error: + result = _failure( + _request_id(value.get("request_id")) + if isinstance(value, Mapping) + else None, + "failed", + "invalid_config", + error, + ) + except _CLIENT_AVAILABILITY_ERRORS: + result = _opaque_failure( + request_id, + "unavailable", + "nokv_open_unavailable", + "NoKV client admission is unavailable", + ) + except (TypeError, ValueError): + result = _opaque_failure( + request_id, + "failed", + "invalid_config", + "NoKV helper configuration is invalid", + ) + else: + sys.stdout.write( + json.dumps( + _response( + request_id, + "ready", + nokv_sdk_version=QUALIFIED_NOKV_SDK_VERSION, + nokv_api_version=QUALIFIED_NOKV_API_VERSION, + ), + sort_keys=True, + separators=(",", ":"), + ) + + "\n" + ) + sys.stdout.flush() + serve(client, sys.stdin, sys.stdout) + return 0 + sys.stdout.write(json.dumps(result, sort_keys=True, separators=(",", ":")) + "\n") + sys.stdout.flush() + return 2 + + +if __name__ == "__main__": # pragma: no cover - exercised by subprocess E2E + raise SystemExit(main()) diff --git a/loopx/control_plane/coordination/nokv_jsonl_transport.ts b/loopx/control_plane/coordination/nokv_jsonl_transport.ts new file mode 100644 index 0000000000..ad5516c7d0 --- /dev/null +++ b/loopx/control_plane/coordination/nokv_jsonl_transport.ts @@ -0,0 +1,346 @@ +import { randomUUID } from "node:crypto"; +import { + spawn, + type ChildProcessWithoutNullStreams, + type SpawnOptionsWithoutStdio, +} from "node:child_process"; +import { once } from "node:events"; + +import type { JsonObject } from "../effect_program.ts"; +import { + NoKVTransportProtocolError, + NoKVTransportUnavailableError, + type NoKVBlobCasRequest, + type NoKVBlobCasResult, + type NoKVBlobReadResult, + type NoKVBlobTransport, + type NoKVStoreIdentityResult, + type NoKVTransportFailure, +} from "./nokv_authority_store.ts"; +import { + isAuthorityJsonObject, +} from "./authority_store_codec.ts"; + +const DEFAULT_REQUEST_TIMEOUT_MS = 30_000; +const DEFAULT_MAX_RESPONSE_BYTES = 32 * 1024 * 1024; + +export type NoKVJsonLinesProcessFactory = ( + command: string, + args: readonly string[], + options: SpawnOptionsWithoutStdio, +) => ChildProcessWithoutNullStreams; + +export interface NoKVJsonLinesTransportOptions { + /** Explicit command plus arguments, for example `[python, helper.py]`. */ + argv: readonly string[]; + /** Sent only over stdin in the helper's open handshake. */ + config: JsonObject; + cwd?: string; + env?: NodeJS.ProcessEnv; + request_timeout_ms?: number; + max_response_bytes?: number; + process_factory?: NoKVJsonLinesProcessFactory; +} + +interface PendingResponse { + resolve(value: JsonObject): void; + reject(error: Error): void; + timer: NodeJS.Timeout; +} + +function positiveSafeInteger(value: unknown, name: string): number { + if (!Number.isSafeInteger(value) || (value as number) < 1) { + throw new NoKVTransportProtocolError(`${name} must be a positive safe integer`); + } + return value as number; +} + +function requiredString(value: unknown, name: string): string { + if (typeof value !== "string" || value.trim() !== value || value.length === 0) { + throw new NoKVTransportProtocolError(`${name} must be a non-empty trimmed string`); + } + return value; +} + +function responseFailure(value: JsonObject): NoKVTransportFailure { + const status = value.status; + if (status !== "unavailable" && status !== "failed") { + throw new NoKVTransportProtocolError("helper response status is invalid"); + } + return { + status, + reason_code: requiredString(value.reason_code, "helper reason code"), + reason: requiredString(value.reason, "helper reason"), + }; +} + +function canonicalBase64(value: unknown): Uint8Array { + if (typeof value !== "string") { + throw new NoKVTransportProtocolError("helper bytes_base64 must be a string"); + } + const bytes = Buffer.from(value, "base64"); + if (bytes.toString("base64") !== value) { + throw new NoKVTransportProtocolError("helper bytes_base64 is not canonical base64"); + } + return bytes; +} + +/** + * Reusable process connection to the NoKV Python SDK helper. + * + * Construction is intentionally asynchronous and requires an explicit argv; + * no runtime path selects NoKV merely by importing this module. + */ +export class NoKVJsonLinesTransport implements NoKVBlobTransport { + private readonly child: ChildProcessWithoutNullStreams; + private readonly requestTimeoutMs: number; + private readonly maxResponseBytes: number; + private readonly pending = new Map(); + private stdoutBuffer = Buffer.alloc(0); + private terminalError: Error | null = null; + private closing = false; + + private constructor(options: NoKVJsonLinesTransportOptions) { + if (!Array.isArray(options.argv) || options.argv.length === 0) { + throw new NoKVTransportProtocolError("NoKV helper argv must not be empty"); + } + for (const value of options.argv) { + requiredString(value, "NoKV helper argv entry"); + } + this.requestTimeoutMs = options.request_timeout_ms ?? DEFAULT_REQUEST_TIMEOUT_MS; + this.maxResponseBytes = options.max_response_bytes ?? DEFAULT_MAX_RESPONSE_BYTES; + positiveSafeInteger(this.requestTimeoutMs, "NoKV helper request timeout"); + positiveSafeInteger(this.maxResponseBytes, "NoKV helper max response bytes"); + const factory = options.process_factory ?? ((command, args, spawnOptions) => + spawn(command, args, { ...spawnOptions, stdio: ["pipe", "pipe", "pipe"] })); + const [command, ...args] = options.argv; + try { + this.child = factory(command!, args, { + cwd: options.cwd, + env: options.env, + }); + } catch { + throw new NoKVTransportUnavailableError("NoKV helper failed to start"); + } + this.child.stdout.on("data", (chunk: Buffer | string) => { + this.onStdout(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + }); + // Drain stderr so a noisy SDK cannot block, but never promote arbitrary + // provider output into LoopX errors where endpoints or credentials could + // escape the provider boundary. + this.child.stderr.on("data", () => {}); + this.child.stdin.on("error", () => { + if (!this.closing) this.failUnavailable("NoKV helper stdin failed"); + }); + this.child.on("error", () => { + this.failUnavailable("NoKV helper failed to start"); + }); + this.child.on("exit", (code, signal) => { + if (this.closing && this.pending.size === 0) return; + this.failUnavailable( + `NoKV helper disconnected (code=${String(code)}, signal=${String(signal)})`, + false, + ); + }); + } + + static async open(options: NoKVJsonLinesTransportOptions): Promise { + const transport = new NoKVJsonLinesTransport(options); + let response: JsonObject; + try { + response = await transport.exchange("open", { config: options.config }); + if (response.status === "unavailable") { + throw new NoKVTransportUnavailableError( + requiredString(response.reason, "helper open reason"), + ); + } + if (response.status !== "ready") { + throw new NoKVTransportProtocolError( + response.status === "failed" && typeof response.reason === "string" + ? response.reason + : "NoKV helper did not acknowledge its open handshake", + ); + } + return transport; + } catch (error) { + await transport.close(); + throw error; + } + } + + private fail(error: Error, terminate: boolean): void { + if (this.terminalError === null) this.terminalError = error; + for (const pending of this.pending.values()) { + clearTimeout(pending.timer); + pending.reject(this.terminalError); + } + this.pending.clear(); + if (terminate && this.child.exitCode === null && this.child.signalCode === null) { + this.child.kill(); + } + } + + private failUnavailable(message: string, terminate = true): void { + this.fail(new NoKVTransportUnavailableError(message), terminate); + } + + private failProtocol(message: string): void { + this.fail(new NoKVTransportProtocolError(message), true); + } + + private onStdout(chunk: Buffer): void { + if (this.terminalError !== null) return; + this.stdoutBuffer = Buffer.concat([this.stdoutBuffer, chunk]); + while (true) { + const newline = this.stdoutBuffer.indexOf(0x0a); + if (newline < 0) break; + if (newline > this.maxResponseBytes) { + this.failProtocol("NoKV helper response exceeded max_response_bytes"); + return; + } + const line = this.stdoutBuffer.subarray(0, newline); + this.stdoutBuffer = this.stdoutBuffer.subarray(newline + 1); + if (line.byteLength === 0) { + this.failProtocol("NoKV helper emitted an empty response line"); + return; + } + let value: unknown; + try { + value = JSON.parse(line.toString("utf8")); + } catch (error) { + this.failProtocol( + `NoKV helper emitted invalid JSON: ${ + error instanceof Error ? error.message : "invalid JSON" + }`, + ); + return; + } + if (!isAuthorityJsonObject(value)) { + this.failProtocol("NoKV helper response must be an object"); + return; + } + const requestId = value.request_id; + if (typeof requestId !== "string") { + this.failProtocol("NoKV helper response omitted request_id"); + return; + } + const pending = this.pending.get(requestId); + if (!pending) { + this.failProtocol("NoKV helper responded with an unknown request_id"); + return; + } + this.pending.delete(requestId); + clearTimeout(pending.timer); + pending.resolve(value); + } + if (this.stdoutBuffer.byteLength > this.maxResponseBytes) { + this.failProtocol("NoKV helper response exceeded max_response_bytes"); + } + } + + private async exchange(operation: string, values: JsonObject): Promise { + if (this.terminalError) throw this.terminalError; + if (this.closing) { + throw new NoKVTransportUnavailableError("NoKV helper transport is closed"); + } + const requestId = randomUUID(); + const response = new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.pending.delete(requestId); + const error = new NoKVTransportUnavailableError( + `NoKV helper request ${operation} timed out`, + ); + reject(error); + this.fail(error, true); + }, this.requestTimeoutMs); + this.pending.set(requestId, { resolve, reject, timer }); + }); + const line = JSON.stringify({ request_id: requestId, operation, ...values }) + "\n"; + try { + this.child.stdin.write(line); + } catch { + this.failUnavailable("NoKV helper request write failed"); + } + return await response; + } + + async storeIdentity(workbench: string): Promise { + const response = await this.exchange("store_identity", { workbench }); + if (response.status === "available") { + return { + status: "available", + store_identity: requiredString( + response.store_identity, + "helper store identity", + ), + }; + } + return responseFailure(response); + } + + async readBlob(workbench: string, path: string): Promise { + const response = await this.exchange("read_blob", { workbench, path }); + if (response.status === "missing") return { status: "missing" }; + if (response.status === "loaded") { + return { + status: "loaded", + bytes: canonicalBase64(response.bytes_base64), + generation: positiveSafeInteger(response.generation, "helper read generation"), + }; + } + return responseFailure(response); + } + + async casPublishBlob(request: NoKVBlobCasRequest): Promise { + const response = await this.exchange("cas_publish_blob", { + workbench: request.workbench, + path: request.path, + expected_generation: request.expected_generation, + bytes_base64: Buffer.from(request.bytes).toString("base64"), + operation_id: request.operation_id, + artifact_revision_id: request.artifact_revision_id, + }); + if (response.status === "applied") { + return { + status: "applied", + generation: positiveSafeInteger(response.generation, "helper publish generation"), + }; + } + if (response.status === "conflict") { + const current = response.current_generation; + return { + status: "conflict", + current_generation: current === null + ? null + : positiveSafeInteger(current, "helper conflict generation"), + }; + } + if (response.status === "ambiguous") { + return { + status: "ambiguous", + reason_code: requiredString(response.reason_code, "helper reason code"), + reason: requiredString(response.reason, "helper reason"), + }; + } + const failure = responseFailure(response); + return { + status: "failed", + reason_code: failure.reason_code, + reason: failure.reason, + }; + } + + async close(): Promise { + if (this.closing) return; + this.closing = true; + if (this.child.exitCode !== null || this.child.signalCode !== null) return; + this.child.stdin.end(); + await Promise.race([ + once(this.child, "exit"), + new Promise((resolve) => setTimeout(resolve, 1_000)), + ]); + if (this.child.exitCode === null && this.child.signalCode === null) { + this.child.kill(); + } + } +} diff --git a/tests/control_plane_ts/nokv_authority_store.test.ts b/tests/control_plane_ts/nokv_authority_store.test.ts new file mode 100644 index 0000000000..675d9c73e2 --- /dev/null +++ b/tests/control_plane_ts/nokv_authority_store.test.ts @@ -0,0 +1,312 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + NoKVAuthorityStore, + type NoKVBlobCasRequest, + type NoKVBlobCasResult, + type NoKVBlobReadResult, + type NoKVBlobTransport, + type NoKVStoreIdentityResult, +} from "../../loopx/control_plane/coordination/nokv_authority_store.ts"; +import { + authorityStoreCommitFixture as commit, + registerAuthorityStoreConformance, +} from "./authority_store_conformance.ts"; + +type PublishFault = + | "ambiguous_before" + | "terminal_ambiguous_before" + | "ambiguous_after" + | "ambiguous_after_then_read_unavailable" + | "failed" + | null; + +interface FakeNoKVBackend { + identity: string; + blob: { bytes: Uint8Array; generation: number } | null; + identityUnavailable: boolean; + readUnavailable: number; + publishFault: PublishFault; + casRequests: NoKVBlobCasRequest[]; + terminalPhysicalIds: Set; +} + +function fakeBackend(): FakeNoKVBackend { + return { + identity: `nokv:authority-workbench:${"a".repeat(32)}`, + blob: null, + identityUnavailable: false, + readUnavailable: 0, + publishFault: null, + casRequests: [], + terminalPhysicalIds: new Set(), + }; +} + +class FakeNoKVTransport implements NoKVBlobTransport { + readonly backend: FakeNoKVBackend; + + constructor(backend: FakeNoKVBackend) { + this.backend = backend; + } + + async storeIdentity(_workbench: string): Promise { + if (this.backend.identityUnavailable) { + return { + status: "unavailable", + reason_code: "injected_identity_unavailable", + reason: "identity lookup unavailable", + }; + } + return { status: "available", store_identity: this.backend.identity }; + } + + async readBlob(_workbench: string, _path: string): Promise { + if (this.backend.readUnavailable > 0) { + this.backend.readUnavailable -= 1; + return { + status: "unavailable", + reason_code: "injected_read_unavailable", + reason: "blob read unavailable", + }; + } + return this.backend.blob + ? { + status: "loaded", + bytes: this.backend.blob.bytes.slice(), + generation: this.backend.blob.generation, + } + : { status: "missing" }; + } + + async casPublishBlob(request: NoKVBlobCasRequest): Promise { + this.backend.casRequests.push({ ...request, bytes: request.bytes.slice() }); + if ( + this.backend.terminalPhysicalIds.has(request.operation_id) || + this.backend.terminalPhysicalIds.has(request.artifact_revision_id) + ) { + return { + status: "ambiguous", + reason_code: "injected_terminal_identity_spent", + reason: "physical publication identity is terminal", + }; + } + const current = this.backend.blob?.generation ?? null; + if (current !== request.expected_generation) { + return { status: "conflict", current_generation: current }; + } + if (this.backend.publishFault === "failed") { + this.backend.publishFault = null; + return { + status: "failed", + reason_code: "injected_publish_rejected", + reason: "publish rejected before SDK call", + }; + } + if (this.backend.publishFault === "ambiguous_before") { + this.backend.publishFault = null; + return { + status: "ambiguous", + reason_code: "injected_lost_response", + reason: "publish outcome unknown", + }; + } + if (this.backend.publishFault === "terminal_ambiguous_before") { + this.backend.publishFault = null; + this.backend.terminalPhysicalIds.add(request.operation_id); + this.backend.terminalPhysicalIds.add(request.artifact_revision_id); + return { + status: "ambiguous", + reason_code: "injected_terminal_identity_spent", + reason: "physical publication identity failed terminally", + }; + } + const generation = (request.expected_generation ?? 0) + 1; + this.backend.blob = { bytes: request.bytes.slice(), generation }; + if ( + this.backend.publishFault === "ambiguous_after" || + this.backend.publishFault === "ambiguous_after_then_read_unavailable" + ) { + if (this.backend.publishFault === "ambiguous_after_then_read_unavailable") { + this.backend.readUnavailable += 1; + } + this.backend.publishFault = null; + return { + status: "ambiguous", + reason_code: "injected_lost_response", + reason: "publish response was lost", + }; + } + return { status: "applied", generation }; + } +} + +function store(backend: FakeNoKVBackend, tenantId = "tenant-a", goalId = "goal-a") { + return new NoKVAuthorityStore(new FakeNoKVTransport(backend), { + tenant_id: tenantId, + goal_id: goalId, + workbench: "authority-workbench", + }); +} + +registerAuthorityStoreConformance("NoKV single-envelope provider", async () => { + const backend = fakeBackend(); + return { store: store(backend), contender: store(backend) }; +}); + +test("NoKV provider uses a deterministic CLI-readable metadata path", () => { + const backend = fakeBackend(); + const first = store(backend); + const same = store(backend); + const otherTenant = store(backend, "tenant-b"); + + assert.equal(first.path, same.path); + assert.match(first.path, /^metadata\/loopx-authority\/[0-9a-f]{32}\.json$/); + assert.notEqual(first.path, otherTenant.path); +}); + +test("NoKV provider keeps proven missing distinct from identity and read unavailability", async () => { + const backend = fakeBackend(); + const provider = store(backend); + assert.deepEqual(await provider.loadAuthority(), { status: "missing" }); + + backend.readUnavailable = 1; + const readUnavailable = await provider.loadAuthority(); + assert.equal(readUnavailable.status, "unavailable"); + if (readUnavailable.status === "unavailable") { + assert.equal(readUnavailable.reason_code, "injected_read_unavailable"); + } + + backend.identityUnavailable = true; + const identityUnavailable = await provider.loadAuthority(); + assert.equal(identityUnavailable.status, "unavailable"); + assert.equal((await provider.storeIdentity()).status, "unavailable"); +}); + +test("NoKV provider reconciles a lost success from the embedded operation receipt", async () => { + const backend = fakeBackend(); + const provider = store(backend); + backend.publishFault = "ambiguous_after"; + + const applied = await provider.commitAuthority(commit(null, "operation-a", 1, 7)); + assert.equal(applied.status, "applied"); + const receipt = await provider.readReceipt("operation-a"); + assert.equal(receipt.status, "found"); + if (receipt.status === "found") assert.equal(receipt.receipts[0]?.lease_epoch, 7); +}); + +test("NoKV provider leaves an outcome ambiguous until readback becomes available", async () => { + const backend = fakeBackend(); + const provider = store(backend); + backend.publishFault = "ambiguous_after_then_read_unavailable"; + + const unknown = await provider.commitAuthority(commit(null, "operation-a", 1, 8)); + assert.equal(unknown.status, "ambiguous"); + const receipt = await provider.readReceipt("operation-a"); + assert.equal(receipt.status, "found"); + if (receipt.status === "found") assert.equal(receipt.receipts[0]?.lease_epoch, 8); +}); + +test("NoKV provider does not invent a receipt when an ambiguous publish did not land", async () => { + const backend = fakeBackend(); + const provider = store(backend); + backend.publishFault = "ambiguous_before"; + + const unknown = await provider.commitAuthority(commit(null, "operation-a", 1, 9)); + assert.equal(unknown.status, "ambiguous"); + assert.deepEqual(await provider.readReceipt("operation-a"), { status: "missing" }); + assert.deepEqual(await provider.loadAuthority(), { status: "missing" }); +}); + +test("NoKV provider retries one logical commit with fresh physical identities", async () => { + const backend = fakeBackend(); + const provider = store(backend); + const request = commit(null, "operation-a", 1, 1); + backend.publishFault = "terminal_ambiguous_before"; + + assert.equal((await provider.commitAuthority(request)).status, "ambiguous"); + assert.equal((await provider.commitAuthority(request)).status, "applied"); + assert.equal(backend.casRequests.length, 2); + assert.notEqual( + backend.casRequests[0]?.operation_id, + backend.casRequests[1]?.operation_id, + ); + assert.notEqual( + backend.casRequests[0]?.artifact_revision_id, + backend.casRequests[1]?.artifact_revision_id, + ); + assert.match(backend.casRequests[0]!.operation_id, /^[0-9a-f]{32}$/); + assert.match(backend.casRequests[0]!.artifact_revision_id, /^[0-9a-f]{32}$/); + assert.notEqual( + backend.casRequests[0]?.operation_id, + backend.casRequests[0]?.artifact_revision_id, + ); +}); + +test("NoKV provider fences restored bytes with a different workspace incarnation", async () => { + const backend = fakeBackend(); + const original = store(backend); + const applied = await original.commitAuthority(commit(null, "operation-a", 1, 1)); + assert.equal(applied.status, "applied"); + const callsBeforeRestore = backend.casRequests.length; + + backend.identity = `nokv:authority-workbench:${"b".repeat(32)}`; + const restored = store(backend); + const loaded = await restored.loadAuthority(); + assert.equal(loaded.status, "failed"); + if (loaded.status === "failed") assert.match(loaded.reason, /lineage mismatch/); + + const rejected = await restored.commitAuthority( + commit( + applied.status === "applied" ? applied.provider_revision : null, + "operation-b", + 2, + 2, + ), + ); + assert.equal(rejected.status, "failed"); + assert.equal(backend.casRequests.length, callsBeforeRestore); +}); + +test("NoKV provider fails closed when persisted generation and bytes diverge", async () => { + const backend = fakeBackend(); + const provider = store(backend); + assert.equal( + (await provider.commitAuthority(commit(null, "operation-a", 1, 1))).status, + "applied", + ); + backend.blob!.generation += 1; + + const loaded = await provider.loadAuthority(); + assert.equal(loaded.status, "failed"); + if (loaded.status === "failed") assert.match(loaded.reason, /storage generation/); +}); + +test("NoKV provider treats non-UTF8 persisted bytes as a protocol failure", async () => { + const backend = fakeBackend(); + backend.blob = { bytes: Uint8Array.of(0xff), generation: 1 }; + + const loaded = await store(backend).loadAuthority(); + assert.equal(loaded.status, "failed"); + if (loaded.status === "failed") { + assert.equal(loaded.reason_code, "provider_protocol_violation"); + } +}); + +test("NoKV provider enforces its candidate envelope capacity before CAS", async () => { + const backend = fakeBackend(); + const provider = new NoKVAuthorityStore(new FakeNoKVTransport(backend), { + tenant_id: "tenant-a", + goal_id: "goal-a", + workbench: "authority-workbench", + max_envelope_bytes: 64, + }); + + const result = await provider.commitAuthority(commit(null, "operation-a", 1, 1)); + assert.equal(result.status, "failed"); + if (result.status === "failed") { + assert.equal(result.reason_code, "authority_envelope_too_large"); + } + assert.equal(backend.casRequests.length, 0); +}); diff --git a/tests/control_plane_ts/nokv_jsonl_transport.test.ts b/tests/control_plane_ts/nokv_jsonl_transport.test.ts new file mode 100644 index 0000000000..478725e199 --- /dev/null +++ b/tests/control_plane_ts/nokv_jsonl_transport.test.ts @@ -0,0 +1,174 @@ +import assert from "node:assert/strict"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +import { + NoKVAuthorityStore, + NoKVTransportProtocolError, + NoKVTransportUnavailableError, +} from "../../loopx/control_plane/coordination/nokv_authority_store.ts"; +import { NoKVJsonLinesTransport } from "../../loopx/control_plane/coordination/nokv_jsonl_transport.ts"; +import { registerAuthorityStoreConformance } from "./authority_store_conformance.ts"; + +const PYTHON = process.env.LOOPX_TEST_PYTHON ?? "python3"; +const FAULT_HELPER = fileURLToPath( + new URL("../fixtures/nokv_jsonl_fake_helper.py", import.meta.url), +); +const SDK_HELPER = fileURLToPath( + new URL("../../loopx/control_plane/coordination/nokv_jsonl_helper.py", import.meta.url), +); +const FAKE_SDK_ROOT = fileURLToPath( + new URL("../fixtures/nokv_fake_sdk", import.meta.url), +); + +async function openSdkHelper() { + return await NoKVJsonLinesTransport.open({ + argv: [PYTHON, SDK_HELPER], + config: { + root_id: "0".repeat(32), + routing: { + kind: "etcd", + endpoints: ["http://127.0.0.1:2379"], + key_prefix: "/nokv/control", + lease_ttl_seconds: 10, + }, + object_store: { kind: "memory" }, + }, + env: { + ...process.env, + PYTHONPATH: process.env.PYTHONPATH + ? `${FAKE_SDK_ROOT}:${process.env.PYTHONPATH}` + : FAKE_SDK_ROOT, + }, + request_timeout_ms: 2_000, + }); +} + +async function openFaultHelper(mode: string, maxResponseBytes?: number) { + return await NoKVJsonLinesTransport.open({ + argv: [PYTHON, FAULT_HELPER, mode], + config: {}, + request_timeout_ms: 2_000, + max_response_bytes: maxResponseBytes, + }); +} + +registerAuthorityStoreConformance("NoKV JSON-lines process", async (t) => { + const transport = await openSdkHelper(); + t.after(async () => await transport.close()); + return { + store: new NoKVAuthorityStore(transport, { + tenant_id: "tenant-a", + goal_id: "goal-a", + workbench: "authority-workbench", + }), + contender: new NoKVAuthorityStore(transport, { + tenant_id: "tenant-a", + goal_id: "goal-a", + workbench: "authority-workbench", + }), + }; +}); + +test("JSON-lines transport starts once and reuses the helper process", async (t) => { + const transport = await openSdkHelper(); + t.after(async () => await transport.close()); + + assert.deepEqual(await transport.storeIdentity("authority-workbench"), { + status: "available", + store_identity: `nokv:authority-workbench:${"a".repeat(32)}`, + }); + assert.deepEqual( + await transport.readBlob("authority-workbench", "metadata/head.json"), + { status: "missing" }, + ); + assert.deepEqual( + await transport.casPublishBlob({ + workbench: "authority-workbench", + path: "metadata/head.json", + expected_generation: null, + bytes: Buffer.from("payload", "utf8"), + operation_id: "a".repeat(32), + artifact_revision_id: "b".repeat(32), + }), + { status: "applied", generation: 1 }, + ); + const loaded = await transport.readBlob("authority-workbench", "metadata/head.json"); + assert.equal(loaded.status, "loaded"); + if (loaded.status === "loaded") { + assert.equal(Buffer.from(loaded.bytes).toString("utf8"), "payload"); + assert.equal(loaded.generation, 1); + } +}); + +test("JSON-lines helper disconnect is typed unavailable", async (t) => { + const transport = await openFaultHelper("disconnect"); + t.after(async () => await transport.close()); + + await assert.rejects( + transport.readBlob("authority-workbench", "metadata/head.json"), + (error: unknown) => { + assert.ok(error instanceof NoKVTransportUnavailableError); + assert.doesNotMatch(error.message, /provider-private-diagnostic/); + assert.match(error.message, /code=17/); + return true; + }, + ); +}); + +test("JSON-lines synchronous start failure is typed and sanitized", async () => { + await assert.rejects( + NoKVJsonLinesTransport.open({ + argv: ["injected-helper"], + config: {}, + process_factory: () => { + throw new Error("private helper path and provider detail"); + }, + }), + (error: unknown) => { + assert.ok(error instanceof NoKVTransportUnavailableError); + assert.equal(error.message, "NoKV helper failed to start"); + return true; + }, + ); +}); + +test("JSON-lines invalid response is a protocol failure", async (t) => { + const transport = await openFaultHelper("invalid"); + t.after(async () => await transport.close()); + + await assert.rejects( + transport.readBlob("authority-workbench", "metadata/head.json"), + NoKVTransportProtocolError, + ); +}); + +test("JSON-lines response limit fails closed as a protocol error", async (t) => { + const transport = await openFaultHelper("oversized", 256); + t.after(async () => await transport.close()); + + await assert.rejects( + transport.readBlob("authority-workbench", "metadata/head.json"), + (error: unknown) => { + assert.ok(error instanceof NoKVTransportProtocolError); + assert.match(error.message, /max_response_bytes/); + return true; + }, + ); +}); + +test("NoKV AuthorityStore preserves helper protocol failure as failed, not missing", async (t) => { + const transport = await openFaultHelper("invalid"); + t.after(async () => await transport.close()); + const store = new NoKVAuthorityStore(transport, { + tenant_id: "tenant-a", + goal_id: "goal-a", + workbench: "authority-workbench", + }); + + const loaded = await store.loadAuthority(); + assert.equal(loaded.status, "failed"); + if (loaded.status === "failed") { + assert.equal(loaded.reason_code, "provider_protocol_violation"); + } +}); diff --git a/tests/fixtures/nokv_fake_sdk/nokv/__init__.py b/tests/fixtures/nokv_fake_sdk/nokv/__init__.py new file mode 100644 index 0000000000..4ad7970ef2 --- /dev/null +++ b/tests/fixtures/nokv_fake_sdk/nokv/__init__.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from typing import Any + +__version__ = "0.11.0" +API_VERSION = 1 + + +class RoutingConfig: + @staticmethod + def etcd(endpoints: list[str], key_prefix: str, lease_ttl_seconds: int) -> object: + return ("etcd", endpoints, key_prefix, lease_ttl_seconds) + + @staticmethod + def static(*values: Any) -> object: + return ("static", values) + + +class ObjectStoreConfig: + @staticmethod + def memory() -> object: + return ("memory",) + + @staticmethod + def s3(**values: Any) -> object: + return ("s3", values) + + +class Client: + def __init__(self, **_values: Any) -> None: + self._bytes: bytes | None = None + self._generation: int | None = None + + def find_workspaces(self, **_values: Any) -> dict[str, Any]: + return { + "workspaces": [ + { + "workspace": { + "workbench": "authority-workbench", + "workspace_incarnation_id": "a" * 32, + } + } + ], + "next_cursor": None, + } + + def read(self, workbench: str, path: str) -> dict[str, Any]: + if self._bytes is None or self._generation is None: + raise FileNotFoundError("missing") + return { + "bytes": self._bytes, + "metadata": { + "workbench": workbench, + "path": path, + "workspace_incarnation_id": "a" * 32, + "generation": self._generation, + }, + } + + def publish_bytes( + self, + workbench: str, + path: str, + payload: bytes, + **values: Any, + ) -> dict[str, Any]: + expected = values["expected_generation"] + if expected is None and self._generation is not None: + raise FileExistsError("already exists") + if expected is not None and expected != self._generation: + raise RuntimeError("generation conflict") + self._generation = (self._generation or 0) + 1 + self._bytes = payload + return { + "operation_id": values["operation_id"], + "artifact_revision_id": values["artifact_revision_id"], + "workbench": workbench, + "path": path, + "generation": self._generation, + } diff --git a/tests/fixtures/nokv_jsonl_fake_helper.py b/tests/fixtures/nokv_jsonl_fake_helper.py new file mode 100644 index 0000000000..d5744293e5 --- /dev/null +++ b/tests/fixtures/nokv_jsonl_fake_helper.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import base64 +import json +import sys + + +mode = sys.argv[1] if len(sys.argv) > 1 else "normal" +blob: bytes | None = None +generation: int | None = None + + +def emit(value: object) -> None: + sys.stdout.write(json.dumps(value, separators=(",", ":")) + "\n") + sys.stdout.flush() + + +first = json.loads(sys.stdin.readline()) +emit({"request_id": first["request_id"], "status": "ready"}) + +for line in sys.stdin: + request = json.loads(line) + request_id = request["request_id"] + if mode == "disconnect": + sys.stderr.write("provider-private-diagnostic=must-not-escape\n") + sys.stderr.flush() + raise SystemExit(17) + if mode == "invalid": + emit({"request_id": request_id, "status": "loaded", "generation": True}) + continue + if mode == "oversized": + emit({"request_id": request_id, "status": "missing", "padding": "x" * 4_096}) + continue + operation = request["operation"] + if operation == "store_identity": + emit( + { + "request_id": request_id, + "status": "available", + "store_identity": f"nokv:{request['workbench']}:{'a' * 32}", + } + ) + elif operation == "read_blob": + if blob is None: + emit({"request_id": request_id, "status": "missing"}) + else: + emit( + { + "request_id": request_id, + "status": "loaded", + "bytes_base64": base64.b64encode(blob).decode("ascii"), + "generation": generation, + } + ) + elif operation == "cas_publish_blob": + if request["expected_generation"] != generation: + emit( + { + "request_id": request_id, + "status": "conflict", + "current_generation": generation, + } + ) + else: + blob = base64.b64decode(request["bytes_base64"], validate=True) + generation = (generation or 0) + 1 + emit( + { + "request_id": request_id, + "status": "applied", + "generation": generation, + } + ) + else: + emit( + { + "request_id": request_id, + "status": "failed", + "reason_code": "unknown_operation", + "reason": "unknown operation", + } + ) diff --git a/tests/test_nokv_jsonl_helper.py b/tests/test_nokv_jsonl_helper.py new file mode 100644 index 0000000000..b67bf1d467 --- /dev/null +++ b/tests/test_nokv_jsonl_helper.py @@ -0,0 +1,590 @@ +from __future__ import annotations + +import base64 +import io +import json +import sys +import types +from typing import Any + +import pytest + +from loopx.control_plane.coordination.nokv_jsonl_helper import ( + ClientAdmissionUnavailable, + RequestError, + build_client, + handle_request, + main, + serve, +) + + +class FakeClient: + def __init__(self) -> None: + self.find_pages: list[dict[str, Any]] = [] + self.read_result: dict[str, Any] | BaseException = FileNotFoundError("missing") + self.publish_result: dict[str, Any] | BaseException = publish_result() + self.publish_calls: list[tuple[tuple[Any, ...], dict[str, Any]]] = [] + + def find_workspaces(self, **kwargs: Any) -> dict[str, Any]: + assert kwargs["limit"] == 100 + return self.find_pages.pop(0) + + def read(self, *args: Any) -> dict[str, Any]: + if isinstance(self.read_result, BaseException): + raise self.read_result + return self.read_result + + def publish_bytes(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + self.publish_calls.append((args, kwargs)) + if isinstance(self.publish_result, BaseException): + raise self.publish_result + return self.publish_result + + +def request(operation: str, **values: Any) -> dict[str, Any]: + return {"request_id": "request-a", "operation": operation, **values} + + +def identity_page() -> dict[str, Any]: + return { + "workspaces": [ + { + "workspace": { + "workbench": "authority-workbench", + "workspace_incarnation_id": "c" * 32, + } + } + ], + "next_cursor": None, + } + + +def read_result() -> dict[str, Any]: + return { + "bytes": b"canonical bytes", + "metadata": { + "workbench": "authority-workbench", + "path": "metadata/head.json", + "workspace_incarnation_id": "c" * 32, + "generation": 7, + }, + } + + +def publish_result(*, generation: int = 1) -> dict[str, Any]: + return { + "operation_id": "a" * 32, + "artifact_revision_id": "b" * 32, + "workbench": "authority-workbench", + "path": "metadata/head.json", + "generation": generation, + } + + +def test_store_identity_follows_all_pages_and_binds_the_workspace_incarnation() -> None: + client = FakeClient() + client.find_pages = [ + { + "workspaces": [{"workspace": {"workbench": "other"}}], + "next_cursor": b"page-two", + }, + { + "workspaces": [ + { + "workspace": { + "workbench": "authority-workbench", + "workspace_incarnation_id": "a" * 32, + } + } + ], + "next_cursor": None, + }, + ] + + result = handle_request( + client, + request("store_identity", workbench="authority-workbench"), + ) + assert result == { + "request_id": "request-a", + "status": "available", + "store_identity": f"nokv:authority-workbench:{'a' * 32}", + } + + +def test_store_identity_never_turns_an_outage_or_missing_workspace_into_identity() -> ( + None +): + unavailable = FakeClient() + unavailable.find_pages = [] + + def fail(**_kwargs: Any) -> dict[str, Any]: + raise RuntimeError("route unavailable") + + unavailable.find_workspaces = fail # type: ignore[method-assign] + result = handle_request( + unavailable, + request("store_identity", workbench="authority-workbench"), + ) + assert result["status"] == "unavailable" + assert result["reason_code"] == "nokv_identity_unavailable" + assert result["reason"] == "NoKV identity lookup is unavailable" + assert "route unavailable" not in result["reason"] + + absent = FakeClient() + absent.find_pages = [{"workspaces": [], "next_cursor": None}] + result = handle_request( + absent, + request("store_identity", workbench="authority-workbench"), + ) + assert result["status"] == "failed" + assert result["reason_code"] == "provider_protocol_violation" + + +def test_read_blob_preserves_missing_unavailable_and_generation() -> None: + client = FakeClient() + assert handle_request( + client, + request( + "read_blob", workbench="authority-workbench", path="metadata/head.json" + ), + ) == {"request_id": "request-a", "status": "missing"} + + client.read_result = RuntimeError("server unavailable") + unavailable = handle_request( + client, + request( + "read_blob", workbench="authority-workbench", path="metadata/head.json" + ), + ) + assert unavailable["status"] == "unavailable" + assert unavailable["reason_code"] == "nokv_read_unavailable" + assert unavailable["reason"] == "NoKV blob read is unavailable" + assert "server unavailable" not in unavailable["reason"] + + client.find_pages = [identity_page()] + client.read_result = read_result() + loaded = handle_request( + client, + request( + "read_blob", workbench="authority-workbench", path="metadata/head.json" + ), + ) + assert loaded == { + "request_id": "request-a", + "status": "loaded", + "bytes_base64": base64.b64encode(b"canonical bytes").decode("ascii"), + "generation": 7, + } + + +def test_cas_publish_blob_forwards_exact_generation_bytes_and_identities() -> None: + client = FakeClient() + client.publish_result = publish_result(generation=5) + payload = b'{"head":true}' + result = handle_request( + client, + request( + "cas_publish_blob", + workbench="authority-workbench", + path="metadata/head.json", + expected_generation=4, + bytes_base64=base64.b64encode(payload).decode("ascii"), + operation_id="a" * 32, + artifact_revision_id="b" * 32, + ), + ) + + assert result == { + "request_id": "request-a", + "status": "applied", + "generation": 5, + } + args, kwargs = client.publish_calls[0] + assert args == ("authority-workbench", "metadata/head.json", payload) + assert kwargs == { + "content_type": "application/json", + "expected_generation": 4, + "operation_id": "a" * 32, + "artifact_revision_id": "b" * 32, + } + + +@pytest.mark.parametrize( + ("field", "wrong_value"), + [ + ("workbench", "other-workbench"), + ("path", "metadata/other.json"), + ("workspace_incarnation_id", "d" * 32), + ], +) +def test_read_blob_rejects_sdk_metadata_bound_to_another_object_or_incarnation( + field: str, + wrong_value: object, +) -> None: + client = FakeClient() + client.find_pages = [identity_page()] + client.read_result = read_result() + client.read_result["metadata"][field] = wrong_value + + result = handle_request( + client, + request( + "read_blob", workbench="authority-workbench", path="metadata/head.json" + ), + ) + + assert result["status"] == "failed" + assert result["reason_code"] == "provider_protocol_violation" + + +@pytest.mark.parametrize( + ("field", "wrong_value"), + [ + ("workbench", "other-workbench"), + ("path", "metadata/other.json"), + ("operation_id", "d" * 32), + ("artifact_revision_id", "e" * 32), + ("generation", 6), + ], +) +def test_publish_never_reports_applied_for_an_sdk_result_bound_to_another_write( + field: str, + wrong_value: object, +) -> None: + client = FakeClient() + client.publish_result = publish_result(generation=5) + client.publish_result[field] = wrong_value + + result = handle_request( + client, + request( + "cas_publish_blob", + workbench="authority-workbench", + path="metadata/head.json", + expected_generation=4, + bytes_base64=base64.b64encode(b"{}").decode("ascii"), + operation_id="a" * 32, + artifact_revision_id="b" * 32, + ), + ) + + assert result["status"] == "ambiguous" + assert result["reason_code"] == "provider_protocol_violation" + + +def test_cas_publish_blob_maps_only_proven_collision_to_conflict() -> None: + client = FakeClient() + client.publish_result = FileExistsError("already exists") + conflict = handle_request( + client, + request( + "cas_publish_blob", + workbench="authority-workbench", + path="metadata/head.json", + expected_generation=None, + bytes_base64=base64.b64encode(b"{}").decode("ascii"), + operation_id="a" * 32, + artifact_revision_id="b" * 32, + ), + ) + assert conflict["status"] == "conflict" + assert conflict["current_generation"] is None + + client.publish_result = RuntimeError("generation conflict or lost response") + ambiguous = handle_request( + client, + request( + "cas_publish_blob", + workbench="authority-workbench", + path="metadata/head.json", + expected_generation=1, + bytes_base64=base64.b64encode(b"{}").decode("ascii"), + operation_id="a" * 32, + artifact_revision_id="b" * 32, + ), + ) + assert ambiguous["status"] == "ambiguous" + assert ambiguous["reason_code"] == "nokv_publish_outcome_unknown" + assert ambiguous["reason"] == "NoKV publish outcome is unknown" + assert "lost response" not in ambiguous["reason"] + + client.publish_result = ValueError("post-call conversion exposed an endpoint") + malformed = handle_request( + client, + request( + "cas_publish_blob", + workbench="authority-workbench", + path="metadata/head.json", + expected_generation=1, + bytes_base64=base64.b64encode(b"{}").decode("ascii"), + operation_id="a" * 32, + artifact_revision_id="b" * 32, + ), + ) + assert malformed["status"] == "ambiguous" + assert "endpoint" not in malformed["reason"] + + +def test_invalid_publish_request_fails_before_calling_the_sdk() -> None: + client = FakeClient() + invalid = handle_request( + client, + request( + "cas_publish_blob", + workbench="authority-workbench", + path="metadata/head.json", + expected_generation=True, + bytes_base64="not base64", + operation_id="short", + artifact_revision_id="b" * 32, + ), + ) + assert invalid["status"] == "failed" + assert invalid["reason_code"] == "invalid_request" + assert client.publish_calls == [] + + +def test_json_lines_server_emits_one_typed_response_per_request() -> None: + client = FakeClient() + incoming = io.StringIO( + json.dumps( + request( + "read_blob", + workbench="authority-workbench", + path="metadata/head.json", + ) + ) + + "\n" + + "not-json\n" + ) + outgoing = io.StringIO() + + serve(client, incoming, outgoing) + + rows = [json.loads(line) for line in outgoing.getvalue().splitlines()] + assert rows[0] == {"request_id": "request-a", "status": "missing"} + assert rows[1]["request_id"] is None + assert rows[1]["status"] == "failed" + assert rows[1]["reason_code"] == "invalid_json" + + +def test_static_route_requires_positive_generation_and_epoch_before_sdk_call( + monkeypatch: pytest.MonkeyPatch, +) -> None: + static_calls: list[tuple[Any, ...]] = [] + + class RoutingConfig: + @staticmethod + def static(*args: Any) -> object: + static_calls.append(args) + return object() + + module = types.SimpleNamespace( + __version__="0.11.0", + API_VERSION=1, + Client=lambda **_kwargs: object(), + ObjectStoreConfig=types.SimpleNamespace(memory=lambda: object()), + RoutingConfig=RoutingConfig, + ) + monkeypatch.setitem(sys.modules, "nokv", module) + base = { + "root_id": "a" * 32, + "routing": { + "kind": "static", + "endpoint": "127.0.0.1:7000", + "logical_shard_id": "b" * 32, + "object_namespace_id": "c" * 32, + "placement_generation": 1, + "owner_epoch": 1, + }, + "object_store": {"kind": "memory"}, + } + for field, value in [ + ("placement_generation", None), + ("placement_generation", True), + ("owner_epoch", 0), + ]: + invalid = json.loads(json.dumps(base)) + invalid["routing"][field] = value + with pytest.raises(RequestError): + build_client(invalid) + assert static_calls == [] + + +def test_client_constructor_value_error_is_typed_as_admission_unavailable( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class RoutingConfig: + @staticmethod + def etcd(*_args: Any) -> object: + return object() + + def unavailable_client(**_kwargs: Any) -> object: + raise ValueError("provider endpoint and credential detail") + + module = types.SimpleNamespace( + __version__="0.11.0", + API_VERSION=1, + Client=unavailable_client, + ObjectStoreConfig=types.SimpleNamespace(memory=lambda: object()), + RoutingConfig=RoutingConfig, + ) + monkeypatch.setitem(sys.modules, "nokv", module) + + with pytest.raises(ClientAdmissionUnavailable) as raised: + build_client( + { + "root_id": "a" * 32, + "routing": { + "kind": "etcd", + "endpoints": ["http://unused.invalid"], + "key_prefix": "/nokv/control", + "lease_ttl_seconds": 10, + }, + "object_store": {"kind": "memory"}, + } + ) + assert "endpoint" not in str(raised.value) + + +@pytest.mark.parametrize("unknown_location", ["top", "routing", "object_store"]) +def test_unknown_config_keys_fail_before_any_sdk_object_is_constructed( + monkeypatch: pytest.MonkeyPatch, + unknown_location: str, +) -> None: + construction_calls: list[str] = [] + + class RoutingConfig: + @staticmethod + def etcd(*_args: Any) -> object: + construction_calls.append("routing") + return object() + + class ObjectStoreConfig: + @staticmethod + def memory() -> object: + construction_calls.append("object_store") + return object() + + def client(**_kwargs: Any) -> object: + construction_calls.append("client") + return object() + + module = types.SimpleNamespace( + __version__="0.11.0", + API_VERSION=1, + Client=client, + ObjectStoreConfig=ObjectStoreConfig, + RoutingConfig=RoutingConfig, + ) + monkeypatch.setitem(sys.modules, "nokv", module) + config = { + "root_id": "a" * 32, + "routing": { + "kind": "etcd", + "endpoints": ["http://unused.invalid"], + "key_prefix": "/nokv/control", + "lease_ttl_seconds": 10, + }, + "object_store": {"kind": "memory"}, + } + secret_marker = "must-not-appear" + if unknown_location == "top": + config["routing_typo"] = secret_marker + elif unknown_location == "routing": + config["routing"]["endpoints_typo"] = secret_marker + else: + config["object_store"]["secret_access_key_typo"] = secret_marker + + with pytest.raises(RequestError) as raised: + build_client(config) + + assert construction_calls == [] + assert secret_marker not in str(raised.value) + + +@pytest.mark.parametrize( + ("sdk_version", "api_version"), + [("incompatible-version", 1), ("0.11.0", 999)], +) +def test_sdk_version_or_api_mismatch_fails_before_provider_construction( + monkeypatch: pytest.MonkeyPatch, + sdk_version: str, + api_version: int, +) -> None: + construction_calls: list[str] = [] + module = types.SimpleNamespace( + __version__=sdk_version, + API_VERSION=api_version, + Client=lambda **_kwargs: construction_calls.append("client"), + ObjectStoreConfig=types.SimpleNamespace( + memory=lambda: construction_calls.append("object_store") + ), + RoutingConfig=types.SimpleNamespace( + etcd=lambda *_args: construction_calls.append("routing") + ), + ) + monkeypatch.setitem(sys.modules, "nokv", module) + + with pytest.raises(RequestError) as raised: + build_client( + { + "root_id": "a" * 32, + "routing": { + "kind": "etcd", + "endpoints": ["http://unused.invalid"], + "key_prefix": "/nokv/control", + "lease_ttl_seconds": 10, + }, + "object_store": {"kind": "memory"}, + } + ) + + assert construction_calls == [] + assert "incompatible-version" not in str(raised.value) + assert "999" not in str(raised.value) + + +def test_open_handshake_reports_the_qualified_sdk_contract( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = types.SimpleNamespace( + __version__="0.11.0", + API_VERSION=1, + Client=lambda **_kwargs: object(), + ObjectStoreConfig=types.SimpleNamespace(memory=lambda: object()), + RoutingConfig=types.SimpleNamespace(etcd=lambda *_args: object()), + ) + monkeypatch.setitem(sys.modules, "nokv", module) + incoming = io.StringIO( + json.dumps( + { + "request_id": "open-a", + "operation": "open", + "config": { + "root_id": "a" * 32, + "routing": { + "kind": "etcd", + "endpoints": ["http://unused.invalid"], + "key_prefix": "/nokv/control", + "lease_ttl_seconds": 10, + }, + "object_store": {"kind": "memory"}, + }, + } + ) + + "\n" + ) + outgoing = io.StringIO() + monkeypatch.setattr(sys, "stdin", incoming) + monkeypatch.setattr(sys, "stdout", outgoing) + + assert main() == 0 + assert json.loads(outgoing.getvalue().splitlines()[0]) == { + "request_id": "open-a", + "status": "ready", + "nokv_api_version": 1, + "nokv_sdk_version": "0.11.0", + } diff --git a/tsconfig.control-plane.json b/tsconfig.control-plane.json index 2f54be74a3..7bbdd67644 100644 --- a/tsconfig.control-plane.json +++ b/tsconfig.control-plane.json @@ -22,6 +22,8 @@ "loopx/control_plane/coordination/authority_store_codec.ts", "loopx/control_plane/coordination/file_authority_store.ts", "loopx/control_plane/coordination/local_authority_shadow.ts", + "loopx/control_plane/coordination/nokv_authority_store.ts", + "loopx/control_plane/coordination/nokv_jsonl_transport.ts", "loopx/control_plane/coordination/postgresql_authority_store.ts", "loopx/control_plane/agents/delivery_workspace.ts", "loopx/control_plane/goals/vision_checkpoint.ts", @@ -48,6 +50,7 @@ "loopx/control_plane/work_items/task_lease_acquire.ts", "loopx/control_plane/work_items/task_lease_lifecycle.ts", "loopx/control_plane/work_items/task_lease_acquire_cli.ts", + "examples/nokv-authority-store/live-qualification.ts", "tests/control_plane_ts/effect_program.test.ts", "tests/control_plane_ts/effect_runtime_errors.test.ts", "tests/control_plane_ts/interaction_contract.test.ts", @@ -55,6 +58,9 @@ "tests/control_plane_ts/authority_store.test.ts", "tests/control_plane_ts/local_authority_shadow.test.ts", "tests/control_plane_ts/authority_store_conformance.ts", + "tests/control_plane_ts/nokv_authority_store.test.ts", + "tests/control_plane_ts/nokv_jsonl_transport.test.ts", + "tests/control_plane_ts/nokv_live_qualification_probe.test.ts", "tests/control_plane_ts/postgresql_authority_store.integration.test.ts", "tests/control_plane_ts/delivery_continuity.test.ts", "tests/control_plane_ts/delivery_workspace.test.ts", From 1063fa38b764951f7ab654ee1f7c1baa689a3918 Mon Sep 17 00:00:00 2001 From: wchwawa Date: Wed, 2 Sep 2026 08:05:10 +1000 Subject: [PATCH 05/12] test(authority): qualify live NoKV candidate Signed-off-by: wchwawa --- ...shared-goal-authority-state-provider-v0.md | 24 +- ...-goal-authority-state-provider-v0.zh-CN.md | 20 +- examples/nokv-authority-store/README.md | 116 ++++ .../live-qualification.ts | 564 ++++++++++++++++++ .../nokv_live_qualification_probe.test.ts | 175 ++++++ 5 files changed, 890 insertions(+), 9 deletions(-) create mode 100644 examples/nokv-authority-store/README.md create mode 100644 examples/nokv-authority-store/live-qualification.ts create mode 100644 tests/control_plane_ts/nokv_live_qualification_probe.test.ts diff --git a/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.md b/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.md index 3ae656da0c..305348f493 100644 --- a/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.md +++ b/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.md @@ -3,15 +3,15 @@ - Status: Draft, under maintainer review - Initially proposed by: NoKV Lab - Widened by: LoopX maintainers -- Date: 2026-08-05; revised 2026-09-01 +- Date: 2026-08-05; revised 2026-09-02 - Scope: one provider-neutral LoopX authority contract with built-in file, optional NoKV, and optional PostgreSQL provider profiles, complementing [`host-integration-surface-v0`](../../reference/protocols/host-integration-surface-v0.md) - Source baseline: LoopX `a0c20f1779d273e7aaa4bd3ea166d145d466e6d5` -- Provider API baseline: NoKV `3d75d96965` (0.11.0 line). The Python - `publish_bytes` generation-CAS mapping was exercised once by hand against a - live NoKV stack at that pin (see the example README); the run is evidence for - the mapping only, not part of any merge gate +- Provider API baseline: NoKV `0f1995ebee96048e5d4f9d4745d84c3518c64351` + (release 0.11.0, Python API 1, Holt pinned to 0.8.6). The Stage 2A executable + qualification admits only that SDK contract and this checkout's helper. It + remains candidate evidence, not a merge gate or authority promotion - PostgreSQL baseline: the TypeScript Stage 2B candidate implements the store contract and has passed a real PostgreSQL 16 transaction matrix. No shared authority service, runtime caller, authentication boundary, or authority @@ -1455,6 +1455,20 @@ claim that the complete P0 acceptance gate above passes. Historical latency or fault results are informative only; they are not a durability, recovery, HA, or production qualification claim. +The additional TEST ONLY Stage 2A probe in +`examples/nokv-authority-store/` opens three independent SDK helper processes +and checks fresh create, exact generation update, a one-winner/two-contender +CAS, winner/loser receipt behavior, and fresh-process history readback. Its +executable fixes argv to one absolute Python executable plus this checkout's +production helper; the helper fails closed unless the SDK reports NoKV 0.11.0 +and Python API 1, and validates every read/publish response against its requested +workbench, path, workspace incarnation, operation, revision, and generation. +Only a successful live JSON report is evidence for that single-node run; +deterministic tests are sequence tests only. This LoopX-only candidate changes +neither NoKV main nor its workbench/artifact data model or frozen-oracle-log use, +and it still does not prove HA, restart recovery, capacity, production routing, +or authority-source promotion. + ## Appendix B: Handoff-Mode Decision Record (2026-08-10) This appendix writes down a direction already agreed during the PR #2787 diff --git a/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.zh-CN.md b/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.zh-CN.md index 8e767a087e..16989521ef 100644 --- a/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.zh-CN.md +++ b/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.zh-CN.md @@ -3,14 +3,15 @@ - 状态：Draft，正在接受 maintainer review - 最初提案方：NoKV Lab - 扩展修订方：LoopX maintainer -- 日期：2026-08-05；修订于 2026-09-01 +- 日期：2026-08-05；修订于 2026-09-02 - 范围：一个 provider-neutral 的 LoopX 权威合同，支持内置 file、可选 NoKV 与可选 PostgreSQL provider profile，用来补充 [`host-integration-surface-v0`](../../reference/protocols/host-integration-surface-v0.md) - 源码基线：LoopX `a0c20f1779d273e7aaa4bd3ea166d145d466e6d5` -- Provider API 基线：NoKV `3d75d96965`（0.11.0 线）。Python `publish_bytes` - generation-CAS 映射已在该基线的真实 NoKV stack 上手工跑过一次（见示例 README）； - 该次运行只是映射本身的证据，不属于任何合并门槛 +- Provider API 基线：NoKV `0f1995ebee96048e5d4f9d4745d84c3518c64351` + （release 0.11.0、Python API 1、Holt 固定为 0.8.6）。Stage 2A 的可执行资格 + 验证只接受这份 SDK 合同与本 checkout 的 helper；它仍是候选证据，不是合并门槛 + 或 authority promotion - PostgreSQL 基线：TypeScript Stage 2B candidate 已实现 store contract，且已通过 真实 PostgreSQL 16 transaction matrix；shared authority service、runtime caller、 authentication boundary 与 authority promotion 均尚未交付 @@ -1168,6 +1169,17 @@ migration/promotion、service recovery 或 HA。 验收门通过。历史 latency 或 fault 结果只具有参考意义，不构成 durability、recovery、 HA 或 production qualification 声明。 +`examples/nokv-authority-store/` 还包含一个 TEST ONLY 的 Stage 2A probe：它会 +打开三个相互独立的 SDK helper 进程，验证 fresh create、精确 generation update、 +两个竞争者恰一胜出的 CAS、胜负双方的 receipt 行为，以及新进程对完整 history 的 +回读。该可执行入口把 argv 固定为一个绝对 Python executable 加本 checkout 的生产 +helper；helper 只接受 NoKV 0.11.0 / Python API 1，并逐项核对 read/publish 回包与 +请求的 workbench、path、workspace incarnation、operation、revision、generation +是否绑定。只有成功的 live JSON report 才是该次单节点运行的证据；确定性测试只证明 +场景序列。这个纯 LoopX 候选既不修改 NoKV main，也不改变其 workbench/artifact +数据模型或 frozen-oracle-log 用途；它仍不证明 HA、重启恢复、容量、生产路由或 +authority-source promotion。 + ## 附录 B：交接模式决策记录（2026-08-10） 本附录把 PR #2787 评审中已同意的方向落成文字，作为实施前置条件的一部分。 diff --git a/examples/nokv-authority-store/README.md b/examples/nokv-authority-store/README.md new file mode 100644 index 0000000000..ab4e9f0238 --- /dev/null +++ b/examples/nokv-authority-store/README.md @@ -0,0 +1,116 @@ +# NoKV authority-store live qualification (TEST ONLY) + +This directory contains an explicit, write-producing qualification probe for +the candidate `NoKVAuthorityStore`. It is **TEST ONLY**. LoopX does not select +NoKV by importing this code, and a successful run does not flip the authority +source for any LoopX runtime. + +The integration priority remains: + +1. the native, full NoKV CLI as the primary operator and production surface; +2. the NoKV Python SDK as the secondary programmable surface; and +3. optional sidecars only as adapters around those surfaces, never as the main + authority API. + +This probe intentionally exercises the current Python SDK bridge because it is +the available raw byte-CAS seam. It always starts this checkout's production +`NoKVJsonLinesTransport` and `nokv_jsonl_helper.py`; it has no fake, skip, or +"unverified but successful" CLI path. The helper admits exactly NoKV SDK +`0.11.0` / Python API `1`, and the successful report repeats both values. + +## What it proves + +Against one **already existing** NoKV workbench, the probe starts three +independent helper processes and verifies: + +- the selected tenant/goal path is initially absent and can be created; +- the stored path generation advances from 1 to 2 under exact generation CAS; +- two writes released together against generation 2 produce exactly one + generation-3 winner and one typed conflict; +- the losing operation does not acquire a durable receipt; and +- a third, freshly opened transport reads the winning envelope, its complete + three-entry history, and the retained winner receipt. + +If the SDK, helper, workbench, backend, CAS, or independent readback cannot be +proved, the process exits nonzero. The normal test suite uses deterministic +fakes only to test this sequence and does **not** count as live evidence. + +The probe does not prove HA, failover, restart recovery, capacity, performance, +or a production authority migration. It also does not create a workbench. A +green single-node run is valid single-node storage evidence only. + +## Inputs + +Use a current NoKV Python environment. Keep the client configuration in an +ignored local file; do not commit credentials. Static routing is valid for a +single-node NoKV deployment—etcd is not required by this probe. The following +shape is illustrative: + +```json +{ + "root_id": "00000000000000000000000000000000", + "routing": { + "kind": "static", + "endpoint": "127.0.0.1:7412", + "logical_shard_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "object_namespace_id": "cccccccccccccccccccccccccccccccc", + "placement_generation": 1, + "owner_epoch": 1 + }, + "object_store": { + "kind": "s3", + "bucket": "qualification-bucket", + "region": "us-east-1", + "root": "/loopx-qualification", + "endpoint": "http://127.0.0.1:9000", + "access_key_id": "set-in-your-ignored-local-file", + "secret_access_key": "set-in-your-ignored-local-file", + "virtual_host_style": false, + "skip_signature": false + } +} +``` + +Configuration objects are exact-key contracts. An unknown top-level, routing, +or object-store key fails before an SDK routing config, object-store config, or +client is constructed. In particular, a misspelled explicit credential cannot +silently fall through to NoKV's ambient provider chain. Intentionally omitted +optional S3 credential fields retain the NoKV SDK's normal behavior. + +Pass only the absolute path to the Python executable that resolves the qualified +NoKV SDK. The probe itself fixes the second and only other argv entry to the +production helper in this checkout; callers cannot supply a wrapper argument or +alternate helper path: + +```text +/path/to/nokv-python-environment/bin/python +``` + +Choose a fresh tenant/goal pair for every run. The probe refuses to overwrite +an existing authority envelope and deliberately leaves its three-generation +test envelope behind for inspection. Use a disposable qualification namespace +or remove it later with the native NoKV CLI according to that environment's +retention policy. + +## Run + +From the LoopX repository root: + +```bash +node --no-warnings --experimental-strip-types \ + examples/nokv-authority-store/live-qualification.ts \ + --execute-live \ + --config-json /path/to/ignored/nokv-client.json \ + --python-executable /path/to/nokv-python-environment/bin/python \ + --tenant-id qualification-tenant-20260902 \ + --goal-id qualification-goal-20260902-01 \ + --workbench existing-qualification-workbench +``` + +`--execute-live` is mandatory and is checked before any helper starts. Exit 0 +means every listed live check passed. Any unavailable, failed, ambiguous, +unfenced, pre-existing, or unreadable state exits nonzero with a compact JSON +reason; provider stderr, endpoints, credentials, and raw SDK errors are not +copied into that result. A successful JSON report includes +`"nokv_sdk_version":"0.11.0"` and `"nokv_api_version":1`; those fields are an +exact helper-admission claim, not an HA or production-readiness claim. diff --git a/examples/nokv-authority-store/live-qualification.ts b/examples/nokv-authority-store/live-qualification.ts new file mode 100644 index 0000000000..b6c2f32600 --- /dev/null +++ b/examples/nokv-authority-store/live-qualification.ts @@ -0,0 +1,564 @@ +#!/usr/bin/env -S node --no-warnings --experimental-strip-types + +import { randomUUID } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { isAbsolute, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; + +import type { JsonObject } from "../../loopx/control_plane/effect_program.ts"; +import type { + AuthorityStoreCommit, + AuthorityStoreCommitResult, +} from "../../loopx/control_plane/coordination/authority_store.ts"; +import { + canonicalAuthorityObject, +} from "../../loopx/control_plane/coordination/authority_store_codec.ts"; +import { + NoKVAuthorityStore, + NoKVTransportProtocolError, + NoKVTransportUnavailableError, + type NoKVBlobCasRequest, + type NoKVBlobCasResult, + type NoKVBlobReadResult, + type NoKVBlobTransport, + type NoKVStoreIdentityResult, +} from "../../loopx/control_plane/coordination/nokv_authority_store.ts"; +import { + NoKVJsonLinesTransport, +} from "../../loopx/control_plane/coordination/nokv_jsonl_transport.ts"; + +const REPORT_SCHEMA = "loopx_nokv_authority_live_qualification_v0"; +export const QUALIFIED_NOKV_SDK_VERSION = "0.11.0"; +export const QUALIFIED_NOKV_API_VERSION = 1; +const PRODUCTION_HELPER = fileURLToPath( + new URL("../../loopx/control_plane/coordination/nokv_jsonl_helper.py", import.meta.url), +); +const COMPETITION_BARRIER_TIMEOUT_MS = 10_000; + +export class QualificationFailure extends Error { + readonly reasonCode: string; + + constructor(reasonCode: string, message: string) { + super(message); + this.reasonCode = reasonCode; + } +} + +export interface QualificationTransport extends NoKVBlobTransport { + close(): Promise; +} + +export interface QualificationOptions { + python_executable: string; + client_config: JsonObject; + tenant_id: string; + goal_id: string; + workbench: string; + request_timeout_ms?: number; +} + +export interface QualificationReport { + schema_version: typeof REPORT_SCHEMA; + ok: true; + checks: readonly { id: string; status: "passed" }[]; + final_generation: number; + final_cursor: string; + durable_test_data_left: true; + authority_source_changed: false; + availability_or_ha_proven: false; + nokv_sdk_version: typeof QUALIFIED_NOKV_SDK_VERSION; + nokv_api_version: typeof QUALIFIED_NOKV_API_VERSION; +} + +export interface QualificationSequenceResult { + checks: readonly { id: string; status: "passed" }[]; + final_generation: number; + final_cursor: string; +} + +export interface QualificationCliArguments { + configJsonPath: string; + pythonExecutable: string; + tenantId: string; + goalId: string; + workbench: string; + requestTimeoutMs?: number; +} + +type QualificationTransportFactory = () => Promise; + +function fail(reasonCode: string, message: string): never { + throw new QualificationFailure(reasonCode, message); +} + +function requiredString(value: unknown, name: string): string { + if (typeof value !== "string" || value.trim() !== value || value.length === 0) { + return fail("invalid_arguments", `${name} must be a non-empty trimmed string`); + } + return value; +} + +/** Build the only helper argv accepted by the destructive live probe. */ +export function productionHelperArgv(pythonExecutable: string): readonly [string, string] { + const executable = requiredString(pythonExecutable, "Python executable"); + if (!isAbsolute(executable)) { + return fail("invalid_arguments", "Python executable must be an absolute path"); + } + return [executable, PRODUCTION_HELPER]; +} + +function positiveSafeInteger(value: unknown, name: string): number { + const parsed = typeof value === "string" && /^[1-9]\d*$/.test(value) + ? Number(value) + : value; + if (!Number.isSafeInteger(parsed) || (parsed as number) < 1) { + return fail("invalid_arguments", `${name} must be a positive safe integer`); + } + return parsed as number; +} + +function expect( + condition: unknown, + reasonCode: string, + message: string, +): asserts condition { + if (!condition) fail(reasonCode, message); +} + +function commit( + expectedProviderRevision: string | null, + runId: string, + operationId: string, + sequence: number, + candidate: string, +): AuthorityStoreCommit { + return { + expected_provider_revision: expectedProviderRevision, + operation_id: operationId, + events: [{ + schema_version: "loopx_nokv_authority_qualification_event_v0", + run_id: runId, + sequence, + candidate, + }], + next_projection: { + schema_version: "loopx_nokv_authority_qualification_head_v0", + run_id: runId, + sequence, + candidate, + }, + receipts: [{ + schema_version: "loopx_nokv_authority_qualification_receipt_v0", + run_id: runId, + operation_id: operationId, + sequence, + candidate, + }], + }; +} + +class OneShotBarrier { + private arrivals = 0; + private readonly released: Promise; + private release!: () => void; + private readonly timer: NodeJS.Timeout; + + constructor() { + this.released = new Promise((resolveBarrier) => { + this.release = resolveBarrier; + }); + this.timer = setTimeout(() => { + this.release(); + }, COMPETITION_BARRIER_TIMEOUT_MS); + this.timer.unref(); + } + + async arrive(): Promise { + this.arrivals += 1; + if (this.arrivals === 2) { + clearTimeout(this.timer); + this.release(); + } + await this.released; + if (this.arrivals !== 2) { + fail("competition_barrier_failed", "both competitors did not reach the NoKV CAS"); + } + } +} + +class BarrierTransport implements NoKVBlobTransport { + readonly inner: NoKVBlobTransport; + readonly barrier: OneShotBarrier; + + constructor(inner: NoKVBlobTransport, barrier: OneShotBarrier) { + this.inner = inner; + this.barrier = barrier; + } + + async storeIdentity(workbench: string): Promise { + return await this.inner.storeIdentity(workbench); + } + + async readBlob(workbench: string, path: string): Promise { + return await this.inner.readBlob(workbench, path); + } + + async casPublishBlob(request: NoKVBlobCasRequest): Promise { + await this.barrier.arrive(); + return await this.inner.casPublishBlob(request); + } +} + +function applied( + result: AuthorityStoreCommitResult, + reasonCode: string, +): Extract { + expect(result.status === "applied", reasonCode, "authority commit was not applied"); + return result; +} + +async function rawGeneration( + transport: NoKVBlobTransport, + store: NoKVAuthorityStore, + expectedGeneration: number, + reasonCode: string, +): Promise { + const result = await transport.readBlob(store.workbench, store.path); + expect( + result.status === "loaded" && result.generation === expectedGeneration, + reasonCode, + `authority envelope did not read back at generation ${expectedGeneration}`, + ); +} + +/** + * Exercise the destructive sequence against three independent handles. + * + * This lower-level export intentionally does not return a live qualification + * report: tests may supply a deterministic transport factory, which proves the + * sequence but cannot prove a reachable NoKV backend. + */ +export async function exerciseQualificationSequence( + options: QualificationOptions, + openTransport: QualificationTransportFactory, +): Promise { + const tenantId = requiredString(options.tenant_id, "tenant id"); + const goalId = requiredString(options.goal_id, "goal id"); + const workbench = requiredString(options.workbench, "workbench"); + const opened: QualificationTransport[] = []; + const open = async (): Promise => { + const transport = await openTransport(); + opened.push(transport); + return transport; + }; + const checks: { id: string; status: "passed" }[] = []; + const passed = (id: string): void => { + checks.push({ id, status: "passed" }); + }; + const runId = randomUUID(); + const operationIds = { + create: `${runId}:create`, + advance: `${runId}:advance`, + contenderA: `${runId}:contender-a`, + contenderB: `${runId}:contender-b`, + }; + + try { + const firstTransport = await open(); + const secondTransport = await open(); + const first = new NoKVAuthorityStore(firstTransport, { + tenant_id: tenantId, + goal_id: goalId, + workbench, + }); + const second = new NoKVAuthorityStore(secondTransport, { + tenant_id: tenantId, + goal_id: goalId, + workbench, + }); + + const [firstIdentity, secondIdentity] = await Promise.all([ + first.storeIdentity(), + second.storeIdentity(), + ]); + expect( + firstIdentity.status === "available" && + secondIdentity.status === "available" && + firstIdentity.store_identity === secondIdentity.store_identity, + "workbench_identity_failed", + "independent transports did not resolve the same existing workbench", + ); + passed("existing_workbench_identity"); + + const [firstInitial, secondInitial] = await Promise.all([ + first.loadAuthority(), + second.loadAuthority(), + ]); + expect( + firstInitial.status === "missing" && secondInitial.status === "missing", + "qualification_target_not_fresh", + "qualification requires a fresh tenant and goal target", + ); + passed("fresh_authority_target"); + + const created = applied( + await first.commitAuthority( + commit(null, runId, operationIds.create, 1, "create"), + ), + "create_failed", + ); + passed("create_applied"); + await rawGeneration(firstTransport, first, 1, "create_generation_failed"); + passed("create_generation_one"); + + const advanced = applied( + await second.commitAuthority( + commit(created.provider_revision, runId, operationIds.advance, 2, "advance"), + ), + "generation_cas_failed", + ); + passed("generation_cas_applied"); + await rawGeneration(firstTransport, first, 2, "generation_two_readback_failed"); + passed("generation_two_readback"); + + const barrier = new OneShotBarrier(); + const contenderA = new NoKVAuthorityStore( + new BarrierTransport(firstTransport, barrier), + { tenant_id: tenantId, goal_id: goalId, workbench }, + ); + const contenderB = new NoKVAuthorityStore( + new BarrierTransport(secondTransport, barrier), + { tenant_id: tenantId, goal_id: goalId, workbench }, + ); + const race = await Promise.all([ + contenderA.commitAuthority( + commit(advanced.provider_revision, runId, operationIds.contenderA, 3, "a"), + ), + contenderB.commitAuthority( + commit(advanced.provider_revision, runId, operationIds.contenderB, 3, "b"), + ), + ]); + const winnerIndex = race.findIndex((result) => result.status === "applied"); + const loserIndex = race.findIndex((result) => result.status === "conflict"); + expect( + winnerIndex >= 0 && loserIndex >= 0 && winnerIndex !== loserIndex && + race.filter((result) => result.status === "applied").length === 1 && + race.filter((result) => result.status === "conflict").length === 1, + "competition_not_fenced", + "competing generation CAS did not produce exactly one winner", + ); + const winner = race[winnerIndex]!; + const loser = race[loserIndex]!; + expect( + winner.status === "applied" && + loser.status === "conflict" && + loser.conflict_kind === "provider_revision_mismatch" && + loser.current_provider_revision === winner.provider_revision && + loser.current_cursor === "3", + "competition_not_fenced", + "competition conflict was not bound to the winning generation", + ); + passed("competing_generation_cas_one_winner"); + await rawGeneration(secondTransport, second, 3, "competition_generation_failed"); + passed("competition_did_not_double_advance"); + + await Promise.all([firstTransport.close(), secondTransport.close()]); + + const readbackTransport = await open(); + const readback = new NoKVAuthorityStore(readbackTransport, { + tenant_id: tenantId, + goal_id: goalId, + workbench, + }); + const loaded = await readback.loadAuthority(); + expect( + loaded.status === "loaded" && + loaded.provider_revision === winner.provider_revision && + loaded.cursor === "3" && + loaded.head.run_id === runId && + loaded.head.sequence === 3, + "independent_readback_failed", + "a fresh transport did not read the winning authority envelope", + ); + await rawGeneration( + readbackTransport, + readback, + 3, + "independent_readback_failed", + ); + const history = await readback.scanCommitted(null, 10); + expect( + history.status === "page" && + history.transactions.length === 3 && + history.next_cursor === "3" && + history.has_more === false, + "independent_readback_failed", + "a fresh transport did not read the complete committed history", + ); + passed("independent_transport_readback"); + + const winnerOperation = winnerIndex === 0 + ? operationIds.contenderA + : operationIds.contenderB; + const loserOperation = winnerIndex === 0 + ? operationIds.contenderB + : operationIds.contenderA; + const [winnerReceipt, loserReceipt] = await Promise.all([ + readback.readReceipt(winnerOperation), + readback.readReceipt(loserOperation), + ]); + expect( + winnerReceipt.status === "found" && winnerReceipt.cursor === "3", + "winner_receipt_missing", + "the winning operation receipt was not retained", + ); + passed("winner_receipt_retained"); + expect( + loserReceipt.status === "missing", + "loser_receipt_present", + "the losing operation unexpectedly acquired a durable receipt", + ); + passed("loser_receipt_absent"); + + return { + checks, + final_generation: 3, + final_cursor: "3", + }; + } finally { + await Promise.allSettled(opened.map(async (transport) => await transport.close())); + } +} + +/** Run the live probe only through this checkout's production JSONL transport. */ +export async function qualifyNoKVAuthorityStore( + options: QualificationOptions, +): Promise { + const sequence = await exerciseQualificationSequence(options, async () => + await NoKVJsonLinesTransport.open({ + argv: productionHelperArgv(options.python_executable), + config: options.client_config, + request_timeout_ms: options.request_timeout_ms, + })); + return { + schema_version: REPORT_SCHEMA, + ok: true, + ...sequence, + durable_test_data_left: true, + authority_source_changed: false, + availability_or_ha_proven: false, + nokv_sdk_version: QUALIFIED_NOKV_SDK_VERSION, + nokv_api_version: QUALIFIED_NOKV_API_VERSION, + }; +} + +export function parseQualificationArguments( + argv: readonly string[], +): QualificationCliArguments { + let values: ReturnType["values"]; + try { + values = parseArgs({ + args: [...argv], + strict: true, + allowPositionals: false, + options: { + "execute-live": { type: "boolean", default: false }, + "config-json": { type: "string" }, + "python-executable": { type: "string" }, + "tenant-id": { type: "string" }, + "goal-id": { type: "string" }, + workbench: { type: "string" }, + "request-timeout-ms": { type: "string" }, + }, + }).values; + } catch { + return fail("invalid_arguments", "qualification arguments are invalid"); + } + if (values["execute-live"] !== true) { + return fail( + "live_opt_in_required", + "--execute-live is required because this probe writes durable test data", + ); + } + return { + configJsonPath: requiredString(values["config-json"], "--config-json"), + pythonExecutable: productionHelperArgv( + requiredString(values["python-executable"], "--python-executable"), + )[0], + tenantId: requiredString(values["tenant-id"], "--tenant-id"), + goalId: requiredString(values["goal-id"], "--goal-id"), + workbench: requiredString(values.workbench, "--workbench"), + requestTimeoutMs: values["request-timeout-ms"] === undefined + ? undefined + : positiveSafeInteger(values["request-timeout-ms"], "--request-timeout-ms"), + }; +} + +async function readJson(path: string, reasonCode: string): Promise { + let bytes: string; + try { + bytes = await readFile(path, "utf8"); + } catch { + return fail(reasonCode, "qualification JSON input could not be read"); + } + try { + return JSON.parse(bytes); + } catch { + return fail(reasonCode, "qualification JSON input is invalid"); + } +} + +async function loadQualificationOptions( + cli: QualificationCliArguments, +): Promise { + const configValue = await readJson(cli.configJsonPath, "config_json_invalid"); + let clientConfig: JsonObject; + try { + clientConfig = canonicalAuthorityObject(configValue, "NoKV client config"); + } catch { + return fail("config_json_invalid", "NoKV client config must be strict JSON"); + } + return { + python_executable: cli.pythonExecutable, + client_config: clientConfig, + tenant_id: cli.tenantId, + goal_id: cli.goalId, + workbench: cli.workbench, + request_timeout_ms: cli.requestTimeoutMs, + }; +} + +async function main(): Promise { + try { + const cli = parseQualificationArguments(process.argv.slice(2)); + const options = await loadQualificationOptions(cli); + const report = await qualifyNoKVAuthorityStore(options); + process.stdout.write(`${JSON.stringify(report)}\n`); + return 0; + } catch (error) { + let reasonCode = "qualification_failed"; + let reason = "NoKV authority-store qualification failed"; + if (error instanceof QualificationFailure) { + reasonCode = error.reasonCode; + reason = error.message; + } else if (error instanceof NoKVTransportUnavailableError) { + reasonCode = "nokv_backend_unavailable"; + reason = "NoKV backend or SDK helper is unavailable"; + } else if (error instanceof NoKVTransportProtocolError) { + reasonCode = "nokv_transport_protocol_failed"; + reason = "NoKV SDK helper violated the transport protocol"; + } + process.stderr.write(`${JSON.stringify({ + schema_version: REPORT_SCHEMA, + ok: false, + reason_code: reasonCode, + reason, + })}\n`); + return 1; + } +} + +if (process.argv[1] && resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url))) { + process.exitCode = await main(); +} diff --git a/tests/control_plane_ts/nokv_live_qualification_probe.test.ts b/tests/control_plane_ts/nokv_live_qualification_probe.test.ts new file mode 100644 index 0000000000..efbd29285d --- /dev/null +++ b/tests/control_plane_ts/nokv_live_qualification_probe.test.ts @@ -0,0 +1,175 @@ +import assert from "node:assert/strict"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +import { + exerciseQualificationSequence, + parseQualificationArguments, + productionHelperArgv, + QUALIFIED_NOKV_API_VERSION, + QUALIFIED_NOKV_SDK_VERSION, + QualificationFailure, + type QualificationTransport, +} from "../../examples/nokv-authority-store/live-qualification.ts"; +import type { + NoKVBlobCasRequest, + NoKVBlobCasResult, + NoKVBlobReadResult, + NoKVStoreIdentityResult, +} from "../../loopx/control_plane/coordination/nokv_authority_store.ts"; + +interface Backend { + blob: { bytes: Uint8Array; generation: number } | null; + ignoreCas: boolean; +} + +class FakeQualificationTransport implements QualificationTransport { + readonly backend: Backend; + closed = false; + + constructor(backend: Backend) { + this.backend = backend; + } + + async storeIdentity(workbench: string): Promise { + return { + status: "available", + store_identity: `nokv:${workbench}:${"a".repeat(32)}`, + }; + } + + async readBlob(_workbench: string, _path: string): Promise { + return this.backend.blob + ? { + status: "loaded", + bytes: this.backend.blob.bytes.slice(), + generation: this.backend.blob.generation, + } + : { status: "missing" }; + } + + async casPublishBlob(request: NoKVBlobCasRequest): Promise { + const current = this.backend.blob?.generation ?? null; + if (!this.backend.ignoreCas && current !== request.expected_generation) { + return { status: "conflict", current_generation: current }; + } + const generation = (current ?? 0) + 1; + this.backend.blob = { bytes: request.bytes.slice(), generation }; + return { status: "applied", generation }; + } + + async close(): Promise { + this.closed = true; + } +} + +const BASE_OPTIONS = { + python_executable: "/usr/bin/python3", + client_config: { + root_id: "0".repeat(32), + routing: { kind: "etcd" }, + object_store: { kind: "memory" }, + }, + tenant_id: "qualification-tenant", + goal_id: "qualification-goal", + workbench: "authority-workbench", +} as const; + +test("live qualification requires explicit write opt-in", () => { + assert.throws( + () => parseQualificationArguments([ + "--config-json", "/tmp/client.json", + "--python-executable", "/usr/bin/python3", + "--tenant-id", "qualification-tenant", + "--goal-id", "qualification-goal", + "--workbench", "authority-workbench", + ]), + (error: unknown) => { + assert.ok(error instanceof QualificationFailure); + assert.equal(error.reasonCode, "live_opt_in_required"); + return true; + }, + ); +}); + +test("live qualification fixes argv to one Python executable and the repository helper", () => { + const executable = "/opt/loopx-qualification/bin/python"; + const expectedHelper = fileURLToPath( + new URL("../../loopx/control_plane/coordination/nokv_jsonl_helper.py", import.meta.url), + ); + + assert.deepEqual(productionHelperArgv(executable), [executable, expectedHelper]); + const parsed = parseQualificationArguments([ + "--execute-live", + "--config-json", "/tmp/client.json", + "--python-executable", executable, + "--tenant-id", "qualification-tenant", + "--goal-id", "qualification-goal", + "--workbench", "authority-workbench", + ]); + assert.equal(parsed.pythonExecutable, executable); +}); + +test("live qualification rejects a relative Python executable", () => { + assert.throws( + () => productionHelperArgv("python3"), + (error: unknown) => { + assert.ok(error instanceof QualificationFailure); + assert.equal(error.reasonCode, "invalid_arguments"); + return true; + }, + ); +}); + +test("live qualification evidence names the exact NoKV SDK contract", () => { + assert.equal(QUALIFIED_NOKV_SDK_VERSION, "0.11.0"); + assert.equal(QUALIFIED_NOKV_API_VERSION, 1); +}); + +test("qualification proves create, generation CAS, competition, and fresh readback", async () => { + const backend: Backend = { blob: null, ignoreCas: false }; + const opened: FakeQualificationTransport[] = []; + const report = await exerciseQualificationSequence(BASE_OPTIONS, async () => { + const transport = new FakeQualificationTransport(backend); + opened.push(transport); + return transport; + }); + + assert.equal(report.final_generation, 3); + assert.equal(report.final_cursor, "3"); + assert.deepEqual(report.checks.map((check) => check.status), + Array(report.checks.length).fill("passed")); + assert.equal(opened.length, 3); + assert.ok(opened.every((transport) => transport.closed)); +}); + +test("qualification rejects a backend that does not enforce generation CAS", async () => { + const backend: Backend = { blob: null, ignoreCas: true }; + await assert.rejects( + exerciseQualificationSequence(BASE_OPTIONS, async () => + new FakeQualificationTransport(backend)), + (error: unknown) => { + assert.ok(error instanceof QualificationFailure); + assert.equal(error.reasonCode, "competition_not_fenced"); + return true; + }, + ); +}); + +test("qualification rejects an independent transport that cannot read the envelope", async () => { + const shared: Backend = { blob: null, ignoreCas: false }; + let opened = 0; + await assert.rejects( + exerciseQualificationSequence(BASE_OPTIONS, async () => { + opened += 1; + return new FakeQualificationTransport( + opened === 3 ? { blob: null, ignoreCas: false } : shared, + ); + }), + (error: unknown) => { + assert.ok(error instanceof QualificationFailure); + assert.equal(error.reasonCode, "independent_readback_failed"); + return true; + }, + ); +}); From f74722f65c912ac77aa396c249adfdcad22d0443 Mon Sep 17 00:00:00 2001 From: wchwawa Date: Wed, 2 Sep 2026 08:48:57 +1000 Subject: [PATCH 06/12] fix(authority): preserve Node 22 transport admission Signed-off-by: wchwawa --- .../coordination/nokv_jsonl_transport.ts | 16 ++++++++-- .../nokv_jsonl_transport.test.ts | 29 +++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/loopx/control_plane/coordination/nokv_jsonl_transport.ts b/loopx/control_plane/coordination/nokv_jsonl_transport.ts index ad5516c7d0..2986b4af31 100644 --- a/loopx/control_plane/coordination/nokv_jsonl_transport.ts +++ b/loopx/control_plane/coordination/nokv_jsonl_transport.ts @@ -23,6 +23,7 @@ import { const DEFAULT_REQUEST_TIMEOUT_MS = 30_000; const DEFAULT_MAX_RESPONSE_BYTES = 32 * 1024 * 1024; +const TRANSPORT_CONSTRUCTION_TOKEN = Symbol("NoKVJsonLinesTransport.open"); export type NoKVJsonLinesProcessFactory = ( command: string, @@ -100,7 +101,15 @@ export class NoKVJsonLinesTransport implements NoKVBlobTransport { private terminalError: Error | null = null; private closing = false; - private constructor(options: NoKVJsonLinesTransportOptions) { + constructor( + options: NoKVJsonLinesTransportOptions, + constructionToken: typeof TRANSPORT_CONSTRUCTION_TOKEN, + ) { + if (constructionToken !== TRANSPORT_CONSTRUCTION_TOKEN) { + throw new NoKVTransportProtocolError( + "NoKV JSON-lines transport must be created with open()", + ); + } if (!Array.isArray(options.argv) || options.argv.length === 0) { throw new NoKVTransportProtocolError("NoKV helper argv must not be empty"); } @@ -145,7 +154,10 @@ export class NoKVJsonLinesTransport implements NoKVBlobTransport { } static async open(options: NoKVJsonLinesTransportOptions): Promise { - const transport = new NoKVJsonLinesTransport(options); + const transport = new NoKVJsonLinesTransport( + options, + TRANSPORT_CONSTRUCTION_TOKEN, + ); let response: JsonObject; try { response = await transport.exchange("open", { config: options.config }); diff --git a/tests/control_plane_ts/nokv_jsonl_transport.test.ts b/tests/control_plane_ts/nokv_jsonl_transport.test.ts index 478725e199..54641a31b3 100644 --- a/tests/control_plane_ts/nokv_jsonl_transport.test.ts +++ b/tests/control_plane_ts/nokv_jsonl_transport.test.ts @@ -53,6 +53,35 @@ async function openFaultHelper(mode: string, maxResponseBytes?: number) { }); } +test("JSON-lines transport cannot bypass its open handshake", () => { + let processStarted = false; + const DirectTransport = NoKVJsonLinesTransport as unknown as new ( + options: { + argv: readonly string[]; + config: Record; + process_factory: () => never; + }, + constructionToken: symbol, + ) => NoKVJsonLinesTransport; + + assert.throws( + () => + new DirectTransport( + { + argv: ["injected-helper"], + config: {}, + process_factory: () => { + processStarted = true; + throw new Error("constructor reached process creation"); + }, + }, + Symbol("caller-token"), + ), + /must be created with open\(\)/, + ); + assert.equal(processStarted, false); +}); + registerAuthorityStoreConformance("NoKV JSON-lines process", async (t) => { const transport = await openSdkHelper(); t.after(async () => await transport.close()); From 1e13208945f085e8955478327efc83aa88101e8e Mon Sep 17 00:00:00 2001 From: wchwawa Date: Wed, 2 Sep 2026 08:12:40 +1000 Subject: [PATCH 07/12] feat(turn): gate effects with shared authority Signed-off-by: wchwawa --- loopx/cli_commands/turn.py | 26 + loopx/cli_commands/turn_registration.py | 14 + loopx/control_plane/turn_driver/__init__.py | 16 + .../turn_driver/authority_checkpoint.py | 841 ++++++++++++++++++ loopx/control_plane/turn_driver/executor.py | 180 ++-- loopx/control_plane/turn_driver/settlement.py | 6 +- loopx/control_plane/turn_driver/settlement.ts | 2 + .../control_plane/turn_driver/transaction.py | 15 +- .../turn_driver/turn_journal_effects.ts | 13 +- tests/test_loopx_turn_driver.py | 99 +++ tests/test_loopx_turn_executor.py | 644 ++++++++++++++ tests/test_loopx_turn_transaction.py | 1 + tests/test_turn_authority_checkpoint.py | 246 +++++ 13 files changed, 1997 insertions(+), 106 deletions(-) create mode 100644 loopx/control_plane/turn_driver/authority_checkpoint.py create mode 100644 tests/test_turn_authority_checkpoint.py diff --git a/loopx/cli_commands/turn.py b/loopx/cli_commands/turn.py index 5f482eaccf..cc6b8d98d0 100644 --- a/loopx/cli_commands/turn.py +++ b/loopx/cli_commands/turn.py @@ -2,6 +2,7 @@ import argparse import json +import os from collections.abc import Callable, Mapping from pathlib import Path from typing import Any @@ -39,6 +40,7 @@ TurnRecoveryBlockedError, build_loopx_turn_command_validator, build_loopx_turn_plan, + build_turn_authority_command_guard, codex_cli_session_binding, load_loopx_turn_plan_from_journal, run_codex_cli_host, @@ -239,6 +241,27 @@ def build_turn_decision( "LoopX Turn resume journal belongs to another agent" ) project = Path(args.project).expanduser().resolve() + authority_checkpoint_guard = None + if args.authority_guard_command_json: + if os.environ.get("LOOPX_SHARED_AUTHORITY_TEST_ONLY") != "1": + raise ValueError( + "--authority-guard-command-json is TEST ONLY and requires " + "LOOPX_SHARED_AUTHORITY_TEST_ONLY=1" + ) + raw_authority_guard_argv = json.loads( + args.authority_guard_command_json + ) + if not isinstance(raw_authority_guard_argv, list) or not all( + isinstance(item, str) for item in raw_authority_guard_argv + ): + raise ValueError( + "--authority-guard-command-json must be a JSON string array" + ) + authority_checkpoint_guard = build_turn_authority_command_guard( + raw_authority_guard_argv, + project=project, + timeout_seconds=args.authority_guard_timeout_seconds, + ) planned_host = ( payload.get("host") if isinstance(payload.get("host"), dict) else {} ) @@ -948,6 +971,9 @@ def resolve_built_in_session_binding( terminal_closeout_resolver if args.execute else None ), scheduler=scheduler if args.execute else None, + authority_checkpoint_guard=( + authority_checkpoint_guard if args.execute else None + ), ) else: raise ValueError("turn requires the `plan` or `run-once` subcommand") diff --git a/loopx/cli_commands/turn_registration.py b/loopx/cli_commands/turn_registration.py index 11f3224250..ffd290f563 100644 --- a/loopx/cli_commands/turn_registration.py +++ b/loopx/cli_commands/turn_registration.py @@ -119,6 +119,20 @@ def register_turn_commands( default="repair_required", help="Typed recovery disposition when the independent validator rejects the result.", ) + run_once.add_argument( + "--authority-guard-command-json", + help=( + "TEST ONLY: JSON argv array for an authority checkpoint guard. " + "Requires LOOPX_SHARED_AUTHORITY_TEST_ONLY=1. The command reads one " + "checkpoint request from stdin and emits one typed result on stdout." + ), + ) + run_once.add_argument( + "--authority-guard-timeout-seconds", + type=float, + default=10.0, + help="TEST ONLY: timeout for each authority checkpoint guard invocation.", + ) run_once.add_argument( "--codex-bin", default="codex", diff --git a/loopx/control_plane/turn_driver/__init__.py b/loopx/control_plane/turn_driver/__init__.py index 551a22ce8f..0ba446bb16 100644 --- a/loopx/control_plane/turn_driver/__init__.py +++ b/loopx/control_plane/turn_driver/__init__.py @@ -1,5 +1,14 @@ """LoopX Turn decision planning for external agent-loop hosts.""" +from .authority_checkpoint import ( + TURN_AUTHORITY_BINDING_SCHEMA_VERSION, + TURN_AUTHORITY_CHECKPOINT_JOURNAL_SCHEMA_VERSION, + TURN_AUTHORITY_CHECKPOINT_RECEIPT_SCHEMA_VERSION, + TURN_AUTHORITY_CHECKPOINT_REQUEST_SCHEMA_VERSION, + TurnAuthorityCheckpointGuard, + TurnAuthorityCheckpointSession, + build_turn_authority_command_guard, +) from .codex_cli import ( CODEX_CLI_SESSION_SCHEMA_VERSION, codex_cli_result_schema, @@ -57,17 +66,24 @@ "LOOPX_TURN_SESSION_BINDING_SCHEMA_VERSION", "LOOPX_TURN_TASK_VALIDATION_SCHEMA_VERSION", "LOOP_CONTROLLER_DISPOSITION_SCHEMA_VERSION", + "TURN_AUTHORITY_BINDING_SCHEMA_VERSION", + "TURN_AUTHORITY_CHECKPOINT_JOURNAL_SCHEMA_VERSION", + "TURN_AUTHORITY_CHECKPOINT_RECEIPT_SCHEMA_VERSION", + "TURN_AUTHORITY_CHECKPOINT_REQUEST_SCHEMA_VERSION", "VALIDATED_TURN_RECEIPT_SCHEMA_VERSION", "BoundedTurnBudget", "LoopDisposition", "LoopXTurnResultKind", "LoopXTurnRoute", "TurnRecoveryBlockedError", + "TurnAuthorityCheckpointGuard", + "TurnAuthorityCheckpointSession", "ValidatedTurnReceipt", "build_loopx_turn_command_validator", "build_loopx_turn_host_request", "build_loopx_turn_plan", "build_loopx_turn_transaction_plan", + "build_turn_authority_command_guard", "codex_cli_result_schema", "codex_cli_session_binding", "codex_cli_session_id_from_jsonl", diff --git a/loopx/control_plane/turn_driver/authority_checkpoint.py b/loopx/control_plane/turn_driver/authority_checkpoint.py new file mode 100644 index 0000000000..22c7b872b1 --- /dev/null +++ b/loopx/control_plane/turn_driver/authority_checkpoint.py @@ -0,0 +1,841 @@ +"""Optional authority checkpoints around one governed LoopX Turn. + +The Turn driver does not choose or implement a coordination provider here. +Compositions may inject a guard that admits one authority binding before Host +execution and revalidates that exact binding before each durable effect. The +guard is disabled by default, so the existing local Turn path is unchanged. +""" + +from __future__ import annotations + +import json +import math +import re +import subprocess +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from ...authority import validate_public_safe_text +from .driver import selected_turn_todo +from .settlement import ( + completion_writeback_outcome, + invoke_result_effect, + invoke_turn_effect, + terminal_closeout_requirement, + verified_terminal_closeout_effect, +) +from .transaction import LoopXTurnResultKind + + +TURN_AUTHORITY_BINDING_SCHEMA_VERSION = "loopx_turn_authority_binding_v0" +TURN_AUTHORITY_CHECKPOINT_REQUEST_SCHEMA_VERSION = ( + "loopx_turn_authority_checkpoint_request_v0" +) +TURN_AUTHORITY_CHECKPOINT_RECEIPT_SCHEMA_VERSION = ( + "loopx_turn_authority_checkpoint_receipt_v0" +) +TURN_AUTHORITY_CHECKPOINT_JOURNAL_SCHEMA_VERSION = ( + "loopx_turn_authority_checkpoint_journal_v0" +) +TURN_AUTHORITY_CHECKPOINTS = frozenset( + { + "host_admission", + "durable_writeback", + "quota_spend", + "authority_complete", + "terminal_closeout", + "scheduler", + } +) + +_BINDING_FIELDS = frozenset( + { + "schema_version", + "store_identity", + "operation_id", + "receipt_digest", + "authority_revision", + "todo_revision", + "lease_id", + "lease_epoch", + "expires_at", + } +) +_REASON_CODE_RE = re.compile(r"[a-z][a-z0-9_]{0,79}\Z") +_DIGEST_RE = re.compile(r"sha256:[0-9a-f]{64}\Z") +_TIMESTAMP_RE = re.compile( + r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3}Z\Z" +) + +TurnAuthorityCheckpointGuard = Callable[[Mapping[str, Any]], Mapping[str, Any]] +PersistCheckpoint = Callable[[], None] +TurnResultEffect = Callable[..., Mapping[str, Any]] +TurnEffect = Callable[..., Mapping[str, Any]] +TurnScheduler = Callable[[dict[str, Any]], dict[str, Any]] +TurnTerminalCheckpoint = Callable[[Mapping[str, Any]], None] +TurnAdmissionFailure = Callable[[str], Mapping[str, Any]] + + +def build_turn_authority_command_guard( + argv: Sequence[str], + *, + project: Path, + timeout_seconds: float, +) -> TurnAuthorityCheckpointGuard: + """Adapt one argv-only TEST ONLY command to the checkpoint contract. + + The command receives exactly one request JSON object on stdin and must emit + exactly one result JSON object on stdout. Process and decoding failures are + deliberately collapsed into one public typed rejection; stderr and local + provider details never become Turn journal material. + """ + + normalized = tuple(argv) + if not normalized or not all(isinstance(item, str) and item for item in normalized): + raise ValueError("Turn authority guard command must be a non-empty argv array") + if ( + isinstance(timeout_seconds, bool) + or not isinstance(timeout_seconds, (int, float)) + or not math.isfinite(float(timeout_seconds)) + or timeout_seconds <= 0 + ): + raise ValueError("Turn authority guard timeout must be positive") + + def reject() -> dict[str, Any]: + return { + "ok": False, + "reason_code": "authority_guard_unavailable", + "reason": "Turn authority guard command did not return a valid receipt", + } + + def guard(request: Mapping[str, Any]) -> Mapping[str, Any]: + try: + completed = subprocess.run( + normalized, + cwd=project, + input=json.dumps(request, ensure_ascii=False, separators=(",", ":")), + text=True, + capture_output=True, + timeout=timeout_seconds, + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + return reject() + if ( + completed.returncode != 0 + or len(completed.stdout.encode("utf-8")) > 64 * 1024 + ): + return reject() + try: + value = json.loads(completed.stdout) + except json.JSONDecodeError: + return reject() + return dict(value) if isinstance(value, Mapping) else reject() + + return guard + + +def _bounded_public_string(value: Any, *, field: str, limit: int) -> str: + if not isinstance(value, str) or not value.strip() or value != value.strip(): + raise ValueError(f"authority {field} must be a non-empty trimmed string") + if len(value) > limit: + raise ValueError(f"authority {field} exceeds {limit} characters") + validate_public_safe_text(f"turn_authority.{field}", value) + return value + + +def _normalize_binding(value: Any) -> dict[str, Any]: + if not isinstance(value, Mapping) or set(value) != _BINDING_FIELDS: + raise ValueError("authority binding fields do not match the v0 contract") + binding = dict(value) + if binding.get("schema_version") != TURN_AUTHORITY_BINDING_SCHEMA_VERSION: + raise ValueError("authority binding schema is unsupported") + for field, limit in ( + ("store_identity", 240), + ("operation_id", 240), + ("lease_id", 240), + ("expires_at", 80), + ): + binding[field] = _bounded_public_string( + binding.get(field), field=field, limit=limit + ) + if not _TIMESTAMP_RE.fullmatch(binding["expires_at"]): + raise ValueError("authority expires_at must be a UTC millisecond timestamp") + digest = _bounded_public_string( + binding.get("receipt_digest"), field="receipt_digest", limit=71 + ) + if not _DIGEST_RE.fullmatch(digest): + raise ValueError("authority receipt_digest must use sha256:<64 lowercase hex>") + binding["receipt_digest"] = digest + for field, minimum in ( + ("authority_revision", 0), + ("todo_revision", 0), + ("lease_epoch", 1), + ): + item = binding.get(field) + if type(item) is not int or item < minimum: + raise ValueError(f"authority {field} must be an integer >= {minimum}") + return binding + + +def _normalize_rejection(value: Mapping[str, Any]) -> tuple[str, str]: + if set(value) != {"ok", "reason_code", "reason"} or value.get("ok") is not False: + raise ValueError( + "authority guard rejection fields do not match the v0 contract" + ) + reason_code = str(value.get("reason_code") or "") + if not _REASON_CODE_RE.fullmatch(reason_code): + raise ValueError("authority guard reason_code is invalid") + reason = _bounded_public_string(value.get("reason"), field="reason", limit=240) + return reason_code, reason + + +def _normalize_completion( + value: Mapping[str, Any] | None, + *, + expected_todo_id: str, +) -> dict[str, Any]: + if not isinstance(value, Mapping): + raise ValueError("authority completion context is missing") + continuation = value.get("continuation") + required = {"todo_id", "continuation"} + if continuation == "successor": + required.add("successor_todo_ids") + if set(value) != required or value.get("todo_id") != expected_todo_id: + raise ValueError("authority completion context does not match the Turn Todo") + if continuation not in {"active_goal", "no_followup", "successor"}: + raise ValueError("authority completion continuation is invalid") + completion = { + "todo_id": expected_todo_id, + "continuation": continuation, + } + if continuation == "successor": + successors = value.get("successor_todo_ids") + if ( + not isinstance(successors, list) + or not successors + or not all(isinstance(item, str) and item for item in successors) + or len(set(successors)) != len(successors) + ): + raise ValueError("authority completion successor Todo ids are invalid") + completion["successor_todo_ids"] = list(successors) + return completion + + +@dataclass(frozen=True, slots=True) +class TurnAuthorityCheckpointOutcome: + accepted: bool + receipt: dict[str, Any] + binding: dict[str, Any] | None + + def rejected_effect(self) -> dict[str, Any]: + return { + "ok": False, + "appended": False, + "reason": str( + self.receipt.get("reason") or "authority checkpoint rejected" + ), + "authority_checkpoint": self.receipt["checkpoint"], + "authority_reason_code": self.receipt.get("reason_code"), + } + + +class TurnAuthorityCheckpointSession: + """Validate and journal one injected guard without owning its decisions.""" + + def __init__( + self, + guard: TurnAuthorityCheckpointGuard, + *, + goal_id: str, + agent_id: str, + todo_id: str, + turn_key: str, + effect_id: str, + journal: dict[str, Any], + persist: PersistCheckpoint, + ) -> None: + self._guard = guard + self._identity = { + "goal_id": _bounded_public_string(goal_id, field="goal_id", limit=200), + "agent_id": _bounded_public_string(agent_id, field="agent_id", limit=200), + "todo_id": _bounded_public_string(todo_id, field="todo_id", limit=200), + "turn_key": _bounded_public_string(turn_key, field="turn_key", limit=80), + "effect_id": _bounded_public_string( + effect_id, field="effect_id", limit=512 + ), + } + self._journal = journal + self._persist = persist + + @property + def effect_id(self) -> str: + return self._identity["effect_id"] + + @property + def identity(self) -> dict[str, str]: + """Return the public Turn identity supplied to every checkpoint.""" + + return dict(self._identity) + + def _state(self) -> dict[str, Any]: + value = self._journal.get("authority_checkpoint_guard") + if value is None: + value = { + "schema_version": TURN_AUTHORITY_CHECKPOINT_JOURNAL_SCHEMA_VERSION, + "checkpoints": {}, + } + self._journal["authority_checkpoint_guard"] = value + allowed_fields = { + "schema_version", + "checkpoints", + "binding", + "invalid_prior_state", + } + if ( + not isinstance(value, dict) + or not set(value).issubset(allowed_fields) + or value.get("schema_version") + != TURN_AUTHORITY_CHECKPOINT_JOURNAL_SCHEMA_VERSION + or not isinstance(value.get("checkpoints"), dict) + or ( + "invalid_prior_state" in value + and value.get("invalid_prior_state") is not True + ) + ): + raise ValueError("Turn authority checkpoint journal is invalid") + for checkpoint, receipt in value["checkpoints"].items(): + if checkpoint not in TURN_AUTHORITY_CHECKPOINTS or not isinstance( + receipt, Mapping + ): + raise ValueError("Turn authority checkpoint journal is invalid") + status = receipt.get("status") + required = {"schema_version", "checkpoint", "status", "attempt"} + if "effect_ref" in receipt: + required.add("effect_ref") + if status == "rejected": + required.update({"reason_code", "reason"}) + if ( + set(receipt) != required + or receipt.get("schema_version") + != TURN_AUTHORITY_CHECKPOINT_RECEIPT_SCHEMA_VERSION + or receipt.get("checkpoint") != checkpoint + or status not in {"accepted", "rejected"} + or type(receipt.get("attempt")) is not int + or receipt["attempt"] < 1 + ): + raise ValueError("Turn authority checkpoint journal is invalid") + if "effect_ref" in receipt: + _bounded_public_string( + receipt.get("effect_ref"), field="effect_ref", limit=512 + ) + if status == "rejected": + reason_code = receipt.get("reason_code") + if not isinstance(reason_code, str) or not _REASON_CODE_RE.fullmatch( + reason_code + ): + raise ValueError("Turn authority checkpoint journal is invalid") + _bounded_public_string(receipt.get("reason"), field="reason", limit=240) + return value + + def _current_binding(self, state: Mapping[str, Any]) -> dict[str, Any] | None: + value = state.get("binding") + return None if value is None else _normalize_binding(value) + + def _record( + self, + *, + checkpoint: str, + effect_ref: str | None, + accepted: bool, + binding: dict[str, Any] | None, + reason_code: str | None = None, + reason: str | None = None, + ) -> TurnAuthorityCheckpointOutcome: + state = self._state() + checkpoints = state["checkpoints"] + assert isinstance(checkpoints, dict) + previous = checkpoints.get(checkpoint) + attempt = ( + int(previous.get("attempt") or 0) + 1 + if isinstance(previous, Mapping) + else 1 + ) + receipt: dict[str, Any] = { + "schema_version": TURN_AUTHORITY_CHECKPOINT_RECEIPT_SCHEMA_VERSION, + "checkpoint": checkpoint, + "status": "accepted" if accepted else "rejected", + "attempt": attempt, + } + if effect_ref is not None: + receipt["effect_ref"] = effect_ref + if not accepted: + receipt.update(reason_code=reason_code, reason=reason) + checkpoints[checkpoint] = receipt + if binding is not None: + state["binding"] = binding + self._persist() + return TurnAuthorityCheckpointOutcome(accepted, receipt, binding) + + def checkpoint( + self, + checkpoint: str, + *, + effect_ref: str | None = None, + completion: Mapping[str, Any] | None = None, + ) -> TurnAuthorityCheckpointOutcome: + if checkpoint not in TURN_AUTHORITY_CHECKPOINTS: + raise ValueError(f"unsupported Turn authority checkpoint: {checkpoint}") + if checkpoint == "authority_complete": + completion_context = _normalize_completion( + completion, + expected_todo_id=self._identity["todo_id"], + ) + elif completion is not None: + raise ValueError("completion context is only valid at authority_complete") + else: + completion_context = None + try: + state = self._state() + current = self._current_binding(state) + except (TypeError, ValueError) as exc: + self._journal["authority_checkpoint_guard"] = { + "schema_version": TURN_AUTHORITY_CHECKPOINT_JOURNAL_SCHEMA_VERSION, + "checkpoints": {}, + "invalid_prior_state": True, + } + return self._record( + checkpoint=checkpoint, + effect_ref=effect_ref, + accepted=False, + binding=None, + reason_code="authority_journal_invalid", + reason=str(exc), + ) + if checkpoint != "host_admission" and current is None: + return self._record( + checkpoint=checkpoint, + effect_ref=effect_ref, + accepted=False, + binding=None, + reason_code="authority_admission_missing", + reason="Turn has no accepted authority admission binding", + ) + request = { + "schema_version": TURN_AUTHORITY_CHECKPOINT_REQUEST_SCHEMA_VERSION, + "checkpoint": checkpoint, + **self._identity, + "effect_ref": effect_ref, + "authority_binding": current, + } + if completion_context is not None: + request["completion"] = completion_context + try: + raw = self._guard(request) + if not isinstance(raw, Mapping): + raise ValueError("authority guard result must be an object") + result = dict(raw) + if result.get("ok") is True and set(result) == {"ok", "binding"}: + binding = _normalize_binding(result.get("binding")) + if current is not None and binding != current: + return self._record( + checkpoint=checkpoint, + effect_ref=effect_ref, + accepted=False, + binding=current, + reason_code="authority_binding_changed", + reason="authority checkpoint returned a different admission binding", + ) + return self._record( + checkpoint=checkpoint, + effect_ref=effect_ref, + accepted=True, + binding=binding, + ) + reason_code, reason = _normalize_rejection(result) + except Exception as exc: # noqa: BLE001 - injected provider boundary + reason_code = "authority_guard_invalid" + reason = f"authority checkpoint guard failed with {type(exc).__name__}" + return self._record( + checkpoint=checkpoint, + effect_ref=effect_ref, + accepted=False, + binding=current, + reason_code=reason_code, + reason=reason, + ) + + +@dataclass(frozen=True, slots=True) +class TurnAuthoritySettlementEffects: + """Authority-aware callbacks handed to the typed settlement driver.""" + + writeback: TurnEffect + spend: TurnEffect + terminal_closeout: TurnEffect | None + terminal_checkpoint: TurnTerminalCheckpoint | None + terminal_closeout_required: bool + + +class TurnAuthorityCheckpointController: + """Keep one Turn's authority checkpoints adjacent to governed effects.""" + + def __init__( + self, + session: TurnAuthorityCheckpointSession | None, + *, + plan: Mapping[str, Any], + journal: dict[str, Any], + persist: PersistCheckpoint, + ) -> None: + self._session = session + self._plan = plan + self._journal = journal + self._persist = persist + + @property + def enabled(self) -> bool: + return self._session is not None + + def admit_host( + self, + *, + completed_phases: Sequence[str], + failure: TurnAdmissionFailure, + ) -> bool: + """Admit Host before its first effect and persist a typed rejection.""" + + if self._session is None or "typed_result" in completed_phases: + return True + admission = self._session.checkpoint( + "host_admission", + effect_ref=self._session.effect_id, + ) + if admission.accepted: + return True + reason = str(admission.receipt.get("reason") or "authority admission rejected") + rejected = failure(reason) + self._journal.update( + status="failed", + reason=rejected["reason"], + receipt=rejected["receipt"], + completed_phases=[], + result_kind=LoopXTurnResultKind.AUTHORITY_REJECTED.value, + ) + self._persist() + return False + + def _rejected_effect( + self, + checkpoint: str, + *, + effect_ref: str, + completion: Mapping[str, Any] | None = None, + ) -> dict[str, Any] | None: + if self._session is None: + return None + outcome = self._session.checkpoint( + checkpoint, + effect_ref=effect_ref, + completion=completion, + ) + return None if outcome.accepted else outcome.rejected_effect() + + def _writeback_effect( + self, + result: Mapping[str, Any], + *, + completion_intent_error: str | None, + terminal_closeout_required: bool, + writeback: TurnResultEffect, + completion_writeback: TurnResultEffect | None, + ) -> TurnEffect: + def writeback_effect(effect_ref: str) -> Mapping[str, Any]: + if completion_intent_error: + return { + "ok": False, + "appended": False, + "reason": completion_intent_error, + } + rejected = self._rejected_effect( + "durable_writeback", + effect_ref=effect_ref, + ) + if rejected is not None: + return rejected + if ( + result.get("result_kind") + == LoopXTurnResultKind.VALIDATED_COMPLETION.value + and not terminal_closeout_required + ): + if completion_writeback is None: + raise ValueError( + "validated_completion requires a todo lifecycle adapter" + ) + callback_payload = invoke_result_effect( + completion_writeback, + result, + effect_ref, + ) + completion = completion_writeback_outcome( + callback_payload, + plan=self._plan, + ) + if completion is None: + return { + "ok": False, + "appended": False, + "reason": str( + callback_payload.get("reason") + or callback_payload.get("error") + or ( + "todo lifecycle adapter returned an invalid " + "completion outcome" + ) + ), + } + return {**callback_payload, "completion": completion} + return invoke_result_effect(writeback, result, effect_ref) + + return writeback_effect + + def _guarded_effect( + self, + checkpoint: str, + effect: TurnEffect, + ) -> TurnEffect: + if self._session is None: + return effect + + def guarded(effect_ref: str) -> Mapping[str, Any]: + rejected = self._rejected_effect(checkpoint, effect_ref=effect_ref) + return rejected if rejected is not None else effect(effect_ref) + + return guarded + + def _active_completion(self) -> dict[str, Any] | None: + stored = self._journal.get("terminal_closeout") + if not isinstance(stored, Mapping): + stored = self._journal.get("writeback") + return ( + completion_writeback_outcome(stored, plan=self._plan) + if isinstance(stored, Mapping) + else None + ) + + def _spend_effect( + self, + spend: TurnEffect, + *, + result: Mapping[str, Any], + terminal_closeout_required: bool, + ) -> TurnEffect: + if self._session is None: + return spend + + def guarded_spend(effect_ref: str) -> Mapping[str, Any]: + rejected = self._rejected_effect("quota_spend", effect_ref=effect_ref) + if rejected is not None: + return rejected + if ( + result.get("result_kind") + == LoopXTurnResultKind.VALIDATED_COMPLETION.value + and not terminal_closeout_required + ): + completion = self._active_completion() + if completion is None: + return { + "ok": False, + "appended": False, + "reason": ( + "validated completion has no durable authority " + "completion context" + ), + } + rejected = self._rejected_effect( + "authority_complete", + effect_ref=effect_ref, + completion=completion, + ) + if rejected is not None: + return rejected + return invoke_turn_effect(spend, effect_ref) + + return guarded_spend + + def settlement_effects( + self, + *, + result: Mapping[str, Any], + writeback: TurnResultEffect, + completion_writeback: TurnResultEffect | None, + completion_intent: TurnResultEffect | None, + terminal_closeout: TurnResultEffect | None, + spend: TurnEffect, + terminal_checkpoint: TurnTerminalCheckpoint, + ) -> TurnAuthoritySettlementEffects: + """Compose each checkpoint with the effect it fences.""" + + terminal_closeout_required = False + completion_intent_error = None + if result.get("result_kind") == LoopXTurnResultKind.VALIDATED_COMPLETION.value: + if ( + completion_writeback is None + or completion_intent is None + or terminal_closeout is None + ): + raise ValueError( + "validated_completion requires intent, lifecycle writeback, " + "and terminal closeout adapters" + ) + terminal_closeout_required, completion_intent_error = ( + terminal_closeout_requirement( + plan=self._plan, + result=result, + journal=self._journal, + completion_intent=completion_intent, + ) + ) + terminal_effect = None + effective_terminal_checkpoint = None + if terminal_closeout_required: + if terminal_closeout is None: + raise ValueError("terminal closeout adapter is required") + terminal_effect = self._guarded_effect( + "terminal_closeout", + verified_terminal_closeout_effect( + terminal_closeout, + result=result, + plan=self._plan, + ), + ) + effective_terminal_checkpoint = terminal_checkpoint + return TurnAuthoritySettlementEffects( + writeback=self._writeback_effect( + result, + completion_intent_error=completion_intent_error, + terminal_closeout_required=terminal_closeout_required, + writeback=writeback, + completion_writeback=completion_writeback, + ), + spend=self._spend_effect( + spend, + result=result, + terminal_closeout_required=terminal_closeout_required, + ), + terminal_closeout=terminal_effect, + terminal_checkpoint=effective_terminal_checkpoint, + terminal_closeout_required=terminal_closeout_required, + ) + + def run_scheduler( + self, + scheduler: TurnScheduler, + spend_payload: dict[str, Any], + *, + terminal_closeout_required: bool, + ) -> dict[str, Any]: + """Complete terminal authority, then fence the scheduler effect.""" + + if self._session is None: + return scheduler(spend_payload) + rejected = None + if terminal_closeout_required: + terminal_payload = self._journal.get("terminal_closeout") + completion = ( + completion_writeback_outcome(terminal_payload, plan=self._plan) + if isinstance(terminal_payload, Mapping) + else None + ) + if completion is None: + raise RuntimeError( + "terminal completion has no durable authority completion context" + ) + rejected = self._rejected_effect( + "authority_complete", + effect_ref=f"{self._session.effect_id}#terminal_closeout", + completion=completion, + ) + if rejected is None: + rejected = self._rejected_effect( + "scheduler", + effect_ref=f"{self._session.effect_id}#scheduler", + ) + if rejected is not None: + return { + "completed": False, + "acknowledged": False, + "disposition": "authority_checkpoint_rejected", + "reason": rejected.get("reason"), + } + return scheduler(spend_payload) + + +def build_turn_authority_checkpoint_controller( + guard: TurnAuthorityCheckpointGuard | None, + *, + plan: Mapping[str, Any], + transaction_plan: Mapping[str, Any], + journal: dict[str, Any], + turn_key: str, + persist: PersistCheckpoint, +) -> TurnAuthorityCheckpointController: + """Build the default-off authority composition for one durable Turn.""" + + effective_guard = guard + if effective_guard is None and "authority_checkpoint_guard" in journal: + + def reject_missing_guard( + _request: Mapping[str, Any], + ) -> Mapping[str, Any]: + return { + "ok": False, + "reason_code": "authority_guard_missing", + "reason": ( + "Turn was admitted under authority checkpoints but this " + "attempt has no authority guard" + ), + } + + effective_guard = reject_missing_guard + + session = None + if effective_guard is not None: + envelope_value = plan.get("turn_envelope") + envelope = envelope_value if isinstance(envelope_value, Mapping) else {} + selected_todo = selected_turn_todo(envelope) + settlement_value = transaction_plan.get("settlement_plan") + settlement_plan = ( + settlement_value if isinstance(settlement_value, Mapping) else {} + ) + identity_value = settlement_plan.get("identity") + identity = identity_value if isinstance(identity_value, Mapping) else {} + session = TurnAuthorityCheckpointSession( + effective_guard, + goal_id=str(envelope.get("goal_id") or ""), + agent_id=str(envelope.get("agent_id") or ""), + todo_id=str(selected_todo.get("todo_id") or ""), + turn_key=turn_key, + effect_id=str(identity.get("effect_id") or ""), + journal=journal, + persist=persist, + ) + return TurnAuthorityCheckpointController( + session, + plan=plan, + journal=journal, + persist=persist, + ) + + +def authority_journal_projection(journal: Mapping[str, Any]) -> dict[str, Any]: + """Expose only the public authority journal block in execution output.""" + + value = journal.get("authority_checkpoint_guard") + return ( + {"authority_checkpoint_guard": dict(value)} + if isinstance(value, Mapping) + else {} + ) diff --git a/loopx/control_plane/turn_driver/executor.py b/loopx/control_plane/turn_driver/executor.py index 5454a12ce5..467482284c 100644 --- a/loopx/control_plane/turn_driver/executor.py +++ b/loopx/control_plane/turn_driver/executor.py @@ -19,6 +19,7 @@ from ..goals.goal_vision import normalize_goal_vision_packet from ..work_items.delivery_batch_scale import require_delivery_batch_scale from ..work_items.delivery_outcome import require_delivery_outcome +from . import authority_checkpoint as turn_authority from .driver import selected_turn_todo from .host_failure import BuiltInHostError, project_host_failure, record_host_failure from .journal_store import ( @@ -41,14 +42,10 @@ from .settlement import ( TurnEffectResolver, TurnSettlementJournalAdapter, - completion_writeback_outcome, execute_turn_driver_settlement, - invoke_result_effect, - terminal_closeout_requirement, turn_settlement_failure_outcome, turn_settlement_outcome, turn_effect_resolvers, - verified_terminal_closeout_effect, ) from .transaction import ( LOOPX_TURN_EXECUTION_SCHEMA_VERSION, @@ -58,6 +55,7 @@ build_loopx_turn_transaction_plan, validate_loopx_turn_receipt, ) + LOOPX_TURN_HOST_REQUEST_SCHEMA_VERSION = "loopx_turn_host_request_v0" LOOPX_TURN_JOURNAL_INSPECTION_SCHEMA_VERSION = "loopx_turn_journal_inspection_v1" LOOPX_TURN_TASK_VALIDATION_SCHEMA_VERSION = "loopx_turn_task_validation_v0" @@ -821,6 +819,7 @@ def _execution_payload( else {} ), **({"todo_completion": todo_completion} if todo_completion else {}), + **turn_authority.authority_journal_projection(journal), **({"reason": journal.get("reason")} if journal.get("reason") else {}), **project_host_failure(journal), **({"recovery": dict(recovery)} if isinstance(recovery, Mapping) else {}), @@ -1063,8 +1062,7 @@ def _ensure_turn_settlement_plan( execution_mode=str(host_fields.get("execution_mode") or "isolated-headless"), session_action=str(host_fields.get("session_action") or "resume"), turn_instance_id=( - transaction_plan.get("turn_instance_id") - or transaction_plan.get("turn_key") + transaction_plan.get("turn_instance_id") or transaction_plan.get("turn_key") ), ) settlement_plan = built.get("settlement_plan") @@ -1087,88 +1085,28 @@ def _typed_settlement_stage( spend: Spend, effect_resolvers: Mapping[SettlementStepKind, TurnEffectResolver], scheduler: Scheduler, + authority_checkpoints: turn_authority.TurnAuthorityCheckpointController, ) -> dict[str, Any]: transaction_plan = ( plan.get("transaction") if isinstance(plan.get("transaction"), Mapping) else {} ) - terminal_closeout_required = False - completion_intent_error: str | None = None - if result.get("result_kind") == LoopXTurnResultKind.VALIDATED_COMPLETION.value: - if ( - completion_writeback is None - or completion_intent is None - or terminal_closeout is None - ): - raise ValueError( - "validated_completion requires intent, lifecycle writeback, " - "and terminal closeout adapters" - ) - terminal_closeout_required, completion_intent_error = ( - terminal_closeout_requirement( - plan=plan, - result=result, - journal=journal, - completion_intent=completion_intent, - ) - ) - - def writeback_effect(effect_ref: str) -> Mapping[str, Any]: - if completion_intent_error: - return { - "ok": False, - "appended": False, - "reason": completion_intent_error, - } - if ( - result.get("result_kind") == LoopXTurnResultKind.VALIDATED_COMPLETION.value - and not terminal_closeout_required - ): - if completion_writeback is None: - raise ValueError( - "validated_completion requires a todo lifecycle adapter" - ) - callback_payload = invoke_result_effect( - completion_writeback, result, effect_ref - ) - completion_outcome = completion_writeback_outcome( - callback_payload, - plan=plan, - ) - if completion_outcome is None: - return { - "ok": False, - "appended": False, - "reason": str( - callback_payload.get("reason") - or callback_payload.get("error") - or ( - "todo lifecycle adapter returned an invalid " - "completion outcome" - ) - ), - } - return { - **callback_payload, - "completion": completion_outcome, - } - return invoke_result_effect(writeback, result, effect_ref) - journal_adapter = TurnSettlementJournalAdapter( journal, effects, lambda: _write_journal(journal_path, journal), _compact_callback, ) - - terminal_effect = None - terminal_checkpoint = None - if terminal_closeout_required: - assert terminal_closeout is not None - terminal_effect = verified_terminal_closeout_effect( - terminal_closeout, result=result, plan=plan - ) - terminal_checkpoint = journal_adapter.checkpoint_terminal + authority_effects = authority_checkpoints.settlement_effects( + result=result, + writeback=writeback, + completion_writeback=completion_writeback, + completion_intent=completion_intent, + terminal_closeout=terminal_closeout, + spend=spend, + terminal_checkpoint=journal_adapter.checkpoint_terminal, + ) + terminal_closeout_required = authority_effects.terminal_closeout_required settlement_result = execute_turn_driver_settlement( transaction_plan, @@ -1184,8 +1122,8 @@ def writeback_effect(effect_ref: str) -> Mapping[str, Any]: if isinstance(journal.get("quota_spend"), Mapping) else None ), - writeback=writeback_effect, - spend=spend, + writeback=authority_effects.writeback, + spend=authority_effects.spend, checkpoint=journal_adapter.checkpoint, committed_effect_id=_journal_committed_effect_id(journal), terminal_closeout_required=terminal_closeout_required, @@ -1194,8 +1132,8 @@ def writeback_effect(effect_ref: str) -> Mapping[str, Any]: if isinstance(journal.get("terminal_closeout"), Mapping) else None ), - terminal_closeout=terminal_effect, - terminal_checkpoint=terminal_checkpoint, + terminal_closeout=authority_effects.terminal_closeout, + terminal_checkpoint=authority_effects.terminal_checkpoint, prepare=journal_adapter.prepare, abort=journal_adapter.abort, effect_attempts=journal_adapter.effect_attempts, @@ -1243,7 +1181,11 @@ def writeback_effect(effect_ref: str) -> Mapping[str, Any]: spend_payload = dict(settlement_state.quota_spend) _write_journal(journal_path, journal) - scheduler_payload = scheduler(spend_payload) + scheduler_payload = authority_checkpoints.run_scheduler( + scheduler, + spend_payload, + terminal_closeout_required=terminal_closeout_required, + ) journal["scheduler"] = scheduler_payload if scheduler_payload.get("completed") is not True: journal.update( @@ -1298,6 +1240,8 @@ def run_loopx_turn_once( spend_resolver: TurnEffectResolver | None = None, terminal_closeout_resolver: TurnEffectResolver | None = None, scheduler: Scheduler | None = None, + authority_checkpoint_guard: turn_authority.TurnAuthorityCheckpointGuard + | None = None, ) -> dict[str, Any]: if host_runner is not None and host_argv is not None: raise ValueError("run-once accepts either host_argv or host_runner, not both") @@ -1440,6 +1384,37 @@ def finish_recovery(payload: dict[str, Any]) -> dict[str, Any]: payload["recovery"] = dict(journal["recovery_audit"]) return payload + authority_checkpoints = ( + turn_authority.build_turn_authority_checkpoint_controller( + authority_checkpoint_guard, + plan=plan, + transaction_plan=transaction_plan, + journal=journal, + turn_key=turn_key, + persist=lambda: _write_journal(journal_path, journal), + ) + ) + admitted = authority_checkpoints.admit_host( + completed_phases=list(journal.get("completed_phases") or []), + failure=lambda reason: _host_failure( + plan, + kind=LoopXTurnResultKind.AUTHORITY_REJECTED, + completed_phases=[], + failed_phase="authority_admission", + reason=reason, + ), + ) + if not admitted: + return finish_recovery( + _execution_payload( + plan, + journal, + execute=True, + replayed=False, + effects=effects, + ) + ) + result, completed_phases, terminal = _host_result_stage( plan, request, @@ -1475,22 +1450,25 @@ def finish_recovery(payload: dict[str, Any]) -> dict[str, Any]: if terminal is not None: return finish_recovery(terminal) - return finish_recovery(_typed_settlement_stage( - plan, - result, - completed_phases=completed_phases, - journal=journal, - journal_path=journal_path, - effects=effects, - writeback=writeback, - completion_writeback=completion_writeback, - completion_intent=completion_intent, - terminal_closeout=terminal_closeout, - spend=spend, - effect_resolvers=turn_effect_resolvers( - writeback=writeback_resolver, - spend=spend_resolver, - terminal_closeout=terminal_closeout_resolver, - ), - scheduler=scheduler, - )) + return finish_recovery( + _typed_settlement_stage( + plan, + result, + completed_phases=completed_phases, + journal=journal, + journal_path=journal_path, + effects=effects, + writeback=writeback, + completion_writeback=completion_writeback, + completion_intent=completion_intent, + terminal_closeout=terminal_closeout, + spend=spend, + effect_resolvers=turn_effect_resolvers( + writeback=writeback_resolver, + spend=spend_resolver, + terminal_closeout=terminal_closeout_resolver, + ), + scheduler=scheduler, + authority_checkpoints=authority_checkpoints, + ) + ) diff --git a/loopx/control_plane/turn_driver/settlement.py b/loopx/control_plane/turn_driver/settlement.py index 1d2e01886b..383a8b1d2b 100644 --- a/loopx/control_plane/turn_driver/settlement.py +++ b/loopx/control_plane/turn_driver/settlement.py @@ -46,7 +46,7 @@ class TurnSettlementState: TURN_SETTLEMENT_REDUCTION_SCHEMA_VERSION = "loopx_turn_settlement_reduction_v0" -def _invoke_turn_effect(effect: TurnEffect, effect_ref: str) -> Mapping[str, Any]: +def invoke_turn_effect(effect: TurnEffect, effect_ref: str) -> Mapping[str, Any]: """Invoke a provider with a stable ref while retaining zero-arg callbacks.""" try: @@ -470,7 +470,7 @@ def reduce() -> Mapping[str, Any]: "terminal closeout requires an effect provider and checkpoint" ) if should_execute: - observed = dict(_invoke_turn_effect(terminal_closeout, effect_ref)) + observed = dict(invoke_turn_effect(terminal_closeout, effect_ref)) assert observed is not None if committed(observed): terminal_value = observed @@ -485,7 +485,7 @@ def reduce() -> Mapping[str, Any]: continue if should_execute: - observed = dict(_invoke_turn_effect(providers[step_kind], effect_ref)) + observed = dict(invoke_turn_effect(providers[step_kind], effect_ref)) assert observed is not None if not committed(observed): if abort is not None: diff --git a/loopx/control_plane/turn_driver/settlement.ts b/loopx/control_plane/turn_driver/settlement.ts index 3a9b88b42f..7c65bbaac7 100644 --- a/loopx/control_plane/turn_driver/settlement.ts +++ b/loopx/control_plane/turn_driver/settlement.ts @@ -54,6 +54,7 @@ export const TURN_RESULT_KINDS = [ "replan_required", "user_action_required", "wait", + "authority_rejected", "host_failure", "validation_failed", "writeback_failed", @@ -63,6 +64,7 @@ export const TURN_RESULT_KINDS = [ export type TurnResultKind = (typeof TURN_RESULT_KINDS)[number]; const FAILED_TURN_RESULT_KINDS = [ + "authority_rejected", "host_failure", "validation_failed", "writeback_failed", diff --git a/loopx/control_plane/turn_driver/transaction.py b/loopx/control_plane/turn_driver/transaction.py index a8368de465..e3e2fbef7c 100644 --- a/loopx/control_plane/turn_driver/transaction.py +++ b/loopx/control_plane/turn_driver/transaction.py @@ -32,6 +32,7 @@ class LoopXTurnResultKind(str, Enum): REPLAN_REQUIRED = "replan_required" USER_ACTION_REQUIRED = "user_action_required" WAIT = "wait" + AUTHORITY_REJECTED = "authority_rejected" HOST_FAILURE = "host_failure" VALIDATION_FAILED = "validation_failed" WRITEBACK_FAILED = "writeback_failed" @@ -48,6 +49,7 @@ class LoopXTurnResultKind(str, Enum): NO_SPEND_RESULT_KINDS = { LoopXTurnResultKind.USER_ACTION_REQUIRED, LoopXTurnResultKind.WAIT, + LoopXTurnResultKind.AUTHORITY_REJECTED, LoopXTurnResultKind.HOST_FAILURE, LoopXTurnResultKind.VALIDATION_FAILED, LoopXTurnResultKind.WRITEBACK_FAILED, @@ -58,6 +60,7 @@ class LoopXTurnResultKind(str, Enum): LoopXTurnResultKind.WAIT, } FAILURE_PHASES = { + LoopXTurnResultKind.AUTHORITY_REJECTED: "authority_admission", LoopXTurnResultKind.HOST_FAILURE: "host_execute", LoopXTurnResultKind.VALIDATION_FAILED: "validation", LoopXTurnResultKind.WRITEBACK_FAILED: "durable_writeback", @@ -313,7 +316,17 @@ def validate_loopx_turn_receipt( and kind is LoopXTurnResultKind.TERMINAL_CLOSEOUT_FAILED and completed == list(TRANSACTION_PHASES[:5]) ) - if failed_phase and failed_phase != expected_next and not terminal_closeout_failure: + authority_admission_failure = bool( + failed_phase == "authority_admission" + and kind is LoopXTurnResultKind.AUTHORITY_REJECTED + and completed == [] + ) + if ( + failed_phase + and failed_phase != expected_next + and not terminal_closeout_failure + and not authority_admission_failure + ): errors.append("failed_phase must be the next uncompleted transaction phase") if failed_phase and kind not in FAILURE_PHASES: errors.append("failed_phase is only valid for a typed failure result") diff --git a/loopx/control_plane/turn_driver/turn_journal_effects.ts b/loopx/control_plane/turn_driver/turn_journal_effects.ts index 168337957c..5874107235 100644 --- a/loopx/control_plane/turn_driver/turn_journal_effects.ts +++ b/loopx/control_plane/turn_driver/turn_journal_effects.ts @@ -185,7 +185,18 @@ function requireJournalState(journal: JsonObject): JournalState { const nextPhase = transactionPhases[completedPhases.length] ?? null; const terminalCloseoutFailure = failedPhase === "terminal_closeout" && completedPhases.length === 5; - if (!failedPhase || (failedPhase !== nextPhase && !terminalCloseoutFailure)) { + const authorityAdmissionFailure = + failedPhase === "authority_admission" && + completedPhases.length === 0 && + journal.result_kind === "authority_rejected"; + if ( + !failedPhase || + ( + failedPhase !== nextPhase && + !terminalCloseoutFailure && + !authorityAdmissionFailure + ) + ) { conflict("Failed Turn journal must name the next uncompleted phase"); } } diff --git a/tests/test_loopx_turn_driver.py b/tests/test_loopx_turn_driver.py index c649798fd7..225824ed3c 100644 --- a/tests/test_loopx_turn_driver.py +++ b/tests/test_loopx_turn_driver.py @@ -1253,6 +1253,105 @@ def test_turn_run_once_cli_commits_validated_result_and_one_quota_slot( ] +def test_turn_run_once_cli_test_only_authority_guard_is_gated_and_wired( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + project, runtime, registry = _write_live_fixture(tmp_path) + host_project = tmp_path / "guarded-host-workspace" + host_project.mkdir() + guard_log = tmp_path / "authority-checkpoints.jsonl" + host_script = """ +import json +import sys +request = json.load(sys.stdin) +json.dump({ + "schema_version": "loopx_turn_result_v0", + "turn_key": request["turn_key"], + "result_kind": "validated_progress", + "completed_phases": ["host_execute", "typed_result"], + "classification": "fixture_progress", + "recommended_action": "Continue the public fixture", + "next_action": "Run the next public fixture check", + "delivery_batch_scale": "implementation", + "delivery_outcome": "outcome_progress", + "vision_unchanged_reason": "The fixture objective remains unchanged.", + "summary": "One guarded fixture advanced." +}, sys.stdout) +""" + validation_script = "import json, sys; json.load(sys.stdin); raise SystemExit(0)" + guard_script = """ +import json +import pathlib +import sys +request = json.load(sys.stdin) +with pathlib.Path(sys.argv[1]).open("a", encoding="utf-8") as stream: + stream.write(json.dumps({"checkpoint": request["checkpoint"]}) + "\\n") +binding = request.get("authority_binding") or { + "schema_version": "loopx_turn_authority_binding_v0", + "store_identity": "file:00000000000000000000000000000001", + "operation_id": "cli-guard-fixture", + "receipt_digest": "sha256:" + "d" * 64, + "authority_revision": 1, + "todo_revision": 1, + "lease_id": "lease-cli-fixture", + "lease_epoch": 1, + "expires_at": "2030-01-01T00:00:00.000Z" +} +json.dump({"ok": True, "binding": binding}, sys.stdout) +""" + argv = [ + "--registry", + str(registry), + "--runtime-root", + str(runtime), + "--format", + "json", + "turn", + "run-once", + "--goal-id", + "loopx-turn-fixture", + "--agent-id", + "codex-fixture", + "--project", + str(host_project), + "--host-adapter-command-json", + json.dumps([sys.executable, "-c", host_script]), + "--validation-command-json", + json.dumps([sys.executable, "-c", validation_script]), + "--authority-guard-command-json", + json.dumps([sys.executable, "-c", guard_script, str(guard_log)]), + "--scan-root", + str(project), + "--no-global-sync", + "--execute", + ] + + monkeypatch.delenv("LOOPX_SHARED_AUTHORITY_TEST_ONLY", raising=False) + rejected_output = io.StringIO() + with contextlib.redirect_stdout(rejected_output): + rejected_exit = cli_main(argv) + rejected = json.loads(rejected_output.getvalue()) + assert rejected_exit == 1 + assert rejected["effects"]["host_invoked"] is False + assert "TEST ONLY" in rejected["error"] + assert not guard_log.exists() + + monkeypatch.setenv("LOOPX_SHARED_AUTHORITY_TEST_ONLY", "1") + output = io.StringIO() + with contextlib.redirect_stdout(output): + exit_code = cli_main(argv) + payload = json.loads(output.getvalue()) + + assert exit_code == 0, payload + assert payload["status"] == "committed" + assert payload["authority_checkpoint_guard"]["binding"]["lease_epoch"] == 1 + assert [ + json.loads(line)["checkpoint"] + for line in guard_log.read_text(encoding="utf-8").splitlines() + ] == ["host_admission", "durable_writeback", "quota_spend", "scheduler"] + + def test_turn_run_once_cli_completes_selected_todo_after_validation( tmp_path: Path, ) -> None: diff --git a/tests/test_loopx_turn_executor.py b/tests/test_loopx_turn_executor.py index 7cbfcf18ad..8644e7c78f 100644 --- a/tests/test_loopx_turn_executor.py +++ b/tests/test_loopx_turn_executor.py @@ -286,6 +286,51 @@ def _passing_validator( } +def _authority_binding(*, lease_epoch: int = 7) -> dict[str, object]: + return { + "schema_version": "loopx_turn_authority_binding_v0", + "store_identity": "file:00000000000000000000000000000001", + "operation_id": "turn-admission-fixture", + "receipt_digest": "sha256:" + ("a" * 64), + "authority_revision": 8, + "todo_revision": 9, + "lease_id": "lease-fixture", + "lease_epoch": lease_epoch, + "expires_at": "2030-01-01T00:00:00.000Z", + } + + +class _AuthorityGuard: + def __init__( + self, + *, + reject_at: str | None = None, + drift_at: str | None = None, + raise_at: str | None = None, + ) -> None: + self.reject_at = reject_at + self.drift_at = drift_at + self.raise_at = raise_at + self.calls: list[str] = [] + + def __call__(self, request: Mapping[str, object]) -> Mapping[str, object]: + checkpoint = str(request["checkpoint"]) + self.calls.append(checkpoint) + if checkpoint == self.raise_at: + raise OSError("private provider detail must not escape") + if checkpoint == self.reject_at: + return { + "ok": False, + "reason_code": "stale_lease_fence", + "reason": "authority lease generation is no longer current", + } + current = request.get("authority_binding") + binding = dict(current) if isinstance(current, Mapping) else _authority_binding() + if checkpoint == self.drift_at: + binding = _authority_binding(lease_epoch=8) + return {"ok": True, "binding": binding} + + def test_host_result_requires_bounded_public_material_fields() -> None: plan = _plan() result = _host_result(plan) @@ -321,6 +366,548 @@ def test_run_once_preview_has_no_host_or_journal_effects(tmp_path: Path) -> None assert not (tmp_path / "runtime").exists() +def test_authority_admission_rejection_is_journaled_before_host( + tmp_path: Path, +) -> None: + plan = _plan() + guard = _AuthorityGuard(reject_at="host_admission") + calls = {"host": 0, "writeback": 0, "spend": 0, "scheduler": 0} + writeback, spend, scheduler = _callbacks(calls) + + payload = run_loopx_turn_once( + plan, + host_runner=lambda _request: ( + calls.__setitem__("host", calls["host"] + 1) or _host_result(plan) + ), + project=tmp_path, + runtime_root=tmp_path / "runtime", + goal_id="fixture-goal", + timeout_seconds=5, + execute=True, + task_validator=_passing_validator, + writeback=writeback, + spend=spend, + scheduler=scheduler, + authority_checkpoint_guard=guard, + ) + + assert payload["ok"] is False + assert payload["result_kind"] == "authority_rejected" + assert payload["receipt"]["failed_phase"] == "authority_admission" + assert payload["effects"] == { + "host_invoked": False, + "state_written": False, + "quota_spent": False, + "scheduler_acknowledged": False, + } + assert calls == {"host": 0, "writeback": 0, "spend": 0, "scheduler": 0} + assert guard.calls == ["host_admission"] + authority = payload["authority_checkpoint_guard"] + assert authority["checkpoints"]["host_admission"] == { + "schema_version": "loopx_turn_authority_checkpoint_receipt_v0", + "checkpoint": "host_admission", + "status": "rejected", + "attempt": 1, + "effect_ref": plan["transaction"]["settlement_plan"]["identity"]["effect_id"], + "reason_code": "stale_lease_fence", + "reason": "authority lease generation is no longer current", + } + assert _journal(tmp_path / "runtime")["authority_checkpoint_guard"] == authority + + +@pytest.mark.parametrize( + ("reject_at", "expected_status", "expected_calls", "expected_guard_calls"), + ( + ( + "durable_writeback", + "failed", + {"host": 1, "writeback": 0, "spend": 0, "scheduler": 0}, + ["host_admission", "durable_writeback"], + ), + ( + "quota_spend", + "failed", + {"host": 1, "writeback": 1, "spend": 0, "scheduler": 0}, + ["host_admission", "durable_writeback", "quota_spend"], + ), + ( + "scheduler", + "scheduler_action_required", + {"host": 1, "writeback": 1, "spend": 1, "scheduler": 0}, + ["host_admission", "durable_writeback", "quota_spend", "scheduler"], + ), + ), +) +def test_authority_revalidation_stops_each_later_effect( + tmp_path: Path, + reject_at: str, + expected_status: str, + expected_calls: dict[str, int], + expected_guard_calls: list[str], +) -> None: + plan = _plan() + guard = _AuthorityGuard(reject_at=reject_at) + calls = {"host": 0, "writeback": 0, "spend": 0, "scheduler": 0} + writeback, spend, scheduler = _callbacks(calls) + + payload = run_loopx_turn_once( + plan, + host_runner=lambda _request: ( + calls.__setitem__("host", calls["host"] + 1) or _host_result(plan) + ), + project=tmp_path, + runtime_root=tmp_path / "runtime", + goal_id="fixture-goal", + timeout_seconds=5, + execute=True, + task_validator=_passing_validator, + writeback=writeback, + spend=spend, + scheduler=scheduler, + authority_checkpoint_guard=guard, + ) + + assert payload["status"] == expected_status + assert calls == expected_calls + assert guard.calls == expected_guard_calls + checkpoint = payload["authority_checkpoint_guard"]["checkpoints"][reject_at] + assert checkpoint["status"] == "rejected" + assert checkpoint["reason_code"] == "stale_lease_fence" + + +def test_authority_guard_cannot_replace_the_admitted_lease_generation( + tmp_path: Path, +) -> None: + plan = _plan() + guard = _AuthorityGuard(drift_at="durable_writeback") + calls = {"host": 0, "writeback": 0, "spend": 0, "scheduler": 0} + writeback, spend, scheduler = _callbacks(calls) + + payload = run_loopx_turn_once( + plan, + host_runner=lambda _request: ( + calls.__setitem__("host", calls["host"] + 1) or _host_result(plan) + ), + project=tmp_path, + runtime_root=tmp_path / "runtime", + goal_id="fixture-goal", + timeout_seconds=5, + execute=True, + task_validator=_passing_validator, + writeback=writeback, + spend=spend, + scheduler=scheduler, + authority_checkpoint_guard=guard, + ) + + assert payload["result_kind"] == "writeback_failed" + assert calls == {"host": 1, "writeback": 0, "spend": 0, "scheduler": 0} + rejected = payload["authority_checkpoint_guard"]["checkpoints"][ + "durable_writeback" + ] + assert rejected["reason_code"] == "authority_binding_changed" + assert payload["authority_checkpoint_guard"]["binding"]["lease_epoch"] == 7 + + +def test_authority_guard_exception_fails_closed_without_exposing_provider_detail( + tmp_path: Path, +) -> None: + plan = _plan() + guard = _AuthorityGuard(raise_at="host_admission") + calls = {"host": 0, "writeback": 0, "spend": 0, "scheduler": 0} + writeback, spend, scheduler = _callbacks(calls) + + payload = run_loopx_turn_once( + plan, + host_runner=lambda _request: pytest.fail("Host must not run"), + project=tmp_path, + runtime_root=tmp_path / "runtime", + goal_id="fixture-goal", + timeout_seconds=5, + execute=True, + task_validator=_passing_validator, + writeback=writeback, + spend=spend, + scheduler=scheduler, + authority_checkpoint_guard=guard, + ) + + assert payload["reason"] == "authority checkpoint guard failed with OSError" + assert "private provider detail" not in json.dumps(payload) + assert calls == {"host": 0, "writeback": 0, "spend": 0, "scheduler": 0} + + +def test_guarded_turn_cannot_resume_effects_without_its_authority_guard( + tmp_path: Path, +) -> None: + plan = _plan() + guard = _AuthorityGuard(reject_at="durable_writeback") + calls = {"host": 0, "writeback": 0, "spend": 0, "scheduler": 0} + writeback, spend, scheduler = _callbacks(calls) + common = { + "host_runner": lambda _request: ( + calls.__setitem__("host", calls["host"] + 1) or _host_result(plan) + ), + "project": tmp_path, + "runtime_root": tmp_path / "runtime", + "goal_id": "fixture-goal", + "timeout_seconds": 5, + "execute": True, + "task_validator": _passing_validator, + "writeback": writeback, + "spend": spend, + "scheduler": scheduler, + } + + failed = run_loopx_turn_once( + plan, + authority_checkpoint_guard=guard, + **common, + ) + retried = run_loopx_turn_once(plan, retry_failed=True, **common) + + assert failed["result_kind"] == "writeback_failed" + assert retried["result_kind"] == "writeback_failed" + assert retried["authority_checkpoint_guard"]["checkpoints"][ + "durable_writeback" + ]["reason_code"] == "authority_guard_missing" + assert calls == {"host": 1, "writeback": 0, "spend": 0, "scheduler": 0} + + +def test_authority_admission_rejection_can_retry_before_first_host_effect( + tmp_path: Path, +) -> None: + plan = _plan() + guard = _AuthorityGuard(reject_at="host_admission") + calls = {"host": 0, "writeback": 0, "spend": 0, "scheduler": 0} + writeback, spend, scheduler = _callbacks(calls) + common = { + "host_runner": lambda _request: ( + calls.__setitem__("host", calls["host"] + 1) or _host_result(plan) + ), + "project": tmp_path, + "runtime_root": tmp_path / "runtime", + "goal_id": "fixture-goal", + "timeout_seconds": 5, + "execute": True, + "task_validator": _passing_validator, + "writeback": writeback, + "spend": spend, + "scheduler": scheduler, + "authority_checkpoint_guard": guard, + } + + rejected = run_loopx_turn_once(plan, **common) + guard.reject_at = None + recovered = run_loopx_turn_once(plan, retry_failed=True, **common) + + assert rejected["result_kind"] == "authority_rejected" + assert recovered["status"] == "committed" + assert recovered["recovery"]["planned"]["resume_from"] == "host_execute" + assert calls == {"host": 1, "writeback": 1, "spend": 1, "scheduler": 1} + assert recovered["authority_checkpoint_guard"]["checkpoints"][ + "host_admission" + ]["attempt"] == 2 + + +def test_completion_writeback_is_fenced_before_lifecycle_mutation( + tmp_path: Path, +) -> None: + plan = _plan() + guard = _AuthorityGuard(reject_at="durable_writeback") + calls = {"completion": 0, "spend": 0, "terminal": 0, "scheduler": 0} + + payload = run_loopx_turn_once( + plan, + host_runner=lambda _request: _host_result( + plan, kind="validated_completion" + ), + project=tmp_path, + runtime_root=tmp_path / "runtime", + goal_id="fixture-goal", + timeout_seconds=5, + execute=True, + task_validator=_passing_validator, + writeback=lambda _result: pytest.fail("progress writeback must not run"), + completion_writeback=lambda _result: ( + calls.__setitem__("completion", calls["completion"] + 1) + or { + "ok": True, + "appended": True, + "completion": { + "todo_id": "todo_fixture0001", + "continuation": "active_goal", + }, + } + ), + completion_intent=lambda _result: { + "todo_id": "todo_fixture0001", + "continuation": "active_goal", + }, + terminal_closeout=lambda _result: ( + calls.__setitem__("terminal", calls["terminal"] + 1) + or {"ok": True, "appended": True} + ), + spend=lambda: ( + calls.__setitem__("spend", calls["spend"] + 1) + or {"ok": True, "appended": True} + ), + scheduler=lambda _spend: ( + calls.__setitem__("scheduler", calls["scheduler"] + 1) + or {"completed": True, "acknowledged": True} + ), + authority_checkpoint_guard=guard, + ) + + assert payload["result_kind"] == "writeback_failed" + assert calls == {"completion": 0, "spend": 0, "terminal": 0, "scheduler": 0} + assert guard.calls == ["host_admission", "durable_writeback"] + + +def test_validated_completion_closes_authority_before_quota_and_scheduler( + tmp_path: Path, +) -> None: + plan = _plan() + guard = _AuthorityGuard() + calls: list[str] = [] + + payload = run_loopx_turn_once( + plan, + host_runner=lambda _request: _host_result( + plan, kind="validated_completion" + ), + project=tmp_path, + runtime_root=tmp_path / "runtime", + goal_id="fixture-goal", + timeout_seconds=5, + execute=True, + task_validator=_passing_validator, + writeback=lambda _result: pytest.fail("progress writeback must not run"), + completion_writeback=lambda _result: ( + calls.append("completion_writeback") + or { + "ok": True, + "appended": True, + "completion": { + "todo_id": "todo_fixture0001", + "continuation": "active_goal", + }, + } + ), + completion_intent=lambda _result: { + "todo_id": "todo_fixture0001", + "continuation": "active_goal", + }, + terminal_closeout=lambda _result: pytest.fail( + "active-goal completion must not close the goal" + ), + spend=lambda: ( + calls.append("quota_spend") or {"ok": True, "appended": True} + ), + scheduler=lambda _spend: ( + calls.append("scheduler") + or {"completed": True, "acknowledged": True} + ), + authority_checkpoint_guard=guard, + ) + + assert payload["status"] == "committed" + assert calls == ["completion_writeback", "quota_spend", "scheduler"] + assert guard.calls == [ + "host_admission", + "durable_writeback", + "quota_spend", + "authority_complete", + "scheduler", + ] + assert payload["authority_checkpoint_guard"]["checkpoints"][ + "authority_complete" + ]["status"] == "accepted" + + +def test_authority_completion_failure_stops_quota_after_local_writeback( + tmp_path: Path, +) -> None: + plan = _plan() + guard = _AuthorityGuard(reject_at="authority_complete") + calls = {"completion": 0, "spend": 0, "scheduler": 0} + + payload = run_loopx_turn_once( + plan, + host_runner=lambda _request: _host_result( + plan, kind="validated_completion" + ), + project=tmp_path, + runtime_root=tmp_path / "runtime", + goal_id="fixture-goal", + timeout_seconds=5, + execute=True, + task_validator=_passing_validator, + writeback=lambda _result: pytest.fail("progress writeback must not run"), + completion_writeback=lambda _result: ( + calls.__setitem__("completion", calls["completion"] + 1) + or { + "ok": True, + "appended": True, + "completion": { + "todo_id": "todo_fixture0001", + "continuation": "active_goal", + }, + } + ), + completion_intent=lambda _result: { + "todo_id": "todo_fixture0001", + "continuation": "active_goal", + }, + terminal_closeout=lambda _result: pytest.fail( + "active-goal completion must not close the goal" + ), + spend=lambda: ( + calls.__setitem__("spend", calls["spend"] + 1) + or {"ok": True, "appended": True} + ), + scheduler=lambda _spend: ( + calls.__setitem__("scheduler", calls["scheduler"] + 1) + or {"completed": True, "acknowledged": True} + ), + authority_checkpoint_guard=guard, + ) + + assert payload["status"] == "failed" + assert calls == {"completion": 1, "spend": 0, "scheduler": 0} + assert guard.calls == [ + "host_admission", + "durable_writeback", + "quota_spend", + "authority_complete", + ] + assert payload["authority_checkpoint_guard"]["checkpoints"][ + "authority_complete" + ]["reason_code"] == "stale_lease_fence" + + +def test_terminal_completion_closes_authority_after_local_closeout( + tmp_path: Path, +) -> None: + plan = _plan() + guard = _AuthorityGuard() + calls: list[str] = [] + + payload = run_loopx_turn_once( + plan, + host_runner=lambda _request: _host_result( + plan, kind="validated_completion" + ), + project=tmp_path, + runtime_root=tmp_path / "runtime", + goal_id="fixture-goal", + timeout_seconds=5, + execute=True, + task_validator=_passing_validator, + writeback=lambda _result: ( + calls.append("writeback") or {"ok": True, "appended": True} + ), + completion_writeback=lambda _result: pytest.fail( + "terminal completion uses the closeout adapter" + ), + completion_intent=lambda _result: { + "todo_id": "todo_fixture0001", + "continuation": "no_followup", + }, + terminal_closeout=lambda _result: ( + calls.append("terminal_closeout") + or { + "ok": True, + "appended": True, + "completion": { + "todo_id": "todo_fixture0001", + "continuation": "no_followup", + }, + } + ), + spend=lambda: ( + calls.append("quota_spend") or {"ok": True, "appended": True} + ), + scheduler=lambda _spend: ( + calls.append("scheduler") + or {"completed": True, "acknowledged": True} + ), + authority_checkpoint_guard=guard, + ) + + assert payload["status"] == "committed" + assert calls == [ + "writeback", + "quota_spend", + "terminal_closeout", + "scheduler", + ] + assert guard.calls == [ + "host_admission", + "durable_writeback", + "quota_spend", + "terminal_closeout", + "authority_complete", + "scheduler", + ] + + +def test_terminal_authority_completion_failure_holds_scheduler( + tmp_path: Path, +) -> None: + plan = _plan() + guard = _AuthorityGuard(reject_at="authority_complete") + calls = {"writeback": 0, "spend": 0, "terminal": 0, "scheduler": 0} + + payload = run_loopx_turn_once( + plan, + host_runner=lambda _request: _host_result( + plan, kind="validated_completion" + ), + project=tmp_path, + runtime_root=tmp_path / "runtime", + goal_id="fixture-goal", + timeout_seconds=5, + execute=True, + task_validator=_passing_validator, + writeback=lambda _result: ( + calls.__setitem__("writeback", calls["writeback"] + 1) + or {"ok": True, "appended": True} + ), + completion_writeback=lambda _result: pytest.fail( + "terminal completion uses the closeout adapter" + ), + completion_intent=lambda _result: { + "todo_id": "todo_fixture0001", + "continuation": "no_followup", + }, + terminal_closeout=lambda _result: ( + calls.__setitem__("terminal", calls["terminal"] + 1) + or { + "ok": True, + "appended": True, + "completion": { + "todo_id": "todo_fixture0001", + "continuation": "no_followup", + }, + } + ), + spend=lambda: ( + calls.__setitem__("spend", calls["spend"] + 1) + or {"ok": True, "appended": True} + ), + scheduler=lambda _spend: ( + calls.__setitem__("scheduler", calls["scheduler"] + 1) + or {"completed": True, "acknowledged": True} + ), + authority_checkpoint_guard=guard, + ) + + assert payload["status"] == "scheduler_action_required" + assert calls == {"writeback": 1, "spend": 1, "terminal": 1, "scheduler": 0} + assert payload["scheduler"]["disposition"] == "authority_checkpoint_rejected" + + def test_run_once_rejects_oversized_built_in_host_result(tmp_path: Path) -> None: plan = _plan() calls = {"writeback": 0, "spend": 0, "scheduler": 0} @@ -1045,6 +1632,63 @@ def test_terminal_closeout_runs_only_after_matching_spend_receipt( } +def test_terminal_completion_is_fenced_after_spend_and_before_closeout( + tmp_path: Path, +) -> None: + plan = _plan() + guard = _AuthorityGuard(reject_at="terminal_closeout") + events: list[str] = [] + + payload = run_loopx_turn_once( + plan, + host_runner=lambda _request: _host_result( + plan, + kind="validated_completion", + ), + project=tmp_path, + runtime_root=tmp_path / "runtime", + goal_id="fixture-goal", + timeout_seconds=5, + execute=True, + task_validator=_passing_validator, + writeback=lambda _result: ( + events.append("writeback") or {"ok": True, "appended": True} + ), + completion_writeback=lambda _result: pytest.fail( + "terminal completion must not use lifecycle writeback" + ), + completion_intent=lambda _result: { + "todo_id": "todo_fixture0001", + "continuation": "no_followup", + }, + spend=lambda: events.append("spend") or {"ok": True, "appended": True}, + terminal_closeout=lambda _result: ( + events.append("terminal_closeout") + or { + "ok": True, + "appended": True, + "completion": { + "todo_id": "todo_fixture0001", + "continuation": "no_followup", + }, + } + ), + scheduler=lambda _spend: ( + events.append("scheduler") or {"completed": True, "acknowledged": True} + ), + authority_checkpoint_guard=guard, + ) + + assert payload["result_kind"] == "terminal_closeout_failed" + assert events == ["writeback", "spend"] + assert guard.calls == [ + "host_admission", + "durable_writeback", + "quota_spend", + "terminal_closeout", + ] + + def test_terminal_closeout_lost_receipt_retries_without_repeating_effects( tmp_path: Path, ) -> None: diff --git a/tests/test_loopx_turn_transaction.py b/tests/test_loopx_turn_transaction.py index 446e628adf..e6feb965b9 100644 --- a/tests/test_loopx_turn_transaction.py +++ b/tests/test_loopx_turn_transaction.py @@ -127,6 +127,7 @@ def test_receipt_rejects_turn_lineage_drift() -> None: @pytest.mark.parametrize( ("kind", "completed", "failed_phase"), [ + (LoopXTurnResultKind.AUTHORITY_REJECTED, [], "authority_admission"), (LoopXTurnResultKind.HOST_FAILURE, [], "host_execute"), ( LoopXTurnResultKind.VALIDATION_FAILED, diff --git a/tests/test_turn_authority_checkpoint.py b/tests/test_turn_authority_checkpoint.py new file mode 100644 index 0000000000..b79db757d4 --- /dev/null +++ b/tests/test_turn_authority_checkpoint.py @@ -0,0 +1,246 @@ +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + +from loopx.control_plane.turn_driver import ( + TurnAuthorityCheckpointSession, + build_turn_authority_command_guard, +) +from loopx.control_plane.turn_driver.authority_checkpoint import ( + build_turn_authority_checkpoint_controller, +) + + +def _request() -> dict[str, object]: + return { + "schema_version": "loopx_turn_authority_checkpoint_request_v0", + "checkpoint": "host_admission", + "goal_id": "goal-fixture", + "agent_id": "agent-fixture", + "todo_id": "todo-fixture", + "turn_key": "sha256:" + ("b" * 64), + "effect_id": "effect-fixture", + "effect_ref": "effect-fixture", + "authority_binding": None, + } + + +def test_command_guard_uses_json_argv_stdin_and_one_typed_stdout( + tmp_path: Path, +) -> None: + script = """ +import json +import sys +request = json.load(sys.stdin) +assert request["checkpoint"] == "host_admission" +json.dump({ + "ok": True, + "binding": { + "schema_version": "loopx_turn_authority_binding_v0", + "store_identity": "file:00000000000000000000000000000001", + "operation_id": "operation-fixture", + "receipt_digest": "sha256:" + "c" * 64, + "authority_revision": 1, + "todo_revision": 1, + "lease_id": "lease-fixture", + "lease_epoch": 1, + "expires_at": "2030-01-01T00:00:00.000Z" + } +}, sys.stdout) +""" + guard = build_turn_authority_command_guard( + [sys.executable, "-c", script], + project=tmp_path, + timeout_seconds=2, + ) + + result = guard(_request()) + + assert result["ok"] is True + assert result["binding"]["lease_epoch"] == 1 + + +@pytest.mark.parametrize( + "script", + ( + "raise SystemExit(7)", + "import sys; sys.stdout.write('not-json')", + "import json, sys; json.dump(['not-an-object'], sys.stdout)", + ), +) +def test_command_guard_process_failures_collapse_to_public_typed_rejection( + tmp_path: Path, + script: str, +) -> None: + guard = build_turn_authority_command_guard( + [sys.executable, "-c", script], + project=tmp_path, + timeout_seconds=2, + ) + + result = guard(_request()) + + assert result == { + "ok": False, + "reason_code": "authority_guard_unavailable", + "reason": "Turn authority guard command did not return a valid receipt", + } + assert "private" not in json.dumps(result) + + +def test_command_guard_rejects_empty_argv(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="non-empty argv"): + build_turn_authority_command_guard( + [], + project=tmp_path, + timeout_seconds=2, + ) + + +def test_session_identity_copy_and_bad_journal_fail_closed() -> None: + journal: dict[str, object] = { + "authority_checkpoint_guard": { + "schema_version": "loopx_turn_authority_checkpoint_journal_v0", + "checkpoints": {"host_admission": {"attempt": "corrupt"}}, + } + } + persisted = 0 + + def persist() -> None: + nonlocal persisted + persisted += 1 + + session = TurnAuthorityCheckpointSession( + lambda _request: pytest.fail("corrupt journal must fail before provider"), + goal_id="goal-fixture", + agent_id="agent-fixture", + todo_id="todo-fixture", + turn_key="sha256:" + ("e" * 64), + effect_id="effect-fixture", + journal=journal, + persist=persist, + ) + identity = session.identity + identity["agent_id"] = "mutated-copy" + + outcome = session.checkpoint("host_admission", effect_ref="effect-fixture") + + assert session.identity["agent_id"] == "agent-fixture" + assert outcome.accepted is False + assert outcome.receipt["reason_code"] == "authority_journal_invalid" + assert journal["authority_checkpoint_guard"]["invalid_prior_state"] is True + assert persisted == 1 + + +def test_completion_context_rejects_unhashable_successor_before_guard() -> None: + session = TurnAuthorityCheckpointSession( + lambda _request: pytest.fail("invalid completion must not reach guard"), + goal_id="goal-fixture", + agent_id="agent-fixture", + todo_id="todo-fixture", + turn_key="sha256:" + ("a" * 64), + effect_id="effect-fixture", + journal={}, + persist=lambda: pytest.fail("invalid completion must not persist"), + ) + + with pytest.raises(ValueError, match="successor Todo ids"): + session.checkpoint( + "authority_complete", + effect_ref="effect-fixture#quota_spend", + completion={ + "todo_id": "todo-fixture", + "continuation": "successor", + "successor_todo_ids": [{}], + }, + ) + + +def test_default_off_controller_does_not_parse_turn_lineage() -> None: + journal: dict[str, object] = {} + controller = build_turn_authority_checkpoint_controller( + None, + plan={}, + transaction_plan={}, + journal=journal, + turn_key="sha256:" + ("d" * 64), + persist=lambda: pytest.fail("default-off controller must not persist"), + ) + effects = controller.settlement_effects( + result={"result_kind": "validated_progress"}, + writeback=lambda _result, effect_ref: { + "ok": True, + "appended": True, + "effect_ref": effect_ref, + }, + completion_writeback=None, + completion_intent=None, + terminal_closeout=None, + spend=lambda effect_ref: { + "ok": True, + "appended": True, + "effect_ref": effect_ref, + }, + terminal_checkpoint=lambda _payload: pytest.fail( + "non-terminal Turn must not checkpoint closeout" + ), + ) + + assert controller.enabled is False + assert effects.writeback("writeback-ref")["effect_ref"] == "writeback-ref" + assert effects.spend("spend-ref")["effect_ref"] == "spend-ref" + assert effects.terminal_closeout is None + assert effects.terminal_checkpoint is None + assert controller.run_scheduler( + lambda spend: {"completed": True, "spend": spend}, + {"receipt": "quota"}, + terminal_closeout_required=False, + ) == {"completed": True, "spend": {"receipt": "quota"}} + assert journal == {} + + +def test_resumed_authority_turn_without_guard_fails_closed_at_admission() -> None: + journal: dict[str, object] = { + "authority_checkpoint_guard": { + "schema_version": "loopx_turn_authority_checkpoint_journal_v0", + "checkpoints": {}, + } + } + persisted = 0 + + def persist() -> None: + nonlocal persisted + persisted += 1 + + controller = build_turn_authority_checkpoint_controller( + None, + plan={ + "turn_envelope": { + "goal_id": "goal-fixture", + "agent_id": "agent-fixture", + "action": {"selected_todo": {"todo_id": "todo-fixture"}}, + } + }, + transaction_plan={ + "settlement_plan": {"identity": {"effect_id": "effect-fixture"}} + }, + journal=journal, + turn_key="sha256:" + ("f" * 64), + persist=persist, + ) + + admitted = controller.admit_host( + completed_phases=[], + failure=lambda reason: {"reason": reason, "receipt": {"typed": True}}, + ) + + assert admitted is False + assert journal["status"] == "failed" + assert journal["result_kind"] == "authority_rejected" + checkpoint = journal["authority_checkpoint_guard"]["checkpoints"]["host_admission"] + assert checkpoint["reason_code"] == "authority_guard_missing" + assert persisted == 2 From 0d0bfded2a2b4093ebb99c78ed48fb32faf1bed9 Mon Sep 17 00:00:00 2001 From: wchwawa Date: Wed, 2 Sep 2026 08:12:44 +1000 Subject: [PATCH 08/12] test(authority): qualify two-process Turn ownership Signed-off-by: wchwawa --- examples/nokv-shadow-provider/README.md | 24 + .../nokv-shadow-provider/authority_guard.py | 561 ++++++++++++++ .../authority_turn_canary.py | 674 +++++++++++++++++ tests/test_authority_turn_canary.py | 66 ++ tests/test_authority_turn_product_e2e.py | 684 ++++++++++++++++++ 5 files changed, 2009 insertions(+) create mode 100755 examples/nokv-shadow-provider/authority_guard.py create mode 100755 examples/nokv-shadow-provider/authority_turn_canary.py create mode 100644 tests/test_authority_turn_canary.py create mode 100644 tests/test_authority_turn_product_e2e.py diff --git a/examples/nokv-shadow-provider/README.md b/examples/nokv-shadow-provider/README.md index 509331ca83..b96f58288b 100644 --- a/examples/nokv-shadow-provider/README.md +++ b/examples/nokv-shadow-provider/README.md @@ -95,6 +95,14 @@ lease, gate, quota, or scheduling decisions. head through the production `validated_head`. Only the checks in the [evidence note](../../docs/architecture/rfcs/shared-goal-authority-state-provider-v0-evidence.zh-CN.md) are merge evidence for the revised receipt contract. +- `authority_guard.py`: a **TEST ONLY** argv/stdin adapter that composes the + production authority executor with the file provider for Turn checkpoint + qualification. It is not a production provider selector. +- `authority_turn_canary.py`: a deterministic two-process canary. Both LoopX + processes share one coordination head but use isolated Turn journals and + Host workspaces. It proves one pre-expiry Host admission and stale-epoch + settlement fencing after reclaim; it does not claim exactly-once behavior + for arbitrary Host workspace mutations. ## Validation boundary @@ -155,6 +163,22 @@ it is evidence tooling, not a merge gate. python3 examples/nokv-shadow-provider/live_e2e.py ``` +Run the TEST ONLY Turn-composition canary with: + +```bash +python3 examples/nokv-shadow-provider/authority_turn_canary.py +``` + +The canary drives `claim_work`, `renew_work`, expired `reclaim_work`, and +`complete_work` through the production `CoordinationAuthorityExecutor` and a +shared `FileCoordinationProvider`. The file provider makes the test +deterministic; replacing that storage adapter with the existing NoKV provider +is still a separate live qualification step. The CLI exposure is likewise +default-off: `turn run-once --authority-guard-command-json ...` requires the +explicit `LOOPX_SHARED_AUTHORITY_TEST_ONLY=1` environment gate and uses JSON +argv without shell parsing. Scheduler wake-up remains an effect after the +authority checks, never an authorization source. + Run the merge-relevant deterministic regression from the repository root with: ```bash diff --git a/examples/nokv-shadow-provider/authority_guard.py b/examples/nokv-shadow-provider/authority_guard.py new file mode 100755 index 0000000000..a6a1af25b5 --- /dev/null +++ b/examples/nokv-shadow-provider/authority_guard.py @@ -0,0 +1,561 @@ +#!/usr/bin/env python3 +"""TEST ONLY file-provider Turn authority guard. + +This process adapter composes the production ``CoordinationAuthorityExecutor`` +with ``FileCoordinationProvider``. It is qualification wiring, not a shared +production-mode declaration: every invocation reads one Turn checkpoint on +stdin and emits one typed result on stdout. Claim/reclaim and renew therefore +use the same authority core, aggregate, receipts, CAS, and store-lineage fence +as the NoKV provider contract; no parallel lock-based oracle exists here. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import sys +from collections.abc import Mapping +from datetime import datetime +from pathlib import Path +from typing import Any + +sys.path.insert( + 0, + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), +) + +from loopx.control_plane.coordination.executor import ( + CoordinationAuthorityExecutor, + sample_claim_envelope, + sample_work_envelope, +) +from loopx.control_plane.coordination.file_provider import ( + FileCoordinationProvider, +) +from loopx.control_plane.coordination.head import validated_head + + +def _clock(path: Path) -> float: + return float(path.read_text(encoding="utf-8").strip()) + + +def _preconditions(todo: Mapping[str, Any]) -> dict[str, Any]: + eligibility = todo["eligibility"] + return { + field: eligibility[field] + for field in ( + "authorization_projection_revision", + "authorization_projection_digest", + "dependency_revision", + "gate_revision", + ) + } + + +def _operation_id(prefix: str, *parts: str) -> str: + digest = hashlib.sha256("\0".join(parts).encode("utf-8")).hexdigest()[:32] + return f"turn-{prefix}-{digest}" + + +def _receipt_digest(receipt: Mapping[str, Any]) -> str: + payload = json.dumps( + receipt, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def _rejection(reason_code: str, reason: str) -> dict[str, Any]: + return {"ok": False, "reason_code": reason_code, "reason": reason} + + +def _stored_receipt( + head: Mapping[str, Any], operation_id: str +) -> dict[str, Any] | None: + index = head.get("receipt_index") + entry = index.get(operation_id) if isinstance(index, Mapping) else None + receipt = entry.get("original_receipt") if isinstance(entry, Mapping) else None + return dict(receipt) if isinstance(receipt, Mapping) else None + + +def _binding_matches_admission_receipt( + binding: Mapping[str, Any], + receipt: Mapping[str, Any], + *, + agent_id: str, + todo_id: str, +) -> bool: + actor = receipt.get("actor") + expected = { + "operation_id": receipt.get("operation_id"), + "receipt_digest": _receipt_digest(receipt), + "authority_revision": receipt.get("accepted_authority_revision"), + "todo_revision": receipt.get("accepted_todo_revision"), + "lease_id": receipt.get("lease_id"), + "lease_epoch": receipt.get("lease_epoch"), + "expires_at": receipt.get("expires_at"), + } + return bool( + all(binding.get(field) == value for field, value in expected.items()) + and receipt.get("todo_id") == todo_id + and isinstance(actor, Mapping) + and actor.get("agent_id") == agent_id + and receipt.get("command") in {"claim_work", "reclaim_work"} + ) + + +def _completion_operation_id( + request: Mapping[str, Any], binding: Mapping[str, Any] +) -> str: + return _operation_id( + "complete", + str(binding.get("operation_id") or ""), + str(request.get("turn_key") or ""), + str(request.get("effect_id") or ""), + ) + + +def _completion_command( + request: Mapping[str, Any], *, todo_id: str +) -> dict[str, Any] | None: + completion = request.get("completion") + if not isinstance(completion, Mapping): + return None + continuation = completion.get("continuation") + expected_fields = {"todo_id", "continuation"} + if continuation == "successor": + expected_fields.add("successor_todo_ids") + if set(completion) != expected_fields or completion.get("todo_id") != todo_id: + return None + successors = completion.get("successor_todo_ids", []) + if ( + continuation not in {"active_goal", "no_followup", "successor"} + or not isinstance(successors, list) + or not all(isinstance(item, str) and item for item in successors) + or len(set(successors)) != len(successors) + or (continuation == "successor") != bool(successors) + ): + return None + return { + "no_followup": continuation == "no_followup", + "successor_todo_ids": list(successors), + "completion_continuation": continuation, + } + + +def _completion_receipt_matches( + head: Mapping[str, Any], + receipt: Mapping[str, Any] | None, + *, + operation_id: str, + binding: Mapping[str, Any], + agent_id: str, + todo_id: str, + continuation: str, +) -> bool: + if not isinstance(receipt, Mapping): + return False + actor = receipt.get("actor") + todo = head["coordination"]["todos"].get(todo_id) + lease = head["coordination"]["leases"].get(todo_id) + return bool( + receipt.get("operation_id") == operation_id + and receipt.get("command") == "complete_work" + and receipt.get("todo_id") == todo_id + and receipt.get("lease_id") == binding.get("lease_id") + and receipt.get("lease_epoch") == binding.get("lease_epoch") + and receipt.get("completion_continuation") == continuation + and isinstance(actor, Mapping) + and actor.get("agent_id") == agent_id + and isinstance(todo, Mapping) + and todo.get("status") == "done" + and todo.get("completion_continuation") == continuation + and lease is None + ) + + +def _complete_authority( + request: Mapping[str, Any], + binding: Mapping[str, Any], + *, + executor: CoordinationAuthorityExecutor, + provider: FileCoordinationProvider, + clock_path: Path, + goal_id: str, + agent_id: str, + todo_id: str, +) -> dict[str, Any]: + command = _completion_command(request, todo_id=todo_id) + if command is None: + return _rejection( + "authority_completion_invalid", + "authority completion context is invalid", + ) + head_value, _generation = provider.load() + if head_value is None: + return _rejection( + "authority_head_missing", "coordination authority head is missing" + ) + head = validated_head(head_value, goal_id=goal_id) + if binding.get("store_identity") != provider.store_identity(): + return _rejection("store_lineage_mismatch", "authority store lineage changed") + admission_receipt = _stored_receipt(head, str(binding.get("operation_id") or "")) + if admission_receipt is None or not _binding_matches_admission_receipt( + binding, + admission_receipt, + agent_id=agent_id, + todo_id=todo_id, + ): + return _rejection( + "authority_receipt_mismatch", + "authority admission receipt no longer matches", + ) + operation_id = _completion_operation_id(request, binding) + prior = _stored_receipt(head, operation_id) + if prior is None: + todo = head["coordination"]["todos"].get(todo_id) + lease = head["coordination"]["leases"].get(todo_id) + if not isinstance(todo, Mapping) or not isinstance(lease, Mapping): + return _rejection( + "stale_lease_fence", "authority lease is no longer current" + ) + if ( + lease.get("owner") != agent_id + or lease.get("lease_id") != binding.get("lease_id") + or lease.get("lease_epoch") != binding.get("lease_epoch") + ): + return _rejection( + "stale_lease_fence", "authority lease generation changed" + ) + expiry = datetime.fromisoformat(str(lease["expires_at"])) + if _clock(clock_path) >= expiry.timestamp(): + return _rejection( + "lease_expired", "authority lease expired before completion" + ) + expected_revision = int(todo["todo_revision"]) + else: + expected_revision = int(prior["accepted_todo_revision"]) - 1 + outcome = executor.apply( + sample_work_envelope( + goal_id=goal_id, + operation_id=operation_id, + agent_id=agent_id, + device_id=f"turn-canary-{agent_id}", + command={ + "type": "complete_work", + "todo_id": todo_id, + "expected_todo_revision": expected_revision, + "lease_id": str(binding["lease_id"]), + "expected_lease_epoch": int(binding["lease_epoch"]), + "no_followup": command["no_followup"], + "successor_todo_ids": command["successor_todo_ids"], + "evidence": None, + }, + ) + ) + if outcome.get("result") not in {"applied", "already_applied"}: + return _rejection( + "authority_completion_rejected", + "coordination authority refused completion", + ) + completed_value, _completed_generation = provider.load() + if completed_value is None: + return _rejection( + "authority_head_missing", "coordination authority head is missing" + ) + completed = validated_head(completed_value, goal_id=goal_id) + receipt = _stored_receipt(completed, operation_id) + if not _completion_receipt_matches( + completed, + receipt, + operation_id=operation_id, + binding=binding, + agent_id=agent_id, + todo_id=todo_id, + continuation=command["completion_continuation"], + ): + return _rejection( + "authority_completion_mismatch", + "authority completion receipt does not match", + ) + return {"ok": True, "binding": dict(binding)} + + +def _admit( + request: Mapping[str, Any], + *, + executor: CoordinationAuthorityExecutor, + provider: FileCoordinationProvider, + goal_id: str, + agent_id: str, + todo_id: str, + lease_ttl_seconds: int, +) -> dict[str, Any]: + head_value, _generation = provider.load() + if head_value is None: + return _rejection( + "authority_head_missing", "coordination authority head is not initialized" + ) + head = validated_head(head_value, goal_id=goal_id) + todo = head["coordination"]["todos"].get(todo_id) + if not isinstance(todo, Mapping): + return _rejection("authority_todo_missing", "coordination Todo is missing") + operation_id = _operation_id( + "admit", str(request.get("turn_key") or ""), agent_id, todo_id + ) + prior = _stored_receipt(head, operation_id) + if prior is not None: + command_kind = str(prior.get("command") or "") + expected_revision = int(prior["accepted_todo_revision"]) - 1 + else: + command_kind = ( + "claim_work" if todo.get("claimed_by") is None else "reclaim_work" + ) + expected_revision = int(todo["todo_revision"]) + if command_kind == "claim_work": + envelope = sample_claim_envelope( + goal_id=goal_id, + operation_id=operation_id, + agent_id=agent_id, + device_id=f"turn-canary-{agent_id}", + todo_id=todo_id, + expected_todo_revision=expected_revision, + expected_preconditions=_preconditions(todo), + lease_ttl_seconds=lease_ttl_seconds, + ) + elif command_kind == "reclaim_work": + envelope = sample_work_envelope( + goal_id=goal_id, + operation_id=operation_id, + agent_id=agent_id, + device_id=f"turn-canary-{agent_id}", + command={ + "type": "reclaim_work", + "todo_id": todo_id, + "expected_todo_revision": expected_revision, + "expected_preconditions": _preconditions(todo), + "lease_ttl_seconds": lease_ttl_seconds, + }, + ) + else: + return _rejection( + "authority_receipt_invalid", "authority admission receipt is invalid" + ) + outcome = executor.apply(envelope) + if outcome.get("result") not in {"applied", "already_applied"}: + return _rejection( + "authority_admission_rejected", "coordination authority refused admission" + ) + if outcome.get("authorization_status") != "active": + return _rejection( + "authority_admission_inactive", "coordination authority is not active" + ) + receipt = outcome.get("original_receipt") + if not isinstance(receipt, Mapping): + return _rejection( + "authority_receipt_invalid", "authority admission receipt is invalid" + ) + return { + "ok": True, + "binding": { + "schema_version": "loopx_turn_authority_binding_v0", + "store_identity": provider.store_identity(), + "operation_id": str(receipt["operation_id"]), + "receipt_digest": _receipt_digest(receipt), + "authority_revision": int(receipt["accepted_authority_revision"]), + "todo_revision": int(receipt["accepted_todo_revision"]), + "lease_id": str(receipt["lease_id"]), + "lease_epoch": int(receipt["lease_epoch"]), + "expires_at": str(receipt["expires_at"]), + }, + } + + +def _revalidate_and_renew( + request: Mapping[str, Any], + binding: Mapping[str, Any], + *, + executor: CoordinationAuthorityExecutor, + provider: FileCoordinationProvider, + clock_path: Path, + goal_id: str, + agent_id: str, + todo_id: str, + lease_ttl_seconds: int, +) -> dict[str, Any]: + head_value, _generation = provider.load() + if head_value is None: + return _rejection( + "authority_head_missing", "coordination authority head is missing" + ) + head = validated_head(head_value, goal_id=goal_id) + if binding.get("store_identity") != provider.store_identity(): + return _rejection("store_lineage_mismatch", "authority store lineage changed") + admission_receipt = _stored_receipt(head, str(binding.get("operation_id") or "")) + if admission_receipt is None or not _binding_matches_admission_receipt( + binding, + admission_receipt, + agent_id=agent_id, + todo_id=todo_id, + ): + return _rejection( + "authority_receipt_mismatch", + "authority admission receipt no longer matches", + ) + completion_operation_id = _completion_operation_id(request, binding) + completion_receipt = _stored_receipt(head, completion_operation_id) + if _completion_receipt_matches( + head, + completion_receipt, + operation_id=completion_operation_id, + binding=binding, + agent_id=agent_id, + todo_id=todo_id, + continuation=str( + completion_receipt.get("completion_continuation") + if isinstance(completion_receipt, Mapping) + else "" + ), + ): + if request.get("checkpoint") in {"quota_spend", "scheduler"}: + return {"ok": True, "binding": dict(binding)} + return _rejection( + "stale_lease_fence", "authority work is already complete" + ) + todo = head["coordination"]["todos"].get(todo_id) + lease = head["coordination"]["leases"].get(todo_id) + if not isinstance(todo, Mapping) or not isinstance(lease, Mapping): + return _rejection("stale_lease_fence", "authority lease is no longer current") + if ( + lease.get("owner") != agent_id + or lease.get("lease_id") != binding.get("lease_id") + or lease.get("lease_epoch") != binding.get("lease_epoch") + ): + return _rejection("stale_lease_fence", "authority lease generation changed") + expiry = datetime.fromisoformat(str(lease["expires_at"])) + if _clock(clock_path) >= expiry.timestamp(): + return _rejection("lease_expired", "authority lease expired before checkpoint") + checkpoint = str(request.get("checkpoint") or "") + effect_ref = str(request.get("effect_ref") or "") + operation_id = _operation_id( + "renew", str(binding["operation_id"]), checkpoint, effect_ref + ) + prior = _stored_receipt(head, operation_id) + expected_revision = ( + int(prior["accepted_todo_revision"]) - 1 + if prior is not None + else int(todo["todo_revision"]) + ) + outcome = executor.apply( + sample_work_envelope( + goal_id=goal_id, + operation_id=operation_id, + agent_id=agent_id, + device_id=f"turn-canary-{agent_id}", + command={ + "type": "renew_work", + "todo_id": todo_id, + "expected_todo_revision": expected_revision, + "lease_id": str(binding["lease_id"]), + "expected_lease_epoch": int(binding["lease_epoch"]), + "lease_ttl_seconds": lease_ttl_seconds, + }, + ) + ) + if outcome.get("result") not in {"applied", "already_applied"}: + return _rejection("stale_lease_fence", "authority renewal was refused") + if outcome.get("authorization_status") != "active": + return _rejection("lease_expired", "authority lease is not active") + return {"ok": True, "binding": dict(binding)} + + +def evaluate(args: argparse.Namespace, request: Mapping[str, Any]) -> dict[str, Any]: + expected = { + "goal_id": args.goal_id, + "agent_id": args.agent_id, + "todo_id": args.todo_id, + } + if any(request.get(field) != value for field, value in expected.items()): + return _rejection( + "authority_identity_mismatch", + "checkpoint identity does not match guard scope", + ) + provider = FileCoordinationProvider(args.store_directory, args.goal_id) + clock_path = Path(args.clock_file) + executor = CoordinationAuthorityExecutor( + provider, + goal_id=args.goal_id, + now=lambda: _clock(clock_path), + reclaim_grace_seconds=args.reclaim_grace_seconds, + ) + binding = request.get("authority_binding") + if binding is None: + if request.get("checkpoint") != "host_admission": + return _rejection( + "authority_admission_missing", "checkpoint has no admission binding" + ) + return _admit( + request, + executor=executor, + provider=provider, + goal_id=args.goal_id, + agent_id=args.agent_id, + todo_id=args.todo_id, + lease_ttl_seconds=args.lease_ttl_seconds, + ) + if not isinstance(binding, Mapping): + return _rejection("authority_binding_invalid", "authority binding is invalid") + if request.get("checkpoint") == "authority_complete": + return _complete_authority( + request, + binding, + executor=executor, + provider=provider, + clock_path=clock_path, + goal_id=args.goal_id, + agent_id=args.agent_id, + todo_id=args.todo_id, + ) + return _revalidate_and_renew( + request, + binding, + executor=executor, + provider=provider, + clock_path=clock_path, + goal_id=args.goal_id, + agent_id=args.agent_id, + todo_id=args.todo_id, + lease_ttl_seconds=args.lease_ttl_seconds, + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--store-directory", required=True) + parser.add_argument("--clock-file", required=True) + parser.add_argument("--goal-id", required=True) + parser.add_argument("--agent-id", required=True) + parser.add_argument("--todo-id", required=True) + parser.add_argument("--lease-ttl-seconds", type=int, default=2) + parser.add_argument("--reclaim-grace-seconds", type=float, default=30.0) + args = parser.parse_args() + try: + request = json.load(sys.stdin) + if not isinstance(request, Mapping): + raise TypeError("checkpoint request must be an object") + result = evaluate(args, request) + except Exception: # noqa: BLE001 - TEST ONLY process boundary fails closed + result = _rejection( + "authority_guard_unavailable", + "authority guard could not verify current state", + ) + json.dump(result, sys.stdout, sort_keys=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/nokv-shadow-provider/authority_turn_canary.py b/examples/nokv-shadow-provider/authority_turn_canary.py new file mode 100755 index 0000000000..572d6e2896 --- /dev/null +++ b/examples/nokv-shadow-provider/authority_turn_canary.py @@ -0,0 +1,674 @@ +#!/usr/bin/env python3 +"""Deterministic TEST ONLY Turn/coordination authority canary. + +Competing LoopX processes share one file-backed coordination head, but never a +Turn journal or workspace. A separate crash-recovery case reopens only the +same agent's own journal. The file provider is the deterministic provider for +the production ``CoordinationAuthorityExecutor`` contract used by the NoKV +adapter. This proves protocol admission and stale-epoch effect fencing; it +does not claim shared production wiring or exactly-once behavior for arbitrary +Host workspace mutations. +""" + +from __future__ import annotations + +import json +import multiprocessing +import os +import sys +from collections.abc import Mapping +from pathlib import Path +from queue import Empty +from tempfile import TemporaryDirectory +from typing import Any + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, os.fspath(REPOSITORY_ROOT)) + +from loopx.control_plane.coordination.executor import ( + CoordinationAuthorityExecutor, + sample_work_envelope, +) +from loopx.control_plane.coordination.file_provider import ( + FileCoordinationProvider, +) +from loopx.control_plane.coordination.head import ( + bootstrap_head, + validated_head, +) +from loopx.control_plane.turn_driver import ( + build_loopx_turn_plan, + build_turn_authority_command_guard, + run_loopx_turn_once, +) +from loopx.file_lock import exclusive_file_lock + +GOAL_ID = "goal-authority-turn-canary" +TODO_ID = "todo_authority_canary" +AGENTS = ("agent-a", "agent-b") +LEASE_TTL_SECONDS = 2 +RECLAIM_GRACE_SECONDS = 3.0 +CRASH_EXIT_CODE = 73 + + +def _todo() -> dict[str, Any]: + return { + "todo_revision": 7, + "status": "open", + "claimed_by": None, + "eligibility": { + "authorization_projection_revision": 3, + "authorization_projection_digest": "sha256:canary-authority", + "allowed_agent_ids": list(AGENTS), + "dependencies_satisfied": True, + "dependency_revision": 12, + "gates_open": True, + "gate_revision": 5, + }, + "repository": "git:example/authority-canary", + "code_revision": "0123456789abcdef", + "last_lease_epoch": 6, + } + + +def _bootstrap(store: Path) -> None: + provider = FileCoordinationProvider(store, GOAL_ID) + head = bootstrap_head( + GOAL_ID, + {TODO_ID: _todo()}, + store_binding=provider.store_identity(), + ) + outcome = provider.compare_and_put(0, head) + if outcome.get("result") != "applied": + raise RuntimeError(f"canary bootstrap failed: {outcome!r}") + + +def _plan(agent_id: str, scenario: str) -> dict[str, Any]: + return build_loopx_turn_plan( + { + "ok": True, + "schema_version": "loopx_turn_envelope_v0", + "goal_id": GOAL_ID, + "agent_id": agent_id, + "should_run": True, + "effective_action": "normal_run", + "action": { + "must_attempt": True, + "delivery_allowed": True, + "quiet_noop_allowed": False, + "selected_todo": { + "todo_id": TODO_ID, + "text": "Run the deterministic authority canary", + }, + }, + "user": { + "action_required": False, + "open_count": 0, + "notify": "DONT_NOTIFY", + }, + "writeback": {"spend_after_validation": True}, + "scheduler": {"action": "run_now"}, + "action_signature": { + "matches": True, + "source_hash": "sha256:authority-canary", + "envelope_hash": "sha256:authority-canary", + }, + "compaction": {"within_budget": True}, + }, + host="generic-cli", + execution_mode="isolated-headless", + turn_instance_id=f"{scenario}-{agent_id}", + ) + + +def _host_result(plan: Mapping[str, Any]) -> dict[str, Any]: + transaction = plan["transaction"] + return { + "schema_version": "loopx_turn_result_v0", + "turn_key": transaction["turn_key"], + "result_kind": "validated_completion", + "completed_phases": ["host_execute", "typed_result"], + "classification": "authority_canary_completion", + "recommended_action": "Review the deterministic canary receipt.", + "next_action": "Finish the canary qualification.", + "delivery_batch_scale": "implementation", + "delivery_outcome": "outcome_progress", + "vision_unchanged_reason": "The canary objective remains unchanged.", + "summary": "One authority-qualified canary Turn completed.", + } + + +def _append_event(path: Path, agent_id: str, stage: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with exclusive_file_lock(path), path.open("a", encoding="utf-8") as stream: + stream.write(json.dumps({"agent_id": agent_id, "stage": stage}) + "\n") + stream.flush() + os.fsync(stream.fileno()) + + +def _guard_argv(store: Path, clock: Path, agent_id: str) -> list[str]: + return [ + sys.executable, + os.fspath(Path(__file__).with_name("authority_guard.py")), + "--store-directory", + os.fspath(store), + "--clock-file", + os.fspath(clock), + "--goal-id", + GOAL_ID, + "--agent-id", + agent_id, + "--todo-id", + TODO_ID, + "--lease-ttl-seconds", + str(LEASE_TTL_SECONDS), + "--reclaim-grace-seconds", + str(RECLAIM_GRACE_SECONDS), + ] + + +def _worker( + store: Path, + clock: Path, + base: Path, + event_log: Path, + scenario: str, + agent_id: str, + start: Any, + host_started: Any, + release_host: Any, + queue: Any, + crash_before_durable_effect: bool = False, +) -> None: + try: + plan = _plan(agent_id, scenario) + guard = build_turn_authority_command_guard( + _guard_argv(store, clock, agent_id), + project=base, + timeout_seconds=10, + ) + start.wait(timeout=20) + + def host(_request: Mapping[str, Any]) -> dict[str, Any]: + _append_event(event_log, agent_id, "host") + if host_started is not None: + host_started.set() + if release_host is not None and not release_host.wait(timeout=20): + raise RuntimeError("canary Host release timed out") + return _host_result(plan) + + def writeback(_result: Mapping[str, Any]) -> dict[str, Any]: + _append_event(event_log, agent_id, "writeback") + return {"ok": True, "appended": True} + + def completion_writeback(_result: Mapping[str, Any]) -> dict[str, Any]: + _append_event(event_log, agent_id, "writeback") + return { + "ok": True, + "appended": True, + "completion": { + "todo_id": TODO_ID, + "continuation": "no_followup", + }, + } + + def completion_intent(_result: Mapping[str, Any]) -> dict[str, Any]: + return {"todo_id": TODO_ID, "continuation": "no_followup"} + + def terminal_closeout(_result: Mapping[str, Any]) -> dict[str, Any]: + return { + "ok": True, + "appended": True, + "completion": { + "todo_id": TODO_ID, + "continuation": "no_followup", + }, + } + + def spend() -> dict[str, Any]: + _append_event(event_log, agent_id, "quota_spend") + return {"ok": True, "appended": True, "slots": 1} + + def scheduler(_spend: Mapping[str, Any]) -> dict[str, Any]: + _append_event(event_log, agent_id, "scheduler") + return {"completed": True, "acknowledged": False} + + def validate( + _plan: Mapping[str, Any], _result: Mapping[str, Any] + ) -> dict[str, Any]: + if crash_before_durable_effect: + # The Turn driver has already made the typed Host result durable, + # but has not reached its first authority-protected effect. + os._exit(CRASH_EXIT_CODE) + return { + "status": "passed", + "validator_kind": "authority_canary", + "summary": "deterministic canary postcondition passed", + } + + payload = run_loopx_turn_once( + plan, + host_runner=host, + project=base / f"workspace-{scenario}-{agent_id}", + runtime_root=base / f"runtime-{scenario}-{agent_id}", + goal_id=GOAL_ID, + timeout_seconds=15, + execute=True, + task_validator=validate, + writeback=writeback, + completion_writeback=completion_writeback, + completion_intent=completion_intent, + terminal_closeout=terminal_closeout, + spend=spend, + scheduler=scheduler, + authority_checkpoint_guard=guard, + ) + queue.put({"agent_id": agent_id, "payload": payload}) + except BaseException as exc: # noqa: BLE001 - child reports compact failure + queue.put( + { + "agent_id": agent_id, + "error": type(exc).__name__, + "message": str(exc), + } + ) + + +def _read_events(path: Path) -> list[dict[str, str]]: + if not path.exists(): + return [] + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] + + +def _run_workers( + *, + context: multiprocessing.context.BaseContext, + store: Path, + clock: Path, + base: Path, + event_log: Path, + scenario: str, + block_agent_a: bool, +) -> tuple[list[dict[str, Any]], Any, Any, list[Any]]: + start = context.Event() + a_started = context.Event() if block_agent_a else None + release_a = context.Event() if block_agent_a else None + queue = context.Queue() + processes = [] + for agent_id in AGENTS: + process = context.Process( + target=_worker, + args=( + store, + clock, + base, + event_log, + scenario, + agent_id, + start, + a_started if agent_id == "agent-a" else None, + release_a if agent_id == "agent-a" else None, + queue, + ), + ) + processes.append(process) + return [], a_started, release_a, [queue, start, *processes] + + +def _collect(queue: Any, processes: list[Any]) -> list[dict[str, Any]]: + results = [] + for _process in processes: + try: + results.append(queue.get(timeout=30)) + except Empty as exc: + raise RuntimeError("canary child did not report") from exc + for process in processes: + process.join(timeout=10) + if process.is_alive(): + process.terminate() + process.join(timeout=5) + raise RuntimeError("canary child did not exit") + if process.exitcode != 0: + raise RuntimeError(f"canary child exited {process.exitcode}") + if any("error" in result for result in results): + raise RuntimeError(f"canary worker failed: {results!r}") + return results + + +def _receipt_commands(store: Path) -> list[str]: + provider = FileCoordinationProvider(store, GOAL_ID) + head_value, _generation = provider.load() + head = validated_head(head_value, goal_id=GOAL_ID) + return sorted( + str(entry["original_receipt"]["command"]) + for entry in head["receipt_index"].values() + ) + + +def run() -> dict[str, Any]: + context = multiprocessing.get_context("spawn") + with TemporaryDirectory(prefix="loopx-authority-turn-canary-") as raw_base: + base = Path(raw_base) + + race_store = base / "race-store" + race_clock = base / "race-clock" + race_events_path = base / "race-events.jsonl" + race_clock.write_text("1000", encoding="utf-8") + _bootstrap(race_store) + _, _, _, race_parts = _run_workers( + context=context, + store=race_store, + clock=race_clock, + base=base, + event_log=race_events_path, + scenario="race", + block_agent_a=False, + ) + race_queue, race_start, *race_processes = race_parts + for process in race_processes: + process.start() + race_start.set() + race_results = _collect(race_queue, race_processes) + race_committed = [ + result + for result in race_results + if result["payload"].get("status") == "committed" + ] + race_rejected = [ + result + for result in race_results + if result["payload"].get("result_kind") == "authority_rejected" + ] + race_events = _read_events(race_events_path) + if len(race_committed) != 1 or len(race_rejected) != 1: + raise RuntimeError(f"race did not select one authority: {race_results!r}") + if [event["stage"] for event in race_events].count("host") != 1: + raise RuntimeError(f"race invoked more than one Host: {race_events!r}") + race_commands = _receipt_commands(race_store) + + reclaim_store = base / "reclaim-store" + reclaim_clock = base / "reclaim-clock" + reclaim_events_path = base / "reclaim-events.jsonl" + reclaim_clock.write_text("2000", encoding="utf-8") + _bootstrap(reclaim_store) + _, a_started, release_a, reclaim_parts = _run_workers( + context=context, + store=reclaim_store, + clock=reclaim_clock, + base=base, + event_log=reclaim_events_path, + scenario="reclaim", + block_agent_a=True, + ) + reclaim_queue, reclaim_start, *reclaim_processes = reclaim_parts + # Start the old holder first so the test controls the failover epoch. + reclaim_processes[0].start() + reclaim_start.set() + if a_started is None or not a_started.wait(timeout=30): + raise RuntimeError("old holder never reached Host") + # A lease that has merely expired is not yet reclaimable. Exercise a + # separate B Turn just before the explicit skew-grace boundary. + reclaim_clock.write_text("2004.999", encoding="utf-8") + early_queue = context.Queue() + early_start = context.Event() + early_reclaimer = context.Process( + target=_worker, + args=( + reclaim_store, + reclaim_clock, + base, + reclaim_events_path, + "reclaim-before-grace", + "agent-b", + early_start, + None, + None, + early_queue, + ), + ) + early_reclaimer.start() + early_start.set() + early_result = _collect(early_queue, [early_reclaimer])[0] + early_payload = early_result["payload"] + if early_payload.get("result_kind") != "authority_rejected": + raise RuntimeError( + f"reclaimer crossed the grace window early: {early_payload!r}" + ) + early_events = _read_events(reclaim_events_path) + early_reclaimer_host_count = sum( + event["agent_id"] == "agent-b" and event["stage"] == "host" + for event in early_events + ) + if early_reclaimer_host_count != 0: + raise RuntimeError( + "early reclaimer reached Host before authority admission" + ) + + # At expiry plus the complete grace window, B performs a real reclaim. + reclaim_clock.write_text("2005", encoding="utf-8") + reclaim_processes[1].start() + first = reclaim_queue.get(timeout=30) + if first.get("agent_id") != "agent-b": + raise RuntimeError(f"reclaimer did not finish first: {first!r}") + assert release_a is not None + release_a.set() + second = reclaim_queue.get(timeout=30) + reclaim_results = [first, second] + for process in reclaim_processes: + process.join(timeout=10) + if process.exitcode != 0: + raise RuntimeError(f"reclaim child exited {process.exitcode}") + if any("error" in result for result in reclaim_results): + raise RuntimeError(f"reclaim worker failed: {reclaim_results!r}") + by_agent = {result["agent_id"]: result["payload"] for result in reclaim_results} + if by_agent["agent-b"].get("status") != "committed": + raise RuntimeError(f"reclaimer did not commit: {by_agent!r}") + if by_agent["agent-a"].get("result_kind") != "writeback_failed": + raise RuntimeError(f"stale holder was not fenced: {by_agent!r}") + stale_receipt = by_agent["agent-a"]["authority_checkpoint_guard"][ + "checkpoints" + ]["durable_writeback"] + if stale_receipt.get("reason_code") != "stale_lease_fence": + raise RuntimeError(f"stale fence was not typed: {stale_receipt!r}") + reclaim_events = _read_events(reclaim_events_path) + a_effects = [ + event["stage"] for event in reclaim_events if event["agent_id"] == "agent-a" + ] + if a_effects != ["host"]: + raise RuntimeError(f"stale holder emitted a later effect: {a_effects!r}") + reclaim_commands = _receipt_commands(reclaim_store) + + crash_store = base / "crash-store" + crash_clock = base / "crash-clock" + crash_events_path = base / "crash-events.jsonl" + crash_runtime = base / "runtime-crash-resume-agent-a" + crash_clock.write_text("3000", encoding="utf-8") + _bootstrap(crash_store) + crash_queue = context.Queue() + crash_start = context.Event() + crashing = context.Process( + target=_worker, + args=( + crash_store, + crash_clock, + base, + crash_events_path, + "crash-resume", + "agent-a", + crash_start, + None, + None, + crash_queue, + True, + ), + ) + crashing.start() + crash_start.set() + crashing.join(timeout=30) + if crashing.is_alive(): + crashing.terminate() + crashing.join(timeout=5) + raise RuntimeError("crash-injection child did not exit") + if crashing.exitcode != CRASH_EXIT_CODE: + raise RuntimeError( + f"crash injection exited {crashing.exitcode}, expected {CRASH_EXIT_CODE}" + ) + crash_journals = list(crash_runtime.rglob("*.json")) + if len(crash_journals) != 1: + raise RuntimeError( + f"crash scenario journal count drifted: {crash_journals!r}" + ) + before_resume = json.loads(crash_journals[0].read_text(encoding="utf-8")) + if before_resume.get("completed_phases") != ["host_execute", "typed_result"]: + raise RuntimeError( + f"crash did not occur before the first durable effect: {before_resume!r}" + ) + original_binding = before_resume["authority_checkpoint_guard"]["binding"] + + resume_queue = context.Queue() + resume_start = context.Event() + resuming = context.Process( + target=_worker, + args=( + crash_store, + crash_clock, + base, + crash_events_path, + "crash-resume", + "agent-a", + resume_start, + None, + None, + resume_queue, + ), + ) + resuming.start() + resume_start.set() + resumed_result = _collect(resume_queue, [resuming])[0] + resumed_payload = resumed_result["payload"] + if resumed_payload.get("status") != "committed": + raise RuntimeError(f"same-agent Turn recovery failed: {resumed_payload!r}") + resumed_binding = resumed_payload["authority_checkpoint_guard"]["binding"] + crash_events = _read_events(crash_events_path) + crash_event_counts = { + stage: sum(event["stage"] == stage for event in crash_events) + for stage in ("host", "writeback", "quota_spend", "scheduler") + } + crash_commands = _receipt_commands(crash_store) + + unavailable_clock = base / "unavailable-clock" + unavailable_events = base / "unavailable-events.jsonl" + unavailable_parent = base / "provider-unavailable" + unavailable_clock.write_text("4000", encoding="utf-8") + unavailable_parent.write_text("not a directory", encoding="utf-8") + unavailable_queue = context.Queue() + unavailable_start = context.Event() + unavailable = context.Process( + target=_worker, + args=( + unavailable_parent / "store", + unavailable_clock, + base, + unavailable_events, + "provider-unavailable", + "agent-a", + unavailable_start, + None, + None, + unavailable_queue, + ), + ) + unavailable.start() + unavailable_start.set() + unavailable_result = _collect(unavailable_queue, [unavailable])[0]["payload"] + unavailable_receipt = unavailable_result["authority_checkpoint_guard"][ + "checkpoints" + ]["host_admission"] + if ( + unavailable_result.get("result_kind") != "authority_rejected" + or unavailable_receipt.get("reason_code") != "authority_guard_unavailable" + or _read_events(unavailable_events) + ): + raise RuntimeError( + f"provider outage did not fail closed before Host: {unavailable_result!r}" + ) + + if "claim_work" not in race_commands: + raise RuntimeError( + "race did not use CoordinationAuthorityExecutor claim_work" + ) + if "renew_work" not in race_commands: + raise RuntimeError( + "race did not use CoordinationAuthorityExecutor renew_work" + ) + if "reclaim_work" not in reclaim_commands: + raise RuntimeError( + "failover did not use CoordinationAuthorityExecutor reclaim_work" + ) + + return { + "ok": True, + "schema_version": "loopx_authority_turn_canary_v0", + "provider": "FileCoordinationProvider(TEST_ONLY)", + "shared_state": "one CoordinationAuthorityExecutor head per scenario", + "race": { + "committed_agent": race_committed[0]["agent_id"], + "host_count": 1, + "rejected_before_host": True, + "authority_commands": race_commands, + }, + "expiry_reclaim": { + "old_epoch": by_agent["agent-a"]["authority_checkpoint_guard"][ + "binding" + ]["lease_epoch"], + "new_epoch": by_agent["agent-b"]["authority_checkpoint_guard"][ + "binding" + ]["lease_epoch"], + "reclaim_grace_seconds": RECLAIM_GRACE_SECONDS, + "blocked_before_expiry_plus_grace": True, + "early_reclaimer_host_count": early_reclaimer_host_count, + "stale_holder_later_effects": 0, + "authority_commands": reclaim_commands, + }, + "crash_resume": { + "crash_exit_code": CRASH_EXIT_CODE, + "same_agent": resumed_result["agent_id"] == "agent-a", + "same_turn_journal": len(list(crash_runtime.rglob("*.json"))) == 1, + "original_binding_reused": resumed_binding == original_binding, + "host_count": crash_event_counts["host"], + "writeback_count": crash_event_counts["writeback"], + "quota_spend_count": crash_event_counts["quota_spend"], + "scheduler_count": crash_event_counts["scheduler"], + "recovery_host_invoked": resumed_payload["recovery"]["actual"][ + "host_invoked" + ], + "claim_work_count": crash_commands.count("claim_work"), + }, + "provider_unavailable": { + "failed_closed_at_admission": True, + "reason_code": unavailable_receipt["reason_code"], + "host_count": 0, + }, + "boundary": ( + "Protocol/effect checkpoints only; Host workspaces are isolated and " + "this does not claim arbitrary workspace-effect exactly-once." + ), + } + + +def main() -> int: + try: + payload = run() + except Exception as exc: # noqa: BLE001 - canary emits compact failure + payload = { + "ok": False, + "schema_version": "loopx_authority_turn_canary_v0", + "error": type(exc).__name__, + "message": str(exc), + } + print(json.dumps(payload, sort_keys=True)) + return 0 if payload.get("ok") else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_authority_turn_canary.py b/tests/test_authority_turn_canary.py new file mode 100644 index 0000000000..5f8e65dae5 --- /dev/null +++ b/tests/test_authority_turn_canary.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + + +def test_two_process_authority_turn_canary_fences_stale_epoch() -> None: + repository = Path(__file__).resolve().parents[1] + completed = subprocess.run( + [ + sys.executable, + str( + repository + / "examples" + / "nokv-shadow-provider" + / "authority_turn_canary.py" + ), + ], + cwd=repository, + text=True, + capture_output=True, + timeout=90, + check=False, + ) + + assert completed.returncode == 0, completed.stderr or completed.stdout + payload = json.loads(completed.stdout) + assert payload["ok"] is True + assert payload["provider"] == "FileCoordinationProvider(TEST_ONLY)" + assert payload["race"]["host_count"] == 1 + assert payload["race"]["rejected_before_host"] is True + assert {"claim_work", "renew_work", "complete_work"}.issubset( + payload["race"]["authority_commands"] + ) + assert ( + payload["expiry_reclaim"]["new_epoch"] > payload["expiry_reclaim"]["old_epoch"] + ) + assert payload["expiry_reclaim"]["reclaim_grace_seconds"] == 3.0 + assert payload["expiry_reclaim"]["blocked_before_expiry_plus_grace"] is True + assert payload["expiry_reclaim"]["early_reclaimer_host_count"] == 0 + assert payload["expiry_reclaim"]["stale_holder_later_effects"] == 0 + assert {"claim_work", "renew_work", "reclaim_work", "complete_work"}.issubset( + payload["expiry_reclaim"]["authority_commands"] + ) + assert payload["crash_resume"] == { + "crash_exit_code": 73, + "same_agent": True, + "same_turn_journal": True, + "original_binding_reused": True, + "host_count": 1, + "writeback_count": 1, + "quota_spend_count": 1, + "scheduler_count": 1, + "recovery_host_invoked": False, + "claim_work_count": 1, + } + assert payload["provider_unavailable"] == { + "failed_closed_at_admission": True, + "reason_code": "authority_guard_unavailable", + "host_count": 0, + } + assert ( + "does not claim arbitrary workspace-effect exactly-once" in payload["boundary"] + ) diff --git a/tests/test_authority_turn_product_e2e.py b/tests/test_authority_turn_product_e2e.py new file mode 100644 index 0000000000..9c677dcb1e --- /dev/null +++ b/tests/test_authority_turn_product_e2e.py @@ -0,0 +1,684 @@ +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import Any + +from loopx.control_plane.coordination.file_provider import FileCoordinationProvider +from loopx.control_plane.coordination.head import bootstrap_head +from loopx.extensions.runtime import default_extension_state_file, install_extension + + +REPOSITORY = Path(__file__).resolve().parents[1] +GOAL_ID = "goal-authority-product-e2e" +TODO_ID = "todo_authority_product_e2e" +FOLLOWUP_TODO_ID = "todo_authority_product_followup" +AGENT_IDS = ("agent-a", "agent-b") + + +def _write_product_fixture( + root: Path, + *, + include_todo: bool = True, +) -> tuple[Path, Path, Path, Path]: + project = root / "project" + runtime = root / "runtime" + host_project = root / "shared-host-project" + runtime.mkdir(parents=True) + host_project.mkdir() + state = project / ".codex" / "goals" / GOAL_ID / "ACTIVE_GOAL_STATE.md" + state.parent.mkdir(parents=True) + todo_lines = ( + [ + "- [ ] [P0] Advance the authority-qualified product fixture.", + " ", + "", + "- [ ] [P2] Keep the follow-up fixture available.", + " ", + "", + ] + if include_todo + else [] + ) + state.write_text( + "\n".join( + [ + "---", + "status: active", + "updated_at: 2026-09-02T00:00:00+00:00", + "---", + "", + "# Authority Product E2E", + "", + "## Agent Todo", + "", + *todo_lines, + ] + ), + encoding="utf-8", + ) + registry = project / ".loopx" / "registry.json" + registry.parent.mkdir(parents=True) + registry.write_text( + json.dumps( + { + "schema_version": 1, + "common_runtime_root": str(runtime), + "goals": [ + { + "id": GOAL_ID, + "domain": "loopx-authority-product-fixture", + "status": "active", + "repo": str(project), + "state_file": str(state.relative_to(project)), + "adapter": { + "kind": "fixture_v0", + "status": "connected-delivery", + }, + "quota": {"compute": 2.0, "window_hours": 24}, + "coordination": { + "agent_model": "peer_v1", + "registered_agents": list(AGENT_IDS), + "agent_profiles": { + agent_id: { + "schema_version": "agent_profile_v1", + "profile_role": "fixture", + "scope": "public qualification", + } + for agent_id in AGENT_IDS + }, + "write_scope": ["docs/**"], + }, + } + ], + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + return project, runtime, registry, host_project + + +def _bootstrap_authority(store: Path) -> None: + provider = FileCoordinationProvider(store, GOAL_ID) + head = bootstrap_head( + GOAL_ID, + { + TODO_ID: { + "todo_revision": 1, + "status": "open", + "claimed_by": None, + "eligibility": { + "authorization_projection_revision": 1, + "authorization_projection_digest": "sha256:product-e2e", + "allowed_agent_ids": list(AGENT_IDS), + "dependencies_satisfied": True, + "dependency_revision": 1, + "gates_open": True, + "gate_revision": 1, + }, + "repository": "git:example/authority-product-e2e", + "code_revision": "0123456789abcdef", + "last_lease_epoch": 0, + } + }, + store_binding=provider.store_identity(), + ) + assert provider.compare_and_put(0, head)["result"] == "applied" + + +def _cli_env() -> dict[str, str]: + env = dict(os.environ) + env["LOOPX_SHARED_AUTHORITY_TEST_ONLY"] = "1" + python_path = env.get("PYTHONPATH", "") + env["PYTHONPATH"] = os.pathsep.join( + part for part in (str(REPOSITORY), python_path) if part + ) + return env + + +def _run_cli(*args: str, timeout: float = 60) -> tuple[int, dict[str, Any]]: + completed = subprocess.run( + [sys.executable, "-m", "loopx.cli", *args], + cwd=REPOSITORY, + env=_cli_env(), + text=True, + capture_output=True, + timeout=timeout, + check=False, + ) + assert completed.stdout, completed.stderr + return completed.returncode, json.loads(completed.stdout) + + +def _common_cli_args(registry: Path, runtime: Path) -> list[str]: + return [ + "--registry", + str(registry), + "--runtime-root", + str(runtime), + "--format", + "json", + ] + + +def _guard_argv( + *, + store: Path, + clock: Path, + barrier: Path, + barrier_helper: Path, + agent_id: str, +) -> list[str]: + delegate = [ + sys.executable, + str(REPOSITORY / "examples" / "nokv-shadow-provider" / "authority_guard.py"), + "--store-directory", + str(store), + "--clock-file", + str(clock), + "--goal-id", + GOAL_ID, + "--agent-id", + agent_id, + "--todo-id", + TODO_ID, + "--lease-ttl-seconds", + "60", + "--reclaim-grace-seconds", + "3", + ] + return [ + sys.executable, + str(barrier_helper), + str(barrier), + agent_id, + *delegate, + ] + + +HOST_SCRIPT = """ +import json +import os +import pathlib +import sys +import time + +request = json.load(sys.stdin) +agent_id = request["turn_envelope"]["agent_id"] +log_path = pathlib.Path(sys.argv[1]) +descriptor = os.open(log_path, os.O_APPEND | os.O_CREAT | os.O_WRONLY, 0o600) +try: + os.write(descriptor, (json.dumps({"agent_id": agent_id}) + "\\n").encode()) +finally: + os.close(descriptor) +time.sleep(0.75) +json.dump({ + "schema_version": "loopx_turn_result_v0", + "turn_key": request["turn_key"], + "result_kind": "validated_completion", + "completed_phases": ["host_execute", "typed_result"], + "classification": "authority_product_completion", + "recommended_action": "Continue the authority-qualified product fixture.", + "next_action": "Run the next authority-qualified product check.", + "delivery_batch_scale": "implementation", + "delivery_outcome": "outcome_progress", + "vision_unchanged_reason": "The qualification objective remains unchanged.", + "summary": "One product CLI Turn completed after shared-authority admission." +}, sys.stdout) +""" + + +VALIDATOR_SCRIPT = """ +import json +import pathlib +import sys + +result = json.load(sys.stdin) +rows = [json.loads(line) for line in pathlib.Path(sys.argv[1]).read_text().splitlines()] +raise SystemExit(0 if len(rows) == 1 and result["result_kind"] == "validated_completion" else 7) +""" + + +GUARD_BARRIER_SCRIPT = """ +import json +import pathlib +import subprocess +import sys +import time + +raw_request = sys.stdin.read() +request = json.loads(raw_request) +barrier = pathlib.Path(sys.argv[1]) +agent_id = sys.argv[2] +delegate = sys.argv[3:] +if request.get("checkpoint") == "host_admission": + barrier.mkdir(parents=True, exist_ok=True) + (barrier / agent_id).write_text("ready", encoding="utf-8") + deadline = time.monotonic() + 20 + while len(list(barrier.iterdir())) < 2: + if time.monotonic() >= deadline: + raise SystemExit(72) + time.sleep(0.01) +completed = subprocess.run(delegate, input=raw_request, text=True, capture_output=True) +sys.stdout.write(completed.stdout) +raise SystemExit(completed.returncode) +""" + + +def _write_process_helpers(root: Path) -> tuple[Path, Path, Path]: + host = root / "typed_host.py" + validator = root / "independent_validator.py" + guard_barrier = root / "authority_guard_barrier.py" + host.write_text(HOST_SCRIPT, encoding="utf-8") + validator.write_text(VALIDATOR_SCRIPT, encoding="utf-8") + guard_barrier.write_text(GUARD_BARRIER_SCRIPT, encoding="utf-8") + return host, validator, guard_barrier + + +def _configure_live_inbox_signal( + *, project: Path, runtime: Path, registry: Path, root: Path +) -> None: + provider = root / "extension_provider.py" + provider.write_text( + """#!/usr/bin/env python3 +import sys + +raise SystemExit(0 if "--doctor" in sys.argv else 0) +""", + encoding="utf-8", + ) + provider.chmod(0o755) + manifest = root / "lark-extension.toml" + permissions = [ + "lark.collector.manage", + "lark.inbox.read", + "lark.inbox.write", + ] + manifest.write_text( + "\n".join( + [ + 'schema_version = "loopx_extension_manifest_v0"', + 'id = "loopx-lark"', + 'version = "0.0.0-test"', + 'requires_loopx_api = ">=1,<2"', + f"permissions = {json.dumps(permissions)}", + "", + "[runtime]", + 'protocol = "lark_test_activation_v0"', + f"entrypoint = {json.dumps(str(provider))}", + 'doctor_args = ["--doctor"]', + f"required_permissions = {json.dumps(permissions)}", + "timeout_seconds = 5", + "", + ] + ), + encoding="utf-8", + ) + install_extension( + manifest, + state_file=default_extension_state_file(runtime), + execute=True, + ) + + config_relative = Path(".loopx/config/lark/product-e2e.json") + config = project / config_relative + config.parent.mkdir(parents=True) + config.write_text( + json.dumps( + { + "schema_version": "lark_event_inbox_config_v0", + "enabled": True, + "inbox_dir": ".loopx/inbox/live-steering", + "capture_scope": "configured_chat_all", + "reply": { + "enabled": True, + "sender_profile": "product-fixture-bot", + "sender_identity": "bot", + "bot_display_name": "LoopX", + "chat_id": "oc_product_fixture", + "placement_policy": "source_context", + "editorial_style": "bullet_points_preferred", + }, + }, + sort_keys=True, + ), + encoding="utf-8", + ) + digest = "sha256:" + hashlib.sha256(config.read_bytes()).hexdigest() + registry_payload = json.loads(registry.read_text(encoding="utf-8")) + registry_payload["goals"][0]["control_plane"] = { + "lark_event_inbox": { + "enabled": True, + "config_path": str(config_relative), + "config_digest": digest, + } + } + registry.write_text( + json.dumps(registry_payload, indent=2) + "\n", + encoding="utf-8", + ) + + +def test_two_product_cli_agents_share_one_authority_and_settle_once( + tmp_path: Path, +) -> None: + project, runtime, registry, host_project = _write_product_fixture(tmp_path) + store = tmp_path / "authority-store" + clock = tmp_path / "authority-clock" + host_log = tmp_path / "host.jsonl" + host_helper, validator_helper, barrier_helper = _write_process_helpers(tmp_path) + admission_barrier = tmp_path / "admission-barrier" + clock.write_text("1000", encoding="utf-8") + _bootstrap_authority(store) + + status_exit, status = _run_cli( + *_common_cli_args(registry, runtime), + "status", + "--goal-id", + GOAL_ID, + "--scan-root", + str(project), + ) + assert status_exit == 0, status + assert status["ok"] is True + + quota_exit, quota = _run_cli( + *_common_cli_args(registry, runtime), + "quota", + "should-run", + "--goal-id", + GOAL_ID, + "--agent-id", + AGENT_IDS[0], + "--host-surface", + "generic_cli", + "--scheduler-owner", + "outer_controller", + "--execution-mode", + "isolated_headless", + "--scan-root", + str(project), + ) + assert quota_exit == 0, quota + assert quota["should_run"] is True + assert quota["selected_todo"]["todo_id"] == TODO_ID + + plan_exit, plan = _run_cli( + *_common_cli_args(registry, runtime), + "turn", + "plan", + "--goal-id", + GOAL_ID, + "--agent-id", + AGENT_IDS[0], + "--host", + "generic-cli", + "--scheduler-owner", + "outer_controller", + "--execution-mode", + "isolated-headless", + "--scan-root", + str(project), + ) + assert plan_exit == 0, plan + assert plan["route"]["kind"] == "ready_for_host" + envelope = plan["turn_envelope"] + assert envelope["schema_version"] == "loopx_turn_envelope_v0" + assert envelope["action_signature"]["matches"] is True + selected_todo = envelope["action"]["selected_todo"] + assert selected_todo["todo_id"] == TODO_ID + assert selected_todo["selected_by"] == "turn_controller_advisory_primary" + assert ( + envelope["action"]["action_portfolio"]["selection_policy"][ + "requires_explicit_turn_binding" + ] + is True + ) + + processes: list[subprocess.Popen[str]] = [] + argv_by_agent: dict[str, list[str]] = {} + for agent_id in AGENT_IDS: + argv = [ + sys.executable, + "-m", + "loopx.cli", + *_common_cli_args(registry, runtime), + "turn", + "run-once", + "--goal-id", + GOAL_ID, + "--agent-id", + agent_id, + "--turn-instance-id", + f"product-e2e-{agent_id}", + "--project", + str(host_project), + "--host-adapter-command-json", + json.dumps([sys.executable, str(host_helper), str(host_log)]), + "--validation-command-json", + json.dumps([sys.executable, str(validator_helper), str(host_log)]), + "--authority-guard-command-json", + json.dumps( + _guard_argv( + store=store, + clock=clock, + barrier=admission_barrier, + barrier_helper=barrier_helper, + agent_id=agent_id, + ) + ), + "--scan-root", + str(project), + "--no-global-sync", + "--execute", + ] + processes.append( + subprocess.Popen( + argv, + cwd=REPOSITORY, + env=_cli_env(), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + ) + argv_by_agent[agent_id] = argv + + completed = [process.communicate(timeout=60) for process in processes] + results = [json.loads(stdout) for stdout, _stderr in completed] + assert sorted(process.returncode for process in processes) == [0, 1], completed + committed = [result for result in results if result.get("status") == "committed"] + rejected = [ + result + for result in results + if result.get("result_kind") == "authority_rejected" + ] + assert len(committed) == 1, results + assert len(rejected) == 1, results + assert committed[0]["validation"]["status"] == "passed" + assert committed[0]["validation"]["validator_kind"] == "command" + assert committed[0]["receipt"]["status"] == "committed" + assert committed[0]["receipt"]["next_phase"] is None + assert committed[0]["scheduler"]["completed"] is True + assert { + checkpoint: receipt["status"] + for checkpoint, receipt in committed[0]["authority_checkpoint_guard"][ + "checkpoints" + ].items() + } == { + "host_admission": "accepted", + "durable_writeback": "accepted", + "quota_spend": "accepted", + "scheduler": "accepted", + "authority_complete": "accepted", + } + assert committed[0]["effects"] == { + "host_invoked": True, + "state_written": True, + "quota_spent": True, + "scheduler_acknowledged": False, + } + assert rejected[0]["effects"] == { + "host_invoked": False, + "state_written": False, + "quota_spent": False, + "scheduler_acknowledged": False, + } + assert rejected[0]["receipt"]["failed_phase"] == "authority_admission" + assert ( + rejected[0]["authority_checkpoint_guard"]["checkpoints"]["host_admission"][ + "status" + ] + == "rejected" + ) + + host_rows = [json.loads(line) for line in host_log.read_text().splitlines()] + assert len(host_rows) == 1 + index = runtime / "goals" / GOAL_ID / "runs" / "index.jsonl" + run_rows = [json.loads(line) for line in index.read_text().splitlines()] + assert [row["classification"] for row in run_rows] == [ + "authority_product_completion", + "quota_slot_spent", + ] + state = (project / ".codex" / "goals" / GOAL_ID / "ACTIVE_GOAL_STATE.md").read_text( + encoding="utf-8" + ) + assert state.count(f"todo_id={TODO_ID} status=done") == 1 + journals = [ + json.loads(path.read_text(encoding="utf-8")) + for path in (runtime / "goals" / GOAL_ID / "turns").glob("*.json") + ] + assert sorted(journal["status"] for journal in journals) == [ + "committed", + "failed", + ] + + winning_agent_id = committed[0]["receipt"]["lineage"]["agent_id"] + replay_argv = list(argv_by_agent[winning_agent_id]) + turn_instance_index = replay_argv.index("--turn-instance-id") + del replay_argv[turn_instance_index : turn_instance_index + 2] + replay_argv.extend(["--resume-turn-key", committed[0]["resume_turn_key"]]) + provider = FileCoordinationProvider(store, GOAL_ID) + _head_before_replay, generation_before_replay = provider.load() + artifacts_before_replay = { + "host": host_log.read_text(encoding="utf-8"), + "runs": index.read_text(encoding="utf-8"), + "state": state, + "rollout": (runtime / "goals" / GOAL_ID / "rollout-event-log.jsonl").read_text( + encoding="utf-8" + ), + } + replay_completed = subprocess.run( + replay_argv, + cwd=REPOSITORY, + env=_cli_env(), + text=True, + capture_output=True, + timeout=60, + check=False, + ) + replay = json.loads(replay_completed.stdout) + assert replay_completed.returncode == 0, replay + assert replay["status"] == "committed" + assert replay["effects"] == { + "host_invoked": False, + "state_written": False, + "quota_spent": False, + "scheduler_acknowledged": False, + } + _head_after_replay, generation_after_replay = provider.load() + assert generation_after_replay == generation_before_replay + assert host_log.read_text(encoding="utf-8") == artifacts_before_replay["host"] + assert index.read_text(encoding="utf-8") == artifacts_before_replay["runs"] + assert (project / ".codex" / "goals" / GOAL_ID / "ACTIVE_GOAL_STATE.md").read_text( + encoding="utf-8" + ) == artifacts_before_replay["state"] + assert (runtime / "goals" / GOAL_ID / "rollout-event-log.jsonl").read_text( + encoding="utf-8" + ) == artifacts_before_replay["rollout"] + + authority_head, _generation = provider.load() + assert authority_head is not None + authority_todo = authority_head["coordination"]["todos"][TODO_ID] + assert authority_todo["status"] == "done" + assert TODO_ID not in authority_head["coordination"]["leases"] + completion_receipts = [ + entry["original_receipt"] + for entry in authority_head["receipt_index"].values() + if entry["original_receipt"]["command"] == "complete_work" + ] + assert len(completion_receipts) == 1 + assert completion_receipts[0]["actor"]["agent_id"] == winning_agent_id + + +def test_configured_inbox_wake_cannot_grant_todo_or_turn_authority( + tmp_path: Path, +) -> None: + project, runtime, registry, _host_project = _write_product_fixture( + tmp_path, + include_todo=False, + ) + inbox = project / ".loopx" / "inbox" / "live-steering" + inbox.mkdir(parents=True) + (inbox / "pending.json").write_text( + json.dumps( + { + "schema_version": "lark_event_inbox_event_v0", + "event_id": "evt_product_e2e", + "message_id": "om_product_e2e", + "create_time": "2026-09-02T00:00:00Z", + "content": "@LoopX continue this project", + "addressed_to_bot": True, + "attachment_count": 0, + } + ), + encoding="utf-8", + ) + _configure_live_inbox_signal( + project=project, + runtime=runtime, + registry=registry, + root=tmp_path, + ) + + plan_exit, plan = _run_cli( + *_common_cli_args(registry, runtime), + "turn", + "plan", + "--goal-id", + GOAL_ID, + "--agent-id", + AGENT_IDS[0], + "--host", + "generic-cli", + "--scheduler-owner", + "outer_controller", + "--execution-mode", + "isolated-headless", + "--scan-root", + str(project), + ) + # Inbox/steering is an urgency signal, not a Todo or execution-authority + # source. With no durable selected Todo, the product Turn fails closed at + # lineage planning instead of admitting Host work. + assert plan_exit == 1, plan + assert plan["route"]["kind"] == "contract_error" + assert plan["turn_envelope"]["should_run"] is True + assert plan["turn_envelope"]["effective_action"] == "lark_inbox_reply_due" + assert plan["route"]["selected_todo"] is None + assert plan["turn_envelope"]["action"]["selected_todo"] is None + assert plan["route"]["would_invoke_host"] is False + assert plan["effects"]["host_invoked"] is False + assert "authority_checkpoint_guard" not in plan + assert not (runtime / "goals" / GOAL_ID / "turns").exists() From 6b89d7e3cc74d48cdbf7214cfea2ef8c510fe069 Mon Sep 17 00:00:00 2001 From: wchwawa Date: Wed, 2 Sep 2026 08:38:56 +1000 Subject: [PATCH 09/12] fix(turn): fence prepared-effect recovery Signed-off-by: wchwawa --- .../turn_driver/authority_checkpoint.py | 50 ++++ loopx/control_plane/turn_driver/executor.py | 3 +- tests/test_loopx_turn_executor.py | 222 ++++++++++++++---- tests/test_turn_authority_checkpoint.py | 97 ++++++++ 4 files changed, 328 insertions(+), 44 deletions(-) diff --git a/loopx/control_plane/turn_driver/authority_checkpoint.py b/loopx/control_plane/turn_driver/authority_checkpoint.py index 22c7b872b1..221b2b1cd0 100644 --- a/loopx/control_plane/turn_driver/authority_checkpoint.py +++ b/loopx/control_plane/turn_driver/authority_checkpoint.py @@ -18,8 +18,10 @@ from typing import Any from ...authority import validate_public_safe_text +from ..effect_program import SettlementStepKind from .driver import selected_turn_todo from .settlement import ( + TurnEffectResolver, completion_writeback_outcome, invoke_result_effect, invoke_turn_effect, @@ -477,6 +479,7 @@ class TurnAuthoritySettlementEffects: terminal_closeout: TurnEffect | None terminal_checkpoint: TurnTerminalCheckpoint | None terminal_closeout_required: bool + effect_resolvers: Mapping[SettlementStepKind, TurnEffectResolver] class TurnAuthorityCheckpointController: @@ -615,6 +618,51 @@ def guarded(effect_ref: str) -> Mapping[str, Any]: return guarded + def _guarded_resolver( + self, + step_kind: SettlementStepKind, + resolver: TurnEffectResolver, + ) -> TurnEffectResolver: + if self._session is None: + return resolver + checkpoint = step_kind.value + if checkpoint not in { + "durable_writeback", + "quota_spend", + "terminal_closeout", + }: + raise ValueError( + f"unsupported prepared-effect authority step: {checkpoint}" + ) + + def revalidate_then_resolve(effect_ref: str) -> Mapping[str, Any]: + rejected = self._rejected_effect( + checkpoint, + effect_ref=effect_ref, + ) + if rejected is not None: + return { + "kind": "unknown", + "reason": str( + rejected.get("reason") + or "authority rejected prepared-effect recovery" + ), + } + return resolver(effect_ref) + + return revalidate_then_resolve + + def _guarded_resolvers( + self, + resolvers: Mapping[SettlementStepKind, TurnEffectResolver], + ) -> dict[SettlementStepKind, TurnEffectResolver]: + """Fence resolver-side receipt repair before it can write.""" + + return { + step_kind: self._guarded_resolver(step_kind, resolver) + for step_kind, resolver in resolvers.items() + } + def _active_completion(self) -> dict[str, Any] | None: stored = self._journal.get("terminal_closeout") if not isinstance(stored, Mapping): @@ -675,6 +723,7 @@ def settlement_effects( terminal_closeout: TurnResultEffect | None, spend: TurnEffect, terminal_checkpoint: TurnTerminalCheckpoint, + effect_resolvers: Mapping[SettlementStepKind, TurnEffectResolver], ) -> TurnAuthoritySettlementEffects: """Compose each checkpoint with the effect it fences.""" @@ -728,6 +777,7 @@ def settlement_effects( terminal_closeout=terminal_effect, terminal_checkpoint=effective_terminal_checkpoint, terminal_closeout_required=terminal_closeout_required, + effect_resolvers=self._guarded_resolvers(effect_resolvers), ) def run_scheduler( diff --git a/loopx/control_plane/turn_driver/executor.py b/loopx/control_plane/turn_driver/executor.py index 467482284c..aedb1c222a 100644 --- a/loopx/control_plane/turn_driver/executor.py +++ b/loopx/control_plane/turn_driver/executor.py @@ -1105,6 +1105,7 @@ def _typed_settlement_stage( terminal_closeout=terminal_closeout, spend=spend, terminal_checkpoint=journal_adapter.checkpoint_terminal, + effect_resolvers=effect_resolvers, ) terminal_closeout_required = authority_effects.terminal_closeout_required @@ -1137,7 +1138,7 @@ def _typed_settlement_stage( prepare=journal_adapter.prepare, abort=journal_adapter.abort, effect_attempts=journal_adapter.effect_attempts, - effect_resolvers=effect_resolvers, + effect_resolvers=authority_effects.effect_resolvers, turn_result_kind=str(result.get("result_kind") or "") or None, ) diff --git a/tests/test_loopx_turn_executor.py b/tests/test_loopx_turn_executor.py index 8644e7c78f..a8f9300184 100644 --- a/tests/test_loopx_turn_executor.py +++ b/tests/test_loopx_turn_executor.py @@ -325,7 +325,9 @@ def __call__(self, request: Mapping[str, object]) -> Mapping[str, object]: "reason": "authority lease generation is no longer current", } current = request.get("authority_binding") - binding = dict(current) if isinstance(current, Mapping) else _authority_binding() + binding = ( + dict(current) if isinstance(current, Mapping) else _authority_binding() + ) if checkpoint == self.drift_at: binding = _authority_binding(lease_epoch=8) return {"ok": True, "binding": binding} @@ -502,9 +504,7 @@ def test_authority_guard_cannot_replace_the_admitted_lease_generation( assert payload["result_kind"] == "writeback_failed" assert calls == {"host": 1, "writeback": 0, "spend": 0, "scheduler": 0} - rejected = payload["authority_checkpoint_guard"]["checkpoints"][ - "durable_writeback" - ] + rejected = payload["authority_checkpoint_guard"]["checkpoints"]["durable_writeback"] assert rejected["reason_code"] == "authority_binding_changed" assert payload["authority_checkpoint_guard"]["binding"]["lease_epoch"] == 7 @@ -568,9 +568,12 @@ def test_guarded_turn_cannot_resume_effects_without_its_authority_guard( assert failed["result_kind"] == "writeback_failed" assert retried["result_kind"] == "writeback_failed" - assert retried["authority_checkpoint_guard"]["checkpoints"][ - "durable_writeback" - ]["reason_code"] == "authority_guard_missing" + assert ( + retried["authority_checkpoint_guard"]["checkpoints"]["durable_writeback"][ + "reason_code" + ] + == "authority_guard_missing" + ) assert calls == {"host": 1, "writeback": 0, "spend": 0, "scheduler": 0} @@ -605,9 +608,12 @@ def test_authority_admission_rejection_can_retry_before_first_host_effect( assert recovered["status"] == "committed" assert recovered["recovery"]["planned"]["resume_from"] == "host_execute" assert calls == {"host": 1, "writeback": 1, "spend": 1, "scheduler": 1} - assert recovered["authority_checkpoint_guard"]["checkpoints"][ - "host_admission" - ]["attempt"] == 2 + assert ( + recovered["authority_checkpoint_guard"]["checkpoints"]["host_admission"][ + "attempt" + ] + == 2 + ) def test_completion_writeback_is_fenced_before_lifecycle_mutation( @@ -619,9 +625,7 @@ def test_completion_writeback_is_fenced_before_lifecycle_mutation( payload = run_loopx_turn_once( plan, - host_runner=lambda _request: _host_result( - plan, kind="validated_completion" - ), + host_runner=lambda _request: _host_result(plan, kind="validated_completion"), project=tmp_path, runtime_root=tmp_path / "runtime", goal_id="fixture-goal", @@ -673,9 +677,7 @@ def test_validated_completion_closes_authority_before_quota_and_scheduler( payload = run_loopx_turn_once( plan, - host_runner=lambda _request: _host_result( - plan, kind="validated_completion" - ), + host_runner=lambda _request: _host_result(plan, kind="validated_completion"), project=tmp_path, runtime_root=tmp_path / "runtime", goal_id="fixture-goal", @@ -701,12 +703,9 @@ def test_validated_completion_closes_authority_before_quota_and_scheduler( terminal_closeout=lambda _result: pytest.fail( "active-goal completion must not close the goal" ), - spend=lambda: ( - calls.append("quota_spend") or {"ok": True, "appended": True} - ), + spend=lambda: calls.append("quota_spend") or {"ok": True, "appended": True}, scheduler=lambda _spend: ( - calls.append("scheduler") - or {"completed": True, "acknowledged": True} + calls.append("scheduler") or {"completed": True, "acknowledged": True} ), authority_checkpoint_guard=guard, ) @@ -720,9 +719,12 @@ def test_validated_completion_closes_authority_before_quota_and_scheduler( "authority_complete", "scheduler", ] - assert payload["authority_checkpoint_guard"]["checkpoints"][ - "authority_complete" - ]["status"] == "accepted" + assert ( + payload["authority_checkpoint_guard"]["checkpoints"]["authority_complete"][ + "status" + ] + == "accepted" + ) def test_authority_completion_failure_stops_quota_after_local_writeback( @@ -734,9 +736,7 @@ def test_authority_completion_failure_stops_quota_after_local_writeback( payload = run_loopx_turn_once( plan, - host_runner=lambda _request: _host_result( - plan, kind="validated_completion" - ), + host_runner=lambda _request: _host_result(plan, kind="validated_completion"), project=tmp_path, runtime_root=tmp_path / "runtime", goal_id="fixture-goal", @@ -781,9 +781,12 @@ def test_authority_completion_failure_stops_quota_after_local_writeback( "quota_spend", "authority_complete", ] - assert payload["authority_checkpoint_guard"]["checkpoints"][ - "authority_complete" - ]["reason_code"] == "stale_lease_fence" + assert ( + payload["authority_checkpoint_guard"]["checkpoints"]["authority_complete"][ + "reason_code" + ] + == "stale_lease_fence" + ) def test_terminal_completion_closes_authority_after_local_closeout( @@ -795,9 +798,7 @@ def test_terminal_completion_closes_authority_after_local_closeout( payload = run_loopx_turn_once( plan, - host_runner=lambda _request: _host_result( - plan, kind="validated_completion" - ), + host_runner=lambda _request: _host_result(plan, kind="validated_completion"), project=tmp_path, runtime_root=tmp_path / "runtime", goal_id="fixture-goal", @@ -825,12 +826,9 @@ def test_terminal_completion_closes_authority_after_local_closeout( }, } ), - spend=lambda: ( - calls.append("quota_spend") or {"ok": True, "appended": True} - ), + spend=lambda: calls.append("quota_spend") or {"ok": True, "appended": True}, scheduler=lambda _spend: ( - calls.append("scheduler") - or {"completed": True, "acknowledged": True} + calls.append("scheduler") or {"completed": True, "acknowledged": True} ), authority_checkpoint_guard=guard, ) @@ -861,9 +859,7 @@ def test_terminal_authority_completion_failure_holds_scheduler( payload = run_loopx_turn_once( plan, - host_runner=lambda _request: _host_result( - plan, kind="validated_completion" - ), + host_runner=lambda _request: _host_result(plan, kind="validated_completion"), project=tmp_path, runtime_root=tmp_path / "runtime", goal_id="fixture-goal", @@ -1183,8 +1179,7 @@ def host(_request: dict[str, object]) -> dict[str, object]: ) assert inspected["recovery_decision"]["action"] == "blocked" assert ( - inspected["recovery_decision"]["reason"] - == "session_binding_identity_mismatch" + inspected["recovery_decision"]["reason"] == "session_binding_identity_mismatch" ) with pytest.raises(ValueError, match="session binding does not match") as exc_info: run_loopx_turn_once(plan, retry_failed=True, **common) @@ -1315,6 +1310,147 @@ def fail_before_writeback_checkpoint( assert "effect_attempts" not in _journal(tmp_path / "runtime") +def test_reclaimed_authority_fences_prepared_effect_receipt_repair( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + plan = _plan() + calls = {"writeback": 0, "repair": 0, "spend": 0, "scheduler": 0} + provider_records: dict[str, dict[str, object]] = {} + + class ReclaimableGuard(_AuthorityGuard): + reclaimed = False + + def __call__(self, request: Mapping[str, object]) -> Mapping[str, object]: + if self.reclaimed: + self.calls.append(str(request["checkpoint"])) + return { + "ok": False, + "reason_code": "stale_lease_fence", + "reason": "another agent reclaimed the expired authority lease", + } + return super().__call__(request) + + guard = ReclaimableGuard() + + def writeback(_result: dict[str, object], effect_ref: str) -> dict[str, object]: + calls["writeback"] += 1 + payload = {"ok": True, "appended": True, "effect_ref": effect_ref} + provider_records[effect_ref] = payload + return payload + + def spend() -> dict[str, object]: + calls["spend"] += 1 + return {"ok": True, "appended": True} + + def scheduler(_spend: dict[str, object]) -> dict[str, object]: + calls["scheduler"] += 1 + return {"completed": True, "acknowledged": False} + + write_journal = turn_executor._write_journal + + def crash_after_provider_commit( + path: Path, + journal: Mapping[str, object], + ) -> None: + if "writeback" in journal and "quota_spend" not in journal: + raise RuntimeError("injected crash before writeback checkpoint") + write_journal(path, journal) + + monkeypatch.setattr(turn_executor, "_write_journal", crash_after_provider_commit) + with pytest.raises( + RuntimeError, + match="injected crash before writeback checkpoint", + ): + run_loopx_turn_once( + plan, + host_runner=lambda _request: _host_result(plan), + project=tmp_path, + runtime_root=tmp_path / "runtime", + goal_id="fixture-goal", + timeout_seconds=5, + execute=True, + task_validator=_passing_validator, + writeback=writeback, + spend=spend, + scheduler=scheduler, + authority_checkpoint_guard=guard, + ) + + interrupted = _journal(tmp_path / "runtime") + assert interrupted["effect_attempts"]["durable_writeback"]["status"] == "prepared" + assert "writeback" not in interrupted + assert calls == {"writeback": 1, "repair": 0, "spend": 0, "scheduler": 0} + + guard.reclaimed = True + monkeypatch.setattr(turn_executor, "_write_journal", write_journal) + + def repair_resolver(effect_ref: str) -> dict[str, object]: + calls["repair"] += 1 + return {"kind": "committed", "payload": provider_records[effect_ref]} + + resumed = run_loopx_turn_once( + plan, + host_runner=lambda _request: pytest.fail("host must not run during recovery"), + project=tmp_path, + runtime_root=tmp_path / "runtime", + goal_id="fixture-goal", + timeout_seconds=5, + execute=True, + task_validator=_passing_validator, + writeback=writeback, + writeback_resolver=repair_resolver, + spend=spend, + scheduler=scheduler, + authority_checkpoint_guard=guard, + ) + + assert calls == {"writeback": 1, "repair": 0, "spend": 0, "scheduler": 0} + assert resumed["result_kind"] == "writeback_failed" + assert resumed["settlement_result"]["failure"] == { + "kind": "effect_outcome_unknown", + "step_kind": "durable_writeback", + "reason": "another agent reclaimed the expired authority lease", + } + assert ( + resumed["authority_checkpoint_guard"]["checkpoints"]["durable_writeback"][ + "reason_code" + ] + == "stale_lease_fence" + ) + assert guard.calls == ["host_admission", "durable_writeback", "durable_writeback"] + assert ( + _journal(tmp_path / "runtime")["effect_attempts"] + == interrupted["effect_attempts"] + ) + + journal_path = next( + (tmp_path / "runtime" / "goals" / "fixture-goal" / "turns").glob("*.json") + ) + write_journal(journal_path, interrupted) + guard.reclaimed = False + guard.calls.clear() + current = run_loopx_turn_once( + plan, + host_runner=lambda _request: pytest.fail("host must not run during recovery"), + project=tmp_path, + runtime_root=tmp_path / "runtime", + goal_id="fixture-goal", + timeout_seconds=5, + execute=True, + task_validator=_passing_validator, + writeback=writeback, + writeback_resolver=repair_resolver, + spend=spend, + scheduler=scheduler, + authority_checkpoint_guard=guard, + ) + + assert current["status"] == "committed" + assert calls == {"writeback": 1, "repair": 1, "spend": 1, "scheduler": 1} + assert guard.calls == ["durable_writeback", "quota_spend", "scheduler"] + + def test_run_once_legacy_plan_without_settlement_plan_is_upgraded( tmp_path: Path, ) -> None: diff --git a/tests/test_turn_authority_checkpoint.py b/tests/test_turn_authority_checkpoint.py index b79db757d4..27c6d29ca4 100644 --- a/tests/test_turn_authority_checkpoint.py +++ b/tests/test_turn_authority_checkpoint.py @@ -2,10 +2,12 @@ import json import sys +from collections.abc import Mapping from pathlib import Path import pytest +from loopx.control_plane.effect_program import SettlementStepKind from loopx.control_plane.turn_driver import ( TurnAuthorityCheckpointSession, build_turn_authority_command_guard, @@ -162,6 +164,12 @@ def test_completion_context_rejects_unhashable_successor_before_guard() -> None: def test_default_off_controller_does_not_parse_turn_lineage() -> None: journal: dict[str, object] = {} + resolver_calls: list[str] = [] + + def resolver(effect_ref: str) -> dict[str, object]: + resolver_calls.append(effect_ref) + return {"kind": "absent"} + controller = build_turn_authority_checkpoint_controller( None, plan={}, @@ -188,6 +196,7 @@ def test_default_off_controller_does_not_parse_turn_lineage() -> None: terminal_checkpoint=lambda _payload: pytest.fail( "non-terminal Turn must not checkpoint closeout" ), + effect_resolvers={SettlementStepKind.QUOTA_SPEND: resolver}, ) assert controller.enabled is False @@ -200,9 +209,97 @@ def test_default_off_controller_does_not_parse_turn_lineage() -> None: {"receipt": "quota"}, terminal_closeout_required=False, ) == {"completed": True, "spend": {"receipt": "quota"}} + assert effects.effect_resolvers[SettlementStepKind.QUOTA_SPEND]("quota-ref") == { + "kind": "absent" + } + assert resolver_calls == ["quota-ref"] assert journal == {} +@pytest.mark.parametrize( + "step_kind", + ( + SettlementStepKind.DURABLE_WRITEBACK, + SettlementStepKind.QUOTA_SPEND, + SettlementStepKind.TERMINAL_CLOSEOUT, + ), +) +def test_prepared_effect_resolver_is_fenced_before_receipt_repair( + step_kind: SettlementStepKind, +) -> None: + journal: dict[str, object] = {} + guard_calls: list[str] = [] + resolver_calls: list[str] = [] + binding = { + "schema_version": "loopx_turn_authority_binding_v0", + "store_identity": "file:00000000000000000000000000000001", + "operation_id": "operation-fixture", + "receipt_digest": "sha256:" + ("c" * 64), + "authority_revision": 1, + "todo_revision": 1, + "lease_id": "lease-fixture", + "lease_epoch": 1, + "expires_at": "2030-01-01T00:00:00.000Z", + } + + def guard(request: Mapping[str, object]) -> Mapping[str, object]: + checkpoint = str(request["checkpoint"]) + guard_calls.append(checkpoint) + if checkpoint == "host_admission": + return {"ok": True, "binding": binding} + return { + "ok": False, + "reason_code": "stale_lease_fence", + "reason": "another agent reclaimed the expired authority lease", + } + + def resolver(effect_ref: str) -> dict[str, object]: + resolver_calls.append(effect_ref) + return {"kind": "committed", "payload": {"ok": True, "appended": True}} + + controller = build_turn_authority_checkpoint_controller( + guard, + plan={ + "turn_envelope": { + "goal_id": "goal-fixture", + "agent_id": "agent-fixture", + "action": {"selected_todo": {"todo_id": "todo-fixture"}}, + } + }, + transaction_plan={ + "settlement_plan": {"identity": {"effect_id": "effect-fixture"}} + }, + journal=journal, + turn_key="sha256:" + ("b" * 64), + persist=lambda: None, + ) + assert controller.admit_host( + completed_phases=[], + failure=lambda reason: {"reason": reason, "receipt": {}}, + ) + effects = controller.settlement_effects( + result={"result_kind": "validated_progress"}, + writeback=lambda _result: {"ok": True, "appended": True}, + completion_writeback=None, + completion_intent=None, + terminal_closeout=None, + spend=lambda: {"ok": True, "appended": True}, + terminal_checkpoint=lambda _payload: None, + effect_resolvers={step_kind: resolver}, + ) + + resolution = effects.effect_resolvers[step_kind]( + f"effect-fixture#{step_kind.value}" + ) + + assert resolution == { + "kind": "unknown", + "reason": "another agent reclaimed the expired authority lease", + } + assert guard_calls == ["host_admission", step_kind.value] + assert resolver_calls == [] + + def test_resumed_authority_turn_without_guard_fails_closed_at_admission() -> None: journal: dict[str, object] = { "authority_checkpoint_guard": { From 5467f4fca04ae46b6e19bca274aa215c0a695412 Mon Sep 17 00:00:00 2001 From: wchwawa Date: Wed, 2 Sep 2026 08:39:01 +1000 Subject: [PATCH 10/12] test(authority): qualify live NoKV Turn composition Signed-off-by: wchwawa --- examples/nokv-shadow-provider/README.md | 56 ++- .../nokv-shadow-provider/authority_guard.py | 83 +++- tests/test_authority_turn_product_e2e.py | 113 +++++- ...nokv_authority_guard_provider_selection.py | 360 ++++++++++++++++++ 4 files changed, 580 insertions(+), 32 deletions(-) create mode 100644 tests/test_nokv_authority_guard_provider_selection.py diff --git a/examples/nokv-shadow-provider/README.md b/examples/nokv-shadow-provider/README.md index b96f58288b..7b5518d318 100644 --- a/examples/nokv-shadow-provider/README.md +++ b/examples/nokv-shadow-provider/README.md @@ -96,8 +96,10 @@ lease, gate, quota, or scheduling decisions. [evidence note](../../docs/architecture/rfcs/shared-goal-authority-state-provider-v0-evidence.zh-CN.md) are merge evidence for the revised receipt contract. - `authority_guard.py`: a **TEST ONLY** argv/stdin adapter that composes the - production authority executor with the file provider for Turn checkpoint - qualification. It is not a production provider selector. + production authority executor with either the deterministic file provider or + an explicitly configured NoKV provider for Turn checkpoint qualification. It + is not a production provider selector, and NoKV construction never falls + back to file authority. - `authority_turn_canary.py`: a deterministic two-process canary. Both LoopX processes share one coordination head but use isolated Turn journals and Host workspaces. It proves one pre-expiry Host admission and stale-epoch @@ -169,15 +171,40 @@ Run the TEST ONLY Turn-composition canary with: python3 examples/nokv-shadow-provider/authority_turn_canary.py ``` +The opt-in product-chain test repeats its two-CLI-process race against a live +NoKV workbench. Keep the client configuration outside the repository and run +the test with a Python environment that contains the pinned NoKV SDK: + +```bash +LOOPX_SHARED_AUTHORITY_TEST_ONLY=1 \ +NOKV_COORDINATION_LIVE=1 \ +NOKV_CLIENT_CONFIG_JSON="$PWD/.local/nokv-client.json" \ +NOKV_COORDINATION_WORKBENCH=authority-qualification \ +python3 -m pytest -q tests/test_authority_turn_product_e2e.py -k live_nokv +``` + The canary drives `claim_work`, `renew_work`, expired `reclaim_work`, and `complete_work` through the production `CoordinationAuthorityExecutor` and a -shared `FileCoordinationProvider`. The file provider makes the test -deterministic; replacing that storage adapter with the existing NoKV provider -is still a separate live qualification step. The CLI exposure is likewise -default-off: `turn run-once --authority-guard-command-json ...` requires the -explicit `LOOPX_SHARED_AUTHORITY_TEST_ONLY=1` environment gate and uses JSON -argv without shell parsing. Scheduler wake-up remains an effect after the -authority checks, never an authorization source. +shared `FileCoordinationProvider`. The file provider keeps the merge test +deterministic. A separate live qualification may select NoKV by passing the +paired `--nokv-client-config-json` and `--nokv-workbench` guard arguments; the +configuration is admitted through the pinned NoKV SDK builder and the typed +provider factory. File and NoKV arguments are mutually exclusive, and an +incomplete or unavailable NoKV selection fails closed without a file fallback. +The config must be an absolute regular file no larger than 1 MiB. The opt-in +live test removes its random coordination head with the exact generation after +readback, so a shared qualification workbench does not accumulate test state. + +The CLI exposure is default-off: `turn run-once +--authority-guard-command-json ...` requires the explicit +`LOOPX_SHARED_AUTHORITY_TEST_ONLY=1` environment gate and uses JSON argv +without shell parsing. The product-chain regression runs two real LoopX CLI +processes through advisory Todo selection, authority admission, Host, +writeback, quota, authority completion, scheduler evaluation, receipt +readback, and exact Turn replay. Scheduler evaluation/projection is guarded; +wake delivery and scheduler ACK are outside this qualification. Inbox input is +characterized only as an urgency signal: it cannot create a Todo or grant Turn +authority. Run the merge-relevant deterministic regression from the repository root with: @@ -228,8 +255,9 @@ The nine current result tags are `probes.py` deliberately remains offline; use `live_e2e.py` for the real stack. The reference still does not establish multi-host wake delivery, automatic -provider promotion, HA/failover, receipt compaction or GC, production -performance, a dynamic eligibility-projection publisher, non-empty write-scope -overlap enforcement, or a full LoopX state migration. The NoKV storage-plane -issues linked from the RFC remain production-canary holds; a green ordered -single-node exercise does not erase them. +provider promotion, arbitrary Host workspace-effect exactly-once, Host +keepalive/cancellation after lease loss, HA/failover, receipt compaction or GC, +production performance, a dynamic eligibility-projection publisher, non-empty +write-scope overlap enforcement, or a full LoopX state migration. The NoKV +storage-plane issues linked from the RFC remain production-canary holds; a +green ordered single-node exercise does not erase them. diff --git a/examples/nokv-shadow-provider/authority_guard.py b/examples/nokv-shadow-provider/authority_guard.py index a6a1af25b5..6bf5840425 100755 --- a/examples/nokv-shadow-provider/authority_guard.py +++ b/examples/nokv-shadow-provider/authority_guard.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 -"""TEST ONLY file-provider Turn authority guard. +"""TEST ONLY file/NoKV Turn authority guard. This process adapter composes the production ``CoordinationAuthorityExecutor`` -with ``FileCoordinationProvider``. It is qualification wiring, not a shared -production-mode declaration: every invocation reads one Turn checkpoint on -stdin and emits one typed result on stdout. Claim/reclaim and renew therefore -use the same authority core, aggregate, receipts, CAS, and store-lineage fence -as the NoKV provider contract; no parallel lock-based oracle exists here. +with one explicitly selected coordination provider. It is qualification +wiring behind the existing TEST ONLY Turn-controller environment gate, not a +shared production-mode declaration: every invocation reads one Turn checkpoint +on stdin and emits one typed result on stdout. Claim/reclaim and renew +therefore use the same authority core, aggregate, receipts, CAS, and +store-lineage fence; no parallel lock-based oracle exists here. """ from __future__ import annotations @@ -15,6 +16,7 @@ import hashlib import json import os +import stat import sys from collections.abc import Mapping from datetime import datetime @@ -35,6 +37,14 @@ FileCoordinationProvider, ) from loopx.control_plane.coordination.head import validated_head +from loopx.control_plane.coordination.nokv_jsonl_helper import build_client +from provider import ( + NoKVCoordinationProvider, + open_nokv_coordination_provider, +) + +CoordinationProvider = FileCoordinationProvider | NoKVCoordinationProvider +MAX_NOKV_CLIENT_CONFIG_BYTES = 1 << 20 def _clock(path: Path) -> float: @@ -183,7 +193,7 @@ def _complete_authority( binding: Mapping[str, Any], *, executor: CoordinationAuthorityExecutor, - provider: FileCoordinationProvider, + provider: CoordinationProvider, clock_path: Path, goal_id: str, agent_id: str, @@ -228,9 +238,7 @@ def _complete_authority( or lease.get("lease_id") != binding.get("lease_id") or lease.get("lease_epoch") != binding.get("lease_epoch") ): - return _rejection( - "stale_lease_fence", "authority lease generation changed" - ) + return _rejection("stale_lease_fence", "authority lease generation changed") expiry = datetime.fromisoformat(str(lease["expires_at"])) if _clock(clock_path) >= expiry.timestamp(): return _rejection( @@ -289,7 +297,7 @@ def _admit( request: Mapping[str, Any], *, executor: CoordinationAuthorityExecutor, - provider: FileCoordinationProvider, + provider: CoordinationProvider, goal_id: str, agent_id: str, todo_id: str, @@ -380,7 +388,7 @@ def _revalidate_and_renew( binding: Mapping[str, Any], *, executor: CoordinationAuthorityExecutor, - provider: FileCoordinationProvider, + provider: CoordinationProvider, clock_path: Path, goal_id: str, agent_id: str, @@ -423,9 +431,7 @@ def _revalidate_and_renew( ): if request.get("checkpoint") in {"quota_spend", "scheduler"}: return {"ok": True, "binding": dict(binding)} - return _rejection( - "stale_lease_fence", "authority work is already complete" - ) + return _rejection("stale_lease_fence", "authority work is already complete") todo = head["coordination"]["todos"].get(todo_id) lease = head["coordination"]["leases"].get(todo_id) if not isinstance(todo, Mapping) or not isinstance(lease, Mapping): @@ -473,6 +479,47 @@ def _revalidate_and_renew( return {"ok": True, "binding": dict(binding)} +def _coordination_provider(args: argparse.Namespace) -> CoordinationProvider: + store_directory = getattr(args, "store_directory", None) + config_path = getattr(args, "nokv_client_config_json", None) + workbench = getattr(args, "nokv_workbench", None) + nokv_selected = config_path is not None or workbench is not None + if store_directory is not None and nokv_selected: + raise ValueError("authority guard backend selection is invalid") + if (config_path is None) != (workbench is None): + raise ValueError("authority guard NoKV selection is incomplete") + if store_directory is not None: + return FileCoordinationProvider(store_directory, args.goal_id) + if config_path is None or workbench is None: + raise ValueError("authority guard backend is required") + if os.environ.get("LOOPX_SHARED_AUTHORITY_TEST_ONLY") != "1": + raise ValueError("NoKV authority guard requires the TEST ONLY gate") + if ( + not isinstance(config_path, str) + or not config_path + or config_path.strip() != config_path + or not isinstance(workbench, str) + or not workbench + or workbench.strip() != workbench + ): + raise ValueError("authority guard NoKV selection is invalid") + config_file = Path(config_path) + if not config_file.is_absolute(): + raise ValueError("authority guard NoKV config path must be absolute") + with config_file.open("rb") as stream: + if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode): + raise ValueError("authority guard NoKV config must be a regular file") + encoded_config = stream.read(MAX_NOKV_CLIENT_CONFIG_BYTES + 1) + if len(encoded_config) > MAX_NOKV_CLIENT_CONFIG_BYTES: + raise ValueError("authority guard NoKV config is too large") + config = json.loads(encoded_config) + return open_nokv_coordination_provider( + lambda: build_client(config), + workbench, + args.goal_id, + ) + + def evaluate(args: argparse.Namespace, request: Mapping[str, Any]) -> dict[str, Any]: expected = { "goal_id": args.goal_id, @@ -484,7 +531,7 @@ def evaluate(args: argparse.Namespace, request: Mapping[str, Any]) -> dict[str, "authority_identity_mismatch", "checkpoint identity does not match guard scope", ) - provider = FileCoordinationProvider(args.store_directory, args.goal_id) + provider = _coordination_provider(args) clock_path = Path(args.clock_file) executor = CoordinationAuthorityExecutor( provider, @@ -535,7 +582,9 @@ def evaluate(args: argparse.Namespace, request: Mapping[str, Any]) -> dict[str, def main() -> int: parser = argparse.ArgumentParser() - parser.add_argument("--store-directory", required=True) + parser.add_argument("--store-directory") + parser.add_argument("--nokv-client-config-json") + parser.add_argument("--nokv-workbench") parser.add_argument("--clock-file", required=True) parser.add_argument("--goal-id", required=True) parser.add_argument("--agent-id", required=True) diff --git a/tests/test_authority_turn_product_e2e.py b/tests/test_authority_turn_product_e2e.py index 9c677dcb1e..77398b9c8c 100644 --- a/tests/test_authority_turn_product_e2e.py +++ b/tests/test_authority_turn_product_e2e.py @@ -5,9 +5,12 @@ import os import subprocess import sys +import uuid from pathlib import Path from typing import Any +import pytest + from loopx.control_plane.coordination.file_provider import FileCoordinationProvider from loopx.control_plane.coordination.head import bootstrap_head from loopx.extensions.runtime import default_extension_state_file, install_extension @@ -622,7 +625,7 @@ def test_two_product_cli_agents_share_one_authority_and_settle_once( assert completion_receipts[0]["actor"]["agent_id"] == winning_agent_id -def test_configured_inbox_wake_cannot_grant_todo_or_turn_authority( +def test_configured_inbox_signal_cannot_grant_todo_or_turn_authority( tmp_path: Path, ) -> None: project, runtime, registry, _host_project = _write_product_fixture( @@ -682,3 +685,111 @@ def test_configured_inbox_wake_cannot_grant_todo_or_turn_authority( assert plan["effects"]["host_invoked"] is False assert "authority_checkpoint_guard" not in plan assert not (runtime / "goals" / GOAL_ID / "turns").exists() + + +def _remove_live_nokv_head(provider: Any) -> None: + head, generation = provider.load() + if head is None: + return + result = provider.client.remove( + provider.workbench, + provider.head_path, + generation, + ) + if not isinstance(result, dict) or result.get("removed") is not True: + raise AssertionError("live NoKV product E2E did not remove its test head") + + +@pytest.mark.skipif( + os.environ.get("NOKV_COORDINATION_LIVE") != "1", + reason="set NOKV_COORDINATION_LIVE=1 for the opt-in NoKV product E2E", +) +def test_two_product_cli_agents_share_one_live_nokv_authority( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config_path_value = os.environ.get("NOKV_CLIENT_CONFIG_JSON", "") + workbench = os.environ.get("NOKV_COORDINATION_WORKBENCH", "") + config_path = Path(config_path_value) + if not config_path.is_absolute() or not config_path.is_file() or not workbench: + pytest.fail( + "live NoKV product E2E requires an absolute NOKV_CLIENT_CONFIG_JSON " + "and NOKV_COORDINATION_WORKBENCH" + ) + + provider_directory = REPOSITORY / "examples" / "nokv-shadow-provider" + sys.path.insert(0, str(provider_directory)) + try: + from provider import open_nokv_coordination_provider + finally: + sys.path.pop(0) + from loopx.control_plane.coordination.nokv_jsonl_helper import build_client + + config = json.loads(config_path.read_text(encoding="utf-8")) + suffix = uuid.uuid4().hex[:12] + goal_id = f"goal-authority-product-live-{suffix}" + todo_id = f"todo_{suffix}" + followup_todo_id = f"follow_{suffix}" + + def open_provider(_directory: Path, selected_goal_id: str): + return open_nokv_coordination_provider( + lambda: build_client(config), + workbench, + selected_goal_id, + ) + + def live_guard_argv( + *, + store: Path, + clock: Path, + barrier: Path, + barrier_helper: Path, + agent_id: str, + ) -> list[str]: + del store + delegate = [ + sys.executable, + str(provider_directory / "authority_guard.py"), + "--nokv-client-config-json", + str(config_path), + "--nokv-workbench", + workbench, + "--clock-file", + str(clock), + "--goal-id", + goal_id, + "--agent-id", + agent_id, + "--todo-id", + todo_id, + "--lease-ttl-seconds", + "60", + "--reclaim-grace-seconds", + "3", + ] + return [ + sys.executable, + str(barrier_helper), + str(barrier), + agent_id, + *delegate, + ] + + module = sys.modules[__name__] + monkeypatch.setattr(module, "GOAL_ID", goal_id) + monkeypatch.setattr(module, "TODO_ID", todo_id) + monkeypatch.setattr(module, "FOLLOWUP_TODO_ID", followup_todo_id) + monkeypatch.setattr(module, "FileCoordinationProvider", open_provider) + monkeypatch.setattr(module, "_guard_argv", live_guard_argv) + + cleanup_provider = open_provider(Path(), goal_id) + completed = False + try: + test_two_product_cli_agents_share_one_authority_and_settle_once(tmp_path) + completed = True + finally: + try: + _remove_live_nokv_head(cleanup_provider) + except Exception: + if completed: + raise diff --git a/tests/test_nokv_authority_guard_provider_selection.py b/tests/test_nokv_authority_guard_provider_selection.py new file mode 100644 index 0000000000..16c99b3d8e --- /dev/null +++ b/tests/test_nokv_authority_guard_provider_selection.py @@ -0,0 +1,360 @@ +from __future__ import annotations + +import importlib.util +import io +import json +import sys +from pathlib import Path +from types import ModuleType, SimpleNamespace +from typing import Any + +import pytest + + +REPOSITORY = Path(__file__).resolve().parents[1] +EXAMPLE_DIRECTORY = REPOSITORY / "examples" / "nokv-shadow-provider" +GUARD_PATH = EXAMPLE_DIRECTORY / "authority_guard.py" + + +def _load_guard() -> ModuleType: + spec = importlib.util.spec_from_file_location( + "test_nokv_authority_guard", + GUARD_PATH, + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.path.insert(0, str(EXAMPLE_DIRECTORY)) + try: + spec.loader.exec_module(module) + finally: + sys.path.pop(0) + return module + + +def _selection_args(**values: object) -> SimpleNamespace: + return SimpleNamespace( + store_directory=values.get("store_directory"), + nokv_client_config_json=values.get("nokv_client_config_json"), + nokv_workbench=values.get("nokv_workbench"), + goal_id="goal-a", + ) + + +def _base_cli() -> list[str]: + return [ + "authority_guard.py", + "--clock-file", + "unused-clock", + "--goal-id", + "goal-a", + "--agent-id", + "agent-a", + "--todo-id", + "todo-a", + ] + + +def _checkpoint() -> io.StringIO: + return io.StringIO( + json.dumps( + { + "goal_id": "goal-a", + "agent_id": "agent-a", + "todo_id": "todo-a", + "checkpoint": "host_admission", + } + ) + ) + + +def _run_main( + guard: ModuleType, + monkeypatch: pytest.MonkeyPatch, + argv: list[str], +) -> tuple[int, dict[str, Any]]: + stdout = io.StringIO() + monkeypatch.setattr(guard.sys, "argv", argv) + monkeypatch.setattr(guard.sys, "stdin", _checkpoint()) + monkeypatch.setattr(guard.sys, "stdout", stdout) + return guard.main(), json.loads(stdout.getvalue()) + + +def test_file_provider_remains_the_default_explicit_guard_backend( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + guard = _load_guard() + + def unexpected(*_args: object, **_kwargs: object) -> object: + raise AssertionError("file selection must not construct a NoKV client") + + monkeypatch.setattr(guard, "build_client", unexpected, raising=False) + monkeypatch.setattr( + guard, + "open_nokv_coordination_provider", + unexpected, + raising=False, + ) + + provider = guard._coordination_provider( + _selection_args(store_directory=str(tmp_path)) + ) + + assert isinstance(provider, guard.FileCoordinationProvider) + assert provider.directory == tmp_path + assert provider.goal_id == "goal-a" + + +def test_nokv_pair_uses_hardened_client_builder_and_typed_provider_factory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + guard = _load_guard() + monkeypatch.setenv("LOOPX_SHARED_AUTHORITY_TEST_ONLY", "1") + config = { + "root_id": "a" * 32, + "routing": { + "kind": "static", + "endpoint": "127.0.0.1:7412", + "logical_shard_id": "b" * 32, + "object_namespace_id": "c" * 32, + "placement_generation": 1, + "owner_epoch": 1, + }, + "object_store": {"kind": "memory"}, + } + config_path = tmp_path / "nokv-client.json" + config_path.write_text(json.dumps(config), encoding="utf-8") + client = object() + built: list[object] = [] + factory_calls: list[tuple[object, str, str]] = [] + + def build_client(value: object) -> object: + built.append(value) + return client + + provider_module = sys.modules[guard.open_nokv_coordination_provider.__module__] + real_open = provider_module.open_nokv_coordination_provider + + def open_provider(client_factory: Any, workbench: str, goal_id: str) -> object: + provider = real_open(client_factory, workbench, goal_id) + factory_calls.append((provider, workbench, goal_id)) + return provider + + monkeypatch.setattr(guard, "build_client", build_client) + monkeypatch.setattr(guard, "open_nokv_coordination_provider", open_provider) + + provider = guard._coordination_provider( + _selection_args( + nokv_client_config_json=str(config_path), + nokv_workbench="authority-workbench", + ) + ) + + assert isinstance(provider, provider_module.NoKVCoordinationProvider) + assert provider.client is client + assert provider.workbench == "authority-workbench" + assert provider.goal_id == "goal-a" + assert built == [config] + assert factory_calls == [(provider, "authority-workbench", "goal-a")] + + +def test_nokv_selection_requires_the_existing_test_only_environment_gate( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + guard = _load_guard() + config_path = tmp_path / "nokv-client.json" + config_path.write_text("{}", encoding="utf-8") + client_constructions: list[object] = [] + + def build_client(config: object) -> object: + client_constructions.append(config) + return object() + + monkeypatch.delenv("LOOPX_SHARED_AUTHORITY_TEST_ONLY", raising=False) + monkeypatch.setattr(guard, "build_client", build_client) + + return_code, result = _run_main( + guard, + monkeypatch, + [ + *_base_cli(), + "--nokv-client-config-json", + str(config_path), + "--nokv-workbench", + "authority-workbench", + ], + ) + + assert return_code == 0 + assert result["reason_code"] == "authority_guard_unavailable" + assert client_constructions == [] + + +@pytest.mark.parametrize("invalid_path", ["relative-client.json", "/dev/null"]) +def test_nokv_config_requires_an_absolute_regular_file( + invalid_path: str, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + guard = _load_guard() + monkeypatch.setenv("LOOPX_SHARED_AUTHORITY_TEST_ONLY", "1") + monkeypatch.chdir(tmp_path) + (tmp_path / "relative-client.json").write_text("{}", encoding="utf-8") + client_constructions: list[object] = [] + + def build_client(config: object) -> object: + client_constructions.append(config) + return object() + + monkeypatch.setattr(guard, "build_client", build_client) + + return_code, result = _run_main( + guard, + monkeypatch, + [ + *_base_cli(), + "--nokv-client-config-json", + invalid_path, + "--nokv-workbench", + "authority-workbench", + ], + ) + + assert return_code == 0 + assert result["reason_code"] == "authority_guard_unavailable" + assert client_constructions == [] + + +def test_nokv_config_is_bounded_before_json_or_sdk_construction( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + guard = _load_guard() + monkeypatch.setenv("LOOPX_SHARED_AUTHORITY_TEST_ONLY", "1") + config_path = tmp_path / "oversized-client.json" + config_path.write_text( + json.dumps({"padding": "x" * (1 << 20)}), + encoding="utf-8", + ) + client_constructions: list[object] = [] + + def build_client(config: object) -> object: + client_constructions.append(config) + return object() + + monkeypatch.setattr(guard, "build_client", build_client) + + return_code, result = _run_main( + guard, + monkeypatch, + [ + *_base_cli(), + "--nokv-client-config-json", + str(config_path), + "--nokv-workbench", + "authority-workbench", + ], + ) + + assert return_code == 0 + assert result["reason_code"] == "authority_guard_unavailable" + assert client_constructions == [] + + +@pytest.mark.parametrize( + "backend_args", + [ + [ + "--store-directory", + "unused-file-store", + "--nokv-client-config-json", + "configuration-path-must-not-leak", + "--nokv-workbench", + "authority-workbench", + ], + ["--nokv-client-config-json", "configuration-path-must-not-leak"], + ["--nokv-workbench", "authority-workbench"], + [], + ], +) +def test_backend_selection_conflicts_or_missing_pair_fail_closed( + backend_args: list[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + guard = _load_guard() + + return_code, result = _run_main( + guard, + monkeypatch, + [*_base_cli(), *backend_args], + ) + + assert return_code == 0 + assert result == { + "ok": False, + "reason_code": "authority_guard_unavailable", + "reason": "authority guard could not verify current state", + } + assert "configuration-path-must-not-leak" not in json.dumps(result) + + +def test_nokv_sdk_failure_is_sanitized_and_never_falls_back_to_file( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + guard = _load_guard() + monkeypatch.setenv("LOOPX_SHARED_AUTHORITY_TEST_ONLY", "1") + secret = "credential-must-not-leak" + config_path = tmp_path / "sensitive-config.json" + config_path.write_text( + json.dumps( + { + "root_id": "a" * 32, + "routing": { + "kind": "etcd", + "endpoints": ["http://unavailable.invalid"], + "key_prefix": "/nokv/control", + }, + "object_store": { + "kind": "s3", + "bucket": "qualification", + "secret_access_key": secret, + }, + } + ), + encoding="utf-8", + ) + + def unavailable(config: object) -> object: + assert isinstance(config, dict) + raise RuntimeError(f"{secret} from {config_path}") + + def no_file_fallback(*_args: object, **_kwargs: object) -> object: + raise AssertionError("NoKV failure must not fall back to file authority") + + monkeypatch.setattr(guard, "build_client", unavailable, raising=False) + monkeypatch.setattr(guard, "FileCoordinationProvider", no_file_fallback) + + return_code, result = _run_main( + guard, + monkeypatch, + [ + *_base_cli(), + "--nokv-client-config-json", + str(config_path), + "--nokv-workbench", + "authority-workbench", + ], + ) + + encoded = json.dumps(result) + assert return_code == 0 + assert result == { + "ok": False, + "reason_code": "authority_guard_unavailable", + "reason": "authority guard could not verify current state", + } + assert secret not in encoded + assert str(config_path) not in encoded From 7008bf60ba849301b810666874639dda03d75242 Mon Sep 17 00:00:00 2001 From: wchwawa Date: Wed, 2 Sep 2026 08:39:08 +1000 Subject: [PATCH 11/12] chore(authority): close qualification lint gaps Signed-off-by: wchwawa --- .../authority_turn_canary.py | 12 +-- loopx/cli_commands/turn.py | 25 ++--- loopx/control_plane/turn_driver/settlement.py | 6 +- tests/test_loopx_turn_driver.py | 101 +++++++----------- 4 files changed, 55 insertions(+), 89 deletions(-) diff --git a/examples/nokv-shadow-provider/authority_turn_canary.py b/examples/nokv-shadow-provider/authority_turn_canary.py index 572d6e2896..ecef212570 100755 --- a/examples/nokv-shadow-provider/authority_turn_canary.py +++ b/examples/nokv-shadow-provider/authority_turn_canary.py @@ -25,23 +25,19 @@ REPOSITORY_ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, os.fspath(REPOSITORY_ROOT)) -from loopx.control_plane.coordination.executor import ( - CoordinationAuthorityExecutor, - sample_work_envelope, -) -from loopx.control_plane.coordination.file_provider import ( +from loopx.control_plane.coordination.file_provider import ( # noqa: E402 FileCoordinationProvider, ) -from loopx.control_plane.coordination.head import ( +from loopx.control_plane.coordination.head import ( # noqa: E402 bootstrap_head, validated_head, ) -from loopx.control_plane.turn_driver import ( +from loopx.control_plane.turn_driver import ( # noqa: E402 build_loopx_turn_plan, build_turn_authority_command_guard, run_loopx_turn_once, ) -from loopx.file_lock import exclusive_file_lock +from loopx.file_lock import exclusive_file_lock # noqa: E402 GOAL_ID = "goal-authority-turn-canary" TODO_ID = "todo_authority_canary" diff --git a/loopx/cli_commands/turn.py b/loopx/cli_commands/turn.py index cc6b8d98d0..62c1b1a2a0 100644 --- a/loopx/cli_commands/turn.py +++ b/loopx/cli_commands/turn.py @@ -123,6 +123,7 @@ def handle_turn_command( execution_mode=args.execution_mode, scheduler_owner=args.scheduler_owner, ) + def build_turn_decision( *, requested_action_todo_id: str | None = None ) -> dict[str, Any]: @@ -248,9 +249,7 @@ def build_turn_decision( "--authority-guard-command-json is TEST ONLY and requires " "LOOPX_SHARED_AUTHORITY_TEST_ONLY=1" ) - raw_authority_guard_argv = json.loads( - args.authority_guard_command_json - ) + raw_authority_guard_argv = json.loads(args.authority_guard_command_json) if not isinstance(raw_authority_guard_argv, list) or not all( isinstance(item, str) for item in raw_authority_guard_argv ): @@ -662,9 +661,7 @@ def spend(*, effect_ref: str) -> dict[str, object]: replan_obligation_id=settlement_identity.replan_obligation_id, ) if readback is None: - raise RuntimeError( - EXACT_SETTLEMENT_READBACK_NOT_FOUND - ) + raise RuntimeError(EXACT_SETTLEMENT_READBACK_NOT_FOUND) event = readback.spend_event if event is None: append_settlement_event( @@ -724,9 +721,7 @@ def terminal_completion_readback() -> dict[str, object] | None: readback = project_durable_terminal_completion_readback( todo=durable_todo, expected_todo_id=todo_id, - expected_completion_turn_key=( - settlement_identity.turn_instance_id - ), + expected_completion_turn_key=(settlement_identity.turn_instance_id), projection_source=projection_source, existing_todo_ids=existing_todo_ids, ) @@ -754,9 +749,7 @@ def writeback_resolver(effect_ref: str) -> dict[str, object]: replan_obligation_id=settlement_identity.replan_obligation_id, ) if readback is None: - raise RuntimeError( - EXACT_SETTLEMENT_READBACK_NOT_FOUND - ) + raise RuntimeError(EXACT_SETTLEMENT_READBACK_NOT_FOUND) run = readback.writeback_run event = readback.writeback_event if run is None and event is None: @@ -798,9 +791,7 @@ def spend_resolver(effect_ref: str) -> dict[str, object]: replan_obligation_id=settlement_identity.replan_obligation_id, ) if readback is None: - raise RuntimeError( - EXACT_SETTLEMENT_READBACK_NOT_FOUND - ) + raise RuntimeError(EXACT_SETTLEMENT_READBACK_NOT_FOUND) run = readback.spend_run event = readback.spend_event if run is not None and run.get("effect_ref") != effect_ref: @@ -846,9 +837,7 @@ def terminal_closeout_resolver(effect_ref: str) -> dict[str, object]: replan_obligation_id=settlement_identity.replan_obligation_id, ) if readback is None: - raise RuntimeError( - EXACT_SETTLEMENT_READBACK_NOT_FOUND - ) + raise RuntimeError(EXACT_SETTLEMENT_READBACK_NOT_FOUND) event = readback.completion_event completion = terminal_completion_readback() if event is None and completion is None: diff --git a/loopx/control_plane/turn_driver/settlement.py b/loopx/control_plane/turn_driver/settlement.py index 383a8b1d2b..14e4a4cf8c 100644 --- a/loopx/control_plane/turn_driver/settlement.py +++ b/loopx/control_plane/turn_driver/settlement.py @@ -570,4 +570,8 @@ def turn_settlement_failure_outcome( raise RuntimeError( "TypeScript Turn settlement failure has unsupported result_kind" ) from exc - return result_kind, tuple(str(phase) for phase in outcome["completed_phases"]), failed_phase + return ( + result_kind, + tuple(str(phase) for phase in outcome["completed_phases"]), + failed_phase, + ) diff --git a/tests/test_loopx_turn_driver.py b/tests/test_loopx_turn_driver.py index 225824ed3c..74f4793a28 100644 --- a/tests/test_loopx_turn_driver.py +++ b/tests/test_loopx_turn_driver.py @@ -1189,15 +1189,13 @@ def test_turn_run_once_cli_commits_validated_result_and_one_quota_slot( "scheduler_acknowledged": False, } state_path = ( - project - / ".codex" - / "goals" - / "loopx-turn-fixture" - / "ACTIVE_GOAL_STATE.md" + project / ".codex" / "goals" / "loopx-turn-fixture" / "ACTIVE_GOAL_STATE.md" ) assert "Run the next public fixture check" in state_path.read_text(encoding="utf-8") index_path = runtime / "goals" / "loopx-turn-fixture" / "runs" / "index.jsonl" - rows = [json.loads(line) for line in index_path.read_text(encoding="utf-8").splitlines()] + rows = [ + json.loads(line) for line in index_path.read_text(encoding="utf-8").splitlines() + ] assert [row["classification"] for row in rows] == [ "fixture_progress", "quota_slot_spent", @@ -1244,8 +1242,7 @@ def test_turn_run_once_cli_commits_validated_result_and_one_quota_slot( "scheduler_acknowledged": False, } replayed_rows = [ - json.loads(line) - for line in index_path.read_text(encoding="utf-8").splitlines() + json.loads(line) for line in index_path.read_text(encoding="utf-8").splitlines() ] assert [row["classification"] for row in replayed_rows] == [ "fixture_progress", @@ -1428,11 +1425,7 @@ def test_turn_run_once_cli_completes_selected_todo_after_validation( "continuation": "active_goal", } state_path = ( - project - / ".codex" - / "goals" - / "loopx-turn-fixture" - / "ACTIVE_GOAL_STATE.md" + project / ".codex" / "goals" / "loopx-turn-fixture" / "ACTIVE_GOAL_STATE.md" ) state = state_path.read_text(encoding="utf-8") assert "todo_id=todo_fixture0001 status=done" in state @@ -1472,8 +1465,7 @@ def test_turn_run_once_cli_completes_selected_todo_after_validation( # Turn itself stays blocked until that replan creates a runnable successor. assert next_plan["route"]["kind"] == "blocked" assert ( - next_plan["turn_envelope"]["effective_action"] - == "autonomous_replan_required" + next_plan["turn_envelope"]["effective_action"] == "autonomous_replan_required" ) next_obligation = next_plan["turn_envelope"]["replan_action_packet"] assert next_obligation["decision"] == "replan_required" @@ -1616,9 +1608,9 @@ def crash_before_quota_receipt( assert first["error"] == "injected crash before quota receipt" interrupted = _turn_journal(runtime) turn_key = interrupted["turn_key"] - effect_id = interrupted["plan"]["transaction"]["settlement_plan"][ - "identity" - ]["effect_id"] + effect_id = interrupted["plan"]["transaction"]["settlement_plan"]["identity"][ + "effect_id" + ] assert interrupted["effect_attempts"] == { "quota_spend": { "status": "prepared", @@ -1627,10 +1619,11 @@ def crash_before_quota_receipt( } index_path = runtime / "goals" / "loopx-turn-fixture" / "runs" / "index.jsonl" rows = [ - json.loads(line) - for line in index_path.read_text(encoding="utf-8").splitlines() + json.loads(line) for line in index_path.read_text(encoding="utf-8").splitlines() + ] + quota_rows = [ + row for row in rows if row.get("classification") == "quota_slot_spent" ] - quota_rows = [row for row in rows if row.get("classification") == "quota_slot_spent"] assert len(quota_rows) == 1 assert quota_rows[0]["effect_ref"] == f"{effect_id}#quota_spend" assert quota_rows[0]["agent_id"] == "codex-fixture" @@ -1655,16 +1648,13 @@ def crash_before_quota_receipt( assert resumed_exit_code == 0, resumed assert resumed["status"] == "committed" replayed_rows = [ - json.loads(line) - for line in index_path.read_text(encoding="utf-8").splitlines() + json.loads(line) for line in index_path.read_text(encoding="utf-8").splitlines() ] - assert sum( - row.get("classification") == "quota_slot_spent" - for row in replayed_rows - ) == 1 - events_path = ( - runtime / "goals" / "loopx-turn-fixture" / "rollout-event-log.jsonl" + assert ( + sum(row.get("classification") == "quota_slot_spent" for row in replayed_rows) + == 1 ) + events_path = runtime / "goals" / "loopx-turn-fixture" / "rollout-event-log.jsonl" events = [ json.loads(line) for line in events_path.read_text(encoding="utf-8").splitlines() @@ -1776,11 +1766,7 @@ def test_turn_run_once_cli_terminal_recovery_rejects_unowned_completion( turn_key = payload["resume_turn_key"] state_path = ( - project - / ".codex" - / "goals" - / "loopx-turn-fixture" - / "ACTIVE_GOAL_STATE.md" + project / ".codex" / "goals" / "loopx-turn-fixture" / "ACTIVE_GOAL_STATE.md" ) state = state_path.read_text(encoding="utf-8") current_turn_field = f" completion_turn_key={turn_key}" @@ -1795,12 +1781,9 @@ def test_turn_run_once_cli_terminal_recovery_rejects_unowned_completion( encoding="utf-8", ) - event_path = ( - runtime / "goals" / "loopx-turn-fixture" / "rollout-event-log.jsonl" - ) + event_path = runtime / "goals" / "loopx-turn-fixture" / "rollout-event-log.jsonl" events = [ - json.loads(line) - for line in event_path.read_text(encoding="utf-8").splitlines() + json.loads(line) for line in event_path.read_text(encoding="utf-8").splitlines() ] events_without_terminal = [ event @@ -1864,12 +1847,10 @@ def test_turn_run_once_cli_terminal_recovery_rejects_unowned_completion( "reason": "closeout readback failed", } recovered_events = [ - json.loads(line) - for line in event_path.read_text(encoding="utf-8").splitlines() + json.loads(line) for line in event_path.read_text(encoding="utf-8").splitlines() ] assert not any( - event.get("event_kind") == "todo_complete" - and event.get("run_id") == turn_key + event.get("event_kind") == "todo_complete" and event.get("run_id") == turn_key for event in recovered_events ) @@ -2064,11 +2045,7 @@ def test_turn_run_once_cli_rejects_unproven_host_claim_before_writeback( raise SystemExit(0 if pathlib.Path("claimed-artifact.txt").is_file() else 9) """ state_path = ( - project - / ".codex" - / "goals" - / "loopx-turn-fixture" - / "ACTIVE_GOAL_STATE.md" + project / ".codex" / "goals" / "loopx-turn-fixture" / "ACTIVE_GOAL_STATE.md" ) before_state = state_path.read_text(encoding="utf-8") output = io.StringIO() @@ -2134,11 +2111,7 @@ def test_turn_run_once_cli_uses_built_in_codex_host_and_typed_writeback( project, runtime, registry = _write_live_fixture(tmp_path) state_path = ( - project - / ".codex" - / "goals" - / "loopx-turn-fixture" - / "ACTIVE_GOAL_STATE.md" + project / ".codex" / "goals" / "loopx-turn-fixture" / "ACTIVE_GOAL_STATE.md" ) state_path.write_text( state_path.read_text(encoding="utf-8").replace( @@ -2185,19 +2158,27 @@ def adaptive_turn_envelope(*args: object, **kwargs: object) -> dict[str, object] } return envelope - def recording_refresh_state_run(*args: object, **kwargs: object) -> dict[str, object]: + def recording_refresh_state_run( + *args: object, **kwargs: object + ) -> dict[str, object]: refresh_workspace_paths.append(kwargs.get("delivery_workspace_path")) return real_refresh_state_run(*args, **kwargs) - def recording_spend_quota_slot(*args: object, **kwargs: object) -> dict[str, object]: + def recording_spend_quota_slot( + *args: object, **kwargs: object + ) -> dict[str, object]: spend_workspace_paths.append(kwargs.get("workspace_path")) return real_spend_quota_slot(*args, **kwargs) - def recording_update_goal_todo(*args: object, **kwargs: object) -> dict[str, object]: + def recording_update_goal_todo( + *args: object, **kwargs: object + ) -> dict[str, object]: updated_todo_ids.append(str(kwargs.get("todo_id") or "")) return real_update_goal_todo(*args, **kwargs) - def fake_codex_host(request: dict[str, object], **_kwargs: object) -> dict[str, object]: + def fake_codex_host( + request: dict[str, object], **_kwargs: object + ) -> dict[str, object]: return { "schema_version": "loopx_turn_result_v0", "turn_key": request["turn_key"], @@ -2275,11 +2256,7 @@ def fake_codex_host(request: dict[str, object], **_kwargs: object) -> dict[str, [] if result_kind == "validated_progress" else ["todo_fixture0001"] ) state = ( - project - / ".codex" - / "goals" - / "loopx-turn-fixture" - / "ACTIVE_GOAL_STATE.md" + project / ".codex" / "goals" / "loopx-turn-fixture" / "ACTIVE_GOAL_STATE.md" ).read_text(encoding="utf-8") assert "Run one revised public fixture check" in state if result_kind != "validated_progress": From 3189df8f70d08700446892d7d2231cc1631c8374 Mon Sep 17 00:00:00 2001 From: wchwawa Date: Wed, 2 Sep 2026 09:16:42 +1000 Subject: [PATCH 12/12] test(authority): require real peer worktree Signed-off-by: wchwawa --- tests/test_authority_turn_product_e2e.py | 72 ++++++++++++++++++++++-- 1 file changed, 68 insertions(+), 4 deletions(-) diff --git a/tests/test_authority_turn_product_e2e.py b/tests/test_authority_turn_product_e2e.py index 77398b9c8c..327c38c858 100644 --- a/tests/test_authority_turn_product_e2e.py +++ b/tests/test_authority_turn_product_e2e.py @@ -11,6 +11,7 @@ import pytest +from loopx.control_plane.agents.workspace_guard import capture_delivery_workspace from loopx.control_plane.coordination.file_provider import FileCoordinationProvider from loopx.control_plane.coordination.head import bootstrap_head from loopx.extensions.runtime import default_extension_state_file, install_extension @@ -21,6 +22,58 @@ TODO_ID = "todo_authority_product_e2e" FOLLOWUP_TODO_ID = "todo_authority_product_followup" AGENT_IDS = ("agent-a", "agent-b") +TASK_REPOSITORY = "git:example.invalid/loopx/authority-product-e2e" +TASK_REPOSITORY_REMOTE = "https://example.invalid/loopx/authority-product-e2e.git" + + +def _git(project: Path, *args: str) -> str: + env = dict(os.environ) + env["GIT_CONFIG_NOSYSTEM"] = "1" + env["GIT_CONFIG_GLOBAL"] = os.devnull + completed = subprocess.run( + ["git", "-C", str(project), *args], + env=env, + text=True, + capture_output=True, + timeout=20, + check=False, + ) + if completed.returncode != 0: + raise AssertionError(completed.stderr or completed.stdout) + return completed.stdout.strip() + + +def _create_independent_host_worktree(root: Path) -> Path: + source = root / "host-source" + source.mkdir() + _git(source, "init") + _git(source, "remote", "add", "origin", TASK_REPOSITORY_REMOTE) + (source / "README.md").write_text("authority product fixture\n", encoding="utf-8") + _git(source, "add", "README.md") + _git( + source, + "-c", + f"core.hooksPath={os.devnull}", + "-c", + "commit.gpgsign=false", + "-c", + "user.name=LoopX test", + "-c", + "user.email=loopx-test@example.invalid", + "commit", + "-m", + "test: initialize authority product fixture", + ) + worktree = root / "shared-host-project" + _git(source, "worktree", "add", "--detach", str(worktree), "HEAD") + workspace = capture_delivery_workspace( + worktree, + peer_independent_worktree_required=True, + ) + assert workspace is not None + assert workspace["task_repository"] == TASK_REPOSITORY + assert workspace["workspace_kind"] == "independent_git_worktree" + return worktree def _write_product_fixture( @@ -30,9 +83,8 @@ def _write_product_fixture( ) -> tuple[Path, Path, Path, Path]: project = root / "project" runtime = root / "runtime" - host_project = root / "shared-host-project" runtime.mkdir(parents=True) - host_project.mkdir() + host_project = _create_independent_host_worktree(root) state = project / ".codex" / "goals" / GOAL_ID / "ACTIVE_GOAL_STATE.md" state.parent.mkdir(parents=True) todo_lines = ( @@ -40,7 +92,7 @@ def _write_product_fixture( "- [ ] [P0] Advance the authority-qualified product fixture.", " ", "", "- [ ] [P2] Keep the follow-up fixture available.", @@ -130,7 +182,7 @@ def _bootstrap_authority(store: Path) -> None: "gates_open": True, "gate_revision": 1, }, - "repository": "git:example/authority-product-e2e", + "repository": TASK_REPOSITORY, "code_revision": "0123456789abcdef", "last_lease_epoch": 0, } @@ -442,6 +494,13 @@ def test_two_product_cli_agents_share_one_authority_and_settle_once( selected_todo = envelope["action"]["selected_todo"] assert selected_todo["todo_id"] == TODO_ID assert selected_todo["selected_by"] == "turn_controller_advisory_primary" + assert selected_todo["task_repository"] == TASK_REPOSITORY + assert envelope["writeback"]["delivery_workspace_causality"] == { + "schema_version": "delivery_workspace_causality_v0", + "refresh": "delivery_workspace; otherwise --delivery-workspace-path", + "spend": "recorded_delivery_workspace", + "mismatch": "fail_closed", + } assert ( envelope["action"]["action_portfolio"]["selection_policy"][ "requires_explicit_turn_binding" @@ -554,6 +613,11 @@ def test_two_product_cli_agents_share_one_authority_and_settle_once( "authority_product_completion", "quota_slot_spent", ] + assert run_rows[0]["delivery_workspace"]["task_repository"] == TASK_REPOSITORY + assert ( + run_rows[0]["delivery_workspace"]["workspace_kind"] + == "independent_git_worktree" + ) state = (project / ".codex" / "goals" / GOAL_ID / "ACTIVE_GOAL_STATE.md").read_text( encoding="utf-8" )