From 81607131f0c51f6c95dba8f0efacef3c4887b518 Mon Sep 17 00:00:00 2001 From: wchwawa Date: Thu, 3 Sep 2026 00:41:56 +1000 Subject: [PATCH 01/11] feat(authority): add transaction-bound shadow outbox and drain plumbing Stage 2C second half, first increment: the durable, per-partition outbox and the bounded drain that turns each committed entry into exactly one candidate transaction. Nothing is wired into a writer yet; this PR only adds the plumbing, its operator surface, and the tests that pin its crash-window semantics. - local_authority_shadow_projection.py: pure compact/canonical projection rules shared by capture, drain, and the later parity verify (floats are rejected so Python and TypeScript digests cannot disagree). - local_authority_shadow_outbox.py: two-phase prepared/committed entries per lock partition (todos, leases) under /authority-shadow/outbox//, gap-free seq minting across the drain cursor, entry identity bound to the primary bytes, prepared-only resolution (committed_proven_by_readback / abandoned / unproved), and seed/reseed entries. The projector is injected, so the module stays stdlib-only and mypy strict. - local_authority_shadow_outbox.ts: beginLeaseOutboxEntry for the TypeScript lease writers, byte-compatible entry identity, durableWriteJson in effect_runtime_io.ts. - local_authority_shadow.ts: coordination.local_authority_shadow.commit_entry (folds one partition into a loopx_local_authority_shadow_projection_v1 head, operation_id = entry_id, receipt bound to the source transaction, replay only on matching partition digest, no-op transactions for abandoned and unproved resolutions) and coordination.local_authority_shadow.read (head, comparison digest, bounded scan page). - local_authority_shadow_adapter.py: drain_local_authority_shadow_outbox (per-goal drain lock, bounded batch, prepared-only resolution only under a free primary lock, cursor-before-delete ordering, readback verification), local_authority_shadow_status, read_local_authority_shadow, evidence v1 builders with measured flags only. - loopx authority-shadow drain|status CLI (help surface, lazy command registry), mypy strict list, Windows test list. - control_plane/todos/goal_todo_projection.py: the todo list projection (filtered summaries, Markdown/event merge, goal_todo_summaries, project_goal_todo_items) moves out of loopx/todos.py so the capture can project the bytes about to be written from text instead of re-reading the file; loopx/todos.py shrinks and stays within its maintainability ceiling. The observation path from the first half is unchanged and still used by every writer; evidence v0 flags stay false until the writers are wired in the next increment. No parity claim is made anywhere. Signed-off-by: wchwawa --- .github/workflows/python-tests.yml | 1 + loopx/cli.py | 13 + loopx/cli_commands/__init__.py | 6 + loopx/cli_commands/authority_shadow.py | 205 ++++ .../coordination/local_authority_shadow.ts | 661 ++++++++++++- .../local_authority_shadow_adapter.py | 906 +++++++++++++++++- .../local_authority_shadow_outbox.py | 790 +++++++++++++++ .../local_authority_shadow_outbox.ts | 271 ++++++ .../local_authority_shadow_projection.py | 204 ++++ .../control_plane/effect_runtime_handlers.ts | 8 +- loopx/control_plane/effect_runtime_io.ts | 36 + .../todos/goal_todo_projection.py | 27 + loopx/help_surface.py | 1 + pyproject.toml | 2 + .../test_local_authority_shadow_cli_e2e.py | 51 + .../test_local_authority_shadow_drain.py | 479 +++++++++ .../test_local_authority_shadow_outbox.py | 354 +++++++ .../local_authority_shadow_outbox.test.ts | 366 +++++++ tsconfig.control-plane.json | 2 + 19 files changed, 4379 insertions(+), 4 deletions(-) create mode 100644 loopx/cli_commands/authority_shadow.py create mode 100644 loopx/control_plane/coordination/local_authority_shadow_outbox.py create mode 100644 loopx/control_plane/coordination/local_authority_shadow_outbox.ts create mode 100644 loopx/control_plane/coordination/local_authority_shadow_projection.py create mode 100644 tests/control_plane/test_local_authority_shadow_drain.py create mode 100644 tests/control_plane/test_local_authority_shadow_outbox.py create mode 100644 tests/control_plane_ts/local_authority_shadow_outbox.test.ts diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 260c25ea7c..868559ab98 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -124,6 +124,7 @@ jobs: tests/test_file_lock_cross_process.py tests/control_plane/test_coordination_file_provider.py tests/control_plane/test_effect_runtime_integration.py + tests/control_plane/test_local_authority_shadow_outbox.py tests/test_self_update_runtime_activation.py tests/test_windows_install.py diff --git a/loopx/cli.py b/loopx/cli.py index 441f6821a5..8dea8f8510 100644 --- a/loopx/cli.py +++ b/loopx/cli.py @@ -100,6 +100,7 @@ handle_support_control_command, handle_handoff_mode_command, handle_task_lease_command, + handle_authority_shadow_command, handle_version_command, handle_host_mode_plan_command, handle_worker_bridge_command, @@ -140,6 +141,7 @@ register_support_control_commands, register_handoff_mode_command, register_task_lease_command, + register_authority_shadow_command, register_todo_command, register_version_command, register_host_mode_plan_command, @@ -325,6 +327,7 @@ def build_parser() -> LoopXArgumentParser: register_todo_command(sub, add_subcommand_format) register_coordination_shadow_command(sub, add_subcommand_format) register_task_lease_command(sub, add_subcommand_format) + register_authority_shadow_command(sub, add_subcommand_format) register_handoff_mode_command(sub, add_subcommand_format) register_shared_goal_alignment_command(sub, add_subcommand_format) register_quota_command(sub) @@ -814,6 +817,16 @@ def main(argv: list[str] | None = None) -> int: if task_lease_result is not None: return task_lease_result + authority_shadow_result = handle_authority_shadow_command( + args, + registry_path=registry_path, + runtime_root_arg=args.runtime_root, + output_format=output_format, + print_payload=print_payload, + ) + if authority_shadow_result is not None: + return authority_shadow_result + handoff_mode_result = handle_handoff_mode_command( args, registry_path=registry_path, diff --git a/loopx/cli_commands/__init__.py b/loopx/cli_commands/__init__.py index c9be2c6d58..ac76f6242a 100644 --- a/loopx/cli_commands/__init__.py +++ b/loopx/cli_commands/__init__.py @@ -142,6 +142,10 @@ def _load_exports() -> None: handle_support_control_command, register_support_control_commands, ) + from .authority_shadow import ( + handle_authority_shadow_command, + register_authority_shadow_command, + ) from .task_lease import handle_task_lease_command, register_task_lease_command from .todo import handle_todo_command, register_todo_command from .version import handle_version_command, register_version_command @@ -221,6 +225,7 @@ def _load_exports() -> None: "handle_starter_visible_pilot_command", "handle_summary_all_command", "handle_support_control_command", + "handle_authority_shadow_command", "handle_task_lease_command", "handle_todo_command", "handle_version_command", @@ -268,6 +273,7 @@ def _load_exports() -> None: "register_summary_all_command", "register_status_commands", "register_support_control_commands", + "register_authority_shadow_command", "register_task_lease_command", "register_todo_command", "register_version_command", diff --git a/loopx/cli_commands/authority_shadow.py b/loopx/cli_commands/authority_shadow.py new file mode 100644 index 0000000000..9d4a8ceaf3 --- /dev/null +++ b/loopx/cli_commands/authority_shadow.py @@ -0,0 +1,205 @@ +from __future__ import annotations + +import argparse +from collections.abc import Callable +from pathlib import Path + +from ..control_plane.coordination.local_authority_shadow_adapter import ( + CLI_DRAIN_LOCK_TIMEOUT_SECONDS, + drain_local_authority_shadow_outbox, + effective_runtime_root, + local_authority_shadow_status, +) +from ..control_plane.coordination.local_authority_shadow_outbox import OutboxError +from ..file_lock import LockAcquireTimeoutError + + +AUTHORITY_SHADOW_CLI_SCHEMA = "loopx_authority_shadow_cli_v0" +CLI_DRAIN_MAX_ENTRIES = 256 +CLI_DRAIN_BUDGET_SECONDS = 30.0 + +PrintPayload = Callable[ + [dict[str, object], str, Callable[[dict[str, object]], str]], + None, +] + + +def render_authority_shadow_markdown(payload: dict[str, object]) -> str: + lines = [ + "# LoopX Authority Shadow", + "", + f"- ok: `{payload.get('ok')}`", + f"- action: `{payload.get('action')}`", + f"- goal_id: `{payload.get('goal_id')}`", + ] + if payload.get("error"): + lines.append(f"- error: {payload.get('error')}") + if payload.get("error_code"): + lines.append(f"- error_code: `{payload.get('error_code')}`") + if payload.get("action") == "drain": + for key in ( + "outcome", + "reason_code", + "delivered", + "replayed", + "reconciled", + "no_op", + "reseeded", + "pending_after", + "prepared_only_after", + "budget_exhausted", + "last_cursor", + "candidate_readback_verified", + ): + lines.append(f"- {key}: `{payload.get(key)}`") + stopped_at = payload.get("stopped_at") + if isinstance(stopped_at, dict): + lines.append( + "- stopped_at: " + f"`{stopped_at.get('partition')}#{stopped_at.get('seq')}` " + f"→ `{stopped_at.get('outcome')}` ({stopped_at.get('reason_code')})" + ) + elif payload.get("action") == "status": + config = payload.get("config") + if isinstance(config, dict): + lines.append(f"- config: `{config.get('status')}`") + backlog = payload.get("outbox") + if isinstance(backlog, dict): + for partition, facts in backlog.items(): + if isinstance(facts, dict): + lines.append( + f"- outbox.{partition}: committed_pending=" + f"`{facts.get('committed_pending')}` prepared_only=" + f"`{facts.get('prepared_only')}` cursor_last_seq=" + f"`{facts.get('cursor_last_seq')}`" + ) + candidate = payload.get("candidate") + if isinstance(candidate, dict): + lines.append( + f"- candidate: status=`{candidate.get('status')}` cursor=" + f"`{candidate.get('cursor')}` store_identity=`{candidate.get('store_identity')}`" + ) + lines.append(f"- store_bytes: `{payload.get('store_bytes')}`") + lines.append(f"- retention_pressure: `{payload.get('retention_pressure')}`") + return "\n".join(lines) + "\n" + + +def register_authority_shadow_command( + subparsers: argparse._SubParsersAction, + add_subcommand_format: Callable[[argparse.ArgumentParser], None], +) -> None: + parser = subparsers.add_parser( + "authority-shadow", + help=( + "Drain or inspect the transaction-bound local authority shadow outbox " + "for one goal. The candidate store is evidence only; it never decides." + ), + ) + add_subcommand_format(parser) + parser.add_argument( + "authority_shadow_command", + choices=["drain", "status"], + help="drain delivers pending outbox entries; status reports backlog and candidate facts.", + ) + parser.add_argument("--goal-id", required=True, help="Goal id whose shadow outbox to operate on.") + parser.add_argument( + "--max-entries", + type=int, + default=CLI_DRAIN_MAX_ENTRIES, + help=f"Maximum entries one drain pass delivers (default {CLI_DRAIN_MAX_ENTRIES}).", + ) + parser.add_argument( + "--budget-seconds", + type=float, + default=CLI_DRAIN_BUDGET_SECONDS, + help=f"Wall-clock budget for one drain pass (default {CLI_DRAIN_BUDGET_SECONDS:g}).", + ) + parser.add_argument( + "--lock-timeout-seconds", + type=float, + default=CLI_DRAIN_LOCK_TIMEOUT_SECONDS, + help=( + "How long to wait for the per-goal drain lock before reporting drain_deferred " + f"(default {CLI_DRAIN_LOCK_TIMEOUT_SECONDS:g})." + ), + ) + + +def handle_authority_shadow_command( + args: argparse.Namespace, + *, + registry_path: Path, + runtime_root_arg: str | None, + output_format: Callable[..., str], + print_payload: PrintPayload, +) -> int | None: + if args.command != "authority-shadow": + return None + action = str(getattr(args, "authority_shadow_command", None)) + payload: dict[str, object] + try: + # The same resolver every writer hook uses, so drain and status address + # the lineage those hooks wrote. + runtime_root = effective_runtime_root(registry_path, runtime_root_arg) + if action == "drain": + if args.max_entries < 1: + raise ValueError("--max-entries must be at least 1") + result = drain_local_authority_shadow_outbox( + registry_path=registry_path, + runtime_root=runtime_root, + goal_id=args.goal_id, + max_entries=args.max_entries, + budget_seconds=args.budget_seconds, + lock_timeout_seconds=args.lock_timeout_seconds, + ) + payload = { + "schema_version": AUTHORITY_SHADOW_CLI_SCHEMA, + "action": "drain", + **result.to_payload(), + } + else: + payload = { + "schema_version": AUTHORITY_SHADOW_CLI_SCHEMA, + **local_authority_shadow_status( + registry_path=registry_path, + runtime_root=runtime_root, + goal_id=args.goal_id, + ), + } + except OutboxError as exc: + payload = { + "ok": False, + "schema_version": AUTHORITY_SHADOW_CLI_SCHEMA, + "action": action, + "goal_id": args.goal_id, + "error": str(exc), + "error_code": exc.reason_code, + } + except LockAcquireTimeoutError as exc: + payload = { + "ok": False, + "schema_version": AUTHORITY_SHADOW_CLI_SCHEMA, + "action": action, + "goal_id": args.goal_id, + "error": str(exc), + **exc.to_payload(), + } + except Exception as exc: + payload = { + "ok": False, + "schema_version": AUTHORITY_SHADOW_CLI_SCHEMA, + "action": action, + "goal_id": args.goal_id, + "error": str(exc), + "error_code": exc.__class__.__name__, + } + print_payload(payload, output_format(args), render_authority_shadow_markdown) + return 0 if payload.get("ok") else 1 + + +__all__ = [ + "AUTHORITY_SHADOW_CLI_SCHEMA", + "handle_authority_shadow_command", + "register_authority_shadow_command", + "render_authority_shadow_markdown", +] diff --git a/loopx/control_plane/coordination/local_authority_shadow.ts b/loopx/control_plane/coordination/local_authority_shadow.ts index b455cf5e04..d6c93928c9 100644 --- a/loopx/control_plane/coordination/local_authority_shadow.ts +++ b/loopx/control_plane/coordination/local_authority_shadow.ts @@ -1,13 +1,20 @@ +import { createHash } from "node:crypto"; import { join } from "node:path"; import type { JsonObject } from "../effect_program.ts"; import { EffectRuntimeRequestError } from "../effect_runtime_errors.ts"; -import { requireJsonObject, requireNonEmptyString } from "../runtime_decode.ts"; +import { + requireInteger, + requireJsonObject, + requireNonEmptyString, + requireStringLiteral, +} from "../runtime_decode.ts"; import type { AuthorityStore, AuthorityStoreLoadResult, AuthorityStoreReceiptResult, } from "./authority_store.ts"; +import { canonicalAuthorityBytes } from "./authority_store_codec.ts"; import { FileAuthorityStore } from "./file_authority_store.ts"; export const LOCAL_AUTHORITY_SHADOW_REQUEST_SCHEMA = @@ -324,3 +331,655 @@ export async function recordLocalAuthorityShadow( }); } } + +// --------------------------------------------------------------------------- +// Transaction-bound entries (Stage 2C second half). +// +// A drained outbox entry becomes exactly one candidate transaction whose +// operation_id is the entry id, so a receipt names the primary transaction it +// records instead of a post-commit snapshot that may include other writers. +// --------------------------------------------------------------------------- + +export const LOCAL_AUTHORITY_SHADOW_PROJECTION_SCHEMA_V1 = + "loopx_local_authority_shadow_projection_v1"; +export const LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_REQUEST_SCHEMA = + "loopx_local_authority_shadow_commit_entry_request_v0"; +export const LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_RESULT_SCHEMA = + "loopx_local_authority_shadow_commit_entry_result_v0"; +export const LOCAL_AUTHORITY_SHADOW_READ_REQUEST_SCHEMA = + "loopx_local_authority_shadow_read_request_v0"; +export const LOCAL_AUTHORITY_SHADOW_READ_RESULT_SCHEMA = + "loopx_local_authority_shadow_read_result_v0"; +export const LOCAL_AUTHORITY_SHADOW_EVENT_SCHEMA_V1 = "loopx_local_authority_shadow_event_v1"; +export const LOCAL_AUTHORITY_SHADOW_TRANSACTION_RECEIPT_SCHEMA = + "loopx_local_authority_shadow_transaction_receipt_v0"; + +const SHADOW_PARTITIONS = ["todos", "leases"] as const; +const ENTRY_RESOLUTIONS = [ + "committed", + "committed_proven_by_readback", + "abandoned", + "unproved", + "seed", +] as const; +const NO_OP_RESOLUTIONS = new Set(["abandoned", "unproved"]); +const SOURCE_KINDS = ["markdown_active_state", "state_event_log", "task_lease_record"] as const; +const WRITER_RUNTIMES = ["python", "typescript"] as const; +const COMMIT_ENTRY_REQUEST_FIELDS = new Set([ + "schema_version", + "runtime_root", + "goal_id", + "entry", + "partition_projection", + "partition_digest", +]); +const ENTRY_FIELDS = new Set([ + "entry_id", + "partition", + "seq", + "writer", + "source", + "source_root_digest", + "prepared_at", + "committed_at", + "resolution", +]); +const READ_REQUEST_FIELDS = new Set([ + "schema_version", + "runtime_root", + "goal_id", + "scan_after_cursor", + "scan_limit", +]); +const ENTRY_ID_PATTERN = /^local-shadow-tx-[0-9a-f]{64}$/u; +const DIGEST_PATTERN = /^sha256:[a-f0-9]{64}$/u; +const MAX_SCAN_LIMIT = 1000; +const REVISION_RETRY_ATTEMPTS = 3; + +export type ShadowPartition = (typeof SHADOW_PARTITIONS)[number]; +export type ShadowEntryResolution = (typeof ENTRY_RESOLUTIONS)[number]; +export type LocalAuthorityShadowCommitEntryOutcome = + | "delivered" + | "replayed" + | "ambiguous_reconciled" + | "ambiguous_unproved" + | "unavailable" + | "failed" + | "protocol_mismatch" + | "conflict_retry_required"; + +interface ShadowEntryWriter { + runtime: (typeof WRITER_RUNTIMES)[number]; + write_class: string; + operation_id: string | null; +} + +interface ShadowEntrySource { + kind: (typeof SOURCE_KINDS)[number]; + previous_bytes_digest: string | null; + bytes_digest: string | null; + lease: JsonObject | null; + event_id: string | null; +} + +interface ShadowEntry { + entry_id: string; + partition: ShadowPartition; + seq: number; + writer: ShadowEntryWriter; + source: ShadowEntrySource; + source_root_digest: string; + prepared_at: string; + committed_at: string | null; + resolution: ShadowEntryResolution; +} + +interface CommitEntryRequest { + runtime_root: string; + goal_id: string; + entry: ShadowEntry; + partition_projection: JsonObject | null; + partition_digest: string | null; +} + +export interface LocalAuthorityShadowCommitEntryResult extends JsonObject { + schema_version: typeof LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_RESULT_SCHEMA; + outcome: LocalAuthorityShadowCommitEntryOutcome; + reason_code: string | null; + goal_id: string; + entry_id: string; + partition: ShadowPartition; + seq: number; + no_op: boolean; + store_identity: string | null; + provider_revision: string | null; + cursor: string | null; + head_digest: string | null; +} + +interface ReadRequest { + runtime_root: string; + goal_id: string; + scan_after_cursor: string | null; + scan_limit: number; +} + +function requireGoalId(value: unknown): string { + const goalId = requireNonEmptyString(value, "goal_id"); + if (goalId === "." || goalId === ".." || goalId.includes("/") || goalId.includes("\\")) { + throw new EffectRuntimeRequestError( + "Local authority shadow goal id must be a single path segment", + ); + } + return goalId; +} + +function rejectUnexpectedFields( + record: JsonObject, + allowed: Set, + label: string, +): void { + const unexpected = Object.keys(record).filter((field) => !allowed.has(field)); + if (unexpected.length > 0) { + unexpected.sort((left, right) => (left < right ? -1 : left > right ? 1 : 0)); + throw new EffectRuntimeRequestError( + `${label} has unsupported fields: ${unexpected.join(", ")}`, + ); + } +} + +function optionalString(value: unknown, label: string): string | null { + if (value === null || value === undefined) return null; + return requireNonEmptyString(value, label); +} + +function optionalDigest(value: unknown, label: string): string | null { + const digest = optionalString(value, label); + if (digest !== null && !DIGEST_PATTERN.test(digest)) { + throw new EffectRuntimeRequestError(`${label} must be sha256:<64 lowercase hex>`); + } + return digest; +} + +function decodeEntry(value: unknown): ShadowEntry { + const raw = requireJsonObject(value, "entry"); + rejectUnexpectedFields(raw, ENTRY_FIELDS, "Local authority shadow entry"); + const entryId = requireNonEmptyString(raw.entry_id, "entry.entry_id"); + if (!ENTRY_ID_PATTERN.test(entryId)) { + throw new EffectRuntimeRequestError("entry.entry_id must be local-shadow-tx-<64 lowercase hex>"); + } + const seq = requireInteger(raw.seq, "entry.seq"); + if (seq < 1) { + throw new EffectRuntimeRequestError("entry.seq must be a positive integer"); + } + const writer = requireJsonObject(raw.writer, "entry.writer"); + const source = requireJsonObject(raw.source, "entry.source"); + const lease = source.lease === null || source.lease === undefined + ? null + : requireJsonObject(source.lease, "entry.source.lease"); + return { + entry_id: entryId, + partition: requireStringLiteral(raw.partition, SHADOW_PARTITIONS, "entry.partition"), + seq, + writer: { + runtime: requireStringLiteral(writer.runtime, WRITER_RUNTIMES, "entry.writer.runtime"), + write_class: requireNonEmptyString(writer.write_class, "entry.writer.write_class"), + operation_id: optionalString(writer.operation_id, "entry.writer.operation_id"), + }, + source: { + kind: requireStringLiteral(source.kind, SOURCE_KINDS, "entry.source.kind"), + previous_bytes_digest: optionalDigest( + source.previous_bytes_digest, + "entry.source.previous_bytes_digest", + ), + bytes_digest: optionalDigest(source.bytes_digest, "entry.source.bytes_digest"), + lease: lease === null ? null : structuredClone(lease), + event_id: optionalString(source.event_id, "entry.source.event_id"), + }, + source_root_digest: requireNonEmptyString(raw.source_root_digest, "entry.source_root_digest"), + prepared_at: requireNonEmptyString(raw.prepared_at, "entry.prepared_at"), + committed_at: optionalString(raw.committed_at, "entry.committed_at"), + resolution: requireStringLiteral(raw.resolution, ENTRY_RESOLUTIONS, "entry.resolution"), + }; +} + +function decodePartitionProjection( + value: unknown, + partition: ShadowPartition, +): JsonObject | null { + if (value === null || value === undefined) return null; + const projection = requireJsonObject(value, "partition_projection"); + if (partition === "todos") { + if ( + Object.keys(projection).length !== 2 || + typeof projection.handoff_mode !== "string" || + !Array.isArray(projection.todos) + ) { + throw new EffectRuntimeRequestError( + "todos partition projection must be exactly {handoff_mode, todos[]}", + ); + } + } else if (Object.keys(projection).length !== 1 || !Array.isArray(projection.leases)) { + throw new EffectRuntimeRequestError("leases partition projection must be exactly {leases[]}"); + } + return structuredClone(projection); +} + +function decodeCommitEntryRequest(value: unknown): CommitEntryRequest { + const request = requireJsonObject(value, "local authority shadow commit entry request"); + rejectUnexpectedFields( + request, + COMMIT_ENTRY_REQUEST_FIELDS, + "Local authority shadow commit entry request", + ); + if (request.schema_version !== LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_REQUEST_SCHEMA) { + throw new EffectRuntimeRequestError( + "Local authority shadow commit entry request schema mismatch", + ); + } + const entry = decodeEntry(request.entry); + const projection = decodePartitionProjection(request.partition_projection, entry.partition); + const digest = optionalDigest(request.partition_digest, "partition_digest"); + const noOp = NO_OP_RESOLUTIONS.has(entry.resolution); + if (noOp && (projection !== null || digest !== null)) { + throw new EffectRuntimeRequestError( + `entry resolution ${entry.resolution} must not carry a partition projection`, + ); + } + if (!noOp && (projection === null || digest === null)) { + throw new EffectRuntimeRequestError( + `entry resolution ${entry.resolution} requires partition_projection and partition_digest`, + ); + } + return { + runtime_root: requireNonEmptyString(request.runtime_root, "runtime_root"), + goal_id: requireGoalId(request.goal_id), + entry, + partition_projection: projection, + partition_digest: digest, + }; +} + +function decodeReadRequest(value: unknown): ReadRequest { + const request = requireJsonObject(value, "local authority shadow read request"); + rejectUnexpectedFields(request, READ_REQUEST_FIELDS, "Local authority shadow read request"); + if (request.schema_version !== LOCAL_AUTHORITY_SHADOW_READ_REQUEST_SCHEMA) { + throw new EffectRuntimeRequestError("Local authority shadow read request schema mismatch"); + } + const limit = request.scan_limit === undefined ? 0 : requireInteger(request.scan_limit, "scan_limit"); + if (limit < 0 || limit > MAX_SCAN_LIMIT) { + throw new EffectRuntimeRequestError(`scan_limit must be between 0 and ${MAX_SCAN_LIMIT}`); + } + return { + runtime_root: requireNonEmptyString(request.runtime_root, "runtime_root"), + goal_id: requireGoalId(request.goal_id), + scan_after_cursor: optionalString(request.scan_after_cursor, "scan_after_cursor"), + scan_limit: limit, + }; +} + +/** Digest of the fields parity compares; must match Python `head_digest`. */ +export function localAuthorityShadowHeadDigest(head: JsonObject): string { + const view = { + handoff_mode: head.handoff_mode ?? null, + todos: head.todos ?? null, + leases: head.leases ?? null, + }; + return `sha256:${createHash("sha256").update(canonicalAuthorityBytes(view)).digest("hex")}`; +} + +function partitionsOf(head: JsonObject | null): JsonObject { + const raw = head?.partitions; + const partitions: JsonObject = { todos: null, leases: null }; + if (raw !== null && typeof raw === "object" && !Array.isArray(raw)) { + for (const partition of SHADOW_PARTITIONS) { + const marker = (raw as JsonObject)[partition]; + if (marker !== null && typeof marker === "object" && !Array.isArray(marker)) { + partitions[partition] = structuredClone(marker); + } + } + } + return partitions; +} + +/** + * Fold one partition into the candidate head. A v0 head (whole-snapshot + * observation) is accepted as the starting point with no partition markers. + */ +export function composeLocalAuthorityShadowHead( + current: JsonObject | null, + goalId: string, + entry: { partition: ShadowPartition; seq: number }, + projection: JsonObject | null, + digest: string | null, +): JsonObject { + const base = current ?? {}; + let handoffMode: string | null = typeof base.handoff_mode === "string" ? base.handoff_mode : null; + let todos = Array.isArray(base.todos) ? structuredClone(base.todos) : []; + let leases = Array.isArray(base.leases) ? structuredClone(base.leases) : []; + const partitions = partitionsOf(current); + if (projection !== null) { + if (entry.partition === "todos") { + handoffMode = String(projection.handoff_mode); + todos = structuredClone(projection.todos as JsonObject[]); + } else { + leases = structuredClone(projection.leases as JsonObject[]); + } + partitions[entry.partition] = { seq: entry.seq, partition_digest: digest }; + } + return { + schema_version: LOCAL_AUTHORITY_SHADOW_PROJECTION_SCHEMA_V1, + goal_id: goalId, + handoff_mode: handoffMode, + todos, + leases, + partitions, + }; +} + +function transactionReceipt(request: CommitEntryRequest, noOp: boolean): JsonObject { + const { entry } = request; + return { + schema_version: LOCAL_AUTHORITY_SHADOW_TRANSACTION_RECEIPT_SCHEMA, + entry_id: entry.entry_id, + partition: entry.partition, + seq: entry.seq, + write_class: entry.writer.write_class, + writer_runtime: entry.writer.runtime, + writer_operation_id: entry.writer.operation_id, + source_kind: entry.source.kind, + source_bytes_digest: entry.source.bytes_digest, + source_previous_bytes_digest: entry.source.previous_bytes_digest, + source_event_id: entry.source.event_id, + source_lease: entry.source.lease, + source_root_digest: entry.source_root_digest, + partition_digest: request.partition_digest, + resolution: entry.resolution, + no_op: noOp, + prepared_at: entry.prepared_at, + committed_at: entry.committed_at, + drained_at: new Date().toISOString(), + source_transaction_correlated: true, + durable_source_outbox: true, + parity_verdict: "not_evaluated", + primary_authority: "legacy_local", + candidate_read_for_decision: false, + provider_to_local_writes: false, + }; +} + +function transactionEvent(request: CommitEntryRequest, noOp: boolean): JsonObject { + const { entry } = request; + let kind = "source_transaction_delivered"; + if (entry.resolution === "seed") kind = "partition_seeded"; + else if (entry.resolution === "abandoned") kind = "source_transaction_abandoned"; + else if (entry.resolution === "unproved") kind = "source_transaction_unproved"; + return { + schema_version: LOCAL_AUTHORITY_SHADOW_EVENT_SCHEMA_V1, + kind, + partition: entry.partition, + seq: entry.seq, + entry_id: entry.entry_id, + write_class: entry.writer.write_class, + partition_digest: request.partition_digest, + no_op: noOp, + }; +} + +function commitEntryResult( + request: CommitEntryRequest, + outcome: LocalAuthorityShadowCommitEntryOutcome, + options: { + reasonCode?: string | null; + storeIdentity?: string | null; + providerRevision?: string | null; + cursor?: string | null; + headDigest?: string | null; + } = {}, +): LocalAuthorityShadowCommitEntryResult { + return { + schema_version: LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_RESULT_SCHEMA, + outcome, + reason_code: options.reasonCode ?? null, + goal_id: request.goal_id, + entry_id: request.entry.entry_id, + partition: request.entry.partition, + seq: request.entry.seq, + no_op: NO_OP_RESOLUTIONS.has(request.entry.resolution), + store_identity: options.storeIdentity ?? null, + provider_revision: options.providerRevision ?? null, + cursor: options.cursor ?? null, + head_digest: options.headDigest ?? null, + }; +} + +function transactionReceiptMatches( + request: CommitEntryRequest, + result: Extract, +): boolean { + return result.receipts.some((raw) => { + const receipt = raw as Record; + return receipt.schema_version === LOCAL_AUTHORITY_SHADOW_TRANSACTION_RECEIPT_SCHEMA && + receipt.entry_id === request.entry.entry_id && + receipt.partition === request.entry.partition && + receipt.seq === request.entry.seq && + (receipt.partition_digest ?? null) === request.partition_digest && + receipt.primary_authority === "legacy_local" && + receipt.provider_to_local_writes === false; + }); +} + +async function reconcileTransactionReceipt( + store: AuthorityStore, + request: CommitEntryRequest, + storeIdentity: string, + reconciledOutcome: "replayed" | "ambiguous_reconciled", +): Promise { + const result = await store.readReceipt(request.entry.entry_id); + if (result.status === "found" && transactionReceiptMatches(request, result)) { + return commitEntryResult(request, reconciledOutcome, { + storeIdentity, + providerRevision: result.provider_revision, + cursor: result.cursor, + }); + } + if (result.status === "unavailable" || result.status === "failed") { + return commitEntryResult(request, result.status, { + reasonCode: result.reason_code, + storeIdentity, + }); + } + return commitEntryResult( + request, + reconciledOutcome === "ambiguous_reconciled" ? "ambiguous_unproved" : "protocol_mismatch", + { + reasonCode: result.status === "missing" + ? "transaction_receipt_missing" + : "transaction_receipt_mismatch", + storeIdentity, + }, + ); +} + +function openShadowStore( + runtimeRoot: string, + goalId: string, + dependencies: LocalAuthorityShadowDependencies, +): AuthorityStore { + const providerDirectory = join(runtimeRoot, "authority-shadow", "file", goalId); + return (dependencies.openStore ?? ((directory, id) => new FileAuthorityStore(directory, id)))( + providerDirectory, + goalId, + ); +} + +/** + * Commit one drained outbox entry as exactly one candidate transaction. + * + * `operation_id` is the entry id, so a retry after a lost response replays + * onto the same transaction instead of recording the source write twice. + * No-op resolutions (abandoned / unproved) keep the sequence chain auditable + * without changing the compared head fields. + */ +export async function commitLocalAuthorityShadowEntry( + value: unknown, + dependencies: LocalAuthorityShadowDependencies = {}, +): Promise { + const request = decodeCommitEntryRequest(value); + const noOp = NO_OP_RESOLUTIONS.has(request.entry.resolution); + let store: AuthorityStore; + try { + store = openShadowStore(request.runtime_root, request.goal_id, dependencies); + } catch { + return commitEntryResult(request, "unavailable", { + reasonCode: "provider_construction_failed", + }); + } + try { + const identity = await store.storeIdentity(); + if (identity.status !== "available") { + return commitEntryResult(request, identity.status, { reasonCode: identity.reason_code }); + } + const storeIdentity = identity.store_identity; + let lastConflict: LocalAuthorityShadowCommitEntryResult | null = null; + for (let attempt = 0; attempt < REVISION_RETRY_ATTEMPTS; attempt += 1) { + const loaded = await store.loadAuthority(); + if (loaded.status === "unavailable" || loaded.status === "failed") { + return commitEntryResult(request, loaded.status, { + reasonCode: loaded.reason_code, + storeIdentity, + }); + } + const currentHead = loaded.status === "loaded" ? loaded.head : null; + const nextHead = composeLocalAuthorityShadowHead( + currentHead, + request.goal_id, + request.entry, + request.partition_projection, + request.partition_digest, + ); + const headDigest = localAuthorityShadowHeadDigest(nextHead); + const committed = await store.commitAuthority({ + expected_provider_revision: loaded.status === "loaded" ? loaded.provider_revision : null, + operation_id: request.entry.entry_id, + events: [transactionEvent(request, noOp)], + next_projection: nextHead, + receipts: [transactionReceipt(request, noOp)], + }); + if (committed.status === "applied") { + return commitEntryResult(request, "delivered", { + storeIdentity, + providerRevision: committed.provider_revision, + cursor: committed.cursor, + headDigest, + }); + } + if (committed.status === "ambiguous") { + return await reconcileTransactionReceipt( + store, + request, + storeIdentity, + "ambiguous_reconciled", + ); + } + if (committed.status === "failed") { + return commitEntryResult(request, "failed", { + reasonCode: committed.reason_code, + storeIdentity, + }); + } + if (committed.conflict_kind === "operation_id_exists") { + return await reconcileTransactionReceipt(store, request, storeIdentity, "replayed"); + } + lastConflict = commitEntryResult(request, "conflict_retry_required", { + reasonCode: "provider_revision_mismatch", + storeIdentity, + providerRevision: committed.current_provider_revision, + cursor: committed.current_cursor, + }); + } + return lastConflict as LocalAuthorityShadowCommitEntryResult; + } catch { + return commitEntryResult(request, "unavailable", { reasonCode: "provider_call_failed" }); + } +} + +/** + * Read-only view of the candidate store for drain readback and parity: + * head, its comparison digest, and a page of committed transactions with the + * projection reduced to its digest so responses stay bounded. + */ +export async function readLocalAuthorityShadow( + value: unknown, + dependencies: LocalAuthorityShadowDependencies = {}, +): Promise { + const request = decodeReadRequest(value); + const base: JsonObject = { + schema_version: LOCAL_AUTHORITY_SHADOW_READ_RESULT_SCHEMA, + goal_id: request.goal_id, + status: "unavailable", + reason_code: null, + store_identity: null, + provider_revision: null, + cursor: null, + head: null, + head_digest: null, + partitions: null, + scan: null, + }; + let store: AuthorityStore; + try { + store = openShadowStore(request.runtime_root, request.goal_id, dependencies); + } catch { + return { ...base, reason_code: "provider_construction_failed" }; + } + try { + const identity = await store.storeIdentity(); + if (identity.status !== "available") { + return { ...base, status: identity.status, reason_code: identity.reason_code }; + } + const loaded = await store.loadAuthority(); + if (loaded.status === "unavailable" || loaded.status === "failed") { + return { + ...base, + status: loaded.status, + reason_code: loaded.reason_code, + store_identity: identity.store_identity, + }; + } + const result: JsonObject = { + ...base, + status: loaded.status, + store_identity: identity.store_identity, + }; + if (loaded.status === "loaded") { + result.provider_revision = loaded.provider_revision; + result.cursor = loaded.cursor; + result.head = structuredClone(loaded.head); + result.head_digest = localAuthorityShadowHeadDigest(loaded.head); + result.partitions = partitionsOf(loaded.head); + } + if (request.scan_limit > 0) { + const page = await store.scanCommitted(request.scan_after_cursor, request.scan_limit); + if (page.status !== "page") { + return { ...result, status: page.status, reason_code: page.reason_code }; + } + result.scan = { + transactions: page.transactions.map((transaction) => ({ + cursor: transaction.cursor, + provider_revision: transaction.provider_revision, + operation_id: transaction.operation_id, + projection_digest: localAuthorityShadowHeadDigest(transaction.projection), + projection_partitions: partitionsOf(transaction.projection), + events: structuredClone(transaction.events) as JsonObject[], + receipts: structuredClone(transaction.receipts) as JsonObject[], + })), + next_cursor: page.next_cursor, + has_more: page.has_more, + }; + } + return result; + } catch { + return { ...base, reason_code: "provider_call_failed" }; + } +} diff --git a/loopx/control_plane/coordination/local_authority_shadow_adapter.py b/loopx/control_plane/coordination/local_authority_shadow_adapter.py index fed81cb32f..70450e95d8 100644 --- a/loopx/control_plane/coordination/local_authority_shadow_adapter.py +++ b/loopx/control_plane/coordination/local_authority_shadow_adapter.py @@ -10,15 +10,34 @@ import hashlib import json -from collections.abc import Mapping +import time +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +from dataclasses import asdict, dataclass, field from pathlib import Path from typing import Any -from ...file_lock import LockAcquireTimeoutError, exclusive_file_lock +from ...file_lock import ( + LockAcquireTimeoutError, + exclusive_cross_runtime_file_lock, + exclusive_file_lock, + try_exclusive_file_lock, +) from ...history import load_registry from ...paths import resolve_runtime_root from ...registry import find_registry_goal from ..effect_runtime import effect_runtime_result +from . import local_authority_shadow_outbox as outbox +from .local_authority_shadow_projection import ( + LEASE_PARTITION, + PARTITIONS, + TODO_PARTITION, + canonical_value, + head_digest, + partition_digest, + text_digest, + todo_partition_projection, +) LOCAL_AUTHORITY_SHADOW_CONFIG_SCHEMA = "loopx_local_authority_shadow_config_v0" @@ -489,3 +508,886 @@ def observe_todo_local_authority_commit( "observe_todo_local_authority_commit", "validate_local_authority_shadow_change", ] + + +# --------------------------------------------------------------------------- +# Transaction-bound outbox drain (Stage 2C second half plumbing). +# +# Writers record per-partition outbox entries inside the primary lock (see +# ``local_authority_shadow_outbox``). The drain below runs after that lock is +# released and turns each committed entry into exactly one candidate +# transaction. It is bounded, never blocks a writer, and reports what it left +# behind instead of guessing. +# --------------------------------------------------------------------------- + +LOCAL_AUTHORITY_SHADOW_EVIDENCE_SCHEMA_V1 = "loopx_local_authority_shadow_evidence_v1" +LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_REQUEST_SCHEMA = ( + "loopx_local_authority_shadow_commit_entry_request_v0" +) +LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_RESULT_SCHEMA = ( + "loopx_local_authority_shadow_commit_entry_result_v0" +) +LOCAL_AUTHORITY_SHADOW_READ_REQUEST_SCHEMA = "loopx_local_authority_shadow_read_request_v0" +LOCAL_AUTHORITY_SHADOW_READ_RESULT_SCHEMA = "loopx_local_authority_shadow_read_result_v0" +INLINE_DRAIN_MAX_ENTRIES = 16 +INLINE_DRAIN_BUDGET_SECONDS = 2.0 +INLINE_DRAIN_LOCK_TIMEOUT_SECONDS = 0.25 +CLI_DRAIN_LOCK_TIMEOUT_SECONDS = 5.0 +RETENTION_PRESSURE_BYTES = 8 * 1024 * 1024 +_COMMIT_ENTRY_OUTCOMES = { + "delivered", + "replayed", + "ambiguous_reconciled", + "ambiguous_unproved", + "unavailable", + "failed", + "protocol_mismatch", + "conflict_retry_required", +} +_SETTLED_OUTCOMES = {"delivered", "replayed", "ambiguous_reconciled"} +_SEED_WRITE_CLASSES = {"seed", "reseed_after_crash_gap"} +_ENTRY_SOURCE_FIELDS = ("kind", "previous_bytes_digest", "bytes_digest", "lease", "event_id") +_EVIDENCE_V1_OUTCOMES = { + "delivered", + "replayed", + "ambiguous_reconciled", + "pending", + "drain_deferred", + "no_transaction", + "capture_failed", + "ambiguous_unproved", + "unavailable", + "failed", + "protocol_mismatch", + "conflict_retry_required", +} + + +@dataclass +class DrainResult: + """Typed outcome of one bounded drain pass.""" + + goal_id: str + outcome: str = "nothing_pending" + config_enabled: bool = False + delivered: int = 0 + replayed: int = 0 + reconciled: int = 0 + no_op: int = 0 + reseeded: int = 0 + pending_after: int = 0 + prepared_only_after: int = 0 + in_flight_partitions: list[str] = field(default_factory=list) + budget_exhausted: bool = False + stopped_at: dict[str, Any] | None = None + reason_code: str | None = None + store_identity: str | None = None + provider_revision: str | None = None + last_cursor: str | None = None + cursor_before: str | None = None + cursor_after: str | None = None + head_digest: str | None = None + candidate_readback_verified: bool | None = None + entries: list[dict[str, Any]] = field(default_factory=list) + + @property + def ok(self) -> bool: + return self.outcome in {"drained", "nothing_pending"} and self.stopped_at is None + + @property + def drained_count(self) -> int: + return self.delivered + self.replayed + self.reconciled + + def entry_outcome(self, entry_id: str | None) -> dict[str, Any] | None: + if entry_id is None: + return None + for item in self.entries: + if item.get("entry_id") == entry_id: + return item + return None + + def to_payload(self) -> dict[str, Any]: + payload = asdict(self) + payload["ok"] = self.ok + payload["drained_count"] = self.drained_count + return payload + + +def todo_partition_projector( + goal: Mapping[str, Any] | None, + *, + state_path: Path, + rollout_events: list[dict[str, Any]] | None = None, +) -> outbox.TodoPartitionProjector: + """Production projector: parse active-state text into the todos partition.""" + + from ...control_plane.todos.handoff_mode import goal_handoff_mode + from ..todos.goal_todo_projection import project_goal_todo_items + + goal_record = dict(goal) if isinstance(goal, Mapping) else None + events = list(rollout_events or []) + + def project(state_text: str) -> dict[str, Any]: + return todo_partition_projection( + handoff_mode=goal_handoff_mode(state_text), + todos=project_goal_todo_items( + goal_record, + state_text=state_text, + state_path=state_path, + rollout_events=events, + ), + ) + + return project + + +def primary_lock_is_free(target: Path) -> bool: + """Probe a partition's Python primary lock once without waiting.""" + + try: + with try_exclusive_file_lock(target, operation="local_authority_shadow_drain_probe") as held: + return held is not None + except OSError: + return False + + +@dataclass(frozen=True) +class _GoalSources: + goal: dict[str, Any] | None + state_path: Path + lease_dir: Path + + +def _goal_sources( + registry: dict[str, Any], + *, + runtime_root: Path, + goal_id: str, +) -> _GoalSources: + from ...state_refresh import resolve_goal_state + + goal, _project, state_path = resolve_goal_state( + registry=registry, + goal_id=goal_id, + project_override=None, + state_file_override=None, + ) + return _GoalSources( + goal=goal, + state_path=state_path, + lease_dir=outbox.lease_directory(runtime_root, goal_id), + ) + + +def _read_state_text(path: Path) -> str: + return path.read_text(encoding="utf-8") if path.exists() else "" + + +def _event_present(sources: _GoalSources, event_id: str) -> bool: + from ...event_sourced_state import AppendOnlyStateEventStore + from ...status import state_event_log_candidates + + if sources.goal is None: + return False + for candidate in state_event_log_candidates(sources.goal, state_path=sources.state_path): + if not candidate.exists(): + continue + for event in AppendOnlyStateEventStore(candidate).load(): + if isinstance(event, dict) and event.get("event_id") == event_id: + return True + return False + + +def _lease_record(sources: _GoalSources, todo_id: str) -> dict[str, Any] | None: + if not todo_id: + return None + path = sources.lease_dir / f"{todo_id}.json" + if not path.exists(): + return None + raw = json.loads(path.read_text(encoding="utf-8")) + return raw if isinstance(raw, dict) else None + + +def _todo_partition_seed( + *, + registry_path: Path, + runtime_root: Path, + goal_id: str, + sources: _GoalSources, +) -> outbox.SeedSource: + """Full todos-partition snapshot; caller holds the state-file lock.""" + + from ...control_plane.todos.handoff_mode import goal_handoff_mode + from ...todos import list_goal_todos + + state_text = _read_state_text(sources.state_path) + payload = list_goal_todos( + registry_path=registry_path, + goal_id=goal_id, + runtime_root_arg=str(runtime_root), + ) + projection = todo_partition_projection( + handoff_mode=goal_handoff_mode(state_text), + todos=payload.get("todos") or [], + ) + return outbox.SeedSource( + partition=TODO_PARTITION, + projection=projection, + source_bytes_digest=text_digest(state_text), + ) + + +@contextmanager +def _primary_lock_if_free( + partition: str, + *, + runtime_root: Path, + goal_id: str, + sources: _GoalSources, +) -> Iterator[bool]: + """Hold the partition's primary lock only if it is free right now.""" + + if partition == TODO_PARTITION: + with try_exclusive_file_lock( + sources.state_path, + operation="local_authority_shadow_drain_resolve", + ) as held: + yield held is not None + return + from ..work_items.task_lease import task_lease_lock_path + + target = task_lease_lock_path(runtime_root=runtime_root, goal_id=goal_id) + try: + with exclusive_cross_runtime_file_lock( + target, + timeout_seconds=0.0, + operation="local_authority_shadow_drain_resolve", + ): + yield True + except LockAcquireTimeoutError: + yield False + + +def _entry_projection( + entry: outbox.OutboxEntry, + *, + goal_id: str, +) -> tuple[dict[str, Any] | None, str | None]: + """Compact projection and digest for a deliverable entry.""" + + raw = entry.projection() + if raw is None: + return None, None + if entry.partition == LEASE_PARTITION: + projection = outbox.compact_lease_projection(raw, goal_id=goal_id) + return projection, partition_digest(projection) + projection = dict(canonical_value(raw)) + digest = partition_digest(projection) + recorded = entry.recorded_partition_digest() + if recorded is not None and recorded != digest: + raise outbox.OutboxError( + "outbox_file_invalid", + f"entry {entry.entry_id} projection does not match its recorded digest", + ) + return projection, digest + + +def _commit_entry_request( + *, + runtime_root: Path, + goal_id: str, + entry: outbox.OutboxEntry, + resolution: str, + projection: dict[str, Any] | None, + digest: str | None, +) -> dict[str, Any]: + raw_source = entry.prepared.get("source") if isinstance(entry.prepared.get("source"), dict) else {} + writer = entry.prepared.get("writer") if isinstance(entry.prepared.get("writer"), dict) else {} + committed_at = entry.committed.get("committed_at") if entry.committed else None + return { + "schema_version": LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_REQUEST_SCHEMA, + "runtime_root": str(runtime_root), + "goal_id": goal_id, + "entry": { + "entry_id": entry.entry_id, + "partition": entry.partition, + "seq": entry.seq, + "writer": { + "runtime": writer.get("runtime"), + "write_class": writer.get("write_class"), + "operation_id": writer.get("operation_id"), + }, + "source": {key: raw_source.get(key) for key in _ENTRY_SOURCE_FIELDS}, + "source_root_digest": entry.prepared.get("source_root_digest"), + "prepared_at": entry.prepared.get("prepared_at"), + "committed_at": committed_at, + "resolution": resolution, + }, + "partition_projection": projection, + "partition_digest": digest, + } + + +def _valid_commit_entry_result(result: object, entry: outbox.OutboxEntry) -> bool: + if not isinstance(result, dict): + return False + return ( + result.get("schema_version") == LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_RESULT_SCHEMA + and result.get("outcome") in _COMMIT_ENTRY_OUTCOMES + and result.get("entry_id") == entry.entry_id + and result.get("partition") == entry.partition + and result.get("seq") == entry.seq + and isinstance(result.get("no_op"), bool) + ) + + +def read_local_authority_shadow( + *, + runtime_root: Path, + goal_id: str, + scan_after_cursor: str | None = None, + scan_limit: int = 0, +) -> dict[str, Any]: + """Read-only candidate view through the TypeScript store boundary.""" + + result = effect_runtime_result( + "coordination.local_authority_shadow.read", + { + "schema_version": LOCAL_AUTHORITY_SHADOW_READ_REQUEST_SCHEMA, + "runtime_root": str(runtime_root), + "goal_id": goal_id, + "scan_after_cursor": scan_after_cursor, + "scan_limit": scan_limit, + }, + timeout=15.0, + ) + if ( + not isinstance(result, dict) + or result.get("schema_version") != LOCAL_AUTHORITY_SHADOW_READ_RESULT_SCHEMA + or result.get("goal_id") != goal_id + ): + raise RuntimeError("local authority shadow read result is invalid") + return dict(result) + + +class _DrainBudget: + def __init__(self, *, max_entries: int, budget_seconds: float) -> None: + self._max_entries = max(1, max_entries) + self._deadline = time.monotonic() + max(0.0, budget_seconds) + self.consumed = 0 + + def exhausted(self) -> bool: + return self.consumed >= self._max_entries or time.monotonic() >= self._deadline + + +class _PartitionDrainer: + """Drain one partition in sequence order; all state lives on ``result``.""" + + def __init__( + self, + *, + registry_path: Path, + runtime_root: Path, + goal_id: str, + partition: str, + sources: _GoalSources, + result: DrainResult, + budget: _DrainBudget, + ) -> None: + self._registry_path = registry_path + self._runtime_root = runtime_root + self._goal_id = goal_id + self._partition = partition + self._sources = sources + self._result = result + self._budget = budget + self._directory = outbox.partition_directory(runtime_root, goal_id, partition) + self.last_delivered_digest: str | None = None + + def run(self) -> None: + while not self._budget.exhausted(): + entries = outbox.list_entries(self._directory) + if not entries: + return + entry = entries[0] + if entry.is_committed: + settled = self._deliver_committed(entry) + else: + settled = self._resolve_prepared_only(entry) + if not settled: + return + if outbox.list_entries(self._directory): + self._result.budget_exhausted = True + + def _deliver_committed(self, entry: outbox.OutboxEntry) -> bool: + writer = entry.prepared.get("writer") if isinstance(entry.prepared.get("writer"), dict) else {} + resolution = "seed" if writer.get("write_class") in _SEED_WRITE_CLASSES else "committed" + projection, digest = _entry_projection(entry, goal_id=self._goal_id) + if projection is None: + raise outbox.OutboxError( + "outbox_file_invalid", + f"committed entry {entry.entry_id} has no partition projection", + ) + return self._commit(entry, resolution=resolution, projection=projection, digest=digest) + + def _resolve_prepared_only(self, entry: outbox.OutboxEntry) -> bool: + with _primary_lock_if_free( + self._partition, + runtime_root=self._runtime_root, + goal_id=self._goal_id, + sources=self._sources, + ) as held: + if not held: + if self._partition not in self._result.in_flight_partitions: + self._result.in_flight_partitions.append(self._partition) + return False + resolution = outbox.resolve_prepared_only_entry( + entry, + markdown_text_reader=lambda: _read_state_text(self._sources.state_path), + lease_record_reader=lambda todo_id: _lease_record(self._sources, todo_id), + event_presence_reader=lambda event_id: _event_present(self._sources, event_id), + ) + projection: dict[str, Any] | None = None + digest: str | None = None + if resolution == "committed": + projection, digest = _entry_projection(entry, goal_id=self._goal_id) + if projection is None: + resolution = "unproved" + else: + resolution = "committed_proven_by_readback" + if resolution == "unproved": + # The source moved in a way no recorded entry explains; a full + # partition snapshot under the same lock closes the gap. + seed = ( + _todo_partition_seed( + registry_path=self._registry_path, + runtime_root=self._runtime_root, + goal_id=self._goal_id, + sources=self._sources, + ) + if self._partition == TODO_PARTITION + else outbox.lease_seed_source(self._runtime_root, self._goal_id) + ) + outbox.write_seed_entry( + runtime_root=self._runtime_root, + goal_id=self._goal_id, + seed=seed, + write_class="reseed_after_crash_gap", + ) + self._result.reseeded += 1 + return self._commit(entry, resolution=resolution, projection=projection, digest=digest) + + def _commit( + self, + entry: outbox.OutboxEntry, + *, + resolution: str, + projection: dict[str, Any] | None, + digest: str | None, + ) -> bool: + request = _commit_entry_request( + runtime_root=self._runtime_root, + goal_id=self._goal_id, + entry=entry, + resolution=resolution, + projection=projection, + digest=digest, + ) + raw = effect_runtime_result( + "coordination.local_authority_shadow.commit_entry", + request, + timeout=15.0, + ) + self._budget.consumed += 1 + if not _valid_commit_entry_result(raw, entry): + self._result.stopped_at = { + "partition": entry.partition, + "seq": entry.seq, + "entry_id": entry.entry_id, + "outcome": "failed", + "reason_code": "shadow_commit_entry_result_invalid", + } + return False + result = dict(raw) + summary = { + "entry_id": entry.entry_id, + "partition": entry.partition, + "seq": entry.seq, + "resolution": resolution, + "outcome": result["outcome"], + "reason_code": result.get("reason_code"), + "cursor": result.get("cursor"), + "provider_revision": result.get("provider_revision"), + "partition_digest": digest, + } + self._result.entries.append(summary) + if result.get("store_identity"): + self._result.store_identity = str(result["store_identity"]) + if result["outcome"] not in _SETTLED_OUTCOMES: + self._result.stopped_at = { + "partition": entry.partition, + "seq": entry.seq, + "entry_id": entry.entry_id, + "outcome": result["outcome"], + "reason_code": result.get("reason_code"), + } + return False + previous = outbox.read_cursor(self._directory) + cursor_digest = digest + if cursor_digest is None and previous is not None: + cursor_digest = previous.get("last_partition_digest") + outbox.write_cursor( + self._directory, + partition=entry.partition, + last_seq=entry.seq, + last_entry_id=entry.entry_id, + last_partition_digest=cursor_digest, + last_cursor=result.get("cursor"), + last_provider_revision=result.get("provider_revision"), + ) + outbox.remove_entry_files(entry) + if result["outcome"] == "delivered": + self._result.delivered += 1 + elif result["outcome"] == "replayed": + self._result.replayed += 1 + else: + self._result.reconciled += 1 + if result["no_op"]: + self._result.no_op += 1 + elif digest is not None: + self.last_delivered_digest = digest + if result.get("cursor"): + self._result.last_cursor = str(result["cursor"]) + if result.get("provider_revision"): + self._result.provider_revision = str(result["provider_revision"]) + return True + + +def _candidate_cursor(runtime_root: Path, goal_id: str) -> str | None: + """Current candidate cursor, or None when the store has no document yet.""" + + if not (runtime_root / "authority-shadow" / "file" / goal_id).is_dir(): + return None + try: + view = read_local_authority_shadow(runtime_root=runtime_root, goal_id=goal_id) + except Exception: + return None + cursor = view.get("cursor") + return str(cursor) if isinstance(cursor, str) else None + + +def _verify_readback( + result: DrainResult, + *, + runtime_root: Path, + goal_id: str, + delivered_digests: dict[str, str], +) -> None: + try: + view = read_local_authority_shadow(runtime_root=runtime_root, goal_id=goal_id) + except Exception: + result.candidate_readback_verified = False + return + head = view.get("head") + if view.get("status") != "loaded" or not isinstance(head, dict): + result.candidate_readback_verified = False + return + result.store_identity = view.get("store_identity") or result.store_identity + result.head_digest = view.get("head_digest") + result.cursor_after = view.get("cursor") + verified = head_digest(head) == view.get("head_digest") + partitions = head.get("partitions") if isinstance(head.get("partitions"), dict) else {} + for partition, digest in delivered_digests.items(): + marker = partitions.get(partition) if isinstance(partitions, dict) else None + verified = verified and isinstance(marker, dict) and marker.get("partition_digest") == digest + result.candidate_readback_verified = verified + + +def drain_local_authority_shadow_outbox( + *, + registry_path: Path, + runtime_root: Path | None, + goal_id: str, + max_entries: int = INLINE_DRAIN_MAX_ENTRIES, + budget_seconds: float = INLINE_DRAIN_BUDGET_SECONDS, + lock_timeout_seconds: float = INLINE_DRAIN_LOCK_TIMEOUT_SECONDS, +) -> DrainResult: + """Deliver pending outbox entries to the candidate store, one transaction each. + + The drain lock is per goal. A held lock means another drainer is already + at work, so the caller's write stays ``pending`` instead of waiting on it. + """ + + result = DrainResult(goal_id=goal_id) + if not goal_id or goal_id in {".", ".."} or "/" in goal_id or "\\" in goal_id: + result.outcome = "failed" + result.reason_code = "invalid_shadow_goal_id" + return result + try: + registry = load_registry(registry_path) + result.config_enabled = _shadow_config(registry, goal_id) is not None + if runtime_root is None: + runtime_root = resolve_runtime_root(registry, None, registry_path=registry_path) + except Exception: + result.outcome = "failed" + result.reason_code = "invalid_shadow_config" + return result + + summary_before = outbox.outbox_summary(runtime_root, goal_id) + if all( + item["committed_pending"] == 0 and item["prepared_only"] == 0 and item["invalid"] is None + for item in summary_before.values() + ): + result.outcome = "nothing_pending" + return result + + try: + with exclusive_file_lock( + outbox.drain_lock_target(runtime_root, goal_id), + timeout_seconds=lock_timeout_seconds, + operation="local_authority_shadow_drain", + ): + sources = _goal_sources(registry, runtime_root=runtime_root, goal_id=goal_id) + result.cursor_before = _candidate_cursor(runtime_root, goal_id) + budget = _DrainBudget(max_entries=max_entries, budget_seconds=budget_seconds) + delivered_digests: dict[str, str] = {} + for partition in PARTITIONS: + if result.stopped_at is not None: + break + drainer = _PartitionDrainer( + registry_path=registry_path, + runtime_root=runtime_root, + goal_id=goal_id, + partition=partition, + sources=sources, + result=result, + budget=budget, + ) + drainer.run() + if drainer.last_delivered_digest is not None: + delivered_digests[partition] = drainer.last_delivered_digest + if delivered_digests or result.delivered or result.replayed or result.reconciled: + _verify_readback( + result, + runtime_root=runtime_root, + goal_id=goal_id, + delivered_digests=delivered_digests, + ) + except LockAcquireTimeoutError: + result.outcome = "drain_deferred" + result.reason_code = "drain_lock_busy" + except outbox.OutboxError as error: + result.outcome = "stopped" + result.reason_code = error.reason_code + except Exception: + result.outcome = "stopped" + result.reason_code = "shadow_drain_failed" + else: + if result.stopped_at is not None: + result.outcome = "stopped" + result.reason_code = str(result.stopped_at.get("reason_code") or result.stopped_at["outcome"]) + else: + result.outcome = "drained" + summary_after = outbox.outbox_summary(runtime_root, goal_id) + result.pending_after = sum(int(item["committed_pending"]) for item in summary_after.values()) + result.prepared_only_after = sum(int(item["prepared_only"]) for item in summary_after.values()) + return result + + +class _CandidateMissing(Exception): + """The candidate store directory does not exist yet.""" + + +def _store_bytes(runtime_root: Path, goal_id: str) -> int: + directory = runtime_root / "authority-shadow" / "file" / goal_id + if not directory.is_dir(): + return 0 + return sum(path.stat().st_size for path in directory.iterdir() if path.is_file()) + + +def local_authority_shadow_status( + *, + registry_path: Path, + runtime_root: Path | None, + goal_id: str, +) -> dict[str, Any]: + """Operator readback: configuration, outbox backlog, and candidate head facts.""" + + registry = load_registry(registry_path) + goal = find_registry_goal(registry, goal_id) + if not isinstance(goal, dict): + raise ValueError(f"goal {goal_id!r} is not registered") + if runtime_root is None: + runtime_root = resolve_runtime_root(registry, None, registry_path=registry_path) + config = local_authority_shadow_summary(goal) + backlog = outbox.outbox_summary(runtime_root, goal_id) + candidate: dict[str, Any] + try: + if not (runtime_root / "authority-shadow" / "file" / goal_id).is_dir(): + # Reading through the store boundary would mint a store identity; + # a status probe must not create candidate lineage. + raise _CandidateMissing + view = read_local_authority_shadow(runtime_root=runtime_root, goal_id=goal_id) + head = view.get("head") if isinstance(view.get("head"), dict) else None + candidate = { + "status": view.get("status"), + "reason_code": view.get("reason_code"), + "store_identity": view.get("store_identity"), + "provider_revision": view.get("provider_revision"), + "cursor": view.get("cursor"), + "head_digest": view.get("head_digest"), + "head_schema_version": head.get("schema_version") if head else None, + "partitions": view.get("partitions"), + "codec_agreement": (head_digest(head) == view.get("head_digest")) if head else None, + } + except _CandidateMissing: + candidate = { + "status": "missing", + "reason_code": None, + "store_identity": None, + "provider_revision": None, + "cursor": None, + "head_digest": None, + "head_schema_version": None, + "partitions": None, + "codec_agreement": None, + } + except Exception: + candidate = { + "status": "unavailable", + "reason_code": "shadow_read_failed", + "store_identity": None, + "provider_revision": None, + "cursor": None, + "head_digest": None, + "head_schema_version": None, + "partitions": None, + "codec_agreement": None, + } + store_bytes = _store_bytes(runtime_root, goal_id) + return { + "ok": all(item["invalid"] is None for item in backlog.values()), + "action": "status", + "goal_id": goal_id, + "config": config, + "runtime_root_digest": outbox.runtime_root_digest(runtime_root), + "outbox": backlog, + "candidate": candidate, + "store_bytes": store_bytes, + "retention_pressure": store_bytes > RETENTION_PRESSURE_BYTES, + } + + +def capture_evidence( + *, + goal_id: str, + capture: outbox.CaptureOutcome, + drain: DrainResult | None, +) -> dict[str, Any]: + """Evidence v1 attached to a writer payload: capture facts plus drain facts. + + Every flag here is a measured fact of this write. ``source_candidate_compared`` + and ``parity_verdict`` stay negative until the verify step exists. + """ + + if capture.failure is not None: + outcome = "capture_failed" + reason_code: str | None = str(capture.failure.get("reason_code")) + elif capture.entry_id is None: + outcome = "no_transaction" + reason_code = capture.skipped_reason + elif drain is None or drain.outcome == "drain_deferred": + outcome = "drain_deferred" if drain is not None else "pending" + reason_code = drain.reason_code if drain is not None else None + else: + settled = drain.entry_outcome(capture.entry_id) + if settled is None: + outcome = "pending" + reason_code = drain.reason_code + else: + outcome = str(settled["outcome"]) + reason_code = settled.get("reason_code") + return { + "schema_version": LOCAL_AUTHORITY_SHADOW_EVIDENCE_SCHEMA_V1, + "outcome": outcome, + "reason_code": reason_code, + "goal_id": goal_id, + "entry": { + "entry_id": capture.entry_id, + "partition": capture.partition, + "seq": capture.seq, + "partition_digest": capture.partition_digest, + "source_bytes_digest": capture.source_bytes_digest, + }, + "drain": None + if drain is None + else { + "outcome": drain.outcome, + "delivered": drain.delivered, + "replayed": drain.replayed, + "pending_after": drain.pending_after, + "prepared_only_after": drain.prepared_only_after, + "stopped_at": drain.stopped_at, + "last_cursor": drain.last_cursor, + "provider_revision": drain.provider_revision, + "candidate_readback_verified": drain.candidate_readback_verified, + }, + "capture_kind": "source_transaction_outbox", + "source_transaction_correlated": capture.recorded, + "durable_source_outbox": capture.failure is None, + "source_candidate_compared": False, + "parity_verdict": "not_evaluated", + "primary_authority": "legacy_local", + "candidate_provider": "file", + "candidate_read_for_decision": False, + "provider_to_local_writes": False, + "primary_writeback_preserved": True, + "store_identity": drain.store_identity if drain is not None else None, + } + + +def valid_evidence_v1(result: object, *, goal_id: str) -> bool: + """Closed-shape check for evidence v1 as attached to writer payloads.""" + + if not isinstance(result, dict): + return False + entry = result.get("entry") + return ( + result.get("schema_version") == LOCAL_AUTHORITY_SHADOW_EVIDENCE_SCHEMA_V1 + and result.get("outcome") in _EVIDENCE_V1_OUTCOMES + and result.get("goal_id") == goal_id + and isinstance(entry, dict) + and result.get("capture_kind") == "source_transaction_outbox" + and isinstance(result.get("source_transaction_correlated"), bool) + and isinstance(result.get("durable_source_outbox"), bool) + and result.get("source_candidate_compared") is False + and result.get("parity_verdict") == "not_evaluated" + and result.get("primary_authority") == "legacy_local" + and result.get("candidate_provider") == "file" + and result.get("candidate_read_for_decision") is False + and result.get("provider_to_local_writes") is False + and result.get("primary_writeback_preserved") is True + and (result.get("reason_code") is None or isinstance(result.get("reason_code"), str)) + ) + + +__all__ += [ + "CLI_DRAIN_LOCK_TIMEOUT_SECONDS", + "INLINE_DRAIN_BUDGET_SECONDS", + "INLINE_DRAIN_LOCK_TIMEOUT_SECONDS", + "INLINE_DRAIN_MAX_ENTRIES", + "LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_REQUEST_SCHEMA", + "LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_RESULT_SCHEMA", + "LOCAL_AUTHORITY_SHADOW_EVIDENCE_SCHEMA_V1", + "LOCAL_AUTHORITY_SHADOW_READ_REQUEST_SCHEMA", + "LOCAL_AUTHORITY_SHADOW_READ_RESULT_SCHEMA", + "RETENTION_PRESSURE_BYTES", + "DrainResult", + "capture_evidence", + "primary_lock_is_free", + "todo_partition_projector", + "drain_local_authority_shadow_outbox", + "local_authority_shadow_status", + "read_local_authority_shadow", + "valid_evidence_v1", +] diff --git a/loopx/control_plane/coordination/local_authority_shadow_outbox.py b/loopx/control_plane/coordination/local_authority_shadow_outbox.py new file mode 100644 index 0000000000..c059d9fa43 --- /dev/null +++ b/loopx/control_plane/coordination/local_authority_shadow_outbox.py @@ -0,0 +1,790 @@ +"""Durable, transaction-bound outbox for the local authority shadow. + +A legacy writer that changes coordination facts records, inside the lock it +already holds, a two-phase entry for exactly the state that lock guards (one +*partition*): a ``prepared`` file computed from the bytes about to be written, +then a ``committed`` marker after the primary write returns. Nothing in the +lock talks to the TypeScript runtime. A later drain (same process after the +lock, or an operator command) turns each committed entry into exactly one +candidate-store transaction whose ``operation_id`` is the entry id. + +The outbox never changes the primary verdict: every failure here is swallowed +into typed capture evidence and the primary write proceeds unchanged. +""" + +from __future__ import annotations + +import json +import os +import re +import uuid +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from .local_authority_shadow_projection import ( + LEASE_PARTITION, + PARTITIONS, + TODO_PARTITION, + ProjectionValueError, + canonical_value, + lease_partition_projection, + partition_digest, + sha256_digest, + text_digest, +) + + +OUTBOX_ENTRY_SCHEMA = "loopx_local_authority_shadow_outbox_entry_v0" +OUTBOX_COMMIT_SCHEMA = "loopx_local_authority_shadow_outbox_commit_v0" +DRAIN_CURSOR_SCHEMA = "loopx_local_authority_shadow_drain_cursor_v0" +SOURCE_MARKDOWN = "markdown_active_state" +SOURCE_STATE_EVENT_LOG = "state_event_log" +SOURCE_TASK_LEASE = "task_lease_record" +WRITER_RUNTIME_PYTHON = "python" +WRITER_RUNTIME_TYPESCRIPT = "typescript" +ENTRY_ID_PREFIX = "local-shadow-tx-" +_ENTRY_FILE = re.compile( + r"^(?P\d{10})-(?Plocal-shadow-tx-[0-9a-f]{64})\.(?Pprepared|committed)\.json$" +) +_LEASE_FILE = re.compile(r"^[A-Za-z0-9_.-]+\.json$") + + +class OutboxError(RuntimeError): + """Typed outbox failure; never escapes into a primary write.""" + + def __init__(self, reason_code: str, message: str) -> None: + super().__init__(message) + self.reason_code = reason_code + + +TodoPartitionProjector = Callable[[str], dict[str, Any]] +"""Active-state Markdown text -> ``{"handoff_mode": str, "todos": [compact...]}``.""" + + +def utc_now_text() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +def outbox_root(runtime_root: Path, goal_id: str) -> Path: + return runtime_root / "authority-shadow" / "outbox" / goal_id + + +def partition_directory(runtime_root: Path, goal_id: str, partition: str) -> Path: + if partition not in PARTITIONS: + raise OutboxError("invalid_partition", f"unknown outbox partition {partition!r}") + return outbox_root(runtime_root, goal_id) / partition + + +def drain_lock_target(runtime_root: Path, goal_id: str) -> Path: + return outbox_root(runtime_root, goal_id) / "drain" + + +def lease_directory(runtime_root: Path, goal_id: str) -> Path: + return runtime_root / "goals" / goal_id / "task-leases" + + +def entry_identity(*, goal_id: str, partition: str, seq: int, source_ref: str) -> str: + """Bind the entry id to the exact primary bytes (or event) it records.""" + + return ENTRY_ID_PREFIX + sha256_digest( + {"goal_id": goal_id, "partition": partition, "seq": seq, "source_ref": source_ref} + ).removeprefix("sha256:") + + +def entry_file_name(seq: int, entry_id: str, phase: str) -> str: + return f"{seq:010d}-{entry_id}.{phase}.json" + + +def _fsync_directory(directory: Path) -> None: + if os.name == "nt": + return + descriptor = os.open(directory, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def durable_write_json(path: Path, payload: Mapping[str, Any]) -> None: + """Temp file in the same directory, fsync, atomic replace, directory fsync.""" + + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + temporary = path.with_name(f"{path.name}.tmp-{os.getpid()}-{uuid.uuid4().hex}") + data = json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=1).encode("utf-8") + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + view = memoryview(data) + while view: + written = os.write(descriptor, view) + view = view[written:] + os.fsync(descriptor) + finally: + os.close(descriptor) + try: + os.replace(temporary, path) + _fsync_directory(path.parent) + finally: + if temporary.exists(): + temporary.unlink(missing_ok=True) + + +def _as_object(value: object) -> dict[str, Any]: + """Narrow an untyped JSON value to an object; anything else is empty.""" + + return dict(value) if isinstance(value, Mapping) else {} + + +def _load_json(path: Path) -> dict[str, Any]: + raw = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(raw, dict): + raise OutboxError("outbox_file_invalid", f"{path.name} is not a JSON object") + return raw + + +@dataclass(frozen=True, slots=True) +class OutboxEntry: + """One two-phase entry as found on disk.""" + + partition: str + seq: int + entry_id: str + prepared_path: Path + committed_path: Path | None + prepared: dict[str, Any] + committed: dict[str, Any] | None + + @property + def is_committed(self) -> bool: + return self.committed is not None + + @property + def source_ref(self) -> str: + source = _as_object(self.prepared.get("source")) + return str(source.get("bytes_digest") or f"event:{source.get('event_id')}") + + def projection(self) -> dict[str, Any] | None: + """The partition projection recorded for this entry, if any.""" + + for record in (self.committed, self.prepared): + if isinstance(record, dict) and isinstance(record.get("projection"), dict): + return dict(record["projection"]) + return None + + def recorded_partition_digest(self) -> str | None: + for record in (self.committed, self.prepared): + if isinstance(record, dict) and isinstance(record.get("partition_digest"), str): + return str(record["partition_digest"]) + return None + + +def list_entries(directory: Path) -> list[OutboxEntry]: + """All entries of one partition directory, oldest first.""" + + if not directory.is_dir(): + return [] + prepared: dict[tuple[int, str], Path] = {} + committed: dict[tuple[int, str], Path] = {} + for path in directory.iterdir(): + match = _ENTRY_FILE.match(path.name) + if match is None: + continue + key = (int(match.group("seq")), match.group("entry_id")) + if match.group("phase") == "prepared": + prepared[key] = path + else: + committed[key] = path + entries: list[OutboxEntry] = [] + for key in sorted(prepared): + seq, entry_id = key + prepared_path = prepared[key] + prepared_record = _load_json(prepared_path) + if ( + prepared_record.get("schema_version") != OUTBOX_ENTRY_SCHEMA + or prepared_record.get("entry_id") != entry_id + or prepared_record.get("seq") != seq + ): + raise OutboxError("outbox_file_invalid", f"{prepared_path.name} does not match its name") + committed_path = committed.get(key) + committed_record = None + if committed_path is not None: + committed_record = _load_json(committed_path) + if ( + committed_record.get("schema_version") != OUTBOX_COMMIT_SCHEMA + or committed_record.get("entry_id") != entry_id + ): + raise OutboxError( + "outbox_file_invalid", f"{committed_path.name} does not match its entry" + ) + entries.append( + OutboxEntry( + partition=str(prepared_record.get("partition") or directory.name), + seq=seq, + entry_id=entry_id, + prepared_path=prepared_path, + committed_path=committed_path, + prepared=prepared_record, + committed=committed_record, + ) + ) + orphan_markers = sorted(set(committed) - set(prepared)) + if orphan_markers: + seq, entry_id = orphan_markers[0] + raise OutboxError( + "outbox_file_invalid", + f"committed marker without prepared entry: {entry_file_name(seq, entry_id, 'committed')}", + ) + return entries + + +def cursor_path(directory: Path) -> Path: + return directory / "drain-cursor.json" + + +def read_cursor(directory: Path) -> dict[str, Any] | None: + path = cursor_path(directory) + if not path.exists(): + return None + record = _load_json(path) + if record.get("schema_version") != DRAIN_CURSOR_SCHEMA: + raise OutboxError("outbox_file_invalid", "drain cursor schema is unsupported") + return record + + +def write_cursor( + directory: Path, + *, + partition: str, + last_seq: int, + last_entry_id: str, + last_partition_digest: str | None, + last_cursor: str | None, + last_provider_revision: str | None, +) -> None: + durable_write_json( + cursor_path(directory), + { + "schema_version": DRAIN_CURSOR_SCHEMA, + "partition": partition, + "last_seq": last_seq, + "last_entry_id": last_entry_id, + "last_partition_digest": last_partition_digest, + "last_cursor": last_cursor, + "last_provider_revision": last_provider_revision, + "updated_at": utc_now_text(), + }, + ) + + +def next_seq(directory: Path) -> int: + """Gap-free sequence: past the newest file and the drained watermark.""" + + highest = 0 + if directory.is_dir(): + for path in directory.iterdir(): + match = _ENTRY_FILE.match(path.name) + if match is not None: + highest = max(highest, int(match.group("seq"))) + cursor = read_cursor(directory) + if cursor is not None: + highest = max(highest, int(cursor.get("last_seq") or 0)) + return highest + 1 + + +def latest_partition_digest(directory: Path) -> str | None: + """Digest of the newest known partition state (pending entry, else cursor).""" + + entries = list_entries(directory) + for entry in reversed(entries): + digest = entry.recorded_partition_digest() + if digest is not None: + return digest + cursor = read_cursor(directory) + if cursor is not None and isinstance(cursor.get("last_partition_digest"), str): + return str(cursor["last_partition_digest"]) + return None + + +def runtime_root_digest(runtime_root: Path) -> str: + return text_digest(str(runtime_root.resolve(strict=False))) + + +def read_lease_records(directory: Path) -> list[tuple[str, dict[str, Any]]]: + """Top-level lease records of a goal; lifecycle receipts are excluded.""" + + if not directory.is_dir(): + return [] + records: list[tuple[str, dict[str, Any]]] = [] + for path in sorted(directory.iterdir()): + if not path.is_file() or not _LEASE_FILE.match(path.name) or path.name.startswith("."): + continue + raw = json.loads(path.read_text(encoding="utf-8")) + if isinstance(raw, dict): + records.append((path.stem, raw)) + return records + + +def compact_lease_projection( + raw_projection: Mapping[str, Any], *, goal_id: str +) -> dict[str, Any]: + """Compact the TypeScript-written lease partition (``{leases: [{file_stem, record}]}``).""" + + raw_leases = raw_projection.get("leases") + if not isinstance(raw_leases, list): + raise OutboxError("outbox_file_invalid", "lease partition projection must list leases") + records: list[tuple[str, object]] = [] + for item in raw_leases: + if not isinstance(item, dict): + raise OutboxError("outbox_file_invalid", "lease projection item must be an object") + stem = item.get("file_stem") + if not isinstance(stem, str) or not stem: + raise OutboxError("outbox_file_invalid", "lease projection item needs a file_stem") + records.append((stem, item.get("record"))) + try: + return lease_partition_projection(records, goal_id=goal_id) + except ProjectionValueError as error: + raise OutboxError("outbox_file_invalid", str(error)) from error + + +def _writer(write_class: str, *, runtime: str, operation_id: str | None) -> dict[str, Any]: + return {"runtime": runtime, "write_class": write_class, "operation_id": operation_id} + + +def _entry_record( + *, + goal_id: str, + partition: str, + seq: int, + entry_id: str, + writer: Mapping[str, Any], + source: Mapping[str, Any], + source_root_digest: str, + projection: Mapping[str, Any] | None, + digest: str | None, +) -> dict[str, Any]: + return { + "schema_version": OUTBOX_ENTRY_SCHEMA, + "goal_id": goal_id, + "partition": partition, + "seq": seq, + "entry_id": entry_id, + "writer": dict(writer), + "source": dict(source), + "source_root_digest": source_root_digest, + "projection": canonical_value(dict(projection)) if projection is not None else None, + "partition_digest": digest, + "prepared_at": utc_now_text(), + } + + +@dataclass +class CaptureOutcome: + """What the capture did for one primary write (attached to its evidence).""" + + entry_id: str | None = None + partition: str | None = None + seq: int | None = None + partition_digest: str | None = None + source_bytes_digest: str | None = None + skipped_reason: str | None = None + failure: dict[str, Any] | None = None + + @property + def recorded(self) -> bool: + return self.entry_id is not None and self.failure is None + + +class TodoPartitionCapture: + """Two-phase capture of the todos partition inside the active-state lock. + + ``begin`` returns an inert capture when the goal has no shadow binding, so + default-off writers create no directory, lock, or file. + """ + + def __init__( + self, + *, + enabled: bool, + runtime_root: Path | None, + goal_id: str, + state_path: Path | None, + write_class: str, + original_text: str, + projector: TodoPartitionProjector | None, + ) -> None: + self._enabled = enabled + self._runtime_root = runtime_root + self._goal_id = goal_id + self._state_path = state_path + self._write_class = write_class + self._original_digest = text_digest(original_text) + self._projector = projector + self._directory = ( + partition_directory(runtime_root, goal_id, TODO_PARTITION) + if enabled and runtime_root is not None + else None + ) + self._seq: int | None = None + self._entry_id: str | None = None + self._event_id: str | None = None + self.outcome = CaptureOutcome(partition=TODO_PARTITION if enabled else None) + + @classmethod + def begin( + cls, + *, + enabled: bool, + runtime_root: Path | None, + goal_id: str, + state_path: Path | None, + write_class: str, + original_text: str, + projector: TodoPartitionProjector | None, + ) -> TodoPartitionCapture: + """``projector`` maps active-state text to the todos partition projection. + + It is injected so this module stays free of the Markdown parser; the + adapter supplies the production projector. + """ + + return cls( + enabled=enabled, + runtime_root=runtime_root, + goal_id=goal_id, + state_path=state_path, + write_class=write_class, + original_text=original_text, + projector=projector, + ) + + @property + def enabled(self) -> bool: + return self._enabled and self._directory is not None + + def _project(self, state_text: str) -> dict[str, Any]: + if self._projector is None: + raise OutboxError("outbox_prepare_failed", "a todo partition projector is required") + projection = self._projector(state_text) + if set(projection) != {"handoff_mode", "todos"}: + raise OutboxError("outbox_prepare_failed", "projector must return {handoff_mode, todos}") + return projection + + def _fail(self, reason_code: str, error: BaseException) -> None: + self.outcome.failure = { + "reason_code": reason_code, + "error_class": error.__class__.__name__, + } + + def prepare(self, new_text: str, *, event_id: str | None = None) -> None: + """Record the prepared entry for the bytes about to be written. + + For the state-event-log branch pass ``new_text=original`` plus the + event id; the projection is then recorded by ``committed`` after the + append, still inside the same lock. + """ + + if not self.enabled or self._directory is None or self._runtime_root is None: + self.outcome.skipped_reason = "shadow_disabled" + return + try: + if event_id is None: + projection = self._project(new_text) + digest = partition_digest(projection) + if digest == latest_partition_digest(self._directory): + self.outcome.skipped_reason = "partition_unchanged" + return + source_ref = text_digest(new_text) + bytes_digest: str | None = source_ref + source_kind = SOURCE_MARKDOWN + else: + projection = None + digest = None + bytes_digest = None + source_kind = SOURCE_STATE_EVENT_LOG + source_ref = f"event:{event_id}" + seq = next_seq(self._directory) + entry_id = entry_identity( + goal_id=self._goal_id, + partition=TODO_PARTITION, + seq=seq, + source_ref=source_ref, + ) + record = _entry_record( + goal_id=self._goal_id, + partition=TODO_PARTITION, + seq=seq, + entry_id=entry_id, + writer=_writer( + self._write_class, + runtime=WRITER_RUNTIME_PYTHON, + operation_id=event_id, + ), + source={ + "kind": source_kind, + "previous_bytes_digest": self._original_digest, + "bytes_digest": bytes_digest, + "lease": None, + "event_id": event_id, + }, + source_root_digest=runtime_root_digest(self._runtime_root), + projection=projection, + digest=digest, + ) + durable_write_json( + self._directory / entry_file_name(seq, entry_id, "prepared"), + record, + ) + except Exception as error: # noqa: BLE001 - the primary write must proceed + self._fail("outbox_prepare_failed", error) + return + self._seq = seq + self._entry_id = entry_id + self._event_id = event_id + self.outcome.entry_id = entry_id + self.outcome.seq = seq + self.outcome.partition_digest = digest + self.outcome.source_bytes_digest = bytes_digest + + def committed(self, *, projection_from_disk: bool = False) -> None: + """Mark the prepared entry committed after the primary write returned.""" + + if self._seq is None or self._entry_id is None or self._directory is None: + return + if self.outcome.failure is not None: + return + marker: dict[str, Any] = { + "schema_version": OUTBOX_COMMIT_SCHEMA, + "entry_id": self._entry_id, + "committed_at": utc_now_text(), + } + try: + if projection_from_disk: + if self._state_path is None: + raise OutboxError("outbox_commit_marker_failed", "state path is required") + projection = self._project(self._state_path.read_text(encoding="utf-8")) + digest = partition_digest(projection) + if digest == latest_partition_digest(self._directory): + # The event changed nothing the shadow compares; retire the + # prepared entry so no crash-window resolution is needed. + (self._directory / entry_file_name(self._seq, self._entry_id, "prepared")).unlink( + missing_ok=True + ) + self.outcome.entry_id = None + self.outcome.seq = None + self.outcome.skipped_reason = "partition_unchanged" + return + marker["projection"] = canonical_value(projection) + marker["partition_digest"] = digest + self.outcome.partition_digest = digest + durable_write_json( + self._directory / entry_file_name(self._seq, self._entry_id, "committed"), + marker, + ) + except Exception as error: # noqa: BLE001 - the primary write already landed + self._fail("outbox_commit_marker_failed", error) + + +SourceProbe = Callable[[OutboxEntry], str] +"""Return ``committed``, ``abandoned`` or ``unproved`` for a prepared-only entry.""" + + +def resolve_prepared_only_entry( + entry: OutboxEntry, + *, + markdown_text_reader: Callable[[], str] | None, + lease_record_reader: Callable[[str], dict[str, Any] | None] | None, + event_presence_reader: Callable[[str], bool] | None, +) -> str: + """Decide what a prepared entry without a committed marker means. + + The caller must hold the partition's primary lock (or have proven it free), + otherwise the source may still be mid-write. + """ + + source = _as_object(entry.prepared.get("source")) + kind = source.get("kind") + if kind == SOURCE_MARKDOWN and markdown_text_reader is not None: + current_digest = text_digest(markdown_text_reader()) + if current_digest == source.get("bytes_digest"): + return "committed" + if current_digest == source.get("previous_bytes_digest"): + return "abandoned" + return "unproved" + if kind == SOURCE_TASK_LEASE and lease_record_reader is not None: + planned = _as_object(source.get("lease")) + if not planned: + return "unproved" + current = lease_record_reader(str(planned.get("todo_id") or "")) + keys = ("version", "lease_epoch", "status", "updated_at") + if current is not None and all(current.get(key) == planned.get(key) for key in keys): + return "committed" + previous = _as_object(source.get("previous_lease")) + if not previous and current is None: + return "abandoned" + if previous and current is not None and all( + current.get(key) == previous.get(key) for key in keys + ): + return "abandoned" + return "unproved" + if kind == SOURCE_STATE_EVENT_LOG and event_presence_reader is not None: + event_id = source.get("event_id") + if isinstance(event_id, str) and event_id and event_presence_reader(event_id): + # The append landed but the projection was never recorded; only a + # fresh full-partition capture can say what the state now is. + return "unproved" + return "abandoned" + return "unproved" + + +def entries_by_partition(runtime_root: Path, goal_id: str) -> dict[str, list[OutboxEntry]]: + return { + partition: list_entries(partition_directory(runtime_root, goal_id, partition)) + for partition in PARTITIONS + } + + +def outbox_summary(runtime_root: Path, goal_id: str) -> dict[str, Any]: + """Counts per partition for operator readback; never raises on an empty outbox.""" + + summary: dict[str, Any] = {} + for partition in PARTITIONS: + directory = partition_directory(runtime_root, goal_id, partition) + try: + entries = list_entries(directory) + cursor = read_cursor(directory) + invalid: str | None = None + except OutboxError as error: + entries, cursor, invalid = [], None, error.reason_code + summary[partition] = { + "committed_pending": sum(1 for entry in entries if entry.is_committed), + "prepared_only": sum(1 for entry in entries if not entry.is_committed), + "next_seq": (max((entry.seq for entry in entries), default=0) if entries else 0), + "cursor_last_seq": int(cursor.get("last_seq") or 0) if cursor else None, + "cursor_last_entry_id": cursor.get("last_entry_id") if cursor else None, + "invalid": invalid, + } + return summary + + +def remove_entry_files(entry: OutboxEntry) -> None: + entry.prepared_path.unlink(missing_ok=True) + if entry.committed_path is not None: + entry.committed_path.unlink(missing_ok=True) + + +@dataclass(frozen=True, slots=True) +class SeedSource: + """A full-partition snapshot taken under the partition's primary lock.""" + + partition: str + projection: dict[str, Any] + source_bytes_digest: str | None = None + extra_source: dict[str, Any] = field(default_factory=dict) + + +def write_seed_entry( + *, + runtime_root: Path, + goal_id: str, + seed: SeedSource, + write_class: str = "seed", +) -> OutboxEntry: + """Write a committed full-partition entry (seed or reseed); caller holds the lock.""" + + directory = partition_directory(runtime_root, goal_id, seed.partition) + digest = partition_digest(seed.projection) + seq = next_seq(directory) + source_ref: str = seed.source_bytes_digest if seed.source_bytes_digest else f"seed:{digest}" + entry_id = entry_identity(goal_id=goal_id, partition=seed.partition, seq=seq, source_ref=source_ref) + record = _entry_record( + goal_id=goal_id, + partition=seed.partition, + seq=seq, + entry_id=entry_id, + writer=_writer(write_class, runtime=WRITER_RUNTIME_PYTHON, operation_id=None), + source={ + "kind": SOURCE_MARKDOWN if seed.partition == TODO_PARTITION else SOURCE_TASK_LEASE, + "previous_bytes_digest": None, + "bytes_digest": seed.source_bytes_digest, + "lease": None, + "event_id": None, + **dict(seed.extra_source), + }, + source_root_digest=runtime_root_digest(runtime_root), + projection=seed.projection, + digest=digest, + ) + prepared_path = directory / entry_file_name(seq, entry_id, "prepared") + committed_path = directory / entry_file_name(seq, entry_id, "committed") + durable_write_json(prepared_path, record) + marker = { + "schema_version": OUTBOX_COMMIT_SCHEMA, + "entry_id": entry_id, + "committed_at": utc_now_text(), + } + durable_write_json(committed_path, marker) + return OutboxEntry( + partition=seed.partition, + seq=seq, + entry_id=entry_id, + prepared_path=prepared_path, + committed_path=committed_path, + prepared=record, + committed=marker, + ) + + +def lease_seed_source(runtime_root: Path, goal_id: str) -> SeedSource: + """Snapshot the lease partition from disk; caller holds the lease lock.""" + + records = read_lease_records(lease_directory(runtime_root, goal_id)) + try: + projection = lease_partition_projection(records, goal_id=goal_id) + except ProjectionValueError as error: + raise OutboxError("outbox_prepare_failed", str(error)) from error + return SeedSource(partition=LEASE_PARTITION, projection=projection) + + +def iter_committed(entries: Iterable[OutboxEntry]) -> list[OutboxEntry]: + return [entry for entry in entries if entry.is_committed] + + +__all__ = [ + "DRAIN_CURSOR_SCHEMA", + "OUTBOX_COMMIT_SCHEMA", + "OUTBOX_ENTRY_SCHEMA", + "SOURCE_MARKDOWN", + "SOURCE_STATE_EVENT_LOG", + "SOURCE_TASK_LEASE", + "CaptureOutcome", + "OutboxEntry", + "OutboxError", + "SeedSource", + "TodoPartitionCapture", + "TodoPartitionProjector", + "compact_lease_projection", + "drain_lock_target", + "durable_write_json", + "entries_by_partition", + "entry_file_name", + "entry_identity", + "iter_committed", + "latest_partition_digest", + "lease_directory", + "lease_seed_source", + "list_entries", + "next_seq", + "outbox_root", + "outbox_summary", + "partition_directory", + "read_cursor", + "read_lease_records", + "remove_entry_files", + "resolve_prepared_only_entry", + "runtime_root_digest", + "utc_now_text", + "write_cursor", + "write_seed_entry", +] diff --git a/loopx/control_plane/coordination/local_authority_shadow_outbox.ts b/loopx/control_plane/coordination/local_authority_shadow_outbox.ts new file mode 100644 index 0000000000..84a4b8a3c6 --- /dev/null +++ b/loopx/control_plane/coordination/local_authority_shadow_outbox.ts @@ -0,0 +1,271 @@ +import { createHash } from "node:crypto"; +import { readdir, readFile } from "node:fs/promises"; +import { join } from "node:path"; + +import type { JsonObject } from "../effect_program.ts"; +import { durableWriteJson } from "../effect_runtime_io.ts"; +import { canonicalAuthorityBytes } from "./authority_store_codec.ts"; + +/** + * Lease-partition side of the local authority shadow outbox. + * + * The task-lease writers already hold the goal's lease lock when they persist + * a record; this module lets them append a two-phase outbox entry for exactly + * that partition inside the same lock. It never touches the candidate store, + * never blocks, and never throws into the lease write: every failure is + * returned on the capture object so the writer can attach typed evidence. + */ + +export const LOCAL_AUTHORITY_SHADOW_BINDING_SCHEMA = "loopx_local_authority_shadow_binding_v0"; +export const LOCAL_AUTHORITY_SHADOW_OUTBOX_ENTRY_SCHEMA = + "loopx_local_authority_shadow_outbox_entry_v0"; +export const LOCAL_AUTHORITY_SHADOW_OUTBOX_COMMIT_SCHEMA = + "loopx_local_authority_shadow_outbox_commit_v0"; +export const LOCAL_AUTHORITY_SHADOW_DRAIN_CURSOR_SCHEMA = + "loopx_local_authority_shadow_drain_cursor_v0"; +export const LEASE_PARTITION = "leases"; +const ENTRY_FILE = /^(\d{10})-(local-shadow-tx-[0-9a-f]{64})\.(prepared|committed)\.json$/u; +const LEASE_FILE = /^[A-Za-z0-9_.-]+\.json$/u; +const LEASE_SOURCE_FIELDS = ["todo_id", "version", "lease_epoch", "status", "updated_at"] as const; + +export interface LocalAuthorityShadowBinding { + schema_version: typeof LOCAL_AUTHORITY_SHADOW_BINDING_SCHEMA; + mode: "file_one_way"; +} + +/** Decode the optional per-request binding; anything but the exact contract is "absent". */ +export function decodeLocalAuthorityShadowBinding( + value: unknown, +): LocalAuthorityShadowBinding | null { + if (value === null || value === undefined || typeof value !== "object" || Array.isArray(value)) { + return null; + } + const record = value as Record; + const keys = Object.keys(record); + if ( + keys.length !== 2 || + record.schema_version !== LOCAL_AUTHORITY_SHADOW_BINDING_SCHEMA || + record.mode !== "file_one_way" + ) { + return null; + } + return { schema_version: LOCAL_AUTHORITY_SHADOW_BINDING_SCHEMA, mode: "file_one_way" }; +} + +export function sha256Digest(input: Uint8Array | string): string { + return `sha256:${createHash("sha256").update(input).digest("hex")}`; +} + +/** Digest of the exact bytes `atomicWriteJson` persists for a lease record. */ +export function leaseRecordDigest(record: JsonObject): string { + return sha256Digest(`${JSON.stringify(record, null, 2)}\n`); +} + +/** Must stay byte-compatible with the Python `entry_identity` derivation. */ +export function outboxEntryIdentity( + goalId: string, + partition: string, + seq: number, + sourceRef: string, +): string { + const digest = createHash("sha256") + .update(canonicalAuthorityBytes({ + goal_id: goalId, + partition, + seq, + source_ref: sourceRef, + })) + .digest("hex"); + return `local-shadow-tx-${digest}`; +} + +export function outboxPartitionDirectory( + runtimeRoot: string, + goalId: string, + partition: string, +): string { + return join(runtimeRoot, "authority-shadow", "outbox", goalId, partition); +} + +export function outboxEntryFileName(seq: number, entryId: string, phase: "prepared" | "committed"): string { + return `${String(seq).padStart(10, "0")}-${entryId}.${phase}.json`; +} + +function isMissing(error: unknown): boolean { + return (error as NodeJS.ErrnoException | null)?.code === "ENOENT"; +} + +async function nextSeq(directory: string): Promise { + let highest = 0; + try { + for (const name of await readdir(directory)) { + const match = ENTRY_FILE.exec(name); + if (match !== null) highest = Math.max(highest, Number(match[1])); + } + } catch (error) { + if (!isMissing(error)) throw error; + } + try { + const raw: unknown = JSON.parse(await readFile(join(directory, "drain-cursor.json"), "utf8")); + if (raw !== null && typeof raw === "object") { + const lastSeq = (raw as Record).last_seq; + if (typeof lastSeq === "number" && Number.isSafeInteger(lastSeq)) { + highest = Math.max(highest, lastSeq); + } + } + } catch (error) { + if (!isMissing(error)) throw error; + } + return highest + 1; +} + +async function readLeasePartition( + leaseDirectory: string, + plannedStem: string, + plannedLease: JsonObject, +): Promise { + const records = new Map(); + let names: string[] = []; + try { + names = await readdir(leaseDirectory); + } catch (error) { + if (!isMissing(error)) throw error; + } + for (const name of names) { + if (!LEASE_FILE.test(name) || name.startsWith(".")) continue; + const stem = name.slice(0, -".json".length); + if (stem === plannedStem) continue; + const raw: unknown = JSON.parse(await readFile(join(leaseDirectory, name), "utf8")); + if (raw !== null && typeof raw === "object" && !Array.isArray(raw)) { + records.set(stem, raw as JsonObject); + } + } + records.set(plannedStem, plannedLease); + const stems = [...records.keys()]; + stems.sort((left, right) => (left < right ? -1 : left > right ? 1 : 0)); + return stems.map((stem) => ({ file_stem: stem, record: records.get(stem) as JsonObject })); +} + +function leaseSourceFacts(record: JsonObject | null): JsonObject | null { + if (record === null) return null; + const facts: JsonObject = {}; + for (const field of LEASE_SOURCE_FIELDS) { + const value = record[field]; + if (value === undefined) continue; + if (typeof value === "string" || typeof value === "number" || value === null) { + facts[field] = value; + } + } + return facts; +} + +export interface LeaseOutboxCaptureInput { + runtime_root: string; + goal_id: string; + lease_directory: string; + write_class: string; + operation_id: string | null; + previous_lease: JsonObject | null; + planned_lease: JsonObject; +} + +export interface LeaseOutboxCapture { + entry_id: string | null; + seq: number | null; + source_bytes_digest: string | null; + failure: { reason_code: string; error_class: string } | null; + /** Write the committed marker after the lease record landed. Never throws. */ + commit(): Promise; +} + +function failureOf(reasonCode: string, error: unknown): { reason_code: string; error_class: string } { + return { + reason_code: reasonCode, + error_class: error instanceof Error ? error.constructor.name : typeof error, + }; +} + +/** + * Record a prepared lease-partition entry for the record about to be written. + * + * Call inside the lease lock, right before `atomicWriteJson(leasePath, planned)`; + * call `commit()` right after it returns. A returned failure never blocks the + * primary write. + */ +export async function beginLeaseOutboxEntry( + input: LeaseOutboxCaptureInput, +): Promise { + const inert: LeaseOutboxCapture = { + entry_id: null, + seq: null, + source_bytes_digest: null, + failure: null, + async commit(): Promise {}, + }; + const plannedStem = typeof input.planned_lease.todo_id === "string" + ? input.planned_lease.todo_id + : ""; + if (plannedStem.length === 0) { + return { ...inert, failure: { reason_code: "outbox_prepare_failed", error_class: "MissingTodoId" } }; + } + const directory = outboxPartitionDirectory(input.runtime_root, input.goal_id, LEASE_PARTITION); + try { + const projection = { leases: await readLeasePartition(input.lease_directory, plannedStem, input.planned_lease) }; + const bytesDigest = leaseRecordDigest(input.planned_lease); + const seq = await nextSeq(directory); + const entryId = outboxEntryIdentity(input.goal_id, LEASE_PARTITION, seq, bytesDigest); + const entry: JsonObject = { + schema_version: LOCAL_AUTHORITY_SHADOW_OUTBOX_ENTRY_SCHEMA, + goal_id: input.goal_id, + partition: LEASE_PARTITION, + seq, + entry_id: entryId, + writer: { + runtime: "typescript", + write_class: input.write_class, + operation_id: input.operation_id, + }, + source: { + kind: "task_lease_record", + previous_bytes_digest: input.previous_lease === null + ? null + : leaseRecordDigest(input.previous_lease), + bytes_digest: bytesDigest, + lease: leaseSourceFacts(input.planned_lease), + previous_lease: leaseSourceFacts(input.previous_lease), + event_id: null, + }, + source_root_digest: sha256Digest(input.runtime_root), + projection, + partition_digest: null, + prepared_at: new Date().toISOString(), + }; + await durableWriteJson(join(directory, outboxEntryFileName(seq, entryId, "prepared")), entry); + let committed = false; + const capture: LeaseOutboxCapture = { + entry_id: entryId, + seq, + source_bytes_digest: bytesDigest, + failure: null, + async commit(): Promise { + if (committed) return; + committed = true; + try { + await durableWriteJson( + join(directory, outboxEntryFileName(seq, entryId, "committed")), + { + schema_version: LOCAL_AUTHORITY_SHADOW_OUTBOX_COMMIT_SCHEMA, + entry_id: entryId, + committed_at: new Date().toISOString(), + }, + ); + } catch (error) { + capture.failure = failureOf("outbox_commit_marker_failed", error); + } + }, + }; + return capture; + } catch (error) { + return { ...inert, failure: failureOf("outbox_prepare_failed", error) }; + } +} diff --git a/loopx/control_plane/coordination/local_authority_shadow_projection.py b/loopx/control_plane/coordination/local_authority_shadow_projection.py new file mode 100644 index 0000000000..b2b3631c65 --- /dev/null +++ b/loopx/control_plane/coordination/local_authority_shadow_projection.py @@ -0,0 +1,204 @@ +"""Pure projection rules shared by the local authority shadow capture and parity. + +Everything here is a deterministic function of its inputs: no file, lock, +registry, or effect-runtime access. The same compact field sets and canonical +bytes define the source digest, the outbox partition digest, and the candidate +readback comparison, so no two code paths can disagree about what "the same +coordination state" means. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Iterable, Mapping +from typing import Any + + +LOCAL_AUTHORITY_SHADOW_PROJECTION_SCHEMA_V0 = "loopx_local_authority_shadow_projection_v0" +LOCAL_AUTHORITY_SHADOW_PROJECTION_SCHEMA_V1 = "loopx_local_authority_shadow_projection_v1" +TODO_PARTITION = "todos" +LEASE_PARTITION = "leases" +PARTITIONS: tuple[str, ...] = (TODO_PARTITION, LEASE_PARTITION) + +TODO_FIELDS: tuple[str, ...] = ( + "todo_id", + "role", + "status", + "claimed_by", + "bound_agent", + "goal_bound", + "blocks_agent", + "excluded_agents", + "global_gate", + "task_class", + "action_kind", + "required_write_scopes", + "required_capabilities", + "continuation_policy", + "successor_todo_ids", + "no_followup", + "completion_continuation", +) +LEASE_FIELDS: tuple[str, ...] = ( + "todo_id", + "owner", + "idempotency_key", + "write_scopes", + "version", + "lease_epoch", + "acquired_at", + "updated_at", + "expires_at", + "released_at", + "status", +) + + +class ProjectionValueError(ValueError): + """A value cannot be part of a canonical shadow projection.""" + + +def _reject_floats(value: object, path: str) -> None: + # Python `1.0` and JavaScript `1` would canonicalize differently, so a + # float anywhere in a compared projection would manufacture a false + # divergence between the Python source digest and the TypeScript head. + if isinstance(value, bool) or value is None or isinstance(value, (str, int)): + return + if isinstance(value, float): + raise ProjectionValueError(f"float values are not allowed in shadow projections ({path})") + if isinstance(value, Mapping): + for key, item in value.items(): + if not isinstance(key, str): + raise ProjectionValueError(f"non-string key in shadow projection ({path})") + _reject_floats(item, f"{path}.{key}") + return + if isinstance(value, (list, tuple)): + for index, item in enumerate(value): + _reject_floats(item, f"{path}[{index}]") + return + raise ProjectionValueError( + f"unsupported value type {type(value).__name__} in shadow projection ({path})" + ) + + +def canonical_bytes(value: object) -> bytes: + """Sorted-key, minimal-separator UTF-8 JSON; floats and NaN are rejected.""" + + _reject_floats(value, "$") + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +def canonical_value(value: object) -> Any: + """Round-trip a value through canonical JSON so key order is normalized.""" + + return json.loads(canonical_bytes(value)) + + +def sha256_digest(value: object) -> str: + return "sha256:" + hashlib.sha256(canonical_bytes(value)).hexdigest() + + +def text_digest(text: str) -> str: + return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def compact_todo(raw: object) -> dict[str, Any] | None: + """Keep only the coordination facts of one todo item; drop prose.""" + + if not isinstance(raw, Mapping): + return None + todo_id = str(raw.get("todo_id") or "").strip() + if not todo_id: + return None + compact = {field: raw[field] for field in TODO_FIELDS if field in raw} + compact["todo_id"] = todo_id + if "status" not in compact and isinstance(raw.get("done"), bool): + compact["status"] = "done" if raw["done"] else "open" + return dict(canonical_value(compact)) + + +def compact_lease(raw: object, *, goal_id: str, file_stem: str) -> dict[str, Any]: + """Keep only the lease fence facts of one on-disk lease record.""" + + if not isinstance(raw, Mapping): + raise ProjectionValueError("task lease must contain an object") + if raw.get("goal_id") != goal_id or raw.get("todo_id") != file_stem: + raise ProjectionValueError("task lease identity does not match its shadow source") + return dict(canonical_value({field: raw[field] for field in LEASE_FIELDS if field in raw})) + + +def todo_partition_projection( + *, + handoff_mode: str, + todos: Iterable[object], +) -> dict[str, Any]: + """The state guarded by the goal's active-state file lock.""" + + compact = [item for item in (compact_todo(raw) for raw in todos) if item is not None] + compact.sort(key=lambda item: str(item["todo_id"])) + return {"handoff_mode": handoff_mode, "todos": compact} + + +def lease_partition_projection( + records: Iterable[tuple[str, object]], + *, + goal_id: str, +) -> dict[str, Any]: + """The state guarded by the goal's task-lease lock. + + ``records`` pairs each lease file stem with its decoded JSON object. + """ + + leases = [ + compact_lease(raw, goal_id=goal_id, file_stem=stem) + for stem, raw in sorted(records, key=lambda pair: pair[0]) + ] + return {"leases": leases} + + +def partition_digest(projection: Mapping[str, Any]) -> str: + return sha256_digest(dict(projection)) + + +def head_comparison_view(head: Mapping[str, Any]) -> dict[str, Any]: + """The part of a candidate head that parity compares against the source.""" + + return { + "handoff_mode": head.get("handoff_mode"), + "todos": head.get("todos"), + "leases": head.get("leases"), + } + + +def head_digest(head: Mapping[str, Any]) -> str: + return sha256_digest(head_comparison_view(head)) + + +__all__ = [ + "LEASE_FIELDS", + "LEASE_PARTITION", + "LOCAL_AUTHORITY_SHADOW_PROJECTION_SCHEMA_V0", + "LOCAL_AUTHORITY_SHADOW_PROJECTION_SCHEMA_V1", + "PARTITIONS", + "TODO_FIELDS", + "TODO_PARTITION", + "ProjectionValueError", + "canonical_bytes", + "canonical_value", + "compact_lease", + "compact_todo", + "head_comparison_view", + "head_digest", + "lease_partition_projection", + "partition_digest", + "sha256_digest", + "text_digest", + "todo_partition_projection", +] diff --git a/loopx/control_plane/effect_runtime_handlers.ts b/loopx/control_plane/effect_runtime_handlers.ts index 9f14d6162b..7d03efda1d 100644 --- a/loopx/control_plane/effect_runtime_handlers.ts +++ b/loopx/control_plane/effect_runtime_handlers.ts @@ -98,7 +98,11 @@ import { executeTaskLeaseAcquire, } from "./work_items/task_lease_acquire.ts"; import { executeTaskLeaseLifecycle } from "./work_items/task_lease_lifecycle.ts"; -import { recordLocalAuthorityShadow } from "./coordination/local_authority_shadow.ts"; +import { + commitLocalAuthorityShadowEntry, + readLocalAuthorityShadow, + recordLocalAuthorityShadow, +} from "./coordination/local_authority_shadow.ts"; import { evaluateTaskLeaseLifecycleDecision } from "./work_items/task_lease_lifecycle_decision.ts"; import { bootstrapCoordinationRuntimeShadow, @@ -399,6 +403,8 @@ export function createEffectRuntimeHandlers( ["task_lease.write_scopes.overlap", evaluateTaskLeaseWriteScopesOverlap], ["quota.monitor_poll.commit", evaluateQuotaMonitorPollCommit], ["coordination.local_authority_shadow.record", recordLocalAuthorityShadow], + ["coordination.local_authority_shadow.commit_entry", commitLocalAuthorityShadowEntry], + ["coordination.local_authority_shadow.read", readLocalAuthorityShadow], [ "effect.program_from_ordered_steps", (params) => effectProgramFromOrderedSteps( diff --git a/loopx/control_plane/effect_runtime_io.ts b/loopx/control_plane/effect_runtime_io.ts index 64fffcadee..a2e70eeeb4 100644 --- a/loopx/control_plane/effect_runtime_io.ts +++ b/loopx/control_plane/effect_runtime_io.ts @@ -457,3 +457,39 @@ export async function appendJsonLine( await handle.close(); } } + +async function syncDirectoryForDurableWrite(directory: string): Promise { + if (process.platform === "win32") return; + const handle = await open(directory, "r"); + try { + await handle.sync(); + } finally { + await handle.close(); + } +} + +/** + * Write JSON so that a crash cannot leave a torn or unlinked file behind: + * temp file in the same directory, fsync, atomic rename, directory fsync. + */ +export async function durableWriteJson( + path: string, + payload: JsonObject, +): Promise { + const directory = dirname(path); + await mkdir(directory, { recursive: true, mode: 0o700 }); + const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`; + const handle = await open(temporary, "wx", 0o600); + try { + try { + await handle.writeFile(`${JSON.stringify(payload, null, 1)}\n`, "utf8"); + await handle.sync(); + } finally { + await handle.close(); + } + await rename(temporary, path); + await syncDirectoryForDurableWrite(directory); + } finally { + await rm(temporary, { force: true }); + } +} diff --git a/loopx/control_plane/todos/goal_todo_projection.py b/loopx/control_plane/todos/goal_todo_projection.py index 583a39473b..14f3e1abd5 100644 --- a/loopx/control_plane/todos/goal_todo_projection.py +++ b/loopx/control_plane/todos/goal_todo_projection.py @@ -364,12 +364,39 @@ def todo_summaries_from_fields( uncapped_todo_count=uncapped_todo_count, ) +def project_goal_todo_items( + goal: dict[str, Any] | None, + *, + state_text: str, + state_path: Path, + rollout_events: list[dict[str, Any]], +) -> list[dict[str, Any]]: + """Every user and agent todo item projected from one active-state text. + + Same items ``list_goal_todos`` returns without filters, computed from the + caller's text instead of the file so it can run inside the writer's lock. + """ + + return goal_todo_summaries( + goal, + state_text=state_text, + state_path=state_path, + rollout_events=rollout_events, + roles=["user", "agent"], + status=None, + todo_id=None, + agent_id=None, + limit=None, + ).todos + + __all__ = [ "GoalTodoSummaries", "empty_todo_summary", "filtered_todo_summary", "goal_todo_summaries", "merge_todo_projection_fields", + "project_goal_todo_items", "summary_items", "todo_summaries_from_fields", ] diff --git a/loopx/help_surface.py b/loopx/help_surface.py index 343f68bd1e..cc6e142d60 100644 --- a/loopx/help_surface.py +++ b/loopx/help_surface.py @@ -288,6 +288,7 @@ MANPAGE_COMMAND_HELP_ONLY = frozenset( { "archive-runtime", + "authority-shadow", "backup-state", "capability", "chat-endpoint", diff --git a/pyproject.toml b/pyproject.toml index 5880ca09a7..6f5725ebe9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,6 +126,8 @@ files = [ "loopx/control_plane/coordination/executor.py", "loopx/control_plane/coordination/file_provider.py", "loopx/control_plane/coordination/head.py", + "loopx/control_plane/coordination/local_authority_shadow_outbox.py", + "loopx/control_plane/coordination/local_authority_shadow_projection.py", "loopx/control_plane/quota/effect_program.py", "loopx/control_plane/quota/states.py", "loopx/control_plane/runtime/event_store_migration_bridge.py", diff --git a/tests/control_plane/test_local_authority_shadow_cli_e2e.py b/tests/control_plane/test_local_authority_shadow_cli_e2e.py index 8ab7cccd0d..d90875a7fb 100644 --- a/tests/control_plane/test_local_authority_shadow_cli_e2e.py +++ b/tests/control_plane/test_local_authority_shadow_cli_e2e.py @@ -467,3 +467,54 @@ def test_product_cli_runtime_root_override_keeps_one_candidate_lineage( assert not (registry_runtime / "goals" / goal_id / "task-leases").exists() assert store_path.is_relative_to(override_runtime) assert todo_id in state.read_text(encoding="utf-8") + + +def test_product_cli_authority_shadow_status_and_drain_read_without_creating_lineage( + tmp_path: Path, +) -> None: + goal_id = "shadow-cli-drain" + registry, _state, runtime_root = _workspace(tmp_path, goal_id=goal_id) + + default_off = _cli(registry, runtime_root, "authority-shadow", "status", "--goal-id", goal_id) + assert default_off["ok"] is True + assert default_off["schema_version"] == "loopx_authority_shadow_cli_v0" + assert default_off["config"] == {"enabled": False, "mode": None, "status": "disabled"} # type: ignore[index] + assert default_off["candidate"]["status"] == "missing" # type: ignore[index] + assert default_off["store_bytes"] == 0 + assert not (runtime_root / "authority-shadow").exists() + idle = _cli(registry, runtime_root, "authority-shadow", "drain", "--goal-id", goal_id) + assert idle["ok"] is True and idle["outcome"] == "nothing_pending" + assert idle["drained_count"] == 0 and idle["config_enabled"] is False + assert not (runtime_root / "authority-shadow").exists() + + _cli( + registry, + runtime_root, + "configure-goal", + "--goal-id", + goal_id, + "--local-authority-shadow-file", + "--execute", + ) + observed = _add_todo(registry, runtime_root, goal_id=goal_id, text="Observed through the v0 path.") + assert observed["authority_shadow"]["outcome"] == "captured" # type: ignore[index] + + status = _cli(registry, runtime_root, "authority-shadow", "status", "--goal-id", goal_id) + assert status["ok"] is True + assert status["config"]["status"] == "enabled" # type: ignore[index] + assert status["outbox"]["todos"]["committed_pending"] == 0 # type: ignore[index] + assert status["outbox"]["leases"]["prepared_only"] == 0 # type: ignore[index] + candidate = status["candidate"] + assert candidate["status"] == "loaded" # type: ignore[index] + assert candidate["cursor"] == "1" # type: ignore[index] + assert candidate["head_schema_version"] == "loopx_local_authority_shadow_projection_v0" # type: ignore[index] + assert candidate["codec_agreement"] is True # type: ignore[index] + assert candidate["partitions"] == {"todos": None, "leases": None} # type: ignore[index] + assert status["store_bytes"] > 0 and status["retention_pressure"] is False + assert str(runtime_root) not in json.dumps(status) + + drained = _cli(registry, runtime_root, "authority-shadow", "drain", "--goal-id", goal_id) + assert drained["ok"] is True and drained["outcome"] == "nothing_pending" + assert drained["config_enabled"] is True + _store_path, store = _store_document(runtime_root, goal_id) + assert store["cursor"] == "1" diff --git a/tests/control_plane/test_local_authority_shadow_drain.py b/tests/control_plane/test_local_authority_shadow_drain.py new file mode 100644 index 0000000000..8f7b800fcf --- /dev/null +++ b/tests/control_plane/test_local_authority_shadow_drain.py @@ -0,0 +1,479 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from loopx.control_plane.coordination import local_authority_shadow_adapter as adapter +from loopx.control_plane.coordination import local_authority_shadow_outbox as outbox +from loopx.control_plane.coordination.local_authority_shadow_projection import ( + head_digest, + partition_digest, + text_digest, +) +from loopx.file_lock import exclusive_file_lock +from loopx.history import load_registry +from loopx.registry import find_registry_goal +from loopx.todos import add_goal_todo, list_goal_todos + + +GOAL_ID = "goal-drain" + + +def _fixture(tmp_path: Path) -> tuple[Path, Path, Path]: + repo = tmp_path / "repo" + repo.mkdir() + state = repo / "ACTIVE_GOAL_STATE.md" + state.write_text( + "---\n" + f"goal_id: {GOAL_ID}\n" + "handoff_mode: hard_lease\n" + "updated_at: 2026-09-03T00:00:00+00:00\n" + "---\n\n" + "## Agent Todo\n\n", + encoding="utf-8", + ) + runtime_root = tmp_path / "runtime" + registry = tmp_path / "registry.json" + registry.write_text( + json.dumps( + { + "common_runtime_root": str(runtime_root), + "goals": [ + { + "id": GOAL_ID, + "domain": "harness_self_improvement", + "status": "active", + "repo": str(repo), + "state_file": state.name, + "adapter": {"kind": "harness_self_improvement"}, + "coordination": { + "agent_model": "peer_v1", + "registered_agents": ["agent-a"], + }, + } + ], + } + ), + encoding="utf-8", + ) + return registry, state, runtime_root + + +def _record_todo_write( + registry: Path, + state: Path, + runtime_root: Path, + text: str, + *, + mark_committed: bool = True, + write_file: bool = True, +) -> outbox.TodoPartitionCapture: + """Simulate a hooked writer: capture around one add_goal_todo write.""" + + original = state.read_text(encoding="utf-8") + goal = find_registry_goal(load_registry(registry), GOAL_ID) + capture = outbox.TodoPartitionCapture.begin( + enabled=True, + runtime_root=runtime_root, + goal_id=GOAL_ID, + state_path=state, + write_class="todo_add", + original_text=original, + projector=adapter.todo_partition_projector(goal, state_path=state), + ) + if write_file: + result = add_goal_todo( + registry_path=registry, + goal_id=GOAL_ID, + role="agent", + text=text, + task_class="advancement_task", + ) + assert result["ok"] is True + new_text = state.read_text(encoding="utf-8") + else: + # The bytes the writer would have produced, without producing them. + result = add_goal_todo( + registry_path=registry, + goal_id=GOAL_ID, + role="agent", + text=text, + task_class="advancement_task", + ) + assert result["ok"] is True + new_text = state.read_text(encoding="utf-8") + state.write_text(original, encoding="utf-8") + capture.prepare(new_text) + assert capture.outcome.failure is None + assert capture.outcome.entry_id is not None + if mark_committed: + capture.committed() + return capture + + +def _drain(registry: Path, runtime_root: Path, **overrides: object) -> adapter.DrainResult: + return adapter.drain_local_authority_shadow_outbox( + registry_path=registry, + runtime_root=runtime_root, + goal_id=GOAL_ID, + **overrides, # type: ignore[arg-type] + ) + + +def _todo_dir(runtime_root: Path) -> Path: + return outbox.partition_directory(runtime_root, GOAL_ID, "todos") + + +def test_drain_delivers_each_committed_entry_once_in_order_and_verifies_readback(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + captures = [ + _record_todo_write(registry, state, runtime_root, f"Fact {index}") for index in range(3) + ] + assert [capture.outcome.seq for capture in captures] == [1, 2, 3] + + result = _drain(registry, runtime_root) + + assert result.ok is True + assert result.outcome == "drained" + assert result.config_enabled is False + assert (result.delivered, result.replayed, result.no_op) == (3, 0, 0) + assert result.pending_after == 0 and result.prepared_only_after == 0 + assert result.budget_exhausted is False + assert result.candidate_readback_verified is True + assert result.last_cursor == "3" + assert (result.cursor_before, result.cursor_after, result.drained_count) == (None, "3", 3) + payload = result.to_payload() + assert payload["ok"] is True and payload["drained_count"] == 3 and payload["cursor_after"] == "3" + assert [item["outcome"] for item in result.entries] == ["delivered"] * 3 + assert [item["cursor"] for item in result.entries] == ["1", "2", "3"] + assert [item["entry_id"] for item in result.entries] == [ + capture.outcome.entry_id for capture in captures + ] + assert result.store_identity is not None and result.store_identity.startswith("file:") + assert list(_todo_dir(runtime_root).iterdir()) == [_todo_dir(runtime_root) / "drain-cursor.json"] + cursor = outbox.read_cursor(_todo_dir(runtime_root)) + assert cursor is not None + assert cursor["last_seq"] == 3 and cursor["last_entry_id"] == captures[-1].outcome.entry_id + assert cursor["last_partition_digest"] == captures[-1].outcome.partition_digest + + view = adapter.read_local_authority_shadow(runtime_root=runtime_root, goal_id=GOAL_ID, scan_limit=10) + assert view["status"] == "loaded" + head = view["head"] + assert head["schema_version"] == "loopx_local_authority_shadow_projection_v1" + assert head["partitions"]["todos"] == { + "seq": 3, + "partition_digest": captures[-1].outcome.partition_digest, + } + assert head["partitions"]["leases"] is None + listed = list_goal_todos(registry_path=registry, goal_id=GOAL_ID, runtime_root_arg=str(runtime_root)) + assert [todo["todo_id"] for todo in head["todos"]] == sorted( + todo["todo_id"] for todo in listed["todos"] + ) + assert view["head_digest"] == head_digest(head) == result.head_digest + assert [tx["operation_id"] for tx in view["scan"]["transactions"]] == [ + capture.outcome.entry_id for capture in captures + ] + receipts = [tx["receipts"][0] for tx in view["scan"]["transactions"]] + assert all(receipt["source_transaction_correlated"] is True for receipt in receipts) + assert all(receipt["durable_source_outbox"] is True for receipt in receipts) + assert all(receipt["parity_verdict"] == "not_evaluated" for receipt in receipts) + assert [receipt["source_bytes_digest"] for receipt in receipts] == [ + capture.outcome.source_bytes_digest for capture in captures + ] + assert str(runtime_root) not in json.dumps(view) + + again = _drain(registry, runtime_root) + assert again.outcome == "nothing_pending" and again.ok is True + + +def test_drain_replays_when_store_committed_but_cursor_was_not_written( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + registry, state, runtime_root = _fixture(tmp_path) + capture = _record_todo_write(registry, state, runtime_root, "Crash after store commit") + + def crash(*_args: object, **_kwargs: object) -> None: + raise OSError("simulated crash before the drain cursor landed") + + monkeypatch.setattr(outbox, "write_cursor", crash) + first = _drain(registry, runtime_root) + assert first.outcome == "stopped" + assert first.reason_code == "shadow_drain_failed" + assert first.pending_after == 1 + monkeypatch.undo() + + second = _drain(registry, runtime_root) + assert second.ok is True + assert (second.delivered, second.replayed) == (0, 1) + assert second.entries[0]["entry_id"] == capture.outcome.entry_id + assert second.entries[0]["cursor"] == "1" + assert second.pending_after == 0 + assert second.candidate_readback_verified is True + view = adapter.read_local_authority_shadow(runtime_root=runtime_root, goal_id=GOAL_ID, scan_limit=10) + assert len(view["scan"]["transactions"]) == 1 + + +def test_drain_defers_when_another_drainer_holds_the_lock(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + _record_todo_write(registry, state, runtime_root, "Pending behind a drainer") + with exclusive_file_lock( + outbox.drain_lock_target(runtime_root, GOAL_ID), timeout_seconds=1.0, operation="test_hold" + ): + result = _drain(registry, runtime_root, lock_timeout_seconds=0.05) + assert result.outcome == "drain_deferred" + assert result.reason_code == "drain_lock_busy" + assert result.ok is False + assert result.pending_after == 1 + assert not (runtime_root / "authority-shadow" / "file").exists() + + +def test_drain_batch_is_bounded_and_reports_what_it_left(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + for index in range(3): + _record_todo_write(registry, state, runtime_root, f"Bounded {index}") + + first = _drain(registry, runtime_root, max_entries=2) + assert first.outcome == "drained" and first.ok is True + assert first.delivered == 2 + assert first.budget_exhausted is True + assert first.pending_after == 1 + assert first.candidate_readback_verified is True + + second = _drain(registry, runtime_root) + assert second.delivered == 1 and second.pending_after == 0 + assert (second.cursor_before, second.cursor_after) == ("2", "3") + view = adapter.read_local_authority_shadow(runtime_root=runtime_root, goal_id=GOAL_ID) + assert view["cursor"] == "3" + assert view["head"]["partitions"]["todos"]["seq"] == 3 + + +def test_drain_stops_in_order_when_the_store_boundary_misbehaves( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + registry, state, runtime_root = _fixture(tmp_path) + for index in range(3): + _record_todo_write(registry, state, runtime_root, f"Ordered {index}") + real = adapter.effect_runtime_result + calls: list[str] = [] + + def flaky(method: str, params: object, **kwargs: object) -> object: + calls.append(method) + if method == "coordination.local_authority_shadow.commit_entry" and len(calls) == 2: + return {"schema_version": "garbage"} + return real(method, params, **kwargs) + + monkeypatch.setattr(adapter, "effect_runtime_result", flaky) + result = _drain(registry, runtime_root) + assert result.outcome == "stopped" + assert result.delivered == 1 + assert result.stopped_at is not None + assert result.stopped_at["seq"] == 2 + assert result.stopped_at["reason_code"] == "shadow_commit_entry_result_invalid" + assert result.pending_after == 2 + assert [entry.seq for entry in outbox.list_entries(_todo_dir(runtime_root))] == [2, 3] + monkeypatch.undo() + + recovered = _drain(registry, runtime_root) + assert recovered.delivered == 2 and recovered.pending_after == 0 + + +def test_prepared_only_entries_resolve_only_under_a_free_primary_lock(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + proven = _record_todo_write(registry, state, runtime_root, "Marker lost after write", mark_committed=False) + with exclusive_file_lock(state, timeout_seconds=1.0, operation="writer_in_flight"): + busy = _drain(registry, runtime_root) + assert busy.outcome == "drained" + assert busy.in_flight_partitions == ["todos"] + assert busy.prepared_only_after == 1 + assert busy.delivered == 0 + + result = _drain(registry, runtime_root) + assert result.ok is True + assert result.delivered == 1 and result.no_op == 0 + assert result.entries[0]["resolution"] == "committed_proven_by_readback" + assert result.entries[0]["entry_id"] == proven.outcome.entry_id + view = adapter.read_local_authority_shadow(runtime_root=runtime_root, goal_id=GOAL_ID, scan_limit=5) + assert view["head"]["partitions"]["todos"]["partition_digest"] == proven.outcome.partition_digest + receipt = view["scan"]["transactions"][0]["receipts"][0] + assert receipt["resolution"] == "committed_proven_by_readback" + + abandoned = _record_todo_write( + registry, state, runtime_root, "Never landed", mark_committed=False, write_file=False + ) + result = _drain(registry, runtime_root) + assert result.delivered == 1 and result.no_op == 1 + assert result.entries[0]["resolution"] == "abandoned" + assert result.entries[0]["entry_id"] == abandoned.outcome.entry_id + view = adapter.read_local_authority_shadow(runtime_root=runtime_root, goal_id=GOAL_ID, scan_limit=5) + assert view["cursor"] == "2" + assert view["head"]["partitions"]["todos"]["seq"] == 1 + assert view["scan"]["transactions"][1]["events"][0]["kind"] == "source_transaction_abandoned" + + +def test_unexplained_prepared_only_entry_triggers_reseed_under_the_primary_lock(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + _record_todo_write(registry, state, runtime_root, "Baseline") + assert _drain(registry, runtime_root).delivered == 1 + stale = _record_todo_write(registry, state, runtime_root, "Half-recorded", mark_committed=False) + # An unhooked writer edits the file after the crash: neither digest matches. + state.write_text( + state.read_text(encoding="utf-8") + "\n- [ ] (agent) foreign edit \n", + encoding="utf-8", + ) + + result = _drain(registry, runtime_root) + + assert result.ok is True + assert result.reseeded == 1 + assert result.no_op == 1 + assert [item["resolution"] for item in result.entries] == ["unproved", "seed"] + assert result.entries[0]["entry_id"] == stale.outcome.entry_id + view = adapter.read_local_authority_shadow(runtime_root=runtime_root, goal_id=GOAL_ID, scan_limit=10) + kinds = [tx["events"][0]["kind"] for tx in view["scan"]["transactions"]] + assert kinds == ["source_transaction_delivered", "source_transaction_unproved", "partition_seeded"] + listed = list_goal_todos(registry_path=registry, goal_id=GOAL_ID, runtime_root_arg=str(runtime_root)) + assert [todo["todo_id"] for todo in view["head"]["todos"]] == sorted( + todo["todo_id"] for todo in listed["todos"] + ) + assert len(view["head"]["todos"]) == 3 + assert view["head"]["partitions"]["todos"]["seq"] == 3 + seed_receipt = view["scan"]["transactions"][2]["receipts"][0] + assert seed_receipt["write_class"] == "reseed_after_crash_gap" + assert seed_receipt["source_bytes_digest"] == text_digest(state.read_text(encoding="utf-8")) + + +def test_lease_partition_entries_are_compacted_at_drain(tmp_path: Path) -> None: + registry, _state, runtime_root = _fixture(tmp_path) + directory = outbox.partition_directory(runtime_root, GOAL_ID, "leases") + record = { + "goal_id": GOAL_ID, + "todo_id": "todo_00000000000a", + "owner": "agent-a", + "idempotency_key": "k1", + "write_scopes": ["loopx/**"], + "version": 2, + "lease_epoch": 1, + "acquired_at": "2026-09-03T00:00:00+00:00", + "updated_at": "2026-09-03T00:01:00+00:00", + "expires_at": "2026-09-03T00:31:00+00:00", + "released_at": None, + "status": "active", + "lease_path": "/should/never/be/compared", + "acquire_ttl_seconds": 1800, + } + bytes_digest = text_digest(json.dumps(record, indent=2) + "\n") + entry_id = outbox.entry_identity(goal_id=GOAL_ID, partition="leases", seq=1, source_ref=bytes_digest) + outbox.durable_write_json( + directory / outbox.entry_file_name(1, entry_id, "prepared"), + { + "schema_version": outbox.OUTBOX_ENTRY_SCHEMA, + "goal_id": GOAL_ID, + "partition": "leases", + "seq": 1, + "entry_id": entry_id, + "writer": {"runtime": "typescript", "write_class": "task_lease_acquire", "operation_id": "op-1"}, + "source": { + "kind": "task_lease_record", + "previous_bytes_digest": None, + "bytes_digest": bytes_digest, + "lease": {"todo_id": record["todo_id"], "version": 2, "lease_epoch": 1, "status": "active", "updated_at": record["updated_at"]}, + "previous_lease": None, + "event_id": None, + }, + "source_root_digest": outbox.runtime_root_digest(runtime_root), + "projection": {"leases": [{"file_stem": record["todo_id"], "record": record}]}, + "partition_digest": None, + "prepared_at": "2026-09-03T00:01:00.000Z", + }, + ) + outbox.durable_write_json( + directory / outbox.entry_file_name(1, entry_id, "committed"), + {"schema_version": outbox.OUTBOX_COMMIT_SCHEMA, "entry_id": entry_id, "committed_at": "2026-09-03T00:01:00.100Z"}, + ) + + result = _drain(registry, runtime_root) + + assert result.ok is True and result.delivered == 1 + view = adapter.read_local_authority_shadow(runtime_root=runtime_root, goal_id=GOAL_ID, scan_limit=5) + head = view["head"] + assert head["todos"] == [] and head["handoff_mode"] is None + assert head["leases"] == [ + { + "todo_id": record["todo_id"], + "owner": "agent-a", + "idempotency_key": "k1", + "write_scopes": ["loopx/**"], + "version": 2, + "lease_epoch": 1, + "acquired_at": record["acquired_at"], + "updated_at": record["updated_at"], + "expires_at": record["expires_at"], + "released_at": None, + "status": "active", + } + ] + expected_digest = partition_digest({"leases": head["leases"]}) + assert head["partitions"]["leases"] == {"seq": 1, "partition_digest": expected_digest} + assert result.entries[0]["partition_digest"] == expected_digest + assert "/should/never/be/compared" not in json.dumps(view) + + +def test_status_reports_backlog_candidate_and_growth_facts(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + empty = adapter.local_authority_shadow_status(registry_path=registry, runtime_root=runtime_root, goal_id=GOAL_ID) + assert empty["ok"] is True + assert empty["config"]["status"] == "disabled" + assert empty["candidate"]["status"] == "missing" + assert empty["store_bytes"] == 0 and empty["retention_pressure"] is False + assert str(runtime_root) not in json.dumps(empty) + + _record_todo_write(registry, state, runtime_root, "Status fact") + pending = adapter.local_authority_shadow_status(registry_path=registry, runtime_root=runtime_root, goal_id=GOAL_ID) + assert pending["outbox"]["todos"]["committed_pending"] == 1 + assert _drain(registry, runtime_root).delivered == 1 + + drained = adapter.local_authority_shadow_status(registry_path=registry, runtime_root=runtime_root, goal_id=GOAL_ID) + assert drained["outbox"]["todos"] == { + "committed_pending": 0, + "prepared_only": 0, + "next_seq": 0, + "cursor_last_seq": 1, + "cursor_last_entry_id": outbox.read_cursor(_todo_dir(runtime_root))["last_entry_id"], + "invalid": None, + } + assert drained["candidate"]["status"] == "loaded" + assert drained["candidate"]["cursor"] == "1" + assert drained["candidate"]["codec_agreement"] is True + assert drained["candidate"]["head_schema_version"] == "loopx_local_authority_shadow_projection_v1" + assert drained["store_bytes"] > 0 + + +def test_capture_evidence_v1_reports_measured_facts_only(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + capture = _record_todo_write(registry, state, runtime_root, "Evidence fact") + drain = _drain(registry, runtime_root) + evidence = adapter.capture_evidence(goal_id=GOAL_ID, capture=capture.outcome, drain=drain) + assert adapter.valid_evidence_v1(evidence, goal_id=GOAL_ID) + assert evidence["outcome"] == "delivered" + assert evidence["entry"]["entry_id"] == capture.outcome.entry_id + assert evidence["source_transaction_correlated"] is True + assert evidence["durable_source_outbox"] is True + assert evidence["source_candidate_compared"] is False + assert evidence["parity_verdict"] == "not_evaluated" + assert evidence["drain"]["candidate_readback_verified"] is True + + deferred = adapter.DrainResult(goal_id=GOAL_ID, outcome="drain_deferred", reason_code="drain_lock_busy") + assert adapter.capture_evidence(goal_id=GOAL_ID, capture=capture.outcome, drain=deferred)["outcome"] == "drain_deferred" + pending = adapter.capture_evidence(goal_id=GOAL_ID, capture=capture.outcome, drain=None) + assert pending["outcome"] == "pending" and pending["drain"] is None + + skipped = outbox.CaptureOutcome(partition="todos", skipped_reason="partition_unchanged") + assert adapter.capture_evidence(goal_id=GOAL_ID, capture=skipped, drain=None)["outcome"] == "no_transaction" + failed = outbox.CaptureOutcome(partition="todos", failure={"reason_code": "outbox_prepare_failed", "error_class": "OSError"}) + failed_evidence = adapter.capture_evidence(goal_id=GOAL_ID, capture=failed, drain=None) + assert failed_evidence["outcome"] == "capture_failed" + assert failed_evidence["durable_source_outbox"] is False + assert failed_evidence["source_transaction_correlated"] is False + assert adapter.valid_evidence_v1(failed_evidence, goal_id=GOAL_ID) diff --git a/tests/control_plane/test_local_authority_shadow_outbox.py b/tests/control_plane/test_local_authority_shadow_outbox.py new file mode 100644 index 0000000000..330397ab10 --- /dev/null +++ b/tests/control_plane/test_local_authority_shadow_outbox.py @@ -0,0 +1,354 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from loopx.control_plane.coordination import local_authority_shadow_adapter as adapter +from loopx.control_plane.coordination import local_authority_shadow_outbox as outbox +from loopx.control_plane.coordination.local_authority_shadow_projection import ( + LEASE_FIELDS, + ProjectionValueError, + canonical_bytes, + lease_partition_projection, + partition_digest, + sha256_digest, + text_digest, + todo_partition_projection, +) +from loopx.file_lock import exclusive_file_lock +from loopx.history import load_registry +from loopx.registry import find_registry_goal +from loopx.todos import add_goal_todo + + +GOAL_ID = "goal-outbox" + + +def _fixture(tmp_path: Path) -> tuple[Path, Path, Path]: + repo = tmp_path / "repo" + repo.mkdir() + state = repo / "ACTIVE_GOAL_STATE.md" + state.write_text( + "---\n" + f"goal_id: {GOAL_ID}\n" + "handoff_mode: hard_lease\n" + "updated_at: 2026-09-03T00:00:00+00:00\n" + "---\n\n" + "## Agent Todo\n\n", + encoding="utf-8", + ) + runtime_root = tmp_path / "runtime" + registry = tmp_path / "registry.json" + registry.write_text( + json.dumps( + { + "common_runtime_root": str(runtime_root), + "goals": [ + { + "id": GOAL_ID, + "domain": "harness_self_improvement", + "status": "active", + "repo": str(repo), + "state_file": state.name, + "adapter": {"kind": "harness_self_improvement"}, + "coordination": { + "agent_model": "peer_v1", + "registered_agents": ["agent-a"], + }, + } + ], + } + ), + encoding="utf-8", + ) + return registry, state, runtime_root + + +def _capture( + registry: Path, + state: Path, + runtime_root: Path, + *, + original_text: str, + enabled: bool = True, + write_class: str = "todo_add", +) -> outbox.TodoPartitionCapture: + goal = find_registry_goal(load_registry(registry), GOAL_ID) + return outbox.TodoPartitionCapture.begin( + enabled=enabled, + runtime_root=runtime_root, + goal_id=GOAL_ID, + state_path=state, + write_class=write_class, + original_text=original_text, + projector=adapter.todo_partition_projector(goal, state_path=state), + ) + + +def _add_todo(registry: Path, text: str) -> str: + result = add_goal_todo( + registry_path=registry, + goal_id=GOAL_ID, + role="agent", + text=text, + task_class="advancement_task", + ) + assert result["ok"] is True + return str(result["todo_id"]) + + +def _todo_dir(runtime_root: Path) -> Path: + return outbox.partition_directory(runtime_root, GOAL_ID, "todos") + + +def test_capture_records_prepared_then_committed_and_skips_prose_only_writes(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + original = state.read_text(encoding="utf-8") + todo_id = _add_todo(registry, "Bind the shadow to the primary transaction.") + new_text = state.read_text(encoding="utf-8") + + capture = _capture(registry, state, runtime_root, original_text=original) + capture.prepare(new_text) + names = sorted(path.name for path in _todo_dir(runtime_root).iterdir()) + assert len(names) == 1 and names[0].endswith(".prepared.json") + assert capture.outcome.entry_id is not None + assert capture.outcome.seq == 1 + assert capture.outcome.source_bytes_digest == text_digest(new_text) + assert capture.outcome.entry_id == outbox.entry_identity( + goal_id=GOAL_ID, partition="todos", seq=1, source_ref=text_digest(new_text) + ) + + capture.committed() + entries = outbox.list_entries(_todo_dir(runtime_root)) + assert [entry.is_committed for entry in entries] == [True] + entry = entries[0] + assert entry.prepared["schema_version"] == outbox.OUTBOX_ENTRY_SCHEMA + assert entry.committed is not None + assert entry.committed["schema_version"] == outbox.OUTBOX_COMMIT_SCHEMA + assert entry.prepared["source"] == { + "kind": "markdown_active_state", + "previous_bytes_digest": text_digest(original), + "bytes_digest": text_digest(new_text), + "lease": None, + "event_id": None, + } + assert entry.prepared["writer"] == { + "runtime": "python", + "write_class": "todo_add", + "operation_id": None, + } + projection = entry.projection() + assert projection is not None + assert projection["handoff_mode"] == "hard_lease" + assert [item["todo_id"] for item in projection["todos"]] == [todo_id] + assert entry.recorded_partition_digest() == partition_digest(projection) + assert str(runtime_root) not in json.dumps(entry.prepared) + + prose_only = new_text + "\nOperator note that changes no coordination fact.\n" + prose = _capture(registry, state, runtime_root, original_text=new_text, write_class="todo_update") + prose.prepare(prose_only) + prose.committed() + assert prose.outcome.entry_id is None + assert prose.outcome.skipped_reason == "partition_unchanged" + assert len(outbox.list_entries(_todo_dir(runtime_root))) == 1 + + _add_todo(registry, "Second coordination fact.") + third = _capture(registry, state, runtime_root, original_text=new_text) + third.prepare(state.read_text(encoding="utf-8")) + third.committed() + assert third.outcome.seq == 2 + assert [entry.seq for entry in outbox.list_entries(_todo_dir(runtime_root))] == [1, 2] + + +def test_disabled_capture_creates_nothing(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + capture = _capture(registry, state, runtime_root, original_text="", enabled=False) + capture.prepare("# anything") + capture.committed() + assert capture.outcome.skipped_reason == "shadow_disabled" + assert capture.outcome.failure is None + assert not (runtime_root / "authority-shadow").exists() + + +def test_event_branch_records_projection_in_committed_marker(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + empty = state.read_text(encoding="utf-8") + baseline_id = _add_todo(registry, "Baseline coordination fact.") + original = state.read_text(encoding="utf-8") + baseline = _capture(registry, state, runtime_root, original_text=empty) + baseline.prepare(original) + baseline.committed() + assert baseline.outcome.seq == 1 + + # An appended event that changes no compared field retires its own entry. + unchanged = _capture(registry, state, runtime_root, original_text=original, write_class="todo_complete_event_projection") + unchanged.prepare(original, event_id="evt-noop") + assert unchanged.outcome.entry_id is not None + unchanged.committed(projection_from_disk=True) + assert unchanged.outcome.entry_id is None + assert unchanged.outcome.skipped_reason == "partition_unchanged" + assert [entry.seq for entry in outbox.list_entries(_todo_dir(runtime_root))] == [1] + + capture = _capture(registry, state, runtime_root, original_text=original, write_class="todo_complete_event_projection") + capture.prepare(original, event_id="evt-1") + [_baseline, entry] = outbox.list_entries(_todo_dir(runtime_root)) + assert entry.prepared["source"]["kind"] == "state_event_log" + assert entry.prepared["source"]["event_id"] == "evt-1" + assert entry.prepared["projection"] is None + assert entry.prepared["writer"]["operation_id"] == "evt-1" + assert entry.source_ref == "event:evt-1" + assert not entry.is_committed + + todo_id = _add_todo(registry, "Landed by the event append.") + capture.committed(projection_from_disk=True) + [_baseline, entry] = outbox.list_entries(_todo_dir(runtime_root)) + assert entry.is_committed + projection = entry.projection() + assert projection is not None + assert sorted(item["todo_id"] for item in projection["todos"]) == sorted([baseline_id, todo_id]) + assert entry.recorded_partition_digest() == partition_digest(projection) + assert capture.outcome.partition_digest == partition_digest(projection) + + +def test_prepared_only_entries_resolve_from_source_probes(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + original = state.read_text(encoding="utf-8") + _add_todo(registry, "Crash between write and marker.") + new_text = state.read_text(encoding="utf-8") + capture = _capture(registry, state, runtime_root, original_text=original) + capture.prepare(new_text) + [entry] = outbox.list_entries(_todo_dir(runtime_root)) + assert not entry.is_committed + + def resolve(text: str) -> str: + return outbox.resolve_prepared_only_entry( + entry, + markdown_text_reader=lambda: text, + lease_record_reader=None, + event_presence_reader=None, + ) + + assert resolve(new_text) == "committed" + assert resolve(original) == "abandoned" + assert resolve(new_text + "\n- [ ] (agent) foreign edit\n") == "unproved" + + event_entry = outbox.OutboxEntry( + partition="todos", + seq=2, + entry_id="local-shadow-tx-" + "0" * 64, + prepared_path=tmp_path / "unused.prepared.json", + committed_path=None, + prepared={"source": {"kind": "state_event_log", "event_id": "evt-9"}}, + committed=None, + ) + assert outbox.resolve_prepared_only_entry( + event_entry, markdown_text_reader=None, lease_record_reader=None, + event_presence_reader=lambda event_id: event_id == "evt-9", + ) == "unproved" + assert outbox.resolve_prepared_only_entry( + event_entry, markdown_text_reader=None, lease_record_reader=None, + event_presence_reader=lambda _event_id: False, + ) == "abandoned" + + planned = {"todo_id": "todo-a", "version": 2, "lease_epoch": 1, "status": "active", "updated_at": "t2"} + previous = {"todo_id": "todo-a", "version": 1, "lease_epoch": 1, "status": "active", "updated_at": "t1"} + lease_entry = outbox.OutboxEntry( + partition="leases", + seq=1, + entry_id="local-shadow-tx-" + "1" * 64, + prepared_path=tmp_path / "unused.prepared.json", + committed_path=None, + prepared={"source": {"kind": "task_lease_record", "lease": planned, "previous_lease": previous}}, + committed=None, + ) + + def lease_resolve(current: dict[str, object] | None) -> str: + return outbox.resolve_prepared_only_entry( + lease_entry, + markdown_text_reader=None, + lease_record_reader=lambda _todo_id: current, + event_presence_reader=None, + ) + + assert lease_resolve({**planned, "owner": "agent-a"}) == "committed" + assert lease_resolve({**previous, "owner": "agent-a"}) == "abandoned" + assert lease_resolve({**planned, "version": 9}) == "unproved" + assert lease_resolve(None) == "unproved" + + +def test_sequence_advances_past_the_drain_cursor_and_lists_oldest_first(tmp_path: Path) -> None: + _registry, _state, runtime_root = _fixture(tmp_path) + directory = _todo_dir(runtime_root) + directory.mkdir(parents=True) + outbox.write_cursor( + directory, + partition="todos", + last_seq=5, + last_entry_id="local-shadow-tx-" + "a" * 64, + last_partition_digest=None, + last_cursor="5", + last_provider_revision="rev-5", + ) + assert outbox.next_seq(directory) == 6 + seed = outbox.SeedSource(partition="todos", projection={"handoff_mode": "hard_lease", "todos": []}) + first = outbox.write_seed_entry(runtime_root=runtime_root, goal_id=GOAL_ID, seed=seed) + second = outbox.write_seed_entry(runtime_root=runtime_root, goal_id=GOAL_ID, seed=seed) + assert (first.seq, second.seq) == (6, 7) + assert [entry.seq for entry in outbox.list_entries(directory)] == [6, 7] + assert all(entry.is_committed for entry in outbox.list_entries(directory)) + assert outbox.latest_partition_digest(directory) == partition_digest(seed.projection) + summary = outbox.outbox_summary(runtime_root, GOAL_ID) + assert summary["todos"]["committed_pending"] == 2 + assert summary["todos"]["cursor_last_seq"] == 5 + assert summary["leases"] == { + "committed_pending": 0, + "prepared_only": 0, + "next_seq": 0, + "cursor_last_seq": None, + "cursor_last_entry_id": None, + "invalid": None, + } + outbox.remove_entry_files(first) + assert [entry.seq for entry in outbox.list_entries(directory)] == [7] + + +def test_canonical_projection_rejects_floats_and_bad_lease_identity() -> None: + with pytest.raises(ProjectionValueError): + canonical_bytes({"version": 1.0}) + with pytest.raises(ProjectionValueError): + todo_partition_projection(handoff_mode="hard_lease", todos=[{"todo_id": "a", "status": 2.5}]) + assert canonical_bytes({"b": 1, "a": [True, None, "\u00e9"]}) == '{"a":[true,null,"\u00e9"],"b":1}'.encode("utf-8") + assert sha256_digest({"a": 1}) == sha256_digest({"a": 1}) + with pytest.raises(ProjectionValueError): + lease_partition_projection([("todo-a", {"goal_id": "other", "todo_id": "todo-a"})], goal_id=GOAL_ID) + projection = lease_partition_projection( + [("todo-b", {"goal_id": GOAL_ID, "todo_id": "todo-b", "version": 1, "extra": "dropped"}), + ("todo-a", {"goal_id": GOAL_ID, "todo_id": "todo-a", "version": 2, "status": "active"})], + goal_id=GOAL_ID, + ) + assert [lease["todo_id"] for lease in projection["leases"]] == ["todo-a", "todo-b"] + assert set(projection["leases"][1]) <= set(LEASE_FIELDS) + + +def test_capture_failure_is_typed_and_never_raises(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + blocker = runtime_root / "authority-shadow" + blocker.parent.mkdir(parents=True) + blocker.write_text("not a directory", encoding="utf-8") + capture = _capture(registry, state, runtime_root, original_text="") + capture.prepare("---\ngoal_id: goal-outbox\n---\n\n## Agent Todo\n\n- [ ] (agent) x \n") + capture.committed() + assert capture.outcome.entry_id is None + assert capture.outcome.failure is not None + assert capture.outcome.failure["reason_code"] == "outbox_prepare_failed" + + +def test_primary_lock_probe_reports_held_locks(tmp_path: Path) -> None: + target = tmp_path / "ACTIVE_GOAL_STATE.md" + target.write_text("", encoding="utf-8") + assert adapter.primary_lock_is_free(target) is True + with exclusive_file_lock(target, timeout_seconds=1.0, operation="test_hold"): + assert adapter.primary_lock_is_free(target) is False + assert adapter.primary_lock_is_free(target) is True diff --git a/tests/control_plane_ts/local_authority_shadow_outbox.test.ts b/tests/control_plane_ts/local_authority_shadow_outbox.test.ts new file mode 100644 index 0000000000..5e09bb7ddd --- /dev/null +++ b/tests/control_plane_ts/local_authority_shadow_outbox.test.ts @@ -0,0 +1,366 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { promisify } from "node:util"; + +import { FileAuthorityStore } from "../../loopx/control_plane/coordination/file_authority_store.ts"; +import { + LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_RESULT_SCHEMA, + LOCAL_AUTHORITY_SHADOW_PROJECTION_SCHEMA_V1, + LOCAL_AUTHORITY_SHADOW_READ_RESULT_SCHEMA, + LOCAL_AUTHORITY_SHADOW_TRANSACTION_RECEIPT_SCHEMA, + commitLocalAuthorityShadowEntry, + composeLocalAuthorityShadowHead, + localAuthorityShadowHeadDigest, + readLocalAuthorityShadow, +} from "../../loopx/control_plane/coordination/local_authority_shadow.ts"; +import { + LOCAL_AUTHORITY_SHADOW_OUTBOX_COMMIT_SCHEMA, + LOCAL_AUTHORITY_SHADOW_OUTBOX_ENTRY_SCHEMA, + beginLeaseOutboxEntry, + decodeLocalAuthorityShadowBinding, + leaseRecordDigest, + outboxEntryIdentity, +} from "../../loopx/control_plane/coordination/local_authority_shadow_outbox.ts"; + +const execFileAsync = promisify(execFile); +const GOAL = "goal-a"; +const HEX = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +function entryId(seq: number, sourceRef: string): string { + return outboxEntryIdentity(GOAL, "todos", seq, sourceRef); +} + +function todoEntry(seq: number, digest: string, resolution = "committed") { + return { + entry_id: entryId(seq, `sha256:${HEX}`), + partition: "todos", + seq, + writer: { runtime: "python", write_class: "todo_add", operation_id: null }, + source: { + kind: "markdown_active_state", + previous_bytes_digest: null, + bytes_digest: `sha256:${HEX}`, + lease: null, + event_id: null, + }, + source_root_digest: `sha256:${HEX}`, + prepared_at: "2026-09-03T00:00:00.000Z", + committed_at: "2026-09-03T00:00:00.100Z", + resolution, + }; +} + +function commitRequest(root: string, seq: number, todos: object[], resolution = "committed") { + const projection = { handoff_mode: "hard_lease", todos }; + return { + schema_version: "loopx_local_authority_shadow_commit_entry_request_v0", + runtime_root: root, + goal_id: GOAL, + entry: todoEntry(seq, `sha256:${HEX}`, resolution), + partition_projection: projection, + partition_digest: `sha256:${"b".repeat(64)}`, + }; +} + +function noOpRequest(root: string, seq: number, resolution: "abandoned" | "unproved") { + return { + schema_version: "loopx_local_authority_shadow_commit_entry_request_v0", + runtime_root: root, + goal_id: GOAL, + entry: todoEntry(seq, `sha256:${HEX}`, resolution), + partition_projection: null, + partition_digest: null, + }; +} + +async function tempRoot(t: test.TestContext): Promise { + const root = await mkdtemp(join(tmpdir(), "loopx-shadow-outbox-")); + t.after(() => rm(root, { recursive: true, force: true })); + return root; +} + +test("commit_entry folds one partition into a v1 head and binds the receipt to the entry", async (t) => { + const root = await tempRoot(t); + const todos = [{ todo_id: "todo-a", status: "open" }]; + + const result = await commitLocalAuthorityShadowEntry(commitRequest(root, 1, todos)); + + assert.equal(result.schema_version, LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_RESULT_SCHEMA); + assert.equal(result.outcome, "delivered"); + assert.equal(result.no_op, false); + assert.equal(result.cursor, "1"); + const store = new FileAuthorityStore(join(root, "authority-shadow", "file", GOAL), GOAL); + const loaded = await store.loadAuthority(); + assert.equal(loaded.status, "loaded"); + if (loaded.status !== "loaded") return; + assert.equal(loaded.head.schema_version, LOCAL_AUTHORITY_SHADOW_PROJECTION_SCHEMA_V1); + assert.equal(loaded.head.handoff_mode, "hard_lease"); + assert.deepEqual(loaded.head.todos, todos); + assert.deepEqual(loaded.head.leases, []); + assert.deepEqual(loaded.head.partitions, { + todos: { seq: 1, partition_digest: `sha256:${"b".repeat(64)}` }, + leases: null, + }); + assert.equal(result.head_digest, localAuthorityShadowHeadDigest(loaded.head)); + const receipt = await store.readReceipt(entryId(1, `sha256:${HEX}`)); + assert.equal(receipt.status, "found"); + if (receipt.status !== "found") return; + const record = receipt.receipts[0] as Record; + assert.equal(record.schema_version, LOCAL_AUTHORITY_SHADOW_TRANSACTION_RECEIPT_SCHEMA); + assert.equal(record.entry_id, entryId(1, `sha256:${HEX}`)); + assert.equal(record.source_transaction_correlated, true); + assert.equal(record.durable_source_outbox, true); + assert.equal(record.parity_verdict, "not_evaluated"); + assert.equal(record.primary_authority, "legacy_local"); + assert.equal(record.provider_to_local_writes, false); + assert.equal(record.candidate_read_for_decision, false); +}); + +test("commit_entry replays only when the existing receipt carries the same partition digest", async (t) => { + const root = await tempRoot(t); + const todos = [{ todo_id: "todo-a", status: "open" }]; + assert.equal((await commitLocalAuthorityShadowEntry(commitRequest(root, 1, todos))).outcome, "delivered"); + + const replay = await commitLocalAuthorityShadowEntry(commitRequest(root, 1, todos)); + assert.equal(replay.outcome, "replayed"); + assert.equal(replay.cursor, "1"); + + const tampered = commitRequest(root, 1, todos); + tampered.partition_digest = `sha256:${"c".repeat(64)}`; + const mismatch = await commitLocalAuthorityShadowEntry(tampered); + assert.equal(mismatch.outcome, "protocol_mismatch"); + assert.equal(mismatch.reason_code, "transaction_receipt_mismatch"); + + const page = await new FileAuthorityStore(join(root, "authority-shadow", "file", GOAL), GOAL) + .scanCommitted(null, 10); + assert.equal(page.status, "page"); + if (page.status === "page") assert.equal(page.transactions.length, 1); +}); + +test("no-op resolutions keep the sequence auditable without touching compared fields", async (t) => { + const root = await tempRoot(t); + const todos = [{ todo_id: "todo-a", status: "open" }]; + const first = await commitLocalAuthorityShadowEntry(commitRequest(root, 1, todos)); + + const abandoned = await commitLocalAuthorityShadowEntry(noOpRequest(root, 2, "abandoned")); + const unproved = await commitLocalAuthorityShadowEntry(noOpRequest(root, 3, "unproved")); + + assert.equal(abandoned.outcome, "delivered"); + assert.equal(abandoned.no_op, true); + assert.equal(unproved.no_op, true); + assert.equal(unproved.cursor, "3"); + assert.equal(abandoned.head_digest, first.head_digest); + const store = new FileAuthorityStore(join(root, "authority-shadow", "file", GOAL), GOAL); + const page = await store.scanCommitted(null, 10); + assert.equal(page.status, "page"); + if (page.status !== "page") return; + assert.deepEqual( + page.transactions.map((transaction) => (transaction.events[0] as Record).kind), + ["source_transaction_delivered", "source_transaction_abandoned", "source_transaction_unproved"], + ); + const loaded = await store.loadAuthority(); + if (loaded.status === "loaded") { + assert.deepEqual((loaded.head.partitions as Record).todos, { + seq: 1, + partition_digest: `sha256:${"b".repeat(64)}`, + }); + } +}); + +test("commit_entry rejects projection/resolution combinations that would misstate a transaction", async (t) => { + const root = await tempRoot(t); + const withProjection = noOpRequest(root, 1, "abandoned") as Record; + withProjection.partition_projection = { handoff_mode: "hard_lease", todos: [] }; + withProjection.partition_digest = `sha256:${"b".repeat(64)}`; + await assert.rejects(commitLocalAuthorityShadowEntry(withProjection), /must not carry/u); + + const withoutProjection = commitRequest(root, 1, []) as Record; + withoutProjection.partition_projection = null; + withoutProjection.partition_digest = null; + await assert.rejects(commitLocalAuthorityShadowEntry(withoutProjection), /requires partition_projection/u); + + const badId = commitRequest(root, 1, []); + badId.entry.entry_id = "local-shadow:abc"; + await assert.rejects(commitLocalAuthorityShadowEntry(badId), /entry\.entry_id/u); + + const extra = { ...commitRequest(root, 1, []), observation_id: "x" }; + await assert.rejects(commitLocalAuthorityShadowEntry(extra), /unsupported fields/u); +}); + +test("a v0 observation head is accepted as the starting point for partition folds", () => { + const v0 = { + schema_version: "loopx_local_authority_shadow_projection_v0", + goal_id: GOAL, + handoff_mode: "hard_lease", + todos: [{ todo_id: "todo-a", status: "open" }], + leases: [{ todo_id: "todo-a", version: 1 }], + }; + const folded = composeLocalAuthorityShadowHead( + v0, + GOAL, + { partition: "leases", seq: 4 }, + { leases: [] }, + `sha256:${"d".repeat(64)}`, + ); + assert.equal(folded.schema_version, LOCAL_AUTHORITY_SHADOW_PROJECTION_SCHEMA_V1); + assert.equal(folded.handoff_mode, "hard_lease"); + assert.deepEqual(folded.todos, v0.todos); + assert.deepEqual(folded.leases, []); + assert.deepEqual(folded.partitions, { + todos: null, + leases: { seq: 4, partition_digest: `sha256:${"d".repeat(64)}` }, + }); +}); + +test("read returns head, comparison digest, and a bounded scan page", async (t) => { + const root = await tempRoot(t); + const missing = await readLocalAuthorityShadow({ + schema_version: "loopx_local_authority_shadow_read_request_v0", + runtime_root: root, + goal_id: GOAL, + scan_after_cursor: null, + scan_limit: 10, + }); + assert.equal(missing.schema_version, LOCAL_AUTHORITY_SHADOW_READ_RESULT_SCHEMA); + assert.equal(missing.status, "missing"); + assert.equal(missing.head, null); + + const todos = [{ todo_id: "todo-a", status: "open" }]; + await commitLocalAuthorityShadowEntry(commitRequest(root, 1, todos)); + await commitLocalAuthorityShadowEntry(noOpRequest(root, 2, "abandoned")); + const view = await readLocalAuthorityShadow({ + schema_version: "loopx_local_authority_shadow_read_request_v0", + runtime_root: root, + goal_id: GOAL, + scan_after_cursor: null, + scan_limit: 1, + }); + assert.equal(view.status, "loaded"); + assert.equal(view.cursor, "2"); + assert.match(String(view.store_identity), /^file:[0-9a-f]{32}$/u); + assert.equal(view.head_digest, localAuthorityShadowHeadDigest(view.head as Record)); + const scan = view.scan as { transactions: Record[]; next_cursor: string | null; has_more: boolean }; + assert.equal(scan.transactions.length, 1); + assert.equal(scan.has_more, true); + assert.equal(scan.transactions[0].operation_id, entryId(1, `sha256:${HEX}`)); + assert.equal(scan.transactions[0].projection_digest, view.head_digest); + assert.equal("projection" in scan.transactions[0], false); +}); + +test("lease outbox entries are two-phase, durable, and skipped without a binding", async (t) => { + const root = await tempRoot(t); + const leaseDirectory = join(root, "goals", GOAL, "task-leases"); + const other = { goal_id: GOAL, todo_id: "todo-b", version: 1, status: "active" }; + await mkdir(leaseDirectory, { recursive: true }); + await writeFile(join(leaseDirectory, "todo-b.json"), `${JSON.stringify(other, null, 2)}\n`); + const planned = { goal_id: GOAL, todo_id: "todo-a", version: 2, lease_epoch: 1, status: "active", updated_at: "t2" }; + + assert.equal(decodeLocalAuthorityShadowBinding(undefined), null); + assert.equal(decodeLocalAuthorityShadowBinding({ mode: "file_one_way" }), null); + assert.deepEqual( + decodeLocalAuthorityShadowBinding({ + schema_version: "loopx_local_authority_shadow_binding_v0", + mode: "file_one_way", + }), + { schema_version: "loopx_local_authority_shadow_binding_v0", mode: "file_one_way" }, + ); + + const capture = await beginLeaseOutboxEntry({ + runtime_root: root, + goal_id: GOAL, + lease_directory: leaseDirectory, + write_class: "task_lease_acquire", + operation_id: "op-1", + previous_lease: null, + planned_lease: planned, + }); + assert.equal(capture.failure, null); + assert.equal(capture.seq, 1); + assert.equal(capture.source_bytes_digest, leaseRecordDigest(planned)); + assert.equal(capture.entry_id, outboxEntryIdentity(GOAL, "leases", 1, leaseRecordDigest(planned))); + const directory = join(root, "authority-shadow", "outbox", GOAL, "leases"); + let names = await readdir(directory); + assert.deepEqual(names, [`0000000001-${capture.entry_id}.prepared.json`]); + const prepared = JSON.parse(await readFile(join(directory, names[0]), "utf8")); + assert.equal(prepared.schema_version, LOCAL_AUTHORITY_SHADOW_OUTBOX_ENTRY_SCHEMA); + assert.equal(prepared.partition, "leases"); + assert.equal(prepared.partition_digest, null); + assert.deepEqual(prepared.source.lease, { + todo_id: "todo-a", + version: 2, + lease_epoch: 1, + status: "active", + updated_at: "t2", + }); + assert.deepEqual( + prepared.projection.leases.map((item: { file_stem: string }) => item.file_stem), + ["todo-a", "todo-b"], + ); + assert.deepEqual(prepared.projection.leases[0].record, planned); + + await capture.commit(); + assert.equal(capture.failure, null); + names = (await readdir(directory)).sort((a, b) => (a < b ? -1 : 1)); + assert.deepEqual(names, [ + `0000000001-${capture.entry_id}.committed.json`, + `0000000001-${capture.entry_id}.prepared.json`, + ]); + const committed = JSON.parse(await readFile(join(directory, names[0]), "utf8")); + assert.equal(committed.schema_version, LOCAL_AUTHORITY_SHADOW_OUTBOX_COMMIT_SCHEMA); + assert.equal(committed.entry_id, capture.entry_id); + + const second = await beginLeaseOutboxEntry({ + runtime_root: root, + goal_id: GOAL, + lease_directory: leaseDirectory, + write_class: "task_lease_renew", + operation_id: null, + previous_lease: planned, + planned_lease: { ...planned, version: 3, updated_at: "t3" }, + }); + assert.equal(second.seq, 2); + const failing = await beginLeaseOutboxEntry({ + runtime_root: root, + goal_id: GOAL, + lease_directory: leaseDirectory, + write_class: "task_lease_renew", + operation_id: null, + previous_lease: null, + planned_lease: { goal_id: GOAL }, + }); + assert.equal(failing.entry_id, null); + assert.equal(failing.failure?.reason_code, "outbox_prepare_failed"); +}); + +test("entry identity derivation agrees byte-for-byte with the Python outbox module", async () => { + const python = process.env.LOOPX_TEST_PYTHON ?? "python3"; + const script = [ + "from loopx.control_plane.coordination.local_authority_shadow_outbox import entry_identity", + "from loopx.control_plane.coordination.local_authority_shadow_projection import sha256_digest", + `print(entry_identity(goal_id='${GOAL}', partition='leases', seq=7, source_ref='sha256:${HEX}'))`, + "print(sha256_digest({'handoff_mode': 'hard_lease', 'todos': [{'todo_id': 'todo-a', 'status': 'open'}], 'leases': []}))", + ].join("\n"); + let stdout: string; + try { + ({ stdout } = await execFileAsync(python, ["-c", script], { + cwd: join(import.meta.dirname, "..", ".."), + env: { ...process.env, PYTHONPATH: join(import.meta.dirname, "..", "..") }, + })); + } catch { + return; // Python without the loopx package: the Python suite pins the same fixture. + } + const [pythonEntryId, pythonHeadDigest] = stdout.trim().split("\n"); + assert.equal(pythonEntryId, outboxEntryIdentity(GOAL, "leases", 7, `sha256:${HEX}`)); + assert.equal( + pythonHeadDigest, + localAuthorityShadowHeadDigest({ + handoff_mode: "hard_lease", + todos: [{ todo_id: "todo-a", status: "open" }], + leases: [], + }), + ); +}); diff --git a/tsconfig.control-plane.json b/tsconfig.control-plane.json index 4af88a3954..02230c3239 100644 --- a/tsconfig.control-plane.json +++ b/tsconfig.control-plane.json @@ -30,6 +30,7 @@ "loopx/control_plane/coordination/nokv_authority_store.ts", "loopx/control_plane/coordination/nokv_jsonl_transport.ts", "loopx/control_plane/coordination/local_authority_shadow.ts", + "loopx/control_plane/coordination/local_authority_shadow_outbox.ts", "loopx/control_plane/coordination/postgresql_authority_store.ts", "loopx/control_plane/agents/delivery_workspace.ts", "loopx/control_plane/goals/vision_checkpoint.ts", @@ -70,6 +71,7 @@ "tests/control_plane_ts/coordination_runtime_shadow.test.ts", "tests/control_plane_ts/coordination_projection.test.ts", "tests/control_plane_ts/local_authority_runtime.test.ts", + "tests/control_plane_ts/local_authority_shadow_outbox.test.ts", "tests/control_plane_ts/authority_store_conformance.ts", "tests/control_plane_ts/nokv_authority_store.test.ts", "tests/control_plane_ts/nokv_jsonl_transport.test.ts", From 64f21b2be4e0d4f27eab8c6a6c2d6dfd63cadc29 Mon Sep 17 00:00:00 2001 From: wchwawa Date: Thu, 3 Sep 2026 11:52:46 +1000 Subject: [PATCH 02/11] refactor(authority): address SonarCloud findings on the outbox plumbing No behavior change; every shadow, drain, and CLI suite passes unchanged. - Reuse authorityUnicodeCompare instead of nested-ternary sort comparators in local_authority_shadow.ts and local_authority_shadow_outbox.ts. - Split readLocalAuthorityShadow into readResultBase, loadedReadResult, scanTransactionView, and appendScanPage. - Split list_entries into _index_entry_files, _load_prepared_record, and _load_committed_record; split resolve_prepared_only_entry into one resolver per source kind with a shared _lease_matches. - Split drain_local_authority_shadow_outbox into _drain_prelude, _outbox_is_idle, _drain_partitions, _settle_drain_outcome, and _count_backlog. - Split the authority-shadow Markdown renderer into per-action helpers. - Tests: one assertion per line instead of composite `and` assertions; the canonical-digest test compares two differently ordered objects instead of the same expression twice. Signed-off-by: wchwawa --- loopx/cli_commands/authority_shadow.py | 99 ++++++----- .../coordination/local_authority_shadow.ts | 115 +++++++----- .../local_authority_shadow_adapter.py | 161 +++++++++++------ .../local_authority_shadow_outbox.py | 165 ++++++++++-------- .../local_authority_shadow_outbox.ts | 4 +- .../test_local_authority_shadow_cli_e2e.py | 12 +- .../test_local_authority_shadow_drain.py | 43 +++-- .../test_local_authority_shadow_outbox.py | 5 +- 8 files changed, 371 insertions(+), 233 deletions(-) diff --git a/loopx/cli_commands/authority_shadow.py b/loopx/cli_commands/authority_shadow.py index 9d4a8ceaf3..15c4aaf419 100644 --- a/loopx/cli_commands/authority_shadow.py +++ b/loopx/cli_commands/authority_shadow.py @@ -24,6 +24,60 @@ ] +_DRAIN_FIELDS = ( + "outcome", + "reason_code", + "delivered", + "replayed", + "reconciled", + "no_op", + "reseeded", + "pending_after", + "prepared_only_after", + "budget_exhausted", + "last_cursor", + "candidate_readback_verified", +) + + +def _drain_markdown_lines(payload: dict[str, object]) -> list[str]: + lines = [f"- {key}: `{payload.get(key)}`" for key in _DRAIN_FIELDS] + stopped_at = payload.get("stopped_at") + if isinstance(stopped_at, dict): + lines.append( + "- stopped_at: " + f"`{stopped_at.get('partition')}#{stopped_at.get('seq')}` " + f"→ `{stopped_at.get('outcome')}` ({stopped_at.get('reason_code')})" + ) + return lines + + +def _status_markdown_lines(payload: dict[str, object]) -> list[str]: + lines: list[str] = [] + config = payload.get("config") + if isinstance(config, dict): + lines.append(f"- config: `{config.get('status')}`") + backlog = payload.get("outbox") + partitions = backlog.items() if isinstance(backlog, dict) else [] + for partition, facts in partitions: + if isinstance(facts, dict): + lines.append( + f"- outbox.{partition}: committed_pending=" + f"`{facts.get('committed_pending')}` prepared_only=" + f"`{facts.get('prepared_only')}` cursor_last_seq=" + f"`{facts.get('cursor_last_seq')}`" + ) + candidate = payload.get("candidate") + if isinstance(candidate, dict): + lines.append( + f"- candidate: status=`{candidate.get('status')}` cursor=" + f"`{candidate.get('cursor')}` store_identity=`{candidate.get('store_identity')}`" + ) + lines.append(f"- store_bytes: `{payload.get('store_bytes')}`") + lines.append(f"- retention_pressure: `{payload.get('retention_pressure')}`") + return lines + + def render_authority_shadow_markdown(payload: dict[str, object]) -> str: lines = [ "# LoopX Authority Shadow", @@ -37,50 +91,9 @@ def render_authority_shadow_markdown(payload: dict[str, object]) -> str: if payload.get("error_code"): lines.append(f"- error_code: `{payload.get('error_code')}`") if payload.get("action") == "drain": - for key in ( - "outcome", - "reason_code", - "delivered", - "replayed", - "reconciled", - "no_op", - "reseeded", - "pending_after", - "prepared_only_after", - "budget_exhausted", - "last_cursor", - "candidate_readback_verified", - ): - lines.append(f"- {key}: `{payload.get(key)}`") - stopped_at = payload.get("stopped_at") - if isinstance(stopped_at, dict): - lines.append( - "- stopped_at: " - f"`{stopped_at.get('partition')}#{stopped_at.get('seq')}` " - f"→ `{stopped_at.get('outcome')}` ({stopped_at.get('reason_code')})" - ) + lines.extend(_drain_markdown_lines(payload)) elif payload.get("action") == "status": - config = payload.get("config") - if isinstance(config, dict): - lines.append(f"- config: `{config.get('status')}`") - backlog = payload.get("outbox") - if isinstance(backlog, dict): - for partition, facts in backlog.items(): - if isinstance(facts, dict): - lines.append( - f"- outbox.{partition}: committed_pending=" - f"`{facts.get('committed_pending')}` prepared_only=" - f"`{facts.get('prepared_only')}` cursor_last_seq=" - f"`{facts.get('cursor_last_seq')}`" - ) - candidate = payload.get("candidate") - if isinstance(candidate, dict): - lines.append( - f"- candidate: status=`{candidate.get('status')}` cursor=" - f"`{candidate.get('cursor')}` store_identity=`{candidate.get('store_identity')}`" - ) - lines.append(f"- store_bytes: `{payload.get('store_bytes')}`") - lines.append(f"- retention_pressure: `{payload.get('retention_pressure')}`") + lines.extend(_status_markdown_lines(payload)) return "\n".join(lines) + "\n" diff --git a/loopx/control_plane/coordination/local_authority_shadow.ts b/loopx/control_plane/coordination/local_authority_shadow.ts index d6c93928c9..a5760fd7dc 100644 --- a/loopx/control_plane/coordination/local_authority_shadow.ts +++ b/loopx/control_plane/coordination/local_authority_shadow.ts @@ -11,10 +11,11 @@ import { } from "../runtime_decode.ts"; import type { AuthorityStore, + AuthorityStoreCommittedTransaction, AuthorityStoreLoadResult, AuthorityStoreReceiptResult, } from "./authority_store.ts"; -import { canonicalAuthorityBytes } from "./authority_store_codec.ts"; +import { authorityUnicodeCompare, canonicalAuthorityBytes } from "./authority_store_codec.ts"; import { FileAuthorityStore } from "./file_authority_store.ts"; export const LOCAL_AUTHORITY_SHADOW_REQUEST_SCHEMA = @@ -481,7 +482,7 @@ function rejectUnexpectedFields( ): void { const unexpected = Object.keys(record).filter((field) => !allowed.has(field)); if (unexpected.length > 0) { - unexpected.sort((left, right) => (left < right ? -1 : left > right ? 1 : 0)); + unexpected.sort(authorityUnicodeCompare); throw new EffectRuntimeRequestError( `${label} has unsupported fields: ${unexpected.join(", ")}`, ); @@ -904,19 +905,10 @@ export async function commitLocalAuthorityShadowEntry( } } -/** - * Read-only view of the candidate store for drain readback and parity: - * head, its comparison digest, and a page of committed transactions with the - * projection reduced to its digest so responses stay bounded. - */ -export async function readLocalAuthorityShadow( - value: unknown, - dependencies: LocalAuthorityShadowDependencies = {}, -): Promise { - const request = decodeReadRequest(value); - const base: JsonObject = { +function readResultBase(goalId: string): JsonObject { + return { schema_version: LOCAL_AUTHORITY_SHADOW_READ_RESULT_SCHEMA, - goal_id: request.goal_id, + goal_id: goalId, status: "unavailable", reason_code: null, store_identity: null, @@ -927,6 +919,67 @@ export async function readLocalAuthorityShadow( partitions: null, scan: null, }; +} + +function loadedReadResult( + base: JsonObject, + storeIdentity: string, + loaded: Extract, +): JsonObject { + const result: JsonObject = { ...base, status: loaded.status, store_identity: storeIdentity }; + if (loaded.status === "loaded") { + result.provider_revision = loaded.provider_revision; + result.cursor = loaded.cursor; + result.head = structuredClone(loaded.head); + result.head_digest = localAuthorityShadowHeadDigest(loaded.head); + result.partitions = partitionsOf(loaded.head); + } + return result; +} + +/** One committed transaction with its projection reduced to a digest. */ +function scanTransactionView(transaction: AuthorityStoreCommittedTransaction): JsonObject { + return { + cursor: transaction.cursor, + provider_revision: transaction.provider_revision, + operation_id: transaction.operation_id, + projection_digest: localAuthorityShadowHeadDigest(transaction.projection), + projection_partitions: partitionsOf(transaction.projection), + events: structuredClone(transaction.events) as JsonObject[], + receipts: structuredClone(transaction.receipts) as JsonObject[], + }; +} + +async function appendScanPage( + store: AuthorityStore, + request: ReadRequest, + result: JsonObject, +): Promise { + const page = await store.scanCommitted(request.scan_after_cursor, request.scan_limit); + if (page.status !== "page") { + return { ...result, status: page.status, reason_code: page.reason_code }; + } + return { + ...result, + scan: { + transactions: page.transactions.map(scanTransactionView), + next_cursor: page.next_cursor, + has_more: page.has_more, + }, + }; +} + +/** + * Read-only view of the candidate store for drain readback and parity: + * head, its comparison digest, and a page of committed transactions with the + * projection reduced to its digest so responses stay bounded. + */ +export async function readLocalAuthorityShadow( + value: unknown, + dependencies: LocalAuthorityShadowDependencies = {}, +): Promise { + const request = decodeReadRequest(value); + const base = readResultBase(request.goal_id); let store: AuthorityStore; try { store = openShadowStore(request.runtime_root, request.goal_id, dependencies); @@ -947,38 +1000,8 @@ export async function readLocalAuthorityShadow( store_identity: identity.store_identity, }; } - const result: JsonObject = { - ...base, - status: loaded.status, - store_identity: identity.store_identity, - }; - if (loaded.status === "loaded") { - result.provider_revision = loaded.provider_revision; - result.cursor = loaded.cursor; - result.head = structuredClone(loaded.head); - result.head_digest = localAuthorityShadowHeadDigest(loaded.head); - result.partitions = partitionsOf(loaded.head); - } - if (request.scan_limit > 0) { - const page = await store.scanCommitted(request.scan_after_cursor, request.scan_limit); - if (page.status !== "page") { - return { ...result, status: page.status, reason_code: page.reason_code }; - } - result.scan = { - transactions: page.transactions.map((transaction) => ({ - cursor: transaction.cursor, - provider_revision: transaction.provider_revision, - operation_id: transaction.operation_id, - projection_digest: localAuthorityShadowHeadDigest(transaction.projection), - projection_partitions: partitionsOf(transaction.projection), - events: structuredClone(transaction.events) as JsonObject[], - receipts: structuredClone(transaction.receipts) as JsonObject[], - })), - next_cursor: page.next_cursor, - has_more: page.has_more, - }; - } - return result; + const result = loadedReadResult(base, identity.store_identity, loaded); + return request.scan_limit > 0 ? await appendScanPage(store, request, result) : result; } catch { return { ...base, reason_code: "provider_call_failed" }; } diff --git a/loopx/control_plane/coordination/local_authority_shadow_adapter.py b/loopx/control_plane/coordination/local_authority_shadow_adapter.py index 70450e95d8..d70d84c16b 100644 --- a/loopx/control_plane/coordination/local_authority_shadow_adapter.py +++ b/loopx/control_plane/coordination/local_authority_shadow_adapter.py @@ -1103,6 +1103,95 @@ def _verify_readback( result.candidate_readback_verified = verified +def _drain_prelude( + result: DrainResult, + *, + registry_path: Path, + runtime_root: Path | None, + goal_id: str, +) -> tuple[dict[str, Any], Path] | None: + """Validate the goal id and resolve the registry and root; typed failure on error.""" + + if not goal_id or goal_id in {".", ".."} or "/" in goal_id or "\\" in goal_id: + result.outcome = "failed" + result.reason_code = "invalid_shadow_goal_id" + return None + try: + registry = load_registry(registry_path) + result.config_enabled = _shadow_config(registry, goal_id) is not None + resolved = ( + runtime_root + if runtime_root is not None + else resolve_runtime_root(registry, None, registry_path=registry_path) + ) + except Exception: + result.outcome = "failed" + result.reason_code = "invalid_shadow_config" + return None + return registry, resolved + + +def _outbox_is_idle(summary: Mapping[str, Mapping[str, Any]]) -> bool: + return all( + item["committed_pending"] == 0 and item["prepared_only"] == 0 and item["invalid"] is None + for item in summary.values() + ) + + +def _drain_partitions( + result: DrainResult, + *, + registry: dict[str, Any], + registry_path: Path, + runtime_root: Path, + goal_id: str, + max_entries: int, + budget_seconds: float, +) -> None: + """Drain every partition in order under the held drain lock, then read back.""" + + sources = _goal_sources(registry, runtime_root=runtime_root, goal_id=goal_id) + result.cursor_before = _candidate_cursor(runtime_root, goal_id) + budget = _DrainBudget(max_entries=max_entries, budget_seconds=budget_seconds) + delivered_digests: dict[str, str] = {} + for partition in PARTITIONS: + if result.stopped_at is not None: + break + drainer = _PartitionDrainer( + registry_path=registry_path, + runtime_root=runtime_root, + goal_id=goal_id, + partition=partition, + sources=sources, + result=result, + budget=budget, + ) + drainer.run() + if drainer.last_delivered_digest is not None: + delivered_digests[partition] = drainer.last_delivered_digest + if delivered_digests or result.drained_count: + _verify_readback( + result, + runtime_root=runtime_root, + goal_id=goal_id, + delivered_digests=delivered_digests, + ) + + +def _settle_drain_outcome(result: DrainResult) -> None: + if result.stopped_at is not None: + result.outcome = "stopped" + result.reason_code = str(result.stopped_at.get("reason_code") or result.stopped_at["outcome"]) + else: + result.outcome = "drained" + + +def _count_backlog(result: DrainResult, runtime_root: Path, goal_id: str) -> None: + summary_after = outbox.outbox_summary(runtime_root, goal_id) + result.pending_after = sum(int(item["committed_pending"]) for item in summary_after.values()) + result.prepared_only_after = sum(int(item["prepared_only"]) for item in summary_after.values()) + + def drain_local_authority_shadow_outbox( *, registry_path: Path, @@ -1119,60 +1208,30 @@ def drain_local_authority_shadow_outbox( """ result = DrainResult(goal_id=goal_id) - if not goal_id or goal_id in {".", ".."} or "/" in goal_id or "\\" in goal_id: - result.outcome = "failed" - result.reason_code = "invalid_shadow_goal_id" - return result - try: - registry = load_registry(registry_path) - result.config_enabled = _shadow_config(registry, goal_id) is not None - if runtime_root is None: - runtime_root = resolve_runtime_root(registry, None, registry_path=registry_path) - except Exception: - result.outcome = "failed" - result.reason_code = "invalid_shadow_config" + prelude = _drain_prelude( + result, registry_path=registry_path, runtime_root=runtime_root, goal_id=goal_id + ) + if prelude is None: return result - - summary_before = outbox.outbox_summary(runtime_root, goal_id) - if all( - item["committed_pending"] == 0 and item["prepared_only"] == 0 and item["invalid"] is None - for item in summary_before.values() - ): + registry, resolved_root = prelude + if _outbox_is_idle(outbox.outbox_summary(resolved_root, goal_id)): result.outcome = "nothing_pending" return result - try: with exclusive_file_lock( - outbox.drain_lock_target(runtime_root, goal_id), + outbox.drain_lock_target(resolved_root, goal_id), timeout_seconds=lock_timeout_seconds, operation="local_authority_shadow_drain", ): - sources = _goal_sources(registry, runtime_root=runtime_root, goal_id=goal_id) - result.cursor_before = _candidate_cursor(runtime_root, goal_id) - budget = _DrainBudget(max_entries=max_entries, budget_seconds=budget_seconds) - delivered_digests: dict[str, str] = {} - for partition in PARTITIONS: - if result.stopped_at is not None: - break - drainer = _PartitionDrainer( - registry_path=registry_path, - runtime_root=runtime_root, - goal_id=goal_id, - partition=partition, - sources=sources, - result=result, - budget=budget, - ) - drainer.run() - if drainer.last_delivered_digest is not None: - delivered_digests[partition] = drainer.last_delivered_digest - if delivered_digests or result.delivered or result.replayed or result.reconciled: - _verify_readback( - result, - runtime_root=runtime_root, - goal_id=goal_id, - delivered_digests=delivered_digests, - ) + _drain_partitions( + result, + registry=registry, + registry_path=registry_path, + runtime_root=resolved_root, + goal_id=goal_id, + max_entries=max_entries, + budget_seconds=budget_seconds, + ) except LockAcquireTimeoutError: result.outcome = "drain_deferred" result.reason_code = "drain_lock_busy" @@ -1183,14 +1242,8 @@ def drain_local_authority_shadow_outbox( result.outcome = "stopped" result.reason_code = "shadow_drain_failed" else: - if result.stopped_at is not None: - result.outcome = "stopped" - result.reason_code = str(result.stopped_at.get("reason_code") or result.stopped_at["outcome"]) - else: - result.outcome = "drained" - summary_after = outbox.outbox_summary(runtime_root, goal_id) - result.pending_after = sum(int(item["committed_pending"]) for item in summary_after.values()) - result.prepared_only_after = sum(int(item["prepared_only"]) for item in summary_after.values()) + _settle_drain_outcome(result) + _count_backlog(result, resolved_root, goal_id) return result diff --git a/loopx/control_plane/coordination/local_authority_shadow_outbox.py b/loopx/control_plane/coordination/local_authority_shadow_outbox.py index c059d9fa43..c071b92006 100644 --- a/loopx/control_plane/coordination/local_authority_shadow_outbox.py +++ b/loopx/control_plane/coordination/local_authority_shadow_outbox.py @@ -180,62 +180,72 @@ def recorded_partition_digest(self) -> str | None: return None -def list_entries(directory: Path) -> list[OutboxEntry]: - """All entries of one partition directory, oldest first.""" +_EntryKey = tuple[int, str] - if not directory.is_dir(): - return [] - prepared: dict[tuple[int, str], Path] = {} - committed: dict[tuple[int, str], Path] = {} + +def _index_entry_files(directory: Path) -> tuple[dict[_EntryKey, Path], dict[_EntryKey, Path]]: + """Map ``(seq, entry_id)`` to the prepared and committed files present.""" + + prepared: dict[_EntryKey, Path] = {} + committed: dict[_EntryKey, Path] = {} for path in directory.iterdir(): match = _ENTRY_FILE.match(path.name) if match is None: continue key = (int(match.group("seq")), match.group("entry_id")) - if match.group("phase") == "prepared": - prepared[key] = path - else: - committed[key] = path + target = prepared if match.group("phase") == "prepared" else committed + target[key] = path + return prepared, committed + + +def _load_prepared_record(path: Path, *, seq: int, entry_id: str) -> dict[str, Any]: + record = _load_json(path) + if ( + record.get("schema_version") != OUTBOX_ENTRY_SCHEMA + or record.get("entry_id") != entry_id + or record.get("seq") != seq + ): + raise OutboxError("outbox_file_invalid", f"{path.name} does not match its name") + return record + + +def _load_committed_record(path: Path | None, *, entry_id: str) -> dict[str, Any] | None: + if path is None: + return None + record = _load_json(path) + if record.get("schema_version") != OUTBOX_COMMIT_SCHEMA or record.get("entry_id") != entry_id: + raise OutboxError("outbox_file_invalid", f"{path.name} does not match its entry") + return record + + +def list_entries(directory: Path) -> list[OutboxEntry]: + """All entries of one partition directory, oldest first.""" + + if not directory.is_dir(): + return [] + prepared, committed = _index_entry_files(directory) + orphan_markers = sorted(set(committed) - set(prepared)) + if orphan_markers: + seq, entry_id = orphan_markers[0] + raise OutboxError( + "outbox_file_invalid", + f"committed marker without prepared entry: {entry_file_name(seq, entry_id, 'committed')}", + ) entries: list[OutboxEntry] = [] - for key in sorted(prepared): - seq, entry_id = key - prepared_path = prepared[key] - prepared_record = _load_json(prepared_path) - if ( - prepared_record.get("schema_version") != OUTBOX_ENTRY_SCHEMA - or prepared_record.get("entry_id") != entry_id - or prepared_record.get("seq") != seq - ): - raise OutboxError("outbox_file_invalid", f"{prepared_path.name} does not match its name") - committed_path = committed.get(key) - committed_record = None - if committed_path is not None: - committed_record = _load_json(committed_path) - if ( - committed_record.get("schema_version") != OUTBOX_COMMIT_SCHEMA - or committed_record.get("entry_id") != entry_id - ): - raise OutboxError( - "outbox_file_invalid", f"{committed_path.name} does not match its entry" - ) + for seq, entry_id in sorted(prepared): + key = (seq, entry_id) + prepared_record = _load_prepared_record(prepared[key], seq=seq, entry_id=entry_id) entries.append( OutboxEntry( partition=str(prepared_record.get("partition") or directory.name), seq=seq, entry_id=entry_id, - prepared_path=prepared_path, - committed_path=committed_path, + prepared_path=prepared[key], + committed_path=committed.get(key), prepared=prepared_record, - committed=committed_record, + committed=_load_committed_record(committed.get(key), entry_id=entry_id), ) ) - orphan_markers = sorted(set(committed) - set(prepared)) - if orphan_markers: - seq, entry_id = orphan_markers[0] - raise OutboxError( - "outbox_file_invalid", - f"committed marker without prepared entry: {entry_file_name(seq, entry_id, 'committed')}", - ) return entries @@ -590,6 +600,49 @@ def committed(self, *, projection_from_disk: bool = False) -> None: """Return ``committed``, ``abandoned`` or ``unproved`` for a prepared-only entry.""" +_LEASE_FENCE_KEYS = ("version", "lease_epoch", "status", "updated_at") + + +def _resolve_markdown_source(source: Mapping[str, Any], reader: Callable[[], str]) -> str: + current_digest = text_digest(reader()) + if current_digest == source.get("bytes_digest"): + return "committed" + if current_digest == source.get("previous_bytes_digest"): + return "abandoned" + return "unproved" + + +def _lease_matches(current: Mapping[str, Any] | None, expected: Mapping[str, Any]) -> bool: + if current is None or not expected: + return False + return all(current.get(key) == expected.get(key) for key in _LEASE_FENCE_KEYS) + + +def _resolve_lease_source( + source: Mapping[str, Any], + reader: Callable[[str], dict[str, Any] | None], +) -> str: + planned = _as_object(source.get("lease")) + if not planned: + return "unproved" + current = reader(str(planned.get("todo_id") or "")) + if _lease_matches(current, planned): + return "committed" + previous = _as_object(source.get("previous_lease")) + if (not previous and current is None) or _lease_matches(current, previous): + return "abandoned" + return "unproved" + + +def _resolve_event_source(source: Mapping[str, Any], reader: Callable[[str], bool]) -> str: + event_id = source.get("event_id") + if isinstance(event_id, str) and event_id and reader(event_id): + # The append landed but the projection was never recorded; only a + # fresh full-partition capture can say what the state now is. + return "unproved" + return "abandoned" + + def resolve_prepared_only_entry( entry: OutboxEntry, *, @@ -606,35 +659,11 @@ def resolve_prepared_only_entry( source = _as_object(entry.prepared.get("source")) kind = source.get("kind") if kind == SOURCE_MARKDOWN and markdown_text_reader is not None: - current_digest = text_digest(markdown_text_reader()) - if current_digest == source.get("bytes_digest"): - return "committed" - if current_digest == source.get("previous_bytes_digest"): - return "abandoned" - return "unproved" + return _resolve_markdown_source(source, markdown_text_reader) if kind == SOURCE_TASK_LEASE and lease_record_reader is not None: - planned = _as_object(source.get("lease")) - if not planned: - return "unproved" - current = lease_record_reader(str(planned.get("todo_id") or "")) - keys = ("version", "lease_epoch", "status", "updated_at") - if current is not None and all(current.get(key) == planned.get(key) for key in keys): - return "committed" - previous = _as_object(source.get("previous_lease")) - if not previous and current is None: - return "abandoned" - if previous and current is not None and all( - current.get(key) == previous.get(key) for key in keys - ): - return "abandoned" - return "unproved" + return _resolve_lease_source(source, lease_record_reader) if kind == SOURCE_STATE_EVENT_LOG and event_presence_reader is not None: - event_id = source.get("event_id") - if isinstance(event_id, str) and event_id and event_presence_reader(event_id): - # The append landed but the projection was never recorded; only a - # fresh full-partition capture can say what the state now is. - return "unproved" - return "abandoned" + return _resolve_event_source(source, event_presence_reader) return "unproved" diff --git a/loopx/control_plane/coordination/local_authority_shadow_outbox.ts b/loopx/control_plane/coordination/local_authority_shadow_outbox.ts index 84a4b8a3c6..8efd304daf 100644 --- a/loopx/control_plane/coordination/local_authority_shadow_outbox.ts +++ b/loopx/control_plane/coordination/local_authority_shadow_outbox.ts @@ -4,7 +4,7 @@ import { join } from "node:path"; import type { JsonObject } from "../effect_program.ts"; import { durableWriteJson } from "../effect_runtime_io.ts"; -import { canonicalAuthorityBytes } from "./authority_store_codec.ts"; +import { authorityUnicodeCompare, canonicalAuthorityBytes } from "./authority_store_codec.ts"; /** * Lease-partition side of the local authority shadow outbox. @@ -142,7 +142,7 @@ async function readLeasePartition( } records.set(plannedStem, plannedLease); const stems = [...records.keys()]; - stems.sort((left, right) => (left < right ? -1 : left > right ? 1 : 0)); + stems.sort(authorityUnicodeCompare); return stems.map((stem) => ({ file_stem: stem, record: records.get(stem) as JsonObject })); } diff --git a/tests/control_plane/test_local_authority_shadow_cli_e2e.py b/tests/control_plane/test_local_authority_shadow_cli_e2e.py index d90875a7fb..68fe35dbd8 100644 --- a/tests/control_plane/test_local_authority_shadow_cli_e2e.py +++ b/tests/control_plane/test_local_authority_shadow_cli_e2e.py @@ -483,8 +483,10 @@ def test_product_cli_authority_shadow_status_and_drain_read_without_creating_lin assert default_off["store_bytes"] == 0 assert not (runtime_root / "authority-shadow").exists() idle = _cli(registry, runtime_root, "authority-shadow", "drain", "--goal-id", goal_id) - assert idle["ok"] is True and idle["outcome"] == "nothing_pending" - assert idle["drained_count"] == 0 and idle["config_enabled"] is False + assert idle["ok"] is True + assert idle["outcome"] == "nothing_pending" + assert idle["drained_count"] == 0 + assert idle["config_enabled"] is False assert not (runtime_root / "authority-shadow").exists() _cli( @@ -510,11 +512,13 @@ def test_product_cli_authority_shadow_status_and_drain_read_without_creating_lin assert candidate["head_schema_version"] == "loopx_local_authority_shadow_projection_v0" # type: ignore[index] assert candidate["codec_agreement"] is True # type: ignore[index] assert candidate["partitions"] == {"todos": None, "leases": None} # type: ignore[index] - assert status["store_bytes"] > 0 and status["retention_pressure"] is False + assert status["store_bytes"] > 0 + assert status["retention_pressure"] is False assert str(runtime_root) not in json.dumps(status) drained = _cli(registry, runtime_root, "authority-shadow", "drain", "--goal-id", goal_id) - assert drained["ok"] is True and drained["outcome"] == "nothing_pending" + assert drained["ok"] is True + assert drained["outcome"] == "nothing_pending" assert drained["config_enabled"] is True _store_path, store = _store_document(runtime_root, goal_id) assert store["cursor"] == "1" diff --git a/tests/control_plane/test_local_authority_shadow_drain.py b/tests/control_plane/test_local_authority_shadow_drain.py index 8f7b800fcf..1b7ee7eb71 100644 --- a/tests/control_plane/test_local_authority_shadow_drain.py +++ b/tests/control_plane/test_local_authority_shadow_drain.py @@ -139,23 +139,28 @@ def test_drain_delivers_each_committed_entry_once_in_order_and_verifies_readback assert result.outcome == "drained" assert result.config_enabled is False assert (result.delivered, result.replayed, result.no_op) == (3, 0, 0) - assert result.pending_after == 0 and result.prepared_only_after == 0 + assert result.pending_after == 0 + assert result.prepared_only_after == 0 assert result.budget_exhausted is False assert result.candidate_readback_verified is True assert result.last_cursor == "3" assert (result.cursor_before, result.cursor_after, result.drained_count) == (None, "3", 3) payload = result.to_payload() - assert payload["ok"] is True and payload["drained_count"] == 3 and payload["cursor_after"] == "3" + assert payload["ok"] is True + assert payload["drained_count"] == 3 + assert payload["cursor_after"] == "3" assert [item["outcome"] for item in result.entries] == ["delivered"] * 3 assert [item["cursor"] for item in result.entries] == ["1", "2", "3"] assert [item["entry_id"] for item in result.entries] == [ capture.outcome.entry_id for capture in captures ] - assert result.store_identity is not None and result.store_identity.startswith("file:") + assert result.store_identity is not None + assert result.store_identity.startswith("file:") assert list(_todo_dir(runtime_root).iterdir()) == [_todo_dir(runtime_root) / "drain-cursor.json"] cursor = outbox.read_cursor(_todo_dir(runtime_root)) assert cursor is not None - assert cursor["last_seq"] == 3 and cursor["last_entry_id"] == captures[-1].outcome.entry_id + assert cursor["last_seq"] == 3 + assert cursor["last_entry_id"] == captures[-1].outcome.entry_id assert cursor["last_partition_digest"] == captures[-1].outcome.partition_digest view = adapter.read_local_authority_shadow(runtime_root=runtime_root, goal_id=GOAL_ID, scan_limit=10) @@ -185,7 +190,8 @@ def test_drain_delivers_each_committed_entry_once_in_order_and_verifies_readback assert str(runtime_root) not in json.dumps(view) again = _drain(registry, runtime_root) - assert again.outcome == "nothing_pending" and again.ok is True + assert again.outcome == "nothing_pending" + assert again.ok is True def test_drain_replays_when_store_committed_but_cursor_was_not_written( @@ -235,14 +241,16 @@ def test_drain_batch_is_bounded_and_reports_what_it_left(tmp_path: Path) -> None _record_todo_write(registry, state, runtime_root, f"Bounded {index}") first = _drain(registry, runtime_root, max_entries=2) - assert first.outcome == "drained" and first.ok is True + assert first.outcome == "drained" + assert first.ok is True assert first.delivered == 2 assert first.budget_exhausted is True assert first.pending_after == 1 assert first.candidate_readback_verified is True second = _drain(registry, runtime_root) - assert second.delivered == 1 and second.pending_after == 0 + assert second.delivered == 1 + assert second.pending_after == 0 assert (second.cursor_before, second.cursor_after) == ("2", "3") view = adapter.read_local_authority_shadow(runtime_root=runtime_root, goal_id=GOAL_ID) assert view["cursor"] == "3" @@ -276,7 +284,8 @@ def flaky(method: str, params: object, **kwargs: object) -> object: monkeypatch.undo() recovered = _drain(registry, runtime_root) - assert recovered.delivered == 2 and recovered.pending_after == 0 + assert recovered.delivered == 2 + assert recovered.pending_after == 0 def test_prepared_only_entries_resolve_only_under_a_free_primary_lock(tmp_path: Path) -> None: @@ -291,7 +300,8 @@ def test_prepared_only_entries_resolve_only_under_a_free_primary_lock(tmp_path: result = _drain(registry, runtime_root) assert result.ok is True - assert result.delivered == 1 and result.no_op == 0 + assert result.delivered == 1 + assert result.no_op == 0 assert result.entries[0]["resolution"] == "committed_proven_by_readback" assert result.entries[0]["entry_id"] == proven.outcome.entry_id view = adapter.read_local_authority_shadow(runtime_root=runtime_root, goal_id=GOAL_ID, scan_limit=5) @@ -303,7 +313,8 @@ def test_prepared_only_entries_resolve_only_under_a_free_primary_lock(tmp_path: registry, state, runtime_root, "Never landed", mark_committed=False, write_file=False ) result = _drain(registry, runtime_root) - assert result.delivered == 1 and result.no_op == 1 + assert result.delivered == 1 + assert result.no_op == 1 assert result.entries[0]["resolution"] == "abandoned" assert result.entries[0]["entry_id"] == abandoned.outcome.entry_id view = adapter.read_local_authority_shadow(runtime_root=runtime_root, goal_id=GOAL_ID, scan_limit=5) @@ -395,10 +406,12 @@ def test_lease_partition_entries_are_compacted_at_drain(tmp_path: Path) -> None: result = _drain(registry, runtime_root) - assert result.ok is True and result.delivered == 1 + assert result.ok is True + assert result.delivered == 1 view = adapter.read_local_authority_shadow(runtime_root=runtime_root, goal_id=GOAL_ID, scan_limit=5) head = view["head"] - assert head["todos"] == [] and head["handoff_mode"] is None + assert head["todos"] == [] + assert head["handoff_mode"] is None assert head["leases"] == [ { "todo_id": record["todo_id"], @@ -426,7 +439,8 @@ def test_status_reports_backlog_candidate_and_growth_facts(tmp_path: Path) -> No assert empty["ok"] is True assert empty["config"]["status"] == "disabled" assert empty["candidate"]["status"] == "missing" - assert empty["store_bytes"] == 0 and empty["retention_pressure"] is False + assert empty["store_bytes"] == 0 + assert empty["retention_pressure"] is False assert str(runtime_root) not in json.dumps(empty) _record_todo_write(registry, state, runtime_root, "Status fact") @@ -467,7 +481,8 @@ def test_capture_evidence_v1_reports_measured_facts_only(tmp_path: Path) -> None deferred = adapter.DrainResult(goal_id=GOAL_ID, outcome="drain_deferred", reason_code="drain_lock_busy") assert adapter.capture_evidence(goal_id=GOAL_ID, capture=capture.outcome, drain=deferred)["outcome"] == "drain_deferred" pending = adapter.capture_evidence(goal_id=GOAL_ID, capture=capture.outcome, drain=None) - assert pending["outcome"] == "pending" and pending["drain"] is None + assert pending["outcome"] == "pending" + assert pending["drain"] is None skipped = outbox.CaptureOutcome(partition="todos", skipped_reason="partition_unchanged") assert adapter.capture_evidence(goal_id=GOAL_ID, capture=skipped, drain=None)["outcome"] == "no_transaction" diff --git a/tests/control_plane/test_local_authority_shadow_outbox.py b/tests/control_plane/test_local_authority_shadow_outbox.py index 330397ab10..463bc0629f 100644 --- a/tests/control_plane/test_local_authority_shadow_outbox.py +++ b/tests/control_plane/test_local_authority_shadow_outbox.py @@ -112,7 +112,8 @@ def test_capture_records_prepared_then_committed_and_skips_prose_only_writes(tmp capture = _capture(registry, state, runtime_root, original_text=original) capture.prepare(new_text) names = sorted(path.name for path in _todo_dir(runtime_root).iterdir()) - assert len(names) == 1 and names[0].endswith(".prepared.json") + assert len(names) == 1 + assert names[0].endswith(".prepared.json") assert capture.outcome.entry_id is not None assert capture.outcome.seq == 1 assert capture.outcome.source_bytes_digest == text_digest(new_text) @@ -320,7 +321,7 @@ def test_canonical_projection_rejects_floats_and_bad_lease_identity() -> None: with pytest.raises(ProjectionValueError): todo_partition_projection(handoff_mode="hard_lease", todos=[{"todo_id": "a", "status": 2.5}]) assert canonical_bytes({"b": 1, "a": [True, None, "\u00e9"]}) == '{"a":[true,null,"\u00e9"],"b":1}'.encode("utf-8") - assert sha256_digest({"a": 1}) == sha256_digest({"a": 1}) + assert sha256_digest({"b": [1, None], "a": "x"}) == sha256_digest({"a": "x", "b": [1, None]}) with pytest.raises(ProjectionValueError): lease_partition_projection([("todo-a", {"goal_id": "other", "todo_id": "todo-a"})], goal_id=GOAL_ID) projection = lease_partition_projection( From c925ce0d350b2437c589195d7ac12b8c435b41a8 Mon Sep 17 00:00:00 2001 From: wchwawa Date: Thu, 3 Sep 2026 12:04:57 +1000 Subject: [PATCH 03/11] refactor(authority): split the shadow entry commit loop into attempt and settlement helpers SonarCloud flagged commitLocalAuthorityShadowEntry at cognitive complexity 18. attemptCommitEntry performs one load-compose-commit round and settleCommitOutcome maps the store result to a final or retryable outcome; the exported function keeps its contract and retry count. No behavior change. Signed-off-by: wchwawa --- .../coordination/local_authority_shadow.ts | 158 +++++++++++------- 1 file changed, 100 insertions(+), 58 deletions(-) diff --git a/loopx/control_plane/coordination/local_authority_shadow.ts b/loopx/control_plane/coordination/local_authority_shadow.ts index a5760fd7dc..4d93e285fd 100644 --- a/loopx/control_plane/coordination/local_authority_shadow.ts +++ b/loopx/control_plane/coordination/local_authority_shadow.ts @@ -11,6 +11,7 @@ import { } from "../runtime_decode.ts"; import type { AuthorityStore, + AuthorityStoreCommitResult, AuthorityStoreCommittedTransaction, AuthorityStoreLoadResult, AuthorityStoreReceiptResult, @@ -814,6 +815,100 @@ function openShadowStore( ); } +type CommitAttempt = + | { kind: "final"; result: LocalAuthorityShadowCommitEntryResult } + | { kind: "retry"; result: LocalAuthorityShadowCommitEntryResult }; + +async function settleCommitOutcome( + store: AuthorityStore, + request: CommitEntryRequest, + storeIdentity: string, + committed: AuthorityStoreCommitResult, + headDigest: string, +): Promise { + if (committed.status === "applied") { + return { + kind: "final", + result: commitEntryResult(request, "delivered", { + storeIdentity, + providerRevision: committed.provider_revision, + cursor: committed.cursor, + headDigest, + }), + }; + } + if (committed.status === "ambiguous") { + return { + kind: "final", + result: await reconcileTransactionReceipt(store, request, storeIdentity, "ambiguous_reconciled"), + }; + } + if (committed.status === "failed") { + return { + kind: "final", + result: commitEntryResult(request, "failed", { + reasonCode: committed.reason_code, + storeIdentity, + }), + }; + } + if (committed.conflict_kind === "operation_id_exists") { + return { + kind: "final", + result: await reconcileTransactionReceipt(store, request, storeIdentity, "replayed"), + }; + } + return { + kind: "retry", + result: commitEntryResult(request, "conflict_retry_required", { + reasonCode: "provider_revision_mismatch", + storeIdentity, + providerRevision: committed.current_provider_revision, + cursor: committed.current_cursor, + }), + }; +} + +/** One load-compose-commit attempt against the current provider revision. */ +async function attemptCommitEntry( + store: AuthorityStore, + request: CommitEntryRequest, + storeIdentity: string, + noOp: boolean, +): Promise { + const loaded = await store.loadAuthority(); + if (loaded.status === "unavailable" || loaded.status === "failed") { + return { + kind: "final", + result: commitEntryResult(request, loaded.status, { + reasonCode: loaded.reason_code, + storeIdentity, + }), + }; + } + const nextHead = composeLocalAuthorityShadowHead( + loaded.status === "loaded" ? loaded.head : null, + request.goal_id, + request.entry, + request.partition_projection, + request.partition_digest, + ); + const committed = await store.commitAuthority({ + expected_provider_revision: loaded.status === "loaded" ? loaded.provider_revision : null, + operation_id: request.entry.entry_id, + events: [transactionEvent(request, noOp)], + next_projection: nextHead, + receipts: [transactionReceipt(request, noOp)], + }); + return await settleCommitOutcome( + store, + request, + storeIdentity, + committed, + localAuthorityShadowHeadDigest(nextHead), + ); +} + /** * Commit one drained outbox entry as exactly one candidate transaction. * @@ -841,65 +936,12 @@ export async function commitLocalAuthorityShadowEntry( if (identity.status !== "available") { return commitEntryResult(request, identity.status, { reasonCode: identity.reason_code }); } - const storeIdentity = identity.store_identity; - let lastConflict: LocalAuthorityShadowCommitEntryResult | null = null; - for (let attempt = 0; attempt < REVISION_RETRY_ATTEMPTS; attempt += 1) { - const loaded = await store.loadAuthority(); - if (loaded.status === "unavailable" || loaded.status === "failed") { - return commitEntryResult(request, loaded.status, { - reasonCode: loaded.reason_code, - storeIdentity, - }); - } - const currentHead = loaded.status === "loaded" ? loaded.head : null; - const nextHead = composeLocalAuthorityShadowHead( - currentHead, - request.goal_id, - request.entry, - request.partition_projection, - request.partition_digest, - ); - const headDigest = localAuthorityShadowHeadDigest(nextHead); - const committed = await store.commitAuthority({ - expected_provider_revision: loaded.status === "loaded" ? loaded.provider_revision : null, - operation_id: request.entry.entry_id, - events: [transactionEvent(request, noOp)], - next_projection: nextHead, - receipts: [transactionReceipt(request, noOp)], - }); - if (committed.status === "applied") { - return commitEntryResult(request, "delivered", { - storeIdentity, - providerRevision: committed.provider_revision, - cursor: committed.cursor, - headDigest, - }); - } - if (committed.status === "ambiguous") { - return await reconcileTransactionReceipt( - store, - request, - storeIdentity, - "ambiguous_reconciled", - ); - } - if (committed.status === "failed") { - return commitEntryResult(request, "failed", { - reasonCode: committed.reason_code, - storeIdentity, - }); - } - if (committed.conflict_kind === "operation_id_exists") { - return await reconcileTransactionReceipt(store, request, storeIdentity, "replayed"); - } - lastConflict = commitEntryResult(request, "conflict_retry_required", { - reasonCode: "provider_revision_mismatch", - storeIdentity, - providerRevision: committed.current_provider_revision, - cursor: committed.current_cursor, - }); + let attempt: CommitAttempt | null = null; + for (let index = 0; index < REVISION_RETRY_ATTEMPTS; index += 1) { + attempt = await attemptCommitEntry(store, request, identity.store_identity, noOp); + if (attempt.kind === "final") return attempt.result; } - return lastConflict as LocalAuthorityShadowCommitEntryResult; + return (attempt as CommitAttempt).result; } catch { return commitEntryResult(request, "unavailable", { reasonCode: "provider_call_failed" }); } From 9e80c80166f1dd8d360142c3d1657b7ae46d9b3d Mon Sep 17 00:00:00 2001 From: wchwawa Date: Fri, 4 Sep 2026 09:11:36 +1000 Subject: [PATCH 04/11] fix(authority): reclaim cursor-covered outbox residue and report only recorded outbox facts Review findings on 50afe7cba, both reproduced on that head: 1. A crash after the drain cursor was written but between unlinking the prepared and the committed file left a committed-only marker that list_entries() rejected as corruption, wedging drain and status for good. The cursor is written before any unlink, so every entry file at or below cursor.last_seq is settled residue: retired_residue() names it, list_entries() no longer lists or rejects it, reclaim_retired_residue() unlinks it under the drain lock at the start of every partition pass, and status reports it as retired_residue instead of invalid. A committed marker above the cursor is still corruption and still fails closed. Swapping the unlink order alone would only have turned the window into a prepared-only entry and risked a wrong source resolution or reseed. 2. capture_evidence() derived durable_source_outbox from "no failure", so a disabled or unchanged capture reported no_transaction together with durable_source_outbox=true. Both durable_source_outbox and source_transaction_correlated are now capture.recorded: true only when a prepared/committed entry was actually written for this write. Hardening from the same review: a prepared record must bind its directory (goal id and partition), carry a well-formed source_root_digest, name a known writer runtime and source kind, and recompute to its own entry id from its source reference (bytes digest, event id, or seed digest); anything else fails closed before it can reach the candidate. The drain also refuses an entry recorded for a different runtime root (source_root_mismatch). Both runtimes now hash the dot-normalized absolute root without resolving symlinks so the digest agrees across the effect-runtime boundary. Tests cover the crash between the two unlinks, the crash before any unlink (reclaimed without a store call), an orphan marker above the cursor, the foreign-root entry, tampered prepared records, the watermark semantics, and the honest evidence flags for disabled and unchanged captures. Signed-off-by: wchwawa --- loopx/cli_commands/authority_shadow.py | 1 + .../local_authority_shadow_adapter.py | 24 ++- .../local_authority_shadow_outbox.py | 141 ++++++++++++++++-- .../local_authority_shadow_outbox.ts | 4 +- .../test_local_authority_shadow_drain.py | 139 ++++++++++++++++- .../test_local_authority_shadow_outbox.py | 73 +++++++++ .../local_authority_shadow_outbox.test.ts | 4 +- 7 files changed, 367 insertions(+), 19 deletions(-) diff --git a/loopx/cli_commands/authority_shadow.py b/loopx/cli_commands/authority_shadow.py index 15c4aaf419..e62a3b4f67 100644 --- a/loopx/cli_commands/authority_shadow.py +++ b/loopx/cli_commands/authority_shadow.py @@ -32,6 +32,7 @@ "reconciled", "no_op", "reseeded", + "reclaimed_residue", "pending_after", "prepared_only_after", "budget_exhausted", diff --git a/loopx/control_plane/coordination/local_authority_shadow_adapter.py b/loopx/control_plane/coordination/local_authority_shadow_adapter.py index d70d84c16b..fd421ae94c 100644 --- a/loopx/control_plane/coordination/local_authority_shadow_adapter.py +++ b/loopx/control_plane/coordination/local_authority_shadow_adapter.py @@ -575,6 +575,7 @@ class DrainResult: reconciled: int = 0 no_op: int = 0 reseeded: int = 0 + reclaimed_residue: int = 0 pending_after: int = 0 prepared_only_after: int = 0 in_flight_partitions: list[str] = field(default_factory=list) @@ -905,6 +906,10 @@ def __init__( self.last_delivered_digest: str | None = None def run(self) -> None: + # Files the cursor already covers are settled; reclaim them first so a + # crash between the cursor write and the unlinks can never wedge the + # partition. + self._result.reclaimed_residue += outbox.reclaim_retired_residue(self._directory) while not self._budget.exhausted(): entries = outbox.list_entries(self._directory) if not entries: @@ -985,6 +990,14 @@ def _commit( projection: dict[str, Any] | None, digest: str | None, ) -> bool: + expected_root = outbox.runtime_root_digest(self._runtime_root) + if entry.prepared.get("source_root_digest") != expected_root: + # The entry was written for a different runtime root; delivering it + # here would stitch another lineage's transaction into this one. + raise outbox.OutboxError( + "source_root_mismatch", + f"entry {entry.entry_id} was recorded for a different runtime root", + ) request = _commit_entry_request( runtime_root=self._runtime_root, goal_id=self._goal_id, @@ -1133,7 +1146,10 @@ def _drain_prelude( def _outbox_is_idle(summary: Mapping[str, Mapping[str, Any]]) -> bool: return all( - item["committed_pending"] == 0 and item["prepared_only"] == 0 and item["invalid"] is None + item["committed_pending"] == 0 + and item["prepared_only"] == 0 + and item["retired_residue"] == 0 + and item["invalid"] is None for item in summary.values() ) @@ -1378,6 +1394,7 @@ def capture_evidence( "outcome": drain.outcome, "delivered": drain.delivered, "replayed": drain.replayed, + "reclaimed_residue": drain.reclaimed_residue, "pending_after": drain.pending_after, "prepared_only_after": drain.prepared_only_after, "stopped_at": drain.stopped_at, @@ -1386,8 +1403,11 @@ def capture_evidence( "candidate_readback_verified": drain.candidate_readback_verified, }, "capture_kind": "source_transaction_outbox", + # Both flags are measured facts of this write: they are true only when + # a prepared/committed entry was actually recorded for it. A disabled + # or unchanged capture recorded nothing and claims nothing. "source_transaction_correlated": capture.recorded, - "durable_source_outbox": capture.failure is None, + "durable_source_outbox": capture.recorded, "source_candidate_compared": False, "parity_verdict": "not_evaluated", "primary_authority": "legacy_local", diff --git a/loopx/control_plane/coordination/local_authority_shadow_outbox.py b/loopx/control_plane/coordination/local_authority_shadow_outbox.py index c071b92006..2ecbb6268d 100644 --- a/loopx/control_plane/coordination/local_authority_shadow_outbox.py +++ b/loopx/control_plane/coordination/local_authority_shadow_outbox.py @@ -162,8 +162,7 @@ def is_committed(self) -> bool: @property def source_ref(self) -> str: - source = _as_object(self.prepared.get("source")) - return str(source.get("bytes_digest") or f"event:{source.get('event_id')}") + return str(record_source_ref(self.prepared)) def projection(self) -> dict[str, Any] | None: """The partition projection recorded for this entry, if any.""" @@ -198,14 +197,59 @@ def _index_entry_files(directory: Path) -> tuple[dict[_EntryKey, Path], dict[_En return prepared, committed -def _load_prepared_record(path: Path, *, seq: int, entry_id: str) -> dict[str, Any]: +_WRITER_RUNTIMES = frozenset({WRITER_RUNTIME_PYTHON, WRITER_RUNTIME_TYPESCRIPT}) +_SOURCE_KINDS = frozenset({SOURCE_MARKDOWN, SOURCE_STATE_EVENT_LOG, SOURCE_TASK_LEASE}) +_DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$") + + +def _load_prepared_record( + path: Path, + *, + directory: Path, + seq: int, + entry_id: str, +) -> dict[str, Any]: + """Load one prepared record and prove it binds this directory and entry id. + + The file name, the directory (``outbox//``), the record's + own goal/partition fields, and the entry id recomputed from the record's + source reference must all agree; a record that was copied, edited, or + written for another goal fails closed before it can reach the candidate. + """ + record = _load_json(path) - if ( - record.get("schema_version") != OUTBOX_ENTRY_SCHEMA - or record.get("entry_id") != entry_id - or record.get("seq") != seq - ): - raise OutboxError("outbox_file_invalid", f"{path.name} does not match its name") + writer = _as_object(record.get("writer")) + source = _as_object(record.get("source")) + expected_goal = directory.parent.name + expected_partition = directory.name + source_ref = record_source_ref(record) + root_digest = record.get("source_root_digest") + bound = ( + record.get("schema_version") == OUTBOX_ENTRY_SCHEMA + and record.get("entry_id") == entry_id + and record.get("seq") == seq + and record.get("goal_id") == expected_goal + and record.get("partition") == expected_partition + and writer.get("runtime") in _WRITER_RUNTIMES + and isinstance(writer.get("write_class"), str) + and bool(writer.get("write_class")) + and source.get("kind") in _SOURCE_KINDS + and isinstance(root_digest, str) + and _DIGEST_PATTERN.match(root_digest) is not None + and source_ref is not None + and entry_identity( + goal_id=expected_goal, + partition=expected_partition, + seq=seq, + source_ref=source_ref, + ) + == entry_id + ) + if not bound: + raise OutboxError( + "outbox_file_invalid", + f"{path.name} does not bind its directory, identity, and source reference", + ) return record @@ -218,12 +262,56 @@ def _load_committed_record(path: Path | None, *, entry_id: str) -> dict[str, Any return record +def _retired_watermark(directory: Path) -> int: + cursor = read_cursor(directory) + return int(cursor.get("last_seq") or 0) if cursor is not None else 0 + + +def retired_residue(directory: Path) -> list[Path]: + """Entry files at or below the durable cursor. + + The cursor is written before an entry's files are unlinked, so anything it + covers is already settled in the candidate store. A crash between the + cursor write and the unlinks, or between the two unlinks, leaves these + files behind; they are residue to reclaim, never entries to deliver or + markers to reject. + """ + + if not directory.is_dir(): + return [] + watermark = _retired_watermark(directory) + prepared, committed = _index_entry_files(directory) + residue = [ + path + for key, path in [*prepared.items(), *committed.items()] + if key[0] <= watermark + ] + return sorted(residue) + + +def reclaim_retired_residue(directory: Path) -> int: + """Unlink retired residue; the caller must hold the goal's drain lock.""" + + residue = retired_residue(directory) + for path in residue: + path.unlink(missing_ok=True) + return len(residue) + + def list_entries(directory: Path) -> list[OutboxEntry]: - """All entries of one partition directory, oldest first.""" + """All live entries of one partition directory, oldest first. + + Files the durable cursor already covers are retired residue and are not + listed; a committed marker without a prepared entry above the cursor is + real corruption and fails closed. + """ if not directory.is_dir(): return [] + watermark = _retired_watermark(directory) prepared, committed = _index_entry_files(directory) + prepared = {key: path for key, path in prepared.items() if key[0] > watermark} + committed = {key: path for key, path in committed.items() if key[0] > watermark} orphan_markers = sorted(set(committed) - set(prepared)) if orphan_markers: seq, entry_id = orphan_markers[0] @@ -234,7 +322,9 @@ def list_entries(directory: Path) -> list[OutboxEntry]: entries: list[OutboxEntry] = [] for seq, entry_id in sorted(prepared): key = (seq, entry_id) - prepared_record = _load_prepared_record(prepared[key], seq=seq, entry_id=entry_id) + prepared_record = _load_prepared_record( + prepared[key], directory=directory, seq=seq, entry_id=entry_id + ) entries.append( OutboxEntry( partition=str(prepared_record.get("partition") or directory.name), @@ -318,7 +408,30 @@ def latest_partition_digest(directory: Path) -> str | None: def runtime_root_digest(runtime_root: Path) -> str: - return text_digest(str(runtime_root.resolve(strict=False))) + """Digest of the absolute, dot-normalized root; must match the TypeScript writer. + + Symlinks are deliberately not resolved: both runtimes normalize the string + they were given, so a root passed through the effect runtime hashes the + same on either side. + """ + + return text_digest(os.path.abspath(str(runtime_root))) + + +def record_source_ref(record: Mapping[str, Any]) -> str | None: + """The source reference an entry id binds: bytes digest, event id, or seed digest.""" + + source = _as_object(record.get("source")) + bytes_digest = source.get("bytes_digest") + if isinstance(bytes_digest, str) and bytes_digest: + return bytes_digest + event_id = source.get("event_id") + if isinstance(event_id, str) and event_id: + return f"event:{event_id}" + digest = record.get("partition_digest") + if isinstance(digest, str) and digest: + return f"seed:{digest}" + return None def read_lease_records(directory: Path) -> list[tuple[str, dict[str, Any]]]: @@ -689,6 +802,7 @@ def outbox_summary(runtime_root: Path, goal_id: str) -> dict[str, Any]: summary[partition] = { "committed_pending": sum(1 for entry in entries if entry.is_committed), "prepared_only": sum(1 for entry in entries if not entry.is_committed), + "retired_residue": len(retired_residue(directory)), "next_seq": (max((entry.seq for entry in entries), default=0) if entries else 0), "cursor_last_seq": int(cursor.get("last_seq") or 0) if cursor else None, "cursor_last_entry_id": cursor.get("last_entry_id") if cursor else None, @@ -810,8 +924,11 @@ def iter_committed(entries: Iterable[OutboxEntry]) -> list[OutboxEntry]: "partition_directory", "read_cursor", "read_lease_records", + "reclaim_retired_residue", + "record_source_ref", "remove_entry_files", "resolve_prepared_only_entry", + "retired_residue", "runtime_root_digest", "utc_now_text", "write_cursor", diff --git a/loopx/control_plane/coordination/local_authority_shadow_outbox.ts b/loopx/control_plane/coordination/local_authority_shadow_outbox.ts index 8efd304daf..4f3dbaa60c 100644 --- a/loopx/control_plane/coordination/local_authority_shadow_outbox.ts +++ b/loopx/control_plane/coordination/local_authority_shadow_outbox.ts @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; import { readdir, readFile } from "node:fs/promises"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; import type { JsonObject } from "../effect_program.ts"; import { durableWriteJson } from "../effect_runtime_io.ts"; @@ -235,7 +235,7 @@ export async function beginLeaseOutboxEntry( previous_lease: leaseSourceFacts(input.previous_lease), event_id: null, }, - source_root_digest: sha256Digest(input.runtime_root), + source_root_digest: sha256Digest(resolve(input.runtime_root)), projection, partition_digest: null, prepared_at: new Date().toISOString(), diff --git a/tests/control_plane/test_local_authority_shadow_drain.py b/tests/control_plane/test_local_authority_shadow_drain.py index 1b7ee7eb71..592b18d4a3 100644 --- a/tests/control_plane/test_local_authority_shadow_drain.py +++ b/tests/control_plane/test_local_authority_shadow_drain.py @@ -452,6 +452,7 @@ def test_status_reports_backlog_candidate_and_growth_facts(tmp_path: Path) -> No assert drained["outbox"]["todos"] == { "committed_pending": 0, "prepared_only": 0, + "retired_residue": 0, "next_seq": 0, "cursor_last_seq": 1, "cursor_last_entry_id": outbox.read_cursor(_todo_dir(runtime_root))["last_entry_id"], @@ -484,11 +485,145 @@ def test_capture_evidence_v1_reports_measured_facts_only(tmp_path: Path) -> None assert pending["outcome"] == "pending" assert pending["drain"] is None - skipped = outbox.CaptureOutcome(partition="todos", skipped_reason="partition_unchanged") - assert adapter.capture_evidence(goal_id=GOAL_ID, capture=skipped, drain=None)["outcome"] == "no_transaction" + for skipped_reason in ("partition_unchanged", "shadow_disabled"): + skipped = outbox.CaptureOutcome(partition="todos", skipped_reason=skipped_reason) + no_transaction = adapter.capture_evidence(goal_id=GOAL_ID, capture=skipped, drain=None) + assert no_transaction["outcome"] == "no_transaction" + assert no_transaction["reason_code"] == skipped_reason + # Nothing was recorded, so nothing durable or correlated may be claimed. + assert no_transaction["durable_source_outbox"] is False + assert no_transaction["source_transaction_correlated"] is False + assert adapter.valid_evidence_v1(no_transaction, goal_id=GOAL_ID) failed = outbox.CaptureOutcome(partition="todos", failure={"reason_code": "outbox_prepare_failed", "error_class": "OSError"}) failed_evidence = adapter.capture_evidence(goal_id=GOAL_ID, capture=failed, drain=None) assert failed_evidence["outcome"] == "capture_failed" assert failed_evidence["durable_source_outbox"] is False assert failed_evidence["source_transaction_correlated"] is False assert adapter.valid_evidence_v1(failed_evidence, goal_id=GOAL_ID) + + +def _commit_entry_calls(monkeypatch: pytest.MonkeyPatch) -> list[str]: + calls: list[str] = [] + real = adapter.effect_runtime_result + + def counting(method: str, params: object, **kwargs: object) -> object: + calls.append(method) + return real(method, params, **kwargs) + + monkeypatch.setattr(adapter, "effect_runtime_result", counting) + return calls + + +def test_crash_between_the_two_unlinks_leaves_residue_the_next_drain_reclaims( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + registry, state, runtime_root = _fixture(tmp_path) + capture = _record_todo_write(registry, state, runtime_root, "Retired but half-removed") + real_remove = outbox.remove_entry_files + + def crash_between_unlinks(entry: outbox.OutboxEntry) -> None: + entry.prepared_path.unlink(missing_ok=True) + raise OSError("simulated crash between the prepared and committed unlinks") + + monkeypatch.setattr(outbox, "remove_entry_files", crash_between_unlinks) + first = _drain(registry, runtime_root) + assert first.outcome == "stopped" + assert first.reason_code == "shadow_drain_failed" + monkeypatch.setattr(outbox, "remove_entry_files", real_remove) + + # On disk: the cursor covers seq 1 and only the committed marker survives. + marker_name = outbox.entry_file_name(1, str(capture.outcome.entry_id), "committed") + names = sorted(path.name for path in _todo_dir(runtime_root).iterdir()) + assert names == sorted([marker_name, "drain-cursor.json"]) + assert outbox.read_cursor(_todo_dir(runtime_root))["last_seq"] == 1 + # The marker is retired residue, not corruption: listing stays valid. + assert outbox.list_entries(_todo_dir(runtime_root)) == [] + assert [path.name for path in outbox.retired_residue(_todo_dir(runtime_root))] == [marker_name] + summary = outbox.outbox_summary(runtime_root, GOAL_ID)["todos"] + assert summary["invalid"] is None + assert summary["retired_residue"] == 1 + assert summary["committed_pending"] == 0 + status = adapter.local_authority_shadow_status(registry_path=registry, runtime_root=runtime_root, goal_id=GOAL_ID) + assert status["ok"] is True + assert status["outbox"]["todos"]["retired_residue"] == 1 + + calls = _commit_entry_calls(monkeypatch) + second = _drain(registry, runtime_root) + assert second.ok is True + assert second.outcome == "drained" + assert second.reclaimed_residue == 1 + assert (second.delivered, second.replayed) == (0, 0) + assert "coordination.local_authority_shadow.commit_entry" not in calls + assert list(_todo_dir(runtime_root).iterdir()) == [_todo_dir(runtime_root) / "drain-cursor.json"] + view = adapter.read_local_authority_shadow(runtime_root=runtime_root, goal_id=GOAL_ID, scan_limit=5) + assert view["cursor"] == "1" + + # A later write mints seq 2 from the cursor, never reusing the retired seq. + later = _record_todo_write(registry, state, runtime_root, "After the reclaim") + assert later.outcome.seq == 2 + third = _drain(registry, runtime_root) + assert third.delivered == 1 + assert third.reclaimed_residue == 0 + + +def test_crash_after_the_cursor_but_before_any_unlink_is_reclaimed_without_a_store_call( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + registry, state, runtime_root = _fixture(tmp_path) + capture = _record_todo_write(registry, state, runtime_root, "Cursor written, files untouched") + real_remove = outbox.remove_entry_files + + def crash_before_unlinks(entry: outbox.OutboxEntry) -> None: + raise OSError("simulated crash after the cursor write") + + monkeypatch.setattr(outbox, "remove_entry_files", crash_before_unlinks) + assert _drain(registry, runtime_root).outcome == "stopped" + monkeypatch.setattr(outbox, "remove_entry_files", real_remove) + names = sorted(path.name for path in _todo_dir(runtime_root).iterdir()) + entry_id = str(capture.outcome.entry_id) + assert names == [ + outbox.entry_file_name(1, entry_id, "committed"), + outbox.entry_file_name(1, entry_id, "prepared"), + "drain-cursor.json", + ] + assert outbox.list_entries(_todo_dir(runtime_root)) == [] + + calls = _commit_entry_calls(monkeypatch) + result = _drain(registry, runtime_root) + assert result.ok is True + assert result.reclaimed_residue == 2 + assert "coordination.local_authority_shadow.commit_entry" not in calls + assert list(_todo_dir(runtime_root).iterdir()) == [_todo_dir(runtime_root) / "drain-cursor.json"] + + +def test_an_orphan_marker_above_the_cursor_is_still_corruption(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + capture = _record_todo_write(registry, state, runtime_root, "Marker without its prepared file") + (_todo_dir(runtime_root) / outbox.entry_file_name(1, str(capture.outcome.entry_id), "prepared")).unlink() + with pytest.raises(outbox.OutboxError) as raised: + outbox.list_entries(_todo_dir(runtime_root)) + assert raised.value.reason_code == "outbox_file_invalid" + result = _drain(registry, runtime_root) + assert result.outcome == "stopped" + assert result.reason_code == "outbox_file_invalid" + status = adapter.local_authority_shadow_status(registry_path=registry, runtime_root=runtime_root, goal_id=GOAL_ID) + assert status["ok"] is False + assert status["outbox"]["todos"]["invalid"] == "outbox_file_invalid" + + +def test_drain_fails_closed_on_an_entry_recorded_for_another_runtime_root(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + capture = _record_todo_write(registry, state, runtime_root, "Written under a foreign root") + entry_id = str(capture.outcome.entry_id) + prepared_path = _todo_dir(runtime_root) / outbox.entry_file_name(1, entry_id, "prepared") + record = json.loads(prepared_path.read_text(encoding="utf-8")) + record["source_root_digest"] = outbox.runtime_root_digest(tmp_path / "elsewhere") + outbox.durable_write_json(prepared_path, record) + + result = _drain(registry, runtime_root) + + assert result.outcome == "stopped" + assert result.reason_code == "source_root_mismatch" + assert result.delivered == 0 + assert result.pending_after == 1 + assert [entry.seq for entry in outbox.list_entries(_todo_dir(runtime_root))] == [1] diff --git a/tests/control_plane/test_local_authority_shadow_outbox.py b/tests/control_plane/test_local_authority_shadow_outbox.py index 463bc0629f..f3ce05f238 100644 --- a/tests/control_plane/test_local_authority_shadow_outbox.py +++ b/tests/control_plane/test_local_authority_shadow_outbox.py @@ -306,6 +306,7 @@ def test_sequence_advances_past_the_drain_cursor_and_lists_oldest_first(tmp_path assert summary["leases"] == { "committed_pending": 0, "prepared_only": 0, + "retired_residue": 0, "next_seq": 0, "cursor_last_seq": None, "cursor_last_entry_id": None, @@ -353,3 +354,75 @@ def test_primary_lock_probe_reports_held_locks(tmp_path: Path) -> None: with exclusive_file_lock(target, timeout_seconds=1.0, operation="test_hold"): assert adapter.primary_lock_is_free(target) is False assert adapter.primary_lock_is_free(target) is True + + +def test_prepared_records_must_bind_their_directory_identity_and_source(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + original = state.read_text(encoding="utf-8") + _add_todo(registry, "Bound to this goal and partition.") + capture = _capture(registry, state, runtime_root, original_text=original) + capture.prepare(state.read_text(encoding="utf-8")) + capture.committed() + directory = _todo_dir(runtime_root) + [entry] = outbox.list_entries(directory) + assert entry.prepared["goal_id"] == GOAL_ID + assert entry.prepared["partition"] == "todos" + assert entry.prepared["source_root_digest"] == outbox.runtime_root_digest(runtime_root) + assert outbox.record_source_ref(entry.prepared) == text_digest(state.read_text(encoding="utf-8")) + + def tampered(**changes: object) -> None: + record = dict(entry.prepared) + for key, value in changes.items(): + if isinstance(value, dict) and isinstance(record.get(key), dict): + record[key] = {**record[key], **value} + else: + record[key] = value + outbox.durable_write_json(entry.prepared_path, record) + with pytest.raises(outbox.OutboxError) as raised: + outbox.list_entries(directory) + assert raised.value.reason_code == "outbox_file_invalid" + + tampered(goal_id="goal-other") + tampered(partition="leases") + tampered(source={"bytes_digest": text_digest("some other bytes")}) + tampered(source_root_digest="not-a-digest") + tampered(writer={"runtime": "ruby"}) + tampered(source={"kind": "unknown_source"}) + outbox.durable_write_json(entry.prepared_path, entry.prepared) + assert len(outbox.list_entries(directory)) == 1 + + # Seed entries bind their identity through the partition digest instead. + lease_seed = outbox.write_seed_entry( + runtime_root=runtime_root, + goal_id=GOAL_ID, + seed=outbox.lease_seed_source(runtime_root, GOAL_ID), + ) + assert outbox.record_source_ref(lease_seed.prepared) == f"seed:{lease_seed.recorded_partition_digest()}" + lease_directory = outbox.partition_directory(runtime_root, GOAL_ID, "leases") + assert [item.entry_id for item in outbox.list_entries(lease_directory)] == [lease_seed.entry_id] + + +def test_retired_residue_is_defined_by_the_cursor_watermark(tmp_path: Path) -> None: + _registry, _state, runtime_root = _fixture(tmp_path) + directory = _todo_dir(runtime_root) + seed = outbox.SeedSource(partition="todos", projection={"handoff_mode": "hard_lease", "todos": []}) + first = outbox.write_seed_entry(runtime_root=runtime_root, goal_id=GOAL_ID, seed=seed) + second = outbox.write_seed_entry(runtime_root=runtime_root, goal_id=GOAL_ID, seed=seed) + assert outbox.retired_residue(directory) == [] + outbox.write_cursor( + directory, + partition="todos", + last_seq=first.seq, + last_entry_id=first.entry_id, + last_partition_digest=first.recorded_partition_digest(), + last_cursor="1", + last_provider_revision="rev-1", + ) + assert [path.name for path in outbox.retired_residue(directory)] == sorted( + [first.committed_path.name, first.prepared_path.name] # type: ignore[union-attr] + ) + assert [entry.seq for entry in outbox.list_entries(directory)] == [second.seq] + assert outbox.next_seq(directory) == 3 + assert outbox.reclaim_retired_residue(directory) == 2 + assert outbox.retired_residue(directory) == [] + assert [entry.seq for entry in outbox.list_entries(directory)] == [second.seq] diff --git a/tests/control_plane_ts/local_authority_shadow_outbox.test.ts b/tests/control_plane_ts/local_authority_shadow_outbox.test.ts index 5e09bb7ddd..0ec8f49194 100644 --- a/tests/control_plane_ts/local_authority_shadow_outbox.test.ts +++ b/tests/control_plane_ts/local_authority_shadow_outbox.test.ts @@ -2,7 +2,7 @@ import assert from "node:assert/strict"; import { execFile } from "node:child_process"; import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; import test from "node:test"; import { promisify } from "node:util"; @@ -24,6 +24,7 @@ import { decodeLocalAuthorityShadowBinding, leaseRecordDigest, outboxEntryIdentity, + sha256Digest, } from "../../loopx/control_plane/coordination/local_authority_shadow_outbox.ts"; const execFileAsync = promisify(execFile); @@ -289,6 +290,7 @@ test("lease outbox entries are two-phase, durable, and skipped without a binding assert.equal(prepared.schema_version, LOCAL_AUTHORITY_SHADOW_OUTBOX_ENTRY_SCHEMA); assert.equal(prepared.partition, "leases"); assert.equal(prepared.partition_digest, null); + assert.equal(prepared.source_root_digest, sha256Digest(resolve(root))); assert.deepEqual(prepared.source.lease, { todo_id: "todo-a", version: 2, From abef0f930415911912de81216301d15f7543ce47 Mon Sep 17 00:00:00 2001 From: wchwawa Date: Fri, 4 Sep 2026 10:30:18 +1000 Subject: [PATCH 05/11] fix(authority): keep a malformed outbox cursor inside the typed boundary outbox_summary() caught the OutboxError raised by list_entries() or read_cursor() and then re-read the cursor through retired_residue() outside that boundary, so an unsupported or unparseable drain-cursor.json escaped as an exception from `authority-shadow status`, from the drain's idle preflight, and from its backlog count instead of surfacing as the typed `outbox_file_invalid` the summary had already folded it into. The summary now performs one protected read of the cursor and the directory index and derives every count from it: a malformed cursor or entry reports `invalid=outbox_file_invalid` with zero counts and `retired_residue=0`, and nothing is re-read outside the boundary. `_load_json` types JSON and text decode errors as `outbox_file_invalid` as well, so a corrupt entry file is the same typed fact rather than a raw JSONDecodeError. No self-heal: the cursor stays exactly as found. Drain returns `stopped/outbox_file_invalid` without calling the candidate store, unlinking a file, or rewriting the cursor; status reports the typed partition fact and exits 1; a capture against the broken cursor records `outbox_prepare_failed` instead of raising into the primary write. Regressions cover all three surfaces plus the unparseable-entry case. Signed-off-by: wchwawa --- .../local_authority_shadow_outbox.py | 59 +++++++++++----- .../test_local_authority_shadow_drain.py | 69 +++++++++++++++++++ .../test_local_authority_shadow_outbox.py | 34 +++++++++ 3 files changed, 143 insertions(+), 19 deletions(-) diff --git a/loopx/control_plane/coordination/local_authority_shadow_outbox.py b/loopx/control_plane/coordination/local_authority_shadow_outbox.py index 2ecbb6268d..40972806f5 100644 --- a/loopx/control_plane/coordination/local_authority_shadow_outbox.py +++ b/loopx/control_plane/coordination/local_authority_shadow_outbox.py @@ -138,7 +138,10 @@ def _as_object(value: object) -> dict[str, Any]: def _load_json(path: Path) -> dict[str, Any]: - raw = json.loads(path.read_text(encoding="utf-8")) + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except ValueError as error: # JSONDecodeError and UnicodeDecodeError + raise OutboxError("outbox_file_invalid", f"{path.name} is not valid JSON") from error if not isinstance(raw, dict): raise OutboxError("outbox_file_invalid", f"{path.name} is not a JSON object") return raw @@ -262,11 +265,20 @@ def _load_committed_record(path: Path | None, *, entry_id: str) -> dict[str, Any return record -def _retired_watermark(directory: Path) -> int: - cursor = read_cursor(directory) +_EntryIndex = tuple[dict[_EntryKey, Path], dict[_EntryKey, Path]] + + +def _retired_watermark(cursor: dict[str, Any] | None) -> int: return int(cursor.get("last_seq") or 0) if cursor is not None else 0 +def _residue_below(index: _EntryIndex, watermark: int) -> list[Path]: + prepared, committed = index + return sorted( + path for key, path in [*prepared.items(), *committed.items()] if key[0] <= watermark + ) + + def retired_residue(directory: Path) -> list[Path]: """Entry files at or below the durable cursor. @@ -279,14 +291,7 @@ def retired_residue(directory: Path) -> list[Path]: if not directory.is_dir(): return [] - watermark = _retired_watermark(directory) - prepared, committed = _index_entry_files(directory) - residue = [ - path - for key, path in [*prepared.items(), *committed.items()] - if key[0] <= watermark - ] - return sorted(residue) + return _residue_below(_index_entry_files(directory), _retired_watermark(read_cursor(directory))) def reclaim_retired_residue(directory: Path) -> int: @@ -308,8 +313,11 @@ def list_entries(directory: Path) -> list[OutboxEntry]: if not directory.is_dir(): return [] - watermark = _retired_watermark(directory) - prepared, committed = _index_entry_files(directory) + return _live_entries(directory, _index_entry_files(directory), _retired_watermark(read_cursor(directory))) + + +def _live_entries(directory: Path, index: _EntryIndex, watermark: int) -> list[OutboxEntry]: + prepared, committed = index prepared = {key: path for key, path in prepared.items() if key[0] > watermark} committed = {key: path for key, path in committed.items() if key[0] > watermark} orphan_markers = sorted(set(committed) - set(prepared)) @@ -788,21 +796,34 @@ def entries_by_partition(runtime_root: Path, goal_id: str) -> dict[str, list[Out def outbox_summary(runtime_root: Path, goal_id: str) -> dict[str, Any]: - """Counts per partition for operator readback; never raises on an empty outbox.""" + """Counts per partition for operator readback. + + Never raises on an empty or malformed outbox: one protected read of the + cursor and the directory index feeds every count, so a malformed cursor or + entry is reported as ``invalid`` with zero counts and is never re-read + outside that boundary. + """ summary: dict[str, Any] = {} for partition in PARTITIONS: directory = partition_directory(runtime_root, goal_id, partition) + entries: list[OutboxEntry] = [] + cursor: dict[str, Any] | None = None + residue: list[Path] = [] + invalid: str | None = None try: - entries = list_entries(directory) - cursor = read_cursor(directory) - invalid: str | None = None + if directory.is_dir(): + cursor = read_cursor(directory) + index = _index_entry_files(directory) + watermark = _retired_watermark(cursor) + residue = _residue_below(index, watermark) + entries = _live_entries(directory, index, watermark) except OutboxError as error: - entries, cursor, invalid = [], None, error.reason_code + entries, cursor, residue, invalid = [], None, [], error.reason_code summary[partition] = { "committed_pending": sum(1 for entry in entries if entry.is_committed), "prepared_only": sum(1 for entry in entries if not entry.is_committed), - "retired_residue": len(retired_residue(directory)), + "retired_residue": len(residue), "next_seq": (max((entry.seq for entry in entries), default=0) if entries else 0), "cursor_last_seq": int(cursor.get("last_seq") or 0) if cursor else None, "cursor_last_entry_id": cursor.get("last_entry_id") if cursor else None, diff --git a/tests/control_plane/test_local_authority_shadow_drain.py b/tests/control_plane/test_local_authority_shadow_drain.py index 592b18d4a3..3ec076a2ef 100644 --- a/tests/control_plane/test_local_authority_shadow_drain.py +++ b/tests/control_plane/test_local_authority_shadow_drain.py @@ -627,3 +627,72 @@ def test_drain_fails_closed_on_an_entry_recorded_for_another_runtime_root(tmp_pa assert result.delivered == 0 assert result.pending_after == 1 assert [entry.seq for entry in outbox.list_entries(_todo_dir(runtime_root))] == [1] + + +def test_a_malformed_cursor_stops_drain_and_status_typed_and_touches_nothing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + registry, state, runtime_root = _fixture(tmp_path) + _record_todo_write(registry, state, runtime_root, "Before the cursor broke") + directory = _todo_dir(runtime_root) + files_before = sorted(path.name for path in directory.iterdir()) + broken_cursor = {"schema_version": "bad", "last_seq": 1} + outbox.cursor_path(directory).write_text(json.dumps(broken_cursor), encoding="utf-8") + calls = _commit_entry_calls(monkeypatch) + + status = adapter.local_authority_shadow_status( + registry_path=registry, runtime_root=runtime_root, goal_id=GOAL_ID + ) + # ``ok`` is "every partition readable": a typed invalid is reported, not + # hidden behind a green flag, so the CLI exits 1 with the reason in place. + assert status["ok"] is False + assert status["outbox"]["todos"]["invalid"] == "outbox_file_invalid" + assert status["outbox"]["todos"]["retired_residue"] == 0 + assert status["outbox"]["todos"]["committed_pending"] == 0 + assert status["outbox"]["leases"]["invalid"] is None + + result = _drain(registry, runtime_root) + assert (result.outcome, result.reason_code) == ("stopped", "outbox_file_invalid") + assert (result.delivered, result.reclaimed_residue, result.reseeded) == (0, 0, 0) + assert calls == [] + # Nothing is delivered, deleted, or healed: the entry and the broken + # cursor are exactly as the operator left them. + assert sorted(path.name for path in directory.iterdir()) == sorted([*files_before, "drain-cursor.json"]) + assert json.loads(outbox.cursor_path(directory).read_text(encoding="utf-8")) == broken_cursor + assert not (runtime_root / "authority-shadow" / "file" / GOAL_ID).exists() + + +def test_capture_against_a_malformed_cursor_records_a_typed_failure_instead_of_raising( + tmp_path: Path, +) -> None: + registry, state, runtime_root = _fixture(tmp_path) + directory = _todo_dir(runtime_root) + directory.mkdir(parents=True) + outbox.cursor_path(directory).write_text("garbage", encoding="utf-8") + original = state.read_text(encoding="utf-8") + goal = find_registry_goal(load_registry(registry), GOAL_ID) + capture = outbox.TodoPartitionCapture.begin( + enabled=True, + runtime_root=runtime_root, + goal_id=GOAL_ID, + state_path=state, + write_class="todo_add", + original_text=original, + projector=adapter.todo_partition_projector(goal, state_path=state), + ) + result = add_goal_todo( + registry_path=registry, + goal_id=GOAL_ID, + role="agent", + text="Written while the cursor is unreadable", + task_class="advancement_task", + ) + assert result["ok"] is True + + capture.prepare(state.read_text(encoding="utf-8")) + capture.committed() + + assert capture.outcome.entry_id is None + assert capture.outcome.failure == {"reason_code": "outbox_prepare_failed", "error_class": "OutboxError"} + assert [path.name for path in directory.iterdir()] == ["drain-cursor.json"] + assert "Written while the cursor is unreadable" in state.read_text(encoding="utf-8") diff --git a/tests/control_plane/test_local_authority_shadow_outbox.py b/tests/control_plane/test_local_authority_shadow_outbox.py index f3ce05f238..e864a5e2c2 100644 --- a/tests/control_plane/test_local_authority_shadow_outbox.py +++ b/tests/control_plane/test_local_authority_shadow_outbox.py @@ -426,3 +426,37 @@ def test_retired_residue_is_defined_by_the_cursor_watermark(tmp_path: Path) -> N assert outbox.reclaim_retired_residue(directory) == 2 assert outbox.retired_residue(directory) == [] assert [entry.seq for entry in outbox.list_entries(directory)] == [second.seq] + + +def test_a_malformed_cursor_or_entry_is_reported_as_typed_invalid_without_raising(tmp_path: Path) -> None: + _registry, _state, runtime_root = _fixture(tmp_path) + directory = _todo_dir(runtime_root) + directory.mkdir(parents=True) + settled = directory / outbox.entry_file_name(1, "local-shadow-tx-" + "0" * 64, "prepared") + settled.write_text("{}", encoding="utf-8") + invalid_summary = { + "committed_pending": 0, + "prepared_only": 0, + "retired_residue": 0, + "next_seq": 0, + "cursor_last_seq": None, + "cursor_last_entry_id": None, + "invalid": "outbox_file_invalid", + } + for cursor_text in (json.dumps({"schema_version": "bad", "last_seq": 1}), "garbage"): + outbox.cursor_path(directory).write_text(cursor_text, encoding="utf-8") + # Zero counts under a typed invalid: the file the cursor would have + # retired is not counted as residue, because the cursor is not trusted. + assert outbox.outbox_summary(runtime_root, GOAL_ID)["todos"] == invalid_summary + for probe in (outbox.next_seq, outbox.list_entries, outbox.retired_residue): + with pytest.raises(outbox.OutboxError) as raised: + probe(directory) + assert raised.value.reason_code == "outbox_file_invalid" + outbox.cursor_path(directory).unlink() + + # An unparseable entry file is the same typed fact, not a raw decode error. + settled.write_text("{not json", encoding="utf-8") + assert outbox.outbox_summary(runtime_root, GOAL_ID)["todos"]["invalid"] == "outbox_file_invalid" + with pytest.raises(outbox.OutboxError) as raised: + outbox.list_entries(directory) + assert raised.value.reason_code == "outbox_file_invalid" From 4dbd01ac687ef73f515f7cd1cb07690fb292e596 Mon Sep 17 00:00:00 2001 From: huangruiteng Date: Fri, 4 Sep 2026 23:33:19 +0800 Subject: [PATCH 06/11] fix(authority): bind drain cursor to a closed shape Signed-off-by: huangruiteng --- .../local_authority_shadow_outbox.py | 59 +++++----------- .../test_local_authority_shadow_drain.py | 69 ------------------- .../test_local_authority_shadow_outbox.py | 34 --------- 3 files changed, 19 insertions(+), 143 deletions(-) diff --git a/loopx/control_plane/coordination/local_authority_shadow_outbox.py b/loopx/control_plane/coordination/local_authority_shadow_outbox.py index 40972806f5..2ecbb6268d 100644 --- a/loopx/control_plane/coordination/local_authority_shadow_outbox.py +++ b/loopx/control_plane/coordination/local_authority_shadow_outbox.py @@ -138,10 +138,7 @@ def _as_object(value: object) -> dict[str, Any]: def _load_json(path: Path) -> dict[str, Any]: - try: - raw = json.loads(path.read_text(encoding="utf-8")) - except ValueError as error: # JSONDecodeError and UnicodeDecodeError - raise OutboxError("outbox_file_invalid", f"{path.name} is not valid JSON") from error + raw = json.loads(path.read_text(encoding="utf-8")) if not isinstance(raw, dict): raise OutboxError("outbox_file_invalid", f"{path.name} is not a JSON object") return raw @@ -265,20 +262,11 @@ def _load_committed_record(path: Path | None, *, entry_id: str) -> dict[str, Any return record -_EntryIndex = tuple[dict[_EntryKey, Path], dict[_EntryKey, Path]] - - -def _retired_watermark(cursor: dict[str, Any] | None) -> int: +def _retired_watermark(directory: Path) -> int: + cursor = read_cursor(directory) return int(cursor.get("last_seq") or 0) if cursor is not None else 0 -def _residue_below(index: _EntryIndex, watermark: int) -> list[Path]: - prepared, committed = index - return sorted( - path for key, path in [*prepared.items(), *committed.items()] if key[0] <= watermark - ) - - def retired_residue(directory: Path) -> list[Path]: """Entry files at or below the durable cursor. @@ -291,7 +279,14 @@ def retired_residue(directory: Path) -> list[Path]: if not directory.is_dir(): return [] - return _residue_below(_index_entry_files(directory), _retired_watermark(read_cursor(directory))) + watermark = _retired_watermark(directory) + prepared, committed = _index_entry_files(directory) + residue = [ + path + for key, path in [*prepared.items(), *committed.items()] + if key[0] <= watermark + ] + return sorted(residue) def reclaim_retired_residue(directory: Path) -> int: @@ -313,11 +308,8 @@ def list_entries(directory: Path) -> list[OutboxEntry]: if not directory.is_dir(): return [] - return _live_entries(directory, _index_entry_files(directory), _retired_watermark(read_cursor(directory))) - - -def _live_entries(directory: Path, index: _EntryIndex, watermark: int) -> list[OutboxEntry]: - prepared, committed = index + watermark = _retired_watermark(directory) + prepared, committed = _index_entry_files(directory) prepared = {key: path for key, path in prepared.items() if key[0] > watermark} committed = {key: path for key, path in committed.items() if key[0] > watermark} orphan_markers = sorted(set(committed) - set(prepared)) @@ -796,34 +788,21 @@ def entries_by_partition(runtime_root: Path, goal_id: str) -> dict[str, list[Out def outbox_summary(runtime_root: Path, goal_id: str) -> dict[str, Any]: - """Counts per partition for operator readback. - - Never raises on an empty or malformed outbox: one protected read of the - cursor and the directory index feeds every count, so a malformed cursor or - entry is reported as ``invalid`` with zero counts and is never re-read - outside that boundary. - """ + """Counts per partition for operator readback; never raises on an empty outbox.""" summary: dict[str, Any] = {} for partition in PARTITIONS: directory = partition_directory(runtime_root, goal_id, partition) - entries: list[OutboxEntry] = [] - cursor: dict[str, Any] | None = None - residue: list[Path] = [] - invalid: str | None = None try: - if directory.is_dir(): - cursor = read_cursor(directory) - index = _index_entry_files(directory) - watermark = _retired_watermark(cursor) - residue = _residue_below(index, watermark) - entries = _live_entries(directory, index, watermark) + entries = list_entries(directory) + cursor = read_cursor(directory) + invalid: str | None = None except OutboxError as error: - entries, cursor, residue, invalid = [], None, [], error.reason_code + entries, cursor, invalid = [], None, error.reason_code summary[partition] = { "committed_pending": sum(1 for entry in entries if entry.is_committed), "prepared_only": sum(1 for entry in entries if not entry.is_committed), - "retired_residue": len(residue), + "retired_residue": len(retired_residue(directory)), "next_seq": (max((entry.seq for entry in entries), default=0) if entries else 0), "cursor_last_seq": int(cursor.get("last_seq") or 0) if cursor else None, "cursor_last_entry_id": cursor.get("last_entry_id") if cursor else None, diff --git a/tests/control_plane/test_local_authority_shadow_drain.py b/tests/control_plane/test_local_authority_shadow_drain.py index 3ec076a2ef..592b18d4a3 100644 --- a/tests/control_plane/test_local_authority_shadow_drain.py +++ b/tests/control_plane/test_local_authority_shadow_drain.py @@ -627,72 +627,3 @@ def test_drain_fails_closed_on_an_entry_recorded_for_another_runtime_root(tmp_pa assert result.delivered == 0 assert result.pending_after == 1 assert [entry.seq for entry in outbox.list_entries(_todo_dir(runtime_root))] == [1] - - -def test_a_malformed_cursor_stops_drain_and_status_typed_and_touches_nothing( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - registry, state, runtime_root = _fixture(tmp_path) - _record_todo_write(registry, state, runtime_root, "Before the cursor broke") - directory = _todo_dir(runtime_root) - files_before = sorted(path.name for path in directory.iterdir()) - broken_cursor = {"schema_version": "bad", "last_seq": 1} - outbox.cursor_path(directory).write_text(json.dumps(broken_cursor), encoding="utf-8") - calls = _commit_entry_calls(monkeypatch) - - status = adapter.local_authority_shadow_status( - registry_path=registry, runtime_root=runtime_root, goal_id=GOAL_ID - ) - # ``ok`` is "every partition readable": a typed invalid is reported, not - # hidden behind a green flag, so the CLI exits 1 with the reason in place. - assert status["ok"] is False - assert status["outbox"]["todos"]["invalid"] == "outbox_file_invalid" - assert status["outbox"]["todos"]["retired_residue"] == 0 - assert status["outbox"]["todos"]["committed_pending"] == 0 - assert status["outbox"]["leases"]["invalid"] is None - - result = _drain(registry, runtime_root) - assert (result.outcome, result.reason_code) == ("stopped", "outbox_file_invalid") - assert (result.delivered, result.reclaimed_residue, result.reseeded) == (0, 0, 0) - assert calls == [] - # Nothing is delivered, deleted, or healed: the entry and the broken - # cursor are exactly as the operator left them. - assert sorted(path.name for path in directory.iterdir()) == sorted([*files_before, "drain-cursor.json"]) - assert json.loads(outbox.cursor_path(directory).read_text(encoding="utf-8")) == broken_cursor - assert not (runtime_root / "authority-shadow" / "file" / GOAL_ID).exists() - - -def test_capture_against_a_malformed_cursor_records_a_typed_failure_instead_of_raising( - tmp_path: Path, -) -> None: - registry, state, runtime_root = _fixture(tmp_path) - directory = _todo_dir(runtime_root) - directory.mkdir(parents=True) - outbox.cursor_path(directory).write_text("garbage", encoding="utf-8") - original = state.read_text(encoding="utf-8") - goal = find_registry_goal(load_registry(registry), GOAL_ID) - capture = outbox.TodoPartitionCapture.begin( - enabled=True, - runtime_root=runtime_root, - goal_id=GOAL_ID, - state_path=state, - write_class="todo_add", - original_text=original, - projector=adapter.todo_partition_projector(goal, state_path=state), - ) - result = add_goal_todo( - registry_path=registry, - goal_id=GOAL_ID, - role="agent", - text="Written while the cursor is unreadable", - task_class="advancement_task", - ) - assert result["ok"] is True - - capture.prepare(state.read_text(encoding="utf-8")) - capture.committed() - - assert capture.outcome.entry_id is None - assert capture.outcome.failure == {"reason_code": "outbox_prepare_failed", "error_class": "OutboxError"} - assert [path.name for path in directory.iterdir()] == ["drain-cursor.json"] - assert "Written while the cursor is unreadable" in state.read_text(encoding="utf-8") diff --git a/tests/control_plane/test_local_authority_shadow_outbox.py b/tests/control_plane/test_local_authority_shadow_outbox.py index e864a5e2c2..f3ce05f238 100644 --- a/tests/control_plane/test_local_authority_shadow_outbox.py +++ b/tests/control_plane/test_local_authority_shadow_outbox.py @@ -426,37 +426,3 @@ def test_retired_residue_is_defined_by_the_cursor_watermark(tmp_path: Path) -> N assert outbox.reclaim_retired_residue(directory) == 2 assert outbox.retired_residue(directory) == [] assert [entry.seq for entry in outbox.list_entries(directory)] == [second.seq] - - -def test_a_malformed_cursor_or_entry_is_reported_as_typed_invalid_without_raising(tmp_path: Path) -> None: - _registry, _state, runtime_root = _fixture(tmp_path) - directory = _todo_dir(runtime_root) - directory.mkdir(parents=True) - settled = directory / outbox.entry_file_name(1, "local-shadow-tx-" + "0" * 64, "prepared") - settled.write_text("{}", encoding="utf-8") - invalid_summary = { - "committed_pending": 0, - "prepared_only": 0, - "retired_residue": 0, - "next_seq": 0, - "cursor_last_seq": None, - "cursor_last_entry_id": None, - "invalid": "outbox_file_invalid", - } - for cursor_text in (json.dumps({"schema_version": "bad", "last_seq": 1}), "garbage"): - outbox.cursor_path(directory).write_text(cursor_text, encoding="utf-8") - # Zero counts under a typed invalid: the file the cursor would have - # retired is not counted as residue, because the cursor is not trusted. - assert outbox.outbox_summary(runtime_root, GOAL_ID)["todos"] == invalid_summary - for probe in (outbox.next_seq, outbox.list_entries, outbox.retired_residue): - with pytest.raises(outbox.OutboxError) as raised: - probe(directory) - assert raised.value.reason_code == "outbox_file_invalid" - outbox.cursor_path(directory).unlink() - - # An unparseable entry file is the same typed fact, not a raw decode error. - settled.write_text("{not json", encoding="utf-8") - assert outbox.outbox_summary(runtime_root, GOAL_ID)["todos"]["invalid"] == "outbox_file_invalid" - with pytest.raises(outbox.OutboxError) as raised: - outbox.list_entries(directory) - assert raised.value.reason_code == "outbox_file_invalid" From 7232a179da0879939c235fe8868a4a22b95f8392 Mon Sep 17 00:00:00 2001 From: huangruiteng Date: Sat, 5 Sep 2026 00:33:11 +0800 Subject: [PATCH 07/11] feat(authority): bind production writers to runtime shadow Signed-off-by: huangruiteng --- .../coordination/local_authority_shadow.ts | 62 ++++++-- .../local_authority_shadow_adapter.py | 53 +++++-- .../local_authority_shadow_outbox.py | 6 + .../local_authority_shadow_outbox.ts | 9 +- .../local_authority_shadow_projection.py | 55 ++----- .../runtime_shadow_writer_adapter.py | 147 ++++++++++++++++++ .../control_plane/effect_runtime_handlers.ts | 4 +- loopx/control_plane/work_items/task_lease.py | 2 +- .../work_items/task_lease_acquire.ts | 30 +++- .../work_items/task_lease_acquire_adapter.py | 37 +++++ .../work_items/task_lease_lifecycle.ts | 80 +++++++++- loopx/todos.py | 79 ++++++++-- 12 files changed, 471 insertions(+), 93 deletions(-) create mode 100644 loopx/control_plane/coordination/runtime_shadow_writer_adapter.py diff --git a/loopx/control_plane/coordination/local_authority_shadow.ts b/loopx/control_plane/coordination/local_authority_shadow.ts index 4d93e285fd..1ccb21cddc 100644 --- a/loopx/control_plane/coordination/local_authority_shadow.ts +++ b/loopx/control_plane/coordination/local_authority_shadow.ts @@ -17,6 +17,9 @@ import type { AuthorityStoreReceiptResult, } from "./authority_store.ts"; import { authorityUnicodeCompare, canonicalAuthorityBytes } from "./authority_store_codec.ts"; +import { + TODO_CANONICAL_READ_RECORD_FIELDS, +} from "./coordination_projection.ts"; import { FileAuthorityStore } from "./file_authority_store.ts"; export const LOCAL_AUTHORITY_SHADOW_REQUEST_SCHEMA = @@ -343,18 +346,19 @@ export async function recordLocalAuthorityShadow( // --------------------------------------------------------------------------- export const LOCAL_AUTHORITY_SHADOW_PROJECTION_SCHEMA_V1 = - "loopx_local_authority_shadow_projection_v1"; + "loopx_coordination_runtime_shadow_projection_v0"; export const LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_REQUEST_SCHEMA = - "loopx_local_authority_shadow_commit_entry_request_v0"; + "loopx_coordination_runtime_shadow_commit_entry_request_v0"; export const LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_RESULT_SCHEMA = - "loopx_local_authority_shadow_commit_entry_result_v0"; + "loopx_coordination_runtime_shadow_commit_entry_result_v0"; export const LOCAL_AUTHORITY_SHADOW_READ_REQUEST_SCHEMA = - "loopx_local_authority_shadow_read_request_v0"; + "loopx_coordination_runtime_shadow_outbox_read_v0"; export const LOCAL_AUTHORITY_SHADOW_READ_RESULT_SCHEMA = - "loopx_local_authority_shadow_read_result_v0"; -export const LOCAL_AUTHORITY_SHADOW_EVENT_SCHEMA_V1 = "loopx_local_authority_shadow_event_v1"; + "loopx_coordination_runtime_shadow_outbox_read_result_v0"; +export const LOCAL_AUTHORITY_SHADOW_EVENT_SCHEMA_V1 = + "loopx_coordination_runtime_shadow_outbox_event_v0"; export const LOCAL_AUTHORITY_SHADOW_TRANSACTION_RECEIPT_SCHEMA = - "loopx_local_authority_shadow_transaction_receipt_v0"; + "loopx_coordination_runtime_shadow_outbox_receipt_v0"; const SHADOW_PARTITIONS = ["todos", "leases"] as const; const ENTRY_RESOLUTIONS = [ @@ -390,6 +394,7 @@ const READ_REQUEST_FIELDS = new Set([ "schema_version", "runtime_root", "goal_id", + "store_kind", "scan_after_cursor", "scan_limit", ]); @@ -462,6 +467,7 @@ export interface LocalAuthorityShadowCommitEntryResult extends JsonObject { interface ReadRequest { runtime_root: string; goal_id: string; + store_kind: "runtime_shadow" | "legacy_observation"; scan_after_cursor: string | null; scan_limit: number; } @@ -615,6 +621,15 @@ function decodeReadRequest(value: unknown): ReadRequest { return { runtime_root: requireNonEmptyString(request.runtime_root, "runtime_root"), goal_id: requireGoalId(request.goal_id), + store_kind: request.store_kind === undefined || request.store_kind === "runtime_shadow" + ? "runtime_shadow" + : request.store_kind === "legacy_observation" + ? "legacy_observation" + : (() => { + throw new EffectRuntimeRequestError( + "Local authority shadow read store_kind must be runtime_shadow or legacy_observation", + ); + })(), scan_after_cursor: optionalString(request.scan_after_cursor, "scan_after_cursor"), scan_limit: limit, }; @@ -644,6 +659,15 @@ function partitionsOf(head: JsonObject | null): JsonObject { return partitions; } +function todoReadModel(todos: readonly JsonObject[]): JsonObject { + return { + schema_version: "loopx_todo_canonical_read_record_v0", + todo_count: todos.length, + records_sha256: createHash("sha256").update(canonicalAuthorityBytes(todos)).digest("hex"), + contract_fields: [...TODO_CANONICAL_READ_RECORD_FIELDS], + }; +} + /** * Fold one partition into the candidate head. A v0 head (whole-snapshot * observation) is accepted as the starting point with no partition markers. @@ -669,14 +693,17 @@ export function composeLocalAuthorityShadowHead( } partitions[entry.partition] = { seq: entry.seq, partition_digest: digest }; } - return { + const next = { schema_version: LOCAL_AUTHORITY_SHADOW_PROJECTION_SCHEMA_V1, goal_id: goalId, + source_authority: "legacy_markdown_and_task_lease", handoff_mode: handoffMode, todos, leases, + todo_read_model: todoReadModel(todos), partitions, }; + return next; } function transactionReceipt(request: CommitEntryRequest, noOp: boolean): JsonObject { @@ -806,9 +833,12 @@ async function reconcileTransactionReceipt( function openShadowStore( runtimeRoot: string, goalId: string, + storeKind: ReadRequest["store_kind"], dependencies: LocalAuthorityShadowDependencies, ): AuthorityStore { - const providerDirectory = join(runtimeRoot, "authority-shadow", "file", goalId); + const providerDirectory = storeKind === "legacy_observation" + ? join(runtimeRoot, "authority-shadow", "file", goalId) + : join(runtimeRoot, "authority-shadow", "file-v0"); return (dependencies.openStore ?? ((directory, id) => new FileAuthorityStore(directory, id)))( providerDirectory, goalId, @@ -925,7 +955,12 @@ export async function commitLocalAuthorityShadowEntry( const noOp = NO_OP_RESOLUTIONS.has(request.entry.resolution); let store: AuthorityStore; try { - store = openShadowStore(request.runtime_root, request.goal_id, dependencies); + store = openShadowStore( + request.runtime_root, + request.goal_id, + "runtime_shadow", + dependencies, + ); } catch { return commitEntryResult(request, "unavailable", { reasonCode: "provider_construction_failed", @@ -1024,7 +1059,12 @@ export async function readLocalAuthorityShadow( const base = readResultBase(request.goal_id); let store: AuthorityStore; try { - store = openShadowStore(request.runtime_root, request.goal_id, dependencies); + store = openShadowStore( + request.runtime_root, + request.goal_id, + request.store_kind, + dependencies, + ); } catch { return { ...base, reason_code: "provider_construction_failed" }; } diff --git a/loopx/control_plane/coordination/local_authority_shadow_adapter.py b/loopx/control_plane/coordination/local_authority_shadow_adapter.py index fd421ae94c..1bed5207a3 100644 --- a/loopx/control_plane/coordination/local_authority_shadow_adapter.py +++ b/loopx/control_plane/coordination/local_authority_shadow_adapter.py @@ -38,6 +38,7 @@ text_digest, todo_partition_projection, ) +from .runtime_shadow import resolve_coordination_runtime_shadow_config LOCAL_AUTHORITY_SHADOW_CONFIG_SCHEMA = "loopx_local_authority_shadow_config_v0" @@ -522,13 +523,13 @@ def observe_todo_local_authority_commit( LOCAL_AUTHORITY_SHADOW_EVIDENCE_SCHEMA_V1 = "loopx_local_authority_shadow_evidence_v1" LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_REQUEST_SCHEMA = ( - "loopx_local_authority_shadow_commit_entry_request_v0" + "loopx_coordination_runtime_shadow_commit_entry_request_v0" ) LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_RESULT_SCHEMA = ( - "loopx_local_authority_shadow_commit_entry_result_v0" + "loopx_coordination_runtime_shadow_commit_entry_result_v0" ) -LOCAL_AUTHORITY_SHADOW_READ_REQUEST_SCHEMA = "loopx_local_authority_shadow_read_request_v0" -LOCAL_AUTHORITY_SHADOW_READ_RESULT_SCHEMA = "loopx_local_authority_shadow_read_result_v0" +LOCAL_AUTHORITY_SHADOW_READ_REQUEST_SCHEMA = "loopx_coordination_runtime_shadow_outbox_read_v0" +LOCAL_AUTHORITY_SHADOW_READ_RESULT_SCHEMA = "loopx_coordination_runtime_shadow_outbox_read_result_v0" INLINE_DRAIN_MAX_ENTRIES = 16 INLINE_DRAIN_BUDGET_SECONDS = 2.0 INLINE_DRAIN_LOCK_TIMEOUT_SECONDS = 0.25 @@ -846,17 +847,19 @@ def read_local_authority_shadow( *, runtime_root: Path, goal_id: str, + store_kind: str = "runtime_shadow", scan_after_cursor: str | None = None, scan_limit: int = 0, ) -> dict[str, Any]: """Read-only candidate view through the TypeScript store boundary.""" result = effect_runtime_result( - "coordination.local_authority_shadow.read", + "coordination.runtime_shadow.outbox_read", { "schema_version": LOCAL_AUTHORITY_SHADOW_READ_REQUEST_SCHEMA, "runtime_root": str(runtime_root), "goal_id": goal_id, + "store_kind": store_kind, "scan_after_cursor": scan_after_cursor, "scan_limit": scan_limit, }, @@ -1007,7 +1010,7 @@ def _commit( digest=digest, ) raw = effect_runtime_result( - "coordination.local_authority_shadow.commit_entry", + "coordination.runtime_shadow.commit_entry", request, timeout=15.0, ) @@ -1079,7 +1082,8 @@ def _commit( def _candidate_cursor(runtime_root: Path, goal_id: str) -> str | None: """Current candidate cursor, or None when the store has no document yet.""" - if not (runtime_root / "authority-shadow" / "file" / goal_id).is_dir(): + directory = runtime_root / "authority-shadow" / "file-v0" + if not directory.is_dir() or not any(directory.glob("authority-store-*.json")): return None try: view = read_local_authority_shadow(runtime_root=runtime_root, goal_id=goal_id) @@ -1131,7 +1135,12 @@ def _drain_prelude( return None try: registry = load_registry(registry_path) - result.config_enabled = _shadow_config(registry, goal_id) is not None + result.config_enabled = ( + resolve_coordination_runtime_shadow_config( + find_registry_goal(registry, goal_id) + ).enabled + or _shadow_config(registry, goal_id) is not None + ) resolved = ( runtime_root if runtime_root is not None @@ -1267,8 +1276,12 @@ class _CandidateMissing(Exception): """The candidate store directory does not exist yet.""" -def _store_bytes(runtime_root: Path, goal_id: str) -> int: - directory = runtime_root / "authority-shadow" / "file" / goal_id +def _store_bytes(runtime_root: Path, goal_id: str, *, legacy_observation: bool) -> int: + directory = ( + runtime_root / "authority-shadow" / "file" / goal_id + if legacy_observation + else runtime_root / "authority-shadow" / "file-v0" + ) if not directory.is_dir(): return 0 return sum(path.stat().st_size for path in directory.iterdir() if path.is_file()) @@ -1289,14 +1302,24 @@ def local_authority_shadow_status( if runtime_root is None: runtime_root = resolve_runtime_root(registry, None, registry_path=registry_path) config = local_authority_shadow_summary(goal) + legacy_observation = config["enabled"] is True backlog = outbox.outbox_summary(runtime_root, goal_id) candidate: dict[str, Any] try: - if not (runtime_root / "authority-shadow" / "file" / goal_id).is_dir(): + directory = ( + runtime_root / "authority-shadow" / "file" / goal_id + if legacy_observation + else runtime_root / "authority-shadow" / "file-v0" + ) + if not directory.is_dir() or not any(directory.glob("authority-store-*.json")): # Reading through the store boundary would mint a store identity; # a status probe must not create candidate lineage. raise _CandidateMissing - view = read_local_authority_shadow(runtime_root=runtime_root, goal_id=goal_id) + view = read_local_authority_shadow( + runtime_root=runtime_root, + goal_id=goal_id, + store_kind=("legacy_observation" if legacy_observation else "runtime_shadow"), + ) head = view.get("head") if isinstance(view.get("head"), dict) else None candidate = { "status": view.get("status"), @@ -1333,7 +1356,11 @@ def local_authority_shadow_status( "partitions": None, "codec_agreement": None, } - store_bytes = _store_bytes(runtime_root, goal_id) + store_bytes = _store_bytes( + runtime_root, + goal_id, + legacy_observation=legacy_observation, + ) return { "ok": all(item["invalid"] is None for item in backlog.values()), "action": "status", diff --git a/loopx/control_plane/coordination/local_authority_shadow_outbox.py b/loopx/control_plane/coordination/local_authority_shadow_outbox.py index 2ecbb6268d..2aa84c721d 100644 --- a/loopx/control_plane/coordination/local_authority_shadow_outbox.py +++ b/loopx/control_plane/coordination/local_authority_shadow_outbox.py @@ -586,6 +586,12 @@ def begin( def enabled(self) -> bool: return self._enabled and self._directory is not None + def skip(self, reason: str) -> None: + """Record why this writer deliberately did not open a transaction.""" + + if self.enabled and self.outcome.entry_id is None: + self.outcome.skipped_reason = reason + def _project(self, state_text: str) -> dict[str, Any]: if self._projector is None: raise OutboxError("outbox_prepare_failed", "a todo partition projector is required") diff --git a/loopx/control_plane/coordination/local_authority_shadow_outbox.ts b/loopx/control_plane/coordination/local_authority_shadow_outbox.ts index 4f3dbaa60c..5688377508 100644 --- a/loopx/control_plane/coordination/local_authority_shadow_outbox.ts +++ b/loopx/control_plane/coordination/local_authority_shadow_outbox.ts @@ -16,7 +16,8 @@ import { authorityUnicodeCompare, canonicalAuthorityBytes } from "./authority_st * returned on the capture object so the writer can attach typed evidence. */ -export const LOCAL_AUTHORITY_SHADOW_BINDING_SCHEMA = "loopx_local_authority_shadow_binding_v0"; +export const LOCAL_AUTHORITY_SHADOW_BINDING_SCHEMA = + "loopx_coordination_runtime_shadow_binding_v0"; export const LOCAL_AUTHORITY_SHADOW_OUTBOX_ENTRY_SCHEMA = "loopx_local_authority_shadow_outbox_entry_v0"; export const LOCAL_AUTHORITY_SHADOW_OUTBOX_COMMIT_SCHEMA = @@ -30,7 +31,7 @@ const LEASE_SOURCE_FIELDS = ["todo_id", "version", "lease_epoch", "status", "upd export interface LocalAuthorityShadowBinding { schema_version: typeof LOCAL_AUTHORITY_SHADOW_BINDING_SCHEMA; - mode: "file_one_way"; + provider: "file_v0"; } /** Decode the optional per-request binding; anything but the exact contract is "absent". */ @@ -45,11 +46,11 @@ export function decodeLocalAuthorityShadowBinding( if ( keys.length !== 2 || record.schema_version !== LOCAL_AUTHORITY_SHADOW_BINDING_SCHEMA || - record.mode !== "file_one_way" + record.provider !== "file_v0" ) { return null; } - return { schema_version: LOCAL_AUTHORITY_SHADOW_BINDING_SCHEMA, mode: "file_one_way" }; + return { schema_version: LOCAL_AUTHORITY_SHADOW_BINDING_SCHEMA, provider: "file_v0" }; } export function sha256Digest(input: Uint8Array | string): string { diff --git a/loopx/control_plane/coordination/local_authority_shadow_projection.py b/loopx/control_plane/coordination/local_authority_shadow_projection.py index b2b3631c65..0763f08a86 100644 --- a/loopx/control_plane/coordination/local_authority_shadow_projection.py +++ b/loopx/control_plane/coordination/local_authority_shadow_projection.py @@ -1,7 +1,7 @@ """Pure projection rules shared by the local authority shadow capture and parity. Everything here is a deterministic function of its inputs: no file, lock, -registry, or effect-runtime access. The same compact field sets and canonical +registry, or effect-runtime access. The same complete record contracts and canonical bytes define the source digest, the outbox partition digest, and the candidate readback comparison, so no two code paths can disagree about what "the same coordination state" means. @@ -14,45 +14,15 @@ from collections.abc import Iterable, Mapping from typing import Any +from ..todos.todo_summary import TODO_CANONICAL_READ_RECORD_FIELDS, canonical_todo_read_record + LOCAL_AUTHORITY_SHADOW_PROJECTION_SCHEMA_V0 = "loopx_local_authority_shadow_projection_v0" -LOCAL_AUTHORITY_SHADOW_PROJECTION_SCHEMA_V1 = "loopx_local_authority_shadow_projection_v1" TODO_PARTITION = "todos" LEASE_PARTITION = "leases" PARTITIONS: tuple[str, ...] = (TODO_PARTITION, LEASE_PARTITION) -TODO_FIELDS: tuple[str, ...] = ( - "todo_id", - "role", - "status", - "claimed_by", - "bound_agent", - "goal_bound", - "blocks_agent", - "excluded_agents", - "global_gate", - "task_class", - "action_kind", - "required_write_scopes", - "required_capabilities", - "continuation_policy", - "successor_todo_ids", - "no_followup", - "completion_continuation", -) -LEASE_FIELDS: tuple[str, ...] = ( - "todo_id", - "owner", - "idempotency_key", - "write_scopes", - "version", - "lease_epoch", - "acquired_at", - "updated_at", - "expires_at", - "released_at", - "status", -) +TODO_FIELDS: tuple[str, ...] = TODO_CANONICAL_READ_RECORD_FIELDS class ProjectionValueError(ValueError): @@ -110,28 +80,27 @@ def text_digest(text: str) -> str: def compact_todo(raw: object) -> dict[str, Any] | None: - """Keep only the coordination facts of one todo item; drop prose.""" + """Retain the complete versioned Todo consumer record.""" if not isinstance(raw, Mapping): return None todo_id = str(raw.get("todo_id") or "").strip() if not todo_id: return None - compact = {field: raw[field] for field in TODO_FIELDS if field in raw} - compact["todo_id"] = todo_id - if "status" not in compact and isinstance(raw.get("done"), bool): - compact["status"] = "done" if raw["done"] else "open" - return dict(canonical_value(compact)) + try: + return dict(canonical_value(canonical_todo_read_record(dict(raw)))) + except ValueError as error: + raise ProjectionValueError(str(error)) from error def compact_lease(raw: object, *, goal_id: str, file_stem: str) -> dict[str, Any]: - """Keep only the lease fence facts of one on-disk lease record.""" + """Retain the complete versioned lease record after binding its identity.""" if not isinstance(raw, Mapping): raise ProjectionValueError("task lease must contain an object") if raw.get("goal_id") != goal_id or raw.get("todo_id") != file_stem: raise ProjectionValueError("task lease identity does not match its shadow source") - return dict(canonical_value({field: raw[field] for field in LEASE_FIELDS if field in raw})) + return dict(canonical_value(dict(raw))) def todo_partition_projection( @@ -182,10 +151,8 @@ def head_digest(head: Mapping[str, Any]) -> str: __all__ = [ - "LEASE_FIELDS", "LEASE_PARTITION", "LOCAL_AUTHORITY_SHADOW_PROJECTION_SCHEMA_V0", - "LOCAL_AUTHORITY_SHADOW_PROJECTION_SCHEMA_V1", "PARTITIONS", "TODO_FIELDS", "TODO_PARTITION", diff --git a/loopx/control_plane/coordination/runtime_shadow_writer_adapter.py b/loopx/control_plane/coordination/runtime_shadow_writer_adapter.py new file mode 100644 index 0000000000..023310d6a5 --- /dev/null +++ b/loopx/control_plane/coordination/runtime_shadow_writer_adapter.py @@ -0,0 +1,147 @@ +"""Transaction-capture orchestration for legacy Todo and lease writers.""" + +from __future__ import annotations + +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from ...history import load_registry +from ...registry import find_registry_goal +from . import local_authority_shadow_outbox as outbox +from .local_authority_shadow_projection import LEASE_PARTITION +from .runtime_shadow import resolve_coordination_runtime_shadow_config + + +def begin_todo_runtime_shadow_capture( + *, + registry_path: Path, + runtime_root: Path, + goal_id: str, + state_path: Path, + write_class: str, + original_text: str, +) -> outbox.TodoPartitionCapture: + """Create the default-off transaction capture while the Todo lock is held.""" + + try: + registry = load_registry(registry_path) + goal = find_registry_goal(registry, goal_id) + enabled = resolve_coordination_runtime_shadow_config(goal).enabled + from ...rollout_event_log import load_rollout_events, rollout_event_log_path + from ..todos.todo_index import MAX_TODO_INDEX_ROLLOUT_EVENTS_PER_GOAL + from .local_authority_shadow_adapter import todo_partition_projector + + events = load_rollout_events( + rollout_event_log_path(runtime_root, goal_id), + limit=MAX_TODO_INDEX_ROLLOUT_EVENTS_PER_GOAL, + ) + projector = todo_partition_projector( + goal, + state_path=state_path, + rollout_events=events, + ) + except Exception: + enabled = False + projector = None + return outbox.TodoPartitionCapture.begin( + enabled=enabled, + runtime_root=runtime_root, + goal_id=goal_id, + state_path=state_path, + write_class=write_class, + original_text=original_text, + projector=projector, + ) + + +def settle_todo_runtime_shadow_capture( + payload: dict[str, Any], + *, + registry_path: Path, + runtime_root: Path, + goal_id: str, + write_class: str, + capture: outbox.TodoPartitionCapture, +) -> dict[str, Any]: + """Boundedly drain one transaction capture after releasing the Todo lock.""" + + from .local_authority_shadow_adapter import ( + capture_evidence, + drain_local_authority_shadow_outbox, + observe_todo_local_authority_commit, + ) + + drain = ( + drain_local_authority_shadow_outbox( + registry_path=registry_path, + runtime_root=runtime_root, + goal_id=goal_id, + ) + if capture.outcome.entry_id is not None + else None + ) + payload["coordination_runtime_shadow"] = capture_evidence( + goal_id=goal_id, + capture=capture.outcome, + drain=drain, + ) + return observe_todo_local_authority_commit( + payload, + registry_path, + goal_id, + write_class, + runtime_root=runtime_root, + ) + + +def settle_lease_runtime_shadow_capture( + payload: dict[str, Any], + *, + registry_path: Path | None, + runtime_root: Path, + goal_id: str, +) -> dict[str, Any]: + """Turn the TS writer capture receipt into bounded runtime-shadow evidence.""" + + raw = payload.pop("coordination_runtime_shadow_capture", None) + if not isinstance(raw, Mapping) or registry_path is None: + return payload + capture = outbox.CaptureOutcome( + entry_id=str(raw["entry_id"]) if isinstance(raw.get("entry_id"), str) else None, + partition=LEASE_PARTITION, + seq=int(raw["seq"]) if isinstance(raw.get("seq"), int) else None, + source_bytes_digest=( + str(raw["source_bytes_digest"]) + if isinstance(raw.get("source_bytes_digest"), str) + else None + ), + failure=dict(raw["failure"]) if isinstance(raw.get("failure"), Mapping) else None, + ) + from .local_authority_shadow_adapter import ( + capture_evidence, + drain_local_authority_shadow_outbox, + ) + + drain = ( + drain_local_authority_shadow_outbox( + registry_path=registry_path, + runtime_root=runtime_root, + goal_id=goal_id, + ) + if capture.entry_id is not None + else None + ) + payload["coordination_runtime_shadow"] = capture_evidence( + goal_id=goal_id, + capture=capture, + drain=drain, + ) + return payload + + +__all__ = [ + "begin_todo_runtime_shadow_capture", + "settle_lease_runtime_shadow_capture", + "settle_todo_runtime_shadow_capture", +] diff --git a/loopx/control_plane/effect_runtime_handlers.ts b/loopx/control_plane/effect_runtime_handlers.ts index 7d03efda1d..22e33e28e5 100644 --- a/loopx/control_plane/effect_runtime_handlers.ts +++ b/loopx/control_plane/effect_runtime_handlers.ts @@ -403,8 +403,8 @@ export function createEffectRuntimeHandlers( ["task_lease.write_scopes.overlap", evaluateTaskLeaseWriteScopesOverlap], ["quota.monitor_poll.commit", evaluateQuotaMonitorPollCommit], ["coordination.local_authority_shadow.record", recordLocalAuthorityShadow], - ["coordination.local_authority_shadow.commit_entry", commitLocalAuthorityShadowEntry], - ["coordination.local_authority_shadow.read", readLocalAuthorityShadow], + ["coordination.runtime_shadow.commit_entry", commitLocalAuthorityShadowEntry], + ["coordination.runtime_shadow.outbox_read", readLocalAuthorityShadow], [ "effect.program_from_ordered_steps", (params) => effectProgramFromOrderedSteps( diff --git a/loopx/control_plane/work_items/task_lease.py b/loopx/control_plane/work_items/task_lease.py index f46d97eea2..dd977bc23d 100644 --- a/loopx/control_plane/work_items/task_lease.py +++ b/loopx/control_plane/work_items/task_lease.py @@ -428,7 +428,7 @@ def hold_task_lease_mutation_fence( def close(committed: bool, release_lease: bool) -> dict[str, Any] | None: return _execute_native_task_lease_lifecycle( runtime_root=runtime_root, - registry_path=None, + registry_path=registry_path, goal_id=normalized_goal_id, todo_id=normalized_todo_id, operation="fence_close", diff --git a/loopx/control_plane/work_items/task_lease_acquire.ts b/loopx/control_plane/work_items/task_lease_acquire.ts index 5be9f6e924..bf22eb563f 100644 --- a/loopx/control_plane/work_items/task_lease_acquire.ts +++ b/loopx/control_plane/work_items/task_lease_acquire.ts @@ -17,6 +17,11 @@ import { type JsonObject, } from "../effect_program.ts"; import { requireJsonObject } from "../runtime_decode.ts"; +import { + beginLeaseOutboxEntry, + decodeLocalAuthorityShadowBinding, + type LocalAuthorityShadowBinding, +} from "../coordination/local_authority_shadow_outbox.ts"; export const TASK_LEASE_ACQUIRE_REQUEST_SCHEMA_VERSION = "loopx_task_lease_acquire_native_v0"; @@ -74,6 +79,7 @@ interface AcquireRequest { ttl_seconds: number; expected_version: number | null; authority: AuthorityFacts; + runtime_shadow: LocalAuthorityShadowBinding | null; } export interface LeaseRecord extends JsonObject { @@ -441,6 +447,7 @@ function decodeRequest(value: unknown): AcquireRequest { ttl_seconds: normalizeTtl(request.ttl_seconds), expected_version: optionalInteger(request.expected_version, "expected_version"), authority, + runtime_shadow: decodeLocalAuthorityShadowBinding(request.runtime_shadow), }; } @@ -1336,8 +1343,29 @@ async function commitAcquire( }; await dependencies.beforeWrite?.(lease); await revalidateAuthoritySources(request.authority.source_receipts); + const shadowCapture = request.runtime_shadow === null + ? null + : await beginLeaseOutboxEntry({ + runtime_root: request.runtime_root, + goal_id: request.goal_id, + lease_directory: taskLeaseDirectory(request), + write_class: "task_lease_acquire", + operation_id: request.idempotency_key, + previous_lease: existing, + planned_lease: lease, + }); await atomicWriteJson(leasePath, lease); - return successEnvelope(request, lease, leasePath, acquireEffectId(request), false); + await shadowCapture?.commit(); + const response = successEnvelope(request, lease, leasePath, acquireEffectId(request), false); + if (shadowCapture !== null) { + response.coordination_runtime_shadow_capture = { + entry_id: shadowCapture.entry_id, + seq: shadowCapture.seq, + source_bytes_digest: shadowCapture.source_bytes_digest, + failure: shadowCapture.failure, + }; + } + return response; } export async function executeTaskLeaseAcquire( diff --git a/loopx/control_plane/work_items/task_lease_acquire_adapter.py b/loopx/control_plane/work_items/task_lease_acquire_adapter.py index e277030df8..42c4efc4e4 100644 --- a/loopx/control_plane/work_items/task_lease_acquire_adapter.py +++ b/loopx/control_plane/work_items/task_lease_acquire_adapter.py @@ -11,6 +11,7 @@ from ...history import load_registry from ...paths import resolve_runtime_root +from ..coordination.runtime_shadow import resolve_coordination_runtime_shadow_config from ..goals.active_state_event_projection import ( state_event_log_candidates as _state_event_log_candidates, ) @@ -344,6 +345,16 @@ def _finalize_native_acquire_result( result["handoff_mode"] = authority.get("handoff_mode") or HANDOFF_MODE_LEGACY result.pop("settlement", None) if result.get("ok") is True and result.get("acquired") is True: + from ..coordination.runtime_shadow_writer_adapter import ( + settle_lease_runtime_shadow_capture, + ) + + result = settle_lease_runtime_shadow_capture( + result, + registry_path=registry_path, + runtime_root=runtime_root, + goal_id=goal_id, + ) result = _attach_local_authority_shadow( result, registry_path=registry_path, @@ -390,6 +401,13 @@ def execute_native_task_lease_acquire( "expected_version": expected_version, "authority": authority, } + registry = load_registry(registry_path) + goal = _registry_goal(registry, str(goal_id)) + if resolve_coordination_runtime_shadow_config(goal).enabled: + request["runtime_shadow"] = { + "schema_version": "loopx_coordination_runtime_shadow_binding_v0", + "provider": "file_v0", + } payload = _require_native_acquire_shape( effect_runtime_result("task_lease.acquire.native", request, timeout=15.0) ) @@ -630,6 +648,14 @@ def execute_native_task_lease_lifecycle( # CLI argument or persisted authority fact. "current_time": _now.isoformat() if _now is not None else None, } + if registry_path is not None: + registry = load_registry(registry_path) + goal = _registry_goal(registry, str(goal_id)) + if resolve_coordination_runtime_shadow_config(goal).enabled: + request["runtime_shadow"] = { + "schema_version": "loopx_coordination_runtime_shadow_binding_v0", + "provider": "file_v0", + } compacted_todo = _compact_lifecycle_todo(todo, todo_id=str(todo_id)) if compacted_todo is not None: request["todo"] = compacted_todo @@ -660,6 +686,17 @@ def execute_native_task_lease_lifecycle( result["handoff_mode"] = authority["handoff_mode"] if _legacy_provider_projection: result.pop("settlement", None) + if "coordination_runtime_shadow_capture" in result: + from ..coordination.runtime_shadow_writer_adapter import ( + settle_lease_runtime_shadow_capture, + ) + + result = settle_lease_runtime_shadow_capture( + result, + registry_path=registry_path, + runtime_root=runtime_root, + goal_id=str(goal_id), + ) committed_mutation = ( normalized_operation == "renew" and result.get("renewed") is True ) or ( diff --git a/loopx/control_plane/work_items/task_lease_lifecycle.ts b/loopx/control_plane/work_items/task_lease_lifecycle.ts index ee414ca6ba..4764004143 100644 --- a/loopx/control_plane/work_items/task_lease_lifecycle.ts +++ b/loopx/control_plane/work_items/task_lease_lifecycle.ts @@ -58,6 +58,12 @@ import { type TaskLeaseLifecycleDecision, type TaskLeaseLifecycleDecisionInput, } from "./task_lease_lifecycle_decision.ts"; +import { + beginLeaseOutboxEntry, + decodeLocalAuthorityShadowBinding, + type LeaseOutboxCapture, + type LocalAuthorityShadowBinding, +} from "../coordination/local_authority_shadow_outbox.ts"; export const TASK_LEASE_LIFECYCLE_REQUEST_SCHEMA_VERSION = "loopx_task_lease_lifecycle_native_v0"; @@ -104,6 +110,7 @@ interface LifecycleRequest { fence_operation_id: string | null; current_time: Date | null; owner_pid: number | null; + runtime_shadow: LocalAuthorityShadowBinding | null; } interface LifecycleDependencies { @@ -471,6 +478,41 @@ function decodeRequest(value: unknown): LifecycleRequest { fence_operation_id: fenceOperationId, current_time: optionalDate(input.current_time, "current_time"), owner_pid: optionalPositiveInteger(input.owner_pid, "owner_pid"), + runtime_shadow: decodeLocalAuthorityShadowBinding(input.runtime_shadow), + }; +} + +async function captureLeaseWrite( + request: LifecycleRequest, + previous: LeaseRecord | null, + next: LeaseRecord, + writeClass: string, +): Promise> | null> { + if (request.runtime_shadow === null) return null; + return await beginLeaseOutboxEntry({ + runtime_root: request.runtime_root, + goal_id: request.goal_id, + lease_directory: taskLeaseDirectory(request), + write_class: writeClass, + operation_id: request.idempotency_key ?? request.fence_operation_id, + previous_lease: previous, + planned_lease: next, + }); +} + +function attachRuntimeShadowCapture( + response: JsonObject, + capture: LeaseOutboxCapture | null, +): JsonObject { + if (capture === null) return response; + return { + ...response, + coordination_runtime_shadow_capture: { + entry_id: capture.entry_id, + seq: capture.seq, + source_bytes_digest: capture.source_bytes_digest, + failure: capture.failure, + }, }; } @@ -1673,9 +1715,13 @@ async function ordinaryOperation( const response = responseForOrdinary(request, leasePath, next, false, { released: true }); await persistOperationReceipt(request, "prepared", next, response); await dependencies.beforeWrite?.(next); + const shadowCapture = await captureLeaseWrite( + request, existing, next, "task_lease_release", + ); await atomicWriteJson(leasePath, next); + await shadowCapture?.commit(); await persistOperationReceipt(request, "committed", next, response); - return response; + return attachRuntimeShadowCapture(response, shadowCapture); } if (existing === null || decision.next_lease === null) { throw transitionError(request, "invalid_lease_snapshot", existing, leasePath); @@ -1704,9 +1750,13 @@ async function ordinaryOperation( await persistOperationReceipt(request, "prepared", next, response); await dependencies.beforeWrite?.(next); if (request.authority) await revalidateAuthoritySources(request.authority.source_receipts); + const shadowCapture = await captureLeaseWrite( + request, existing, next, `task_lease_${request.operation}`, + ); await atomicWriteJson(leasePath, next); + await shadowCapture?.commit(); await persistOperationReceipt(request, "committed", next, response); - return response; + return attachRuntimeShadowCapture(response, shadowCapture); } function fencePayload( @@ -2212,7 +2262,11 @@ async function fenceVerify( }; await dependencies.beforeWrite?.(lease); await revalidateAuthoritySources(request.authority.source_receipts); + const shadowCapture = await captureLeaseWrite( + request, rawLease, lease, "task_lease_auto_acquire", + ); await atomicWriteJson(leasePath, lease); + await shadowCapture?.commit(); const response = fencePayload( { ...request, owner: request.owner }, lease, @@ -2231,7 +2285,7 @@ async function fenceVerify( base: receipt, }); keep = true; - return response; + return attachRuntimeShadowCapture(response, shadowCapture); } // A response can be lost after an auto-acquire write but before the // receipt reaches the caller. The durable fence receipt proves that the @@ -2558,6 +2612,7 @@ async function fenceClose( if (!claimedOwner || claimedOwner.token !== request.lock_token) { throw new EffectRuntimeConflictError("task lease fence token is no longer held", "fence_token_invalid"); } + let shadowCapture: LeaseOutboxCapture | null = null; if (request.committed && request.release_lease) { const at = lifecycleNow(request, dependencies); const leasePath = leasePathFor(request); @@ -2631,7 +2686,11 @@ async function fenceClose( if (!alreadyReleased) { const next = releasedLease(lease, at); await dependencies.beforeWrite?.(next); + shadowCapture = await captureLeaseWrite( + request, lease, next, "task_lease_fence_close", + ); await atomicWriteJson(leasePath, next); + await shadowCapture?.commit(); released = true; } } @@ -2657,7 +2716,7 @@ async function fenceClose( fenceExpectedLeaseEpoch: request.fence_expected_lease_epoch ?? receipt?.fence_expected_lease_epoch, closeRequestDigest: fenceCloseRequestDigest(request), }); - return response; + return attachRuntimeShadowCapture(response, shadowCapture); } finally { if (claim) { await releaseFileMutationLock( @@ -2682,7 +2741,18 @@ export async function executeTaskLeaseLifecycle( if (request.operation === "fence_close") return await fenceClose(request, dependencies); if (request.operation === "terminal_verify" || request.operation === "holder_verify") { const fence = await fenceVerify(request, dependencies); - return { ok: true, schema_version: "task_lease_v0", action: request.operation, fence, settlement: lifecycleSettlement(request, "committed") }; + const capture = fence.coordination_runtime_shadow_capture; + delete fence.coordination_runtime_shadow_capture; + return { + ok: true, + schema_version: "task_lease_v0", + action: request.operation, + fence, + settlement: lifecycleSettlement(request, "committed"), + ...(capture === undefined + ? {} + : { coordination_runtime_shadow_capture: capture }), + }; } return await withFileMutationLock( legacyCoordinationLeaseLockPath(request.runtime_root, request.goal_id), diff --git a/loopx/todos.py b/loopx/todos.py index c72dcd6728..8fff3fbf25 100644 --- a/loopx/todos.py +++ b/loopx/todos.py @@ -128,9 +128,10 @@ enter_todo_ownership_handoff_gate, resolve_todo_completion_handoff, ) -from .control_plane.coordination.local_authority_shadow_adapter import ( - effective_runtime_root, - observe_todo_local_authority_commit as _shadow_todo, +from .control_plane.coordination.local_authority_shadow_adapter import effective_runtime_root +from .control_plane.coordination.runtime_shadow_writer_adapter import ( + begin_todo_runtime_shadow_capture, + settle_todo_runtime_shadow_capture, ) from .control_plane.work_items.task_lease import ( enter_terminal_todo_lease_fence, @@ -765,6 +766,11 @@ def add_goal_todo( runtime_root=shadow_runtime_root, ), ExitStack() as handoff_gate_stack: original = resolved_state_file.read_text(encoding="utf-8") + shadow_capture = begin_todo_runtime_shadow_capture( + registry_path=registry_path, runtime_root=shadow_runtime_root, + goal_id=goal_id, state_path=resolved_state_file, + write_class="todo_add", original_text=original, + ) lines = original.splitlines() updated_at = now_local() effective_claimed_by = ( @@ -921,7 +927,9 @@ def add_goal_todo( if changed: new_text = replace_updated_at(new_text, updated_at) if changed and not dry_run: + shadow_capture.prepare(new_text) resolved_state_file.write_text(new_text, encoding="utf-8") + shadow_capture.committed() payload = { "ok": True, @@ -974,7 +982,10 @@ def add_goal_todo( write_class="todo_add", state_text=original, ) - return _shadow_todo(payload, registry_path, goal_id, "todo_add", runtime_root=shadow_runtime_root) + return settle_todo_runtime_shadow_capture( + payload, registry_path=registry_path, runtime_root=shadow_runtime_root, + goal_id=goal_id, write_class="todo_add", capture=shadow_capture, + ) def resolve_todo_state( @@ -1085,6 +1096,11 @@ def update_goal_todo( runtime_root=shadow_runtime_root, ), ExitStack() as handoff_gate_stack: original = resolved_state_file.read_text(encoding="utf-8") + shadow_capture = begin_todo_runtime_shadow_capture( + registry_path=registry_path, runtime_root=shadow_runtime_root, + goal_id=goal_id, state_path=resolved_state_file, + write_class="todo_update", original_text=original, + ) lines = original.splitlines() updated_at = now_local() effective_claimed_by = ( @@ -1402,7 +1418,9 @@ def update_goal_todo( if changed: new_text = replace_updated_at(new_text, updated_at) if changed and not dry_run: + shadow_capture.prepare(new_text) resolved_state_file.write_text(new_text, encoding="utf-8") + shadow_capture.committed() write_class = "todo_claim" if claim_only else "todo_update" payload = { "ok": True, @@ -1435,7 +1453,10 @@ def update_goal_todo( write_class=write_class, state_text=original, ) - return _shadow_todo(payload, registry_path, goal_id, write_class, runtime_root=shadow_runtime_root) + return settle_todo_runtime_shadow_capture( + payload, registry_path=registry_path, runtime_root=shadow_runtime_root, + goal_id=goal_id, write_class=write_class, capture=shadow_capture, + ) def complete_goal_todo( @@ -1514,6 +1535,11 @@ def complete_goal_todo( runtime_root=shadow_runtime_root, ), ExitStack() as lease_fence_stack: original = resolved_state_file.read_text(encoding="utf-8") + shadow_capture = begin_todo_runtime_shadow_capture( + registry_path=registry_path, runtime_root=shadow_runtime_root, + goal_id=goal_id, state_path=resolved_state_file, + write_class="todo_complete", original_text=original, + ) lines = original.splitlines() updated_at = now_local() completion_match, completion_todo, event_context = ( @@ -1668,9 +1694,15 @@ def complete_goal_todo( task_lease_fence, committed=bool(event_result.get("changed")) and not dry_run, ) - write_class = "todo_complete_event_projection" - return _shadow_todo( - event_result, registry_path, goal_id, write_class, runtime_root=shadow_runtime_root + # This branch can append multiple state-log events inside the + # event writer. Capturing after that call would be observation, + # not a transaction-bound prepare/commit pair. Keep the gap + # explicit until the event writer owns the outbox boundary. + shadow_capture.skip("event_log_writer_not_bound") + return settle_todo_runtime_shadow_capture( + event_result, registry_path=registry_path, + runtime_root=shadow_runtime_root, goal_id=goal_id, + write_class="todo_complete_event_projection", capture=shadow_capture, ) if not isinstance(completion_state, dict): raise RuntimeError( @@ -1801,7 +1833,9 @@ def complete_goal_todo( if changed: new_text = replace_updated_at(new_text, updated_at) if changed and not dry_run: + shadow_capture.prepare(new_text) resolved_state_file.write_text(new_text, encoding="utf-8") + shadow_capture.committed() release_verified_task_lease_fence( task_lease_fence, committed=changed and not dry_run, @@ -1829,7 +1863,10 @@ def complete_goal_todo( if effective_decision_outcome: result["decision_outcome"] = effective_decision_outcome result["self_merged"] = effective_self_merged - return _shadow_todo(result, registry_path, goal_id, "todo_complete", runtime_root=shadow_runtime_root) + return settle_todo_runtime_shadow_capture( + result, registry_path=registry_path, runtime_root=shadow_runtime_root, + goal_id=goal_id, write_class="todo_complete", capture=shadow_capture, + ) def supersede_goal_todo( *, @@ -1875,6 +1912,11 @@ def supersede_goal_todo( runtime_root=shadow_runtime_root, ), ExitStack() as lease_fence_stack: original = resolved_state_file.read_text(encoding="utf-8") + shadow_capture = begin_todo_runtime_shadow_capture( + registry_path=registry_path, runtime_root=shadow_runtime_root, + goal_id=goal_id, state_path=resolved_state_file, + write_class="todo_supersede", original_text=original, + ) lines = original.splitlines() updated_at = now_local() current_match = find_todo_block(lines, todo_id=todo_id, role=role) @@ -2026,7 +2068,9 @@ def supersede_goal_todo( if changed: new_text = replace_updated_at(new_text, updated_at) if changed and not dry_run: + shadow_capture.prepare(new_text) resolved_state_file.write_text(new_text, encoding="utf-8") + shadow_capture.committed() release_verified_task_lease_fence(task_lease_fence, committed=changed and not dry_run) result = { "ok": True, @@ -2042,7 +2086,10 @@ def supersede_goal_todo( "project": str(resolved_project) if resolved_project else None, "updated_at": updated_at if changed else None, } - return _shadow_todo(result, registry_path, goal_id, "todo_supersede", runtime_root=shadow_runtime_root) + return settle_todo_runtime_shadow_capture( + result, registry_path=registry_path, runtime_root=shadow_runtime_root, + goal_id=goal_id, write_class="todo_supersede", capture=shadow_capture, + ) def archive_completed_todos( @@ -2073,6 +2120,11 @@ def archive_completed_todos( runtime_root=shadow_runtime_root, ): original = resolved_state_file.read_text(encoding="utf-8") + shadow_capture = begin_todo_runtime_shadow_capture( + registry_path=registry_path, runtime_root=shadow_runtime_root, + goal_id=goal_id, state_path=resolved_state_file, + write_class="todo_archive_completed", original_text=original, + ) lines = original.splitlines() archive_result = archive_completed_todo_lines( lines, @@ -2087,7 +2139,9 @@ def archive_completed_todos( if changed: new_text = replace_updated_at(new_text, updated_at) if changed and not dry_run: + shadow_capture.prepare(new_text) resolved_state_file.write_text(new_text, encoding="utf-8") + shadow_capture.committed() result = { "ok": True, @@ -2098,6 +2152,7 @@ def archive_completed_todos( "project": str(resolved_project) if resolved_project else None, "updated_at": updated_at if changed else None, } - return _shadow_todo( - result, registry_path, goal_id, "todo_archive_completed", runtime_root=shadow_runtime_root + return settle_todo_runtime_shadow_capture( + result, registry_path=registry_path, runtime_root=shadow_runtime_root, + goal_id=goal_id, write_class="todo_archive_completed", capture=shadow_capture, ) From 5401aeed0075659a03fb47503de58e93bb45e445 Mon Sep 17 00:00:00 2001 From: huangruiteng Date: Sat, 5 Sep 2026 00:33:11 +0800 Subject: [PATCH 08/11] test(authority): cover runtime shadow writer capture Signed-off-by: huangruiteng --- .../test_local_authority_shadow_drain.py | 30 +-- .../test_local_authority_shadow_outbox.py | 5 +- .../test_runtime_shadow_writer_capture.py | 209 ++++++++++++++++++ .../local_authority_shadow_outbox.test.ts | 22 +- .../task_lease_lifecycle.test.ts | 14 ++ 5 files changed, 244 insertions(+), 36 deletions(-) create mode 100644 tests/control_plane/test_runtime_shadow_writer_capture.py diff --git a/tests/control_plane/test_local_authority_shadow_drain.py b/tests/control_plane/test_local_authority_shadow_drain.py index 592b18d4a3..73b760d4e6 100644 --- a/tests/control_plane/test_local_authority_shadow_drain.py +++ b/tests/control_plane/test_local_authority_shadow_drain.py @@ -166,7 +166,7 @@ def test_drain_delivers_each_committed_entry_once_in_order_and_verifies_readback view = adapter.read_local_authority_shadow(runtime_root=runtime_root, goal_id=GOAL_ID, scan_limit=10) assert view["status"] == "loaded" head = view["head"] - assert head["schema_version"] == "loopx_local_authority_shadow_projection_v1" + assert head["schema_version"] == "loopx_coordination_runtime_shadow_projection_v0" assert head["partitions"]["todos"] == { "seq": 3, "partition_digest": captures[-1].outcome.partition_digest, @@ -268,7 +268,7 @@ def test_drain_stops_in_order_when_the_store_boundary_misbehaves( def flaky(method: str, params: object, **kwargs: object) -> object: calls.append(method) - if method == "coordination.local_authority_shadow.commit_entry" and len(calls) == 2: + if method == "coordination.runtime_shadow.commit_entry" and len(calls) == 2: return {"schema_version": "garbage"} return real(method, params, **kwargs) @@ -355,7 +355,7 @@ def test_unexplained_prepared_only_entry_triggers_reseed_under_the_primary_lock( assert seed_receipt["source_bytes_digest"] == text_digest(state.read_text(encoding="utf-8")) -def test_lease_partition_entries_are_compacted_at_drain(tmp_path: Path) -> None: +def test_lease_partition_entries_retain_complete_records_at_drain(tmp_path: Path) -> None: registry, _state, runtime_root = _fixture(tmp_path) directory = outbox.partition_directory(runtime_root, GOAL_ID, "leases") record = { @@ -412,25 +412,11 @@ def test_lease_partition_entries_are_compacted_at_drain(tmp_path: Path) -> None: head = view["head"] assert head["todos"] == [] assert head["handoff_mode"] is None - assert head["leases"] == [ - { - "todo_id": record["todo_id"], - "owner": "agent-a", - "idempotency_key": "k1", - "write_scopes": ["loopx/**"], - "version": 2, - "lease_epoch": 1, - "acquired_at": record["acquired_at"], - "updated_at": record["updated_at"], - "expires_at": record["expires_at"], - "released_at": None, - "status": "active", - } - ] + assert head["leases"] == [record] expected_digest = partition_digest({"leases": head["leases"]}) assert head["partitions"]["leases"] == {"seq": 1, "partition_digest": expected_digest} assert result.entries[0]["partition_digest"] == expected_digest - assert "/should/never/be/compared" not in json.dumps(view) + assert "/should/never/be/compared" in json.dumps(view) def test_status_reports_backlog_candidate_and_growth_facts(tmp_path: Path) -> None: @@ -461,7 +447,7 @@ def test_status_reports_backlog_candidate_and_growth_facts(tmp_path: Path) -> No assert drained["candidate"]["status"] == "loaded" assert drained["candidate"]["cursor"] == "1" assert drained["candidate"]["codec_agreement"] is True - assert drained["candidate"]["head_schema_version"] == "loopx_local_authority_shadow_projection_v1" + assert drained["candidate"]["head_schema_version"] == "loopx_coordination_runtime_shadow_projection_v0" assert drained["store_bytes"] > 0 @@ -553,7 +539,7 @@ def crash_between_unlinks(entry: outbox.OutboxEntry) -> None: assert second.outcome == "drained" assert second.reclaimed_residue == 1 assert (second.delivered, second.replayed) == (0, 0) - assert "coordination.local_authority_shadow.commit_entry" not in calls + assert "coordination.runtime_shadow.commit_entry" not in calls assert list(_todo_dir(runtime_root).iterdir()) == [_todo_dir(runtime_root) / "drain-cursor.json"] view = adapter.read_local_authority_shadow(runtime_root=runtime_root, goal_id=GOAL_ID, scan_limit=5) assert view["cursor"] == "1" @@ -592,7 +578,7 @@ def crash_before_unlinks(entry: outbox.OutboxEntry) -> None: result = _drain(registry, runtime_root) assert result.ok is True assert result.reclaimed_residue == 2 - assert "coordination.local_authority_shadow.commit_entry" not in calls + assert "coordination.runtime_shadow.commit_entry" not in calls assert list(_todo_dir(runtime_root).iterdir()) == [_todo_dir(runtime_root) / "drain-cursor.json"] diff --git a/tests/control_plane/test_local_authority_shadow_outbox.py b/tests/control_plane/test_local_authority_shadow_outbox.py index f3ce05f238..907dc8b140 100644 --- a/tests/control_plane/test_local_authority_shadow_outbox.py +++ b/tests/control_plane/test_local_authority_shadow_outbox.py @@ -8,7 +8,6 @@ from loopx.control_plane.coordination import local_authority_shadow_adapter as adapter from loopx.control_plane.coordination import local_authority_shadow_outbox as outbox from loopx.control_plane.coordination.local_authority_shadow_projection import ( - LEASE_FIELDS, ProjectionValueError, canonical_bytes, lease_partition_projection, @@ -326,12 +325,12 @@ def test_canonical_projection_rejects_floats_and_bad_lease_identity() -> None: with pytest.raises(ProjectionValueError): lease_partition_projection([("todo-a", {"goal_id": "other", "todo_id": "todo-a"})], goal_id=GOAL_ID) projection = lease_partition_projection( - [("todo-b", {"goal_id": GOAL_ID, "todo_id": "todo-b", "version": 1, "extra": "dropped"}), + [("todo-b", {"goal_id": GOAL_ID, "todo_id": "todo-b", "version": 1, "extra": "retained"}), ("todo-a", {"goal_id": GOAL_ID, "todo_id": "todo-a", "version": 2, "status": "active"})], goal_id=GOAL_ID, ) assert [lease["todo_id"] for lease in projection["leases"]] == ["todo-a", "todo-b"] - assert set(projection["leases"][1]) <= set(LEASE_FIELDS) + assert projection["leases"][1]["extra"] == "retained" def test_capture_failure_is_typed_and_never_raises(tmp_path: Path) -> None: diff --git a/tests/control_plane/test_runtime_shadow_writer_capture.py b/tests/control_plane/test_runtime_shadow_writer_capture.py new file mode 100644 index 0000000000..a37a4919af --- /dev/null +++ b/tests/control_plane/test_runtime_shadow_writer_capture.py @@ -0,0 +1,209 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from loopx.control_plane.coordination import local_authority_shadow_adapter as adapter +from loopx.control_plane.work_items.task_lease import ( + acquire_task_lease, + release_task_lease, + renew_task_lease, + transfer_task_lease, +) +from loopx.todos import add_goal_todo, update_goal_todo + + +GOAL_ID = "runtime-shadow-writer" + + +def _fixture(tmp_path: Path, *, enabled: bool) -> tuple[Path, Path, Path]: + repo = tmp_path / "repo" + repo.mkdir() + state = repo / "ACTIVE_GOAL_STATE.md" + state.write_text( + "---\n" + f"goal_id: {GOAL_ID}\n" + "handoff_mode: hard_lease\n" + "updated_at: 2026-09-04T00:00:00+00:00\n" + "---\n\n## Agent Todo\n\n", + encoding="utf-8", + ) + runtime_root = tmp_path / "runtime" + coordination: dict[str, object] = { + "agent_model": "peer_v1", + "registered_agents": ["agent-a", "agent-b"], + } + if enabled: + coordination["runtime_shadow"] = { + "enabled": True, + "schema_version": "loopx_coordination_runtime_shadow_config_v0", + "provider": "file_v0", + } + registry = tmp_path / "registry.json" + registry.write_text( + json.dumps( + { + "common_runtime_root": str(runtime_root), + "goals": [ + { + "id": GOAL_ID, + "domain": "harness_self_improvement", + "status": "active", + "repo": str(repo), + "state_file": state.name, + "adapter": {"kind": "harness_self_improvement"}, + "coordination": coordination, + } + ], + } + ), + encoding="utf-8", + ) + return registry, state, runtime_root + + +def test_runtime_shadow_todo_writer_captures_full_records_and_reuses_one_store( + tmp_path: Path, +) -> None: + registry, _state, runtime_root = _fixture(tmp_path, enabled=True) + + added = add_goal_todo( + registry_path=registry, + goal_id=GOAL_ID, + role="agent", + text="Retain complete canonical Todo fields.", + task_class="advancement_task", + action_kind="implement", + claimed_by="agent-a", + ) + updated = update_goal_todo( + registry_path=registry, + goal_id=GOAL_ID, + todo_id=str(added["todo_id"]), + note="transaction-bound", + agent_id="agent-a", + ) + + assert added["coordination_runtime_shadow"]["outcome"] == "delivered" + assert updated["coordination_runtime_shadow"]["outcome"] == "delivered" + view = adapter.read_local_authority_shadow( + runtime_root=runtime_root, + goal_id=GOAL_ID, + scan_limit=10, + ) + head = view["head"] + assert head["schema_version"] == "loopx_coordination_runtime_shadow_projection_v0" + assert head["todos"][0]["text"] == "Retain complete canonical Todo fields." + assert head["todos"][0]["note"] == "transaction-bound" + assert head["todo_read_model"]["todo_count"] == 1 + assert view["cursor"] == "2" + assert not (runtime_root / "authority-shadow" / "file" / GOAL_ID).exists() + + +def test_runtime_shadow_todo_writer_is_zero_effect_by_default(tmp_path: Path) -> None: + registry, _state, runtime_root = _fixture(tmp_path, enabled=False) + + result = add_goal_todo( + registry_path=registry, + goal_id=GOAL_ID, + role="agent", + text="Default-off capture.", + task_class="advancement_task", + ) + + assert result["coordination_runtime_shadow"]["outcome"] == "no_transaction" + assert result["coordination_runtime_shadow"]["reason_code"] == "shadow_disabled" + assert not (runtime_root / "authority-shadow").exists() + + +def test_runtime_shadow_native_lease_writers_capture_complete_records(tmp_path: Path) -> None: + registry, _state, runtime_root = _fixture(tmp_path, enabled=True) + added = add_goal_todo( + registry_path=registry, + goal_id=GOAL_ID, + role="agent", + text="Capture the native lease transaction.", + task_class="advancement_task", + ) + todo_id = str(added["todo_id"]) + + acquired = acquire_task_lease( + registry_path=registry, + runtime_root=runtime_root, + goal_id=GOAL_ID, + todo_id=todo_id, + owner="agent-a", + idempotency_key="runtime-shadow-lease", + write_scopes=["loopx/**"], + ttl_seconds=120, + ) + renewed = renew_task_lease( + registry_path=registry, + runtime_root=runtime_root, + goal_id=GOAL_ID, + todo_id=todo_id, + owner="agent-a", + idempotency_key="runtime-shadow-lease", + ttl_seconds=180, + expected_version=int(acquired["lease"]["version"]), + ) + transferred = transfer_task_lease( + registry_path=registry, + runtime_root=runtime_root, + goal_id=GOAL_ID, + todo_id=todo_id, + owner="agent-a", + idempotency_key="runtime-shadow-lease", + new_owner="agent-b", + new_idempotency_key="runtime-shadow-lease-b", + ttl_seconds=180, + expected_version=int(renewed["lease"]["version"]), + ) + released = release_task_lease( + registry_path=registry, + runtime_root=runtime_root, + goal_id=GOAL_ID, + todo_id=todo_id, + owner="agent-b", + idempotency_key="runtime-shadow-lease-b", + expected_version=int(transferred["lease"]["version"]), + ) + + assert acquired["coordination_runtime_shadow"]["outcome"] == "delivered" + assert renewed["coordination_runtime_shadow"]["outcome"] == "delivered" + assert transferred["coordination_runtime_shadow"]["outcome"] == "delivered" + assert released["coordination_runtime_shadow"]["outcome"] == "delivered" + view = adapter.read_local_authority_shadow(runtime_root=runtime_root, goal_id=GOAL_ID) + lease = view["head"]["leases"][0] + assert lease["goal_id"] == GOAL_ID + assert lease["owner"] == "agent-b" + assert lease["idempotency_key"] == "runtime-shadow-lease-b" + assert lease["write_scopes"] == ["loopx/**"] + assert lease["status"] == "released" + + +def test_runtime_shadow_native_lease_writer_is_zero_effect_by_default(tmp_path: Path) -> None: + registry, _state, runtime_root = _fixture(tmp_path, enabled=False) + added = add_goal_todo( + registry_path=registry, + goal_id=GOAL_ID, + role="agent", + text="Do not capture an unconfigured lease.", + task_class="advancement_task", + claimed_by="agent-a", + ) + + acquired = acquire_task_lease( + registry_path=registry, + runtime_root=runtime_root, + goal_id=GOAL_ID, + todo_id=str(added["todo_id"]), + owner="agent-a", + idempotency_key="default-off-lease", + write_scopes=["loopx/**"], + ttl_seconds=120, + ) + + assert "coordination_runtime_shadow_capture" not in acquired + assert "coordination_runtime_shadow" not in acquired + assert not (runtime_root / "authority-shadow").exists() diff --git a/tests/control_plane_ts/local_authority_shadow_outbox.test.ts b/tests/control_plane_ts/local_authority_shadow_outbox.test.ts index 0ec8f49194..b8358971a7 100644 --- a/tests/control_plane_ts/local_authority_shadow_outbox.test.ts +++ b/tests/control_plane_ts/local_authority_shadow_outbox.test.ts @@ -58,7 +58,7 @@ function todoEntry(seq: number, digest: string, resolution = "committed") { function commitRequest(root: string, seq: number, todos: object[], resolution = "committed") { const projection = { handoff_mode: "hard_lease", todos }; return { - schema_version: "loopx_local_authority_shadow_commit_entry_request_v0", + schema_version: "loopx_coordination_runtime_shadow_commit_entry_request_v0", runtime_root: root, goal_id: GOAL, entry: todoEntry(seq, `sha256:${HEX}`, resolution), @@ -69,7 +69,7 @@ function commitRequest(root: string, seq: number, todos: object[], resolution = function noOpRequest(root: string, seq: number, resolution: "abandoned" | "unproved") { return { - schema_version: "loopx_local_authority_shadow_commit_entry_request_v0", + schema_version: "loopx_coordination_runtime_shadow_commit_entry_request_v0", runtime_root: root, goal_id: GOAL, entry: todoEntry(seq, `sha256:${HEX}`, resolution), @@ -94,7 +94,7 @@ test("commit_entry folds one partition into a v1 head and binds the receipt to t assert.equal(result.outcome, "delivered"); assert.equal(result.no_op, false); assert.equal(result.cursor, "1"); - const store = new FileAuthorityStore(join(root, "authority-shadow", "file", GOAL), GOAL); + const store = new FileAuthorityStore(join(root, "authority-shadow", "file-v0"), GOAL); const loaded = await store.loadAuthority(); assert.equal(loaded.status, "loaded"); if (loaded.status !== "loaded") return; @@ -136,7 +136,7 @@ test("commit_entry replays only when the existing receipt carries the same parti assert.equal(mismatch.outcome, "protocol_mismatch"); assert.equal(mismatch.reason_code, "transaction_receipt_mismatch"); - const page = await new FileAuthorityStore(join(root, "authority-shadow", "file", GOAL), GOAL) + const page = await new FileAuthorityStore(join(root, "authority-shadow", "file-v0"), GOAL) .scanCommitted(null, 10); assert.equal(page.status, "page"); if (page.status === "page") assert.equal(page.transactions.length, 1); @@ -155,7 +155,7 @@ test("no-op resolutions keep the sequence auditable without touching compared fi assert.equal(unproved.no_op, true); assert.equal(unproved.cursor, "3"); assert.equal(abandoned.head_digest, first.head_digest); - const store = new FileAuthorityStore(join(root, "authority-shadow", "file", GOAL), GOAL); + const store = new FileAuthorityStore(join(root, "authority-shadow", "file-v0"), GOAL); const page = await store.scanCommitted(null, 10); assert.equal(page.status, "page"); if (page.status !== "page") return; @@ -220,7 +220,7 @@ test("a v0 observation head is accepted as the starting point for partition fold test("read returns head, comparison digest, and a bounded scan page", async (t) => { const root = await tempRoot(t); const missing = await readLocalAuthorityShadow({ - schema_version: "loopx_local_authority_shadow_read_request_v0", + schema_version: "loopx_coordination_runtime_shadow_outbox_read_v0", runtime_root: root, goal_id: GOAL, scan_after_cursor: null, @@ -234,7 +234,7 @@ test("read returns head, comparison digest, and a bounded scan page", async (t) await commitLocalAuthorityShadowEntry(commitRequest(root, 1, todos)); await commitLocalAuthorityShadowEntry(noOpRequest(root, 2, "abandoned")); const view = await readLocalAuthorityShadow({ - schema_version: "loopx_local_authority_shadow_read_request_v0", + schema_version: "loopx_coordination_runtime_shadow_outbox_read_v0", runtime_root: root, goal_id: GOAL, scan_after_cursor: null, @@ -261,13 +261,13 @@ test("lease outbox entries are two-phase, durable, and skipped without a binding const planned = { goal_id: GOAL, todo_id: "todo-a", version: 2, lease_epoch: 1, status: "active", updated_at: "t2" }; assert.equal(decodeLocalAuthorityShadowBinding(undefined), null); - assert.equal(decodeLocalAuthorityShadowBinding({ mode: "file_one_way" }), null); + assert.equal(decodeLocalAuthorityShadowBinding({ provider: "file_v0" }), null); assert.deepEqual( decodeLocalAuthorityShadowBinding({ - schema_version: "loopx_local_authority_shadow_binding_v0", - mode: "file_one_way", + schema_version: "loopx_coordination_runtime_shadow_binding_v0", + provider: "file_v0", }), - { schema_version: "loopx_local_authority_shadow_binding_v0", mode: "file_one_way" }, + { schema_version: "loopx_coordination_runtime_shadow_binding_v0", provider: "file_v0" }, ); const capture = await beginLeaseOutboxEntry({ diff --git a/tests/control_plane_ts/task_lease_lifecycle.test.ts b/tests/control_plane_ts/task_lease_lifecycle.test.ts index e8073a7c3b..4886f5942f 100644 --- a/tests/control_plane_ts/task_lease_lifecycle.test.ts +++ b/tests/control_plane_ts/task_lease_lifecycle.test.ts @@ -598,6 +598,10 @@ test("fence close write failure leaves the lease unchanged and unlocks", async ( test("user-gate auto-acquire returns a persistent fence that can be closed", async (t) => { const root = await workspace(t); + const runtimeShadow = { + schema_version: "loopx_coordination_runtime_shadow_binding_v0", + provider: "file_v0", + }; const facts = await authority(root, { todos: [{ todo_id: "todo_target", @@ -615,10 +619,15 @@ test("user-gate auto-acquire returns a persistent fence that can be closed", asy idempotency_key: null, expected_version: null, allow_user_gate_auto_acquire: true, + runtime_shadow: runtimeShadow, }), { now: () => ACQUIRE_NOW }, ); assert.equal(checked.ok, true); + const autoCapture = checked.coordination_runtime_shadow_capture as Record; + assert.equal(typeof autoCapture.entry_id, "string"); + assert.equal(autoCapture.seq, 1); + assert.equal(autoCapture.failure, null); const fence = checked.fence as Record; assert.equal(fence.auto_acquired, true); assert.equal((await lease(root)).status, "active"); @@ -635,10 +644,15 @@ test("user-gate auto-acquire returns a persistent fence that can be closed", asy fence_owner: "agent-a", fence_idempotency_key: (await lease(root)).idempotency_key, fence_expected_version: (await lease(root)).version, + runtime_shadow: runtimeShadow, }), }, { now: () => new Date("2026-09-01T03:01:00.000Z") }); assert.equal(closed.ok, true); assert.equal(closed.released, true); + const closeCapture = closed.coordination_runtime_shadow_capture as Record; + assert.equal(typeof closeCapture.entry_id, "string"); + assert.equal(closeCapture.seq, 2); + assert.equal(closeCapture.failure, null); assert.equal((await lease(root)).status, "released"); }); From 505c808b17bbba376261fd09ff25091ca788b4dc Mon Sep 17 00:00:00 2001 From: huangruiteng Date: Sat, 5 Sep 2026 00:33:11 +0800 Subject: [PATCH 09/11] docs(authority): align RFC with transaction capture Signed-off-by: huangruiteng --- .../shared-goal-authority-state-provider-v0.md | 18 +++++++++--------- ...d-goal-authority-state-provider-v0.zh-CN.md | 13 +++++++------ 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.md b/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.md index 5f81324c9a..4983f38222 100644 --- a/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.md +++ b/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.md @@ -2134,14 +2134,14 @@ write that cannot preserve the contract. fence, run `promote`, render the projections, and record the declaration of question 11; refuse a rerun whose source digest changed unless the abandoned store is explicitly discarded. -- A transaction-bound capture (question 14): the runtime shadow samples the - source after the commit, from outside the writer's lock, so a concurrent - writer can land inside the sampled projection and a crash between the commit - and the dispatch loses the mirror. The parity-half outbox closes both: the - prepared entry is written inside the lock the writer already holds, the - committed marker after the primary write returns, and a bounded drain turns - each entry into exactly one shadow transaction whose `operation_id` is the - entry id. `qualify` then counts entries, not samples. +- Full transaction-capture qualification (question 14): Todo add, update, + complete, supersede, and archive plus native lease acquire, renew, transfer, + release, auto-acquire, and fence-close now emit prepared/committed outbox + entries around their primary write. The bounded drain commits complete + versioned records into the existing `coordination.runtime_shadow` file-v0 + lineage; it does not create the former second local-shadow candidate. Before + promotion, add sustained mixed-writer parity runs, event-only Todo coverage, + and the selected provider profile's recovery/capacity evidence. - The provider-neutral authority binding, compatibility projection outbox, and conformance rows for file, NoKV, and PostgreSQL. This does not require all providers to promote together; each profile must pass the same contract @@ -2172,6 +2172,6 @@ one document is acceptable only for promotion bootstrap and bounded test goals. | Lane | May start | Scope and exit condition | Dependency | | --- | --- | --- | --- | | P. PostgreSQL provider plane | Now, from current `main` | Keep the existing `AuthorityStore` contract; finish schema migration/install ownership, authenticated service and tenant authorization, restore-incarnation rotation, pool/cancellation/failover behavior, and reviewed indexes, partitioning, retention, and measured capacity. Live PostgreSQL conformance remains mandatory. | Does not depend on #3870 and must not stack on its branch. This lane alone creates no runtime caller or promotion claim. | -| C. Canonical transaction capture | Now, by revising or replacing #3870 | Make the outbox transport complete versioned Todo/lease records into `coordination.runtime_shadow.commit`; retire the duplicate observation/local-shadow lineage and prove omission/explicit-clear behavior. | Can run in parallel with P, but both C and the selected provider profile must finish before parity or promotion integration. | +| C. Canonical transaction capture | In implementation, based on #3870 | Transaction-bound outbox capture now targets the one `coordination.runtime_shadow` lineage and retains complete versioned Todo/lease records. Finish sustained mixed-writer parity, explicit-clear/omission coverage, and event-only Todo recovery evidence. | Can run in parallel with P, but both C and the selected provider profile must finish before parity or promotion integration. | | I. Binding and qualification integration | After P and C | Bind one exact provider lineage, field manifest, source revision, digest, and cursor; run sustained transaction parity and provider-specific recovery/capacity qualification without consulting legacy state for missing fields. | This is the merge point between provider work and capture work. | | F. Promotion and cleanup | After I and explicit maintainer approval | Add provider-first CLI routing, the lock-owning promotion orchestrator, compatibility projection outbox, post-promotion fenced export/rollback, then delete duplicate reference aggregates and flip the reviewed stage/hold declarations. | No profile is eligible until its exact implementation and lineage pass P, C, and I. | diff --git a/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.zh-CN.md b/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.zh-CN.md index 8340d170dc..7c97b00268 100644 --- a/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.zh-CN.md +++ b/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.zh-CN.md @@ -1698,11 +1698,12 @@ integrity chain、确定性 scan 与 recovery readback。物理 profile 可以 片）：取 Todo 与 lease 两把 legacy 锁，要求 `qualify` 在当前 revision 与 digest 上 为 `qualified`，engage fence，执行 `promote`，渲染投影，写入问题 11 的声明；源 digest 已变时拒绝重跑，除非显式丢弃被放弃的 store。 -- 事务绑定的捕获（问题 14）：runtime shadow 在提交之后、写者锁之外采样源，并发写者 - 可能落进被采样的投影，commit 与 dispatch 之间崩溃则丢失镜像。parity 半段的 outbox - 同时关闭两者：prepared entry 在写者已持有的锁内写入，committed 标记在主写返回后 - 写入，有界 drain 把每条 entry 变成恰好一笔 `operation_id` 为 entry id 的 shadow - 事务。此后 `qualify` 数的是 entry，不是采样。 +- 完成事务捕获资格验证（问题 14）：Todo add、update、complete、supersede、archive， + 以及 native lease acquire、renew、transfer、release、auto-acquire、fence-close，现已在 + 主写前后生成 prepared/committed outbox entry。有界 drain 把完整、带版本的 record + 提交到既有 `coordination.runtime_shadow` file-v0 lineage，不再创建第二套 local-shadow + candidate。晋升前仍需补持续 mixed-writer parity、event-only Todo 覆盖和所选 provider + profile 的 recovery/capacity 证据。 - provider-neutral authority binding、兼容投影 outbox，以及 file、NoKV、PostgreSQL 的 conformance row。三个 provider 不必同时晋升，但每个 profile 都必须先通过同一 合同才具备资格。 @@ -1727,6 +1728,6 @@ latency、response size 与 recovery。单文档全量保留只适用于 promoti | Lane | 何时开始 | 范围与退出条件 | 依赖 | | --- | --- | --- | --- | | P. PostgreSQL provider plane | 现在，从当前 `main` 开始 | 保持既有 `AuthorityStore` 合同；完成 schema migration/install ownership、authenticated service 与 tenant authorization、restore-incarnation rotation、pool/cancellation/failover 行为，以及经评审的 index、partition、retention 与实测 capacity。真实 PostgreSQL conformance 始终是强制门禁。 | 不依赖 #3870，也不得叠在其分支上。仅完成本 lane 不产生 runtime caller 或 promotion 声明。 | -| C. Canonical transaction capture | 现在，通过修订或替代 #3870 | 让 outbox 把完整、带版本的 Todo/lease record 传给 `coordination.runtime_shadow.commit`；退役重复 observation/local-shadow lineage，并证明 omission/explicit-clear 行为。 | 可与 P 并行；但 C 与选定 provider profile 都完成后，才能进入 parity 或 promotion 集成。 | +| C. Canonical transaction capture | 实现中，基于 #3870 | transaction-bound outbox 已指向唯一 `coordination.runtime_shadow` lineage，并保留完整带版本的 Todo/lease record；继续完成 sustained mixed-writer parity、explicit-clear/omission 与 event-only Todo recovery 证据。 | 可与 P 并行；但 C 与选定 provider profile 都完成后，才能进入 parity 或 promotion 集成。 | | I. Binding 与资格集成 | P 与 C 完成后 | 绑定一个精确 provider lineage、field manifest、source revision、digest 与 cursor；运行持续 transaction parity，以及 provider-specific recovery/capacity qualification；缺字段时不得查询 legacy state 补齐。 | 这是 provider 工作与 capture 工作的汇合点。 | | F. Promotion 与清理 | I 完成且 maintainer 显式批准后 | 加入 provider-first CLI routing、持锁 promotion orchestrator、兼容投影 outbox、晋升后 fenced export/rollback；随后删除重复 reference aggregate，并翻转经评审的 stage/hold 声明。 | 一个 profile 的精确实现与 lineage 通过 P、C、I 前，不具备晋升资格。 | From 438690ba866f989323e89885c5d785debf502188 Mon Sep 17 00:00:00 2001 From: huangruiteng Date: Sat, 5 Sep 2026 01:27:53 +0800 Subject: [PATCH 10/11] fix(types): restore Python kernel contract checks Signed-off-by: huangruiteng --- loopx/control_plane/agents/agent_scope.py | 57 ++++++++----------- loopx/control_plane/agents/capability_gate.py | 14 ++--- .../scheduler/state_transition_rules.py | 26 ++++----- loopx/control_plane/todos/contract.py | 2 +- loopx/control_plane/todos/decision_scope.py | 3 +- loopx/control_plane/todos/deferred_resume.py | 31 +++++----- loopx/control_plane/todos/handoff_gate.py | 3 +- loopx/control_plane/todos/projection.py | 10 ++-- loopx/control_plane/todos/quota_summary.py | 30 +++++++--- .../control_plane/todos/route_continuation.py | 3 +- .../control_plane/todos/succession_warning.py | 19 +++---- loopx/control_plane/todos/summary_item.py | 6 +- loopx/control_plane/todos/todo_summary.py | 21 ++++--- loopx/control_plane/todos/user_gate.py | 8 ++- .../autonomous_replan_obligation.py | 42 +++++++------- .../work_items/progress_observation.py | 38 ++++++------- .../control_plane/work_items/project_asset.py | 3 +- 17 files changed, 164 insertions(+), 152 deletions(-) diff --git a/loopx/control_plane/agents/agent_scope.py b/loopx/control_plane/agents/agent_scope.py index 9edd1d7d6a..9802ec2f1a 100644 --- a/loopx/control_plane/agents/agent_scope.py +++ b/loopx/control_plane/agents/agent_scope.py @@ -2,7 +2,7 @@ import re from dataclasses import dataclass -from typing import Any +from typing import Any, TypeGuard from .agent_scope_frontier import ( AGENT_LANE_FRONTIER_HINT_SCHEMA_VERSION, @@ -87,15 +87,12 @@ } AGENT_TASK_SCOPE = "goal_all_read_claimed_run_global_read_v0" - - -def _agent_identity_has_scoped_lane(agent_identity: dict[str, Any] | None) -> bool: - return bool( - isinstance(agent_identity, dict) - and normalize_todo_claimed_by(agent_identity.get("agent_id")) +def _agent_identity_has_scoped_lane( + agent_identity: dict[str, Any] | None, +) -> TypeGuard[dict[str, object]]: + return isinstance(agent_identity, dict) and bool( + normalize_todo_claimed_by(agent_identity.get("agent_id")) ) - - def _attach_agent_identity_contracts( *, payload: dict[str, Any], @@ -110,9 +107,7 @@ def _attach_agent_identity_contracts( def _todo_task_class(item: dict[str, Any]) -> str: return todo_item_task_class(item) - - -def _todo_projection_sort_key(item: dict[str, Any]) -> tuple[int, int, int, str]: +def _todo_projection_sort_key(item: dict[str, Any]) -> tuple[int, int]: return todo_projection_sort_key(item) @@ -377,17 +372,12 @@ def _scoped_user_gate_fallback( gates = _open_user_gate_todo_items(user_todo_summary) if not gates or not isinstance(agent_todo_summary, dict): return None - - due_monitor_candidates = ( - agent_todo_summary.get("monitor_due_items") - if isinstance(agent_todo_summary.get("monitor_due_items"), list) - else [] - ) - ready_deferred_candidates = ( - agent_todo_summary.get("deferred_resume_candidates") - if isinstance(agent_todo_summary.get("deferred_resume_candidates"), list) - else [] + raw_due_monitor_candidates = agent_todo_summary.get("monitor_due_items") + due_monitor_candidates = raw_due_monitor_candidates if isinstance(raw_due_monitor_candidates, list) else [] + raw_ready_deferred_candidates = agent_todo_summary.get( + "deferred_resume_candidates" ) + ready_deferred_candidates = raw_ready_deferred_candidates if isinstance(raw_ready_deferred_candidates, list) else [] # An empty capability projection is authoritative for advancement work. # Due monitors are projected separately from advancement candidates and are @@ -404,11 +394,13 @@ def _scoped_user_gate_fallback( *ready_deferred_candidates, ] else: + raw_backlog_items = agent_todo_summary.get("executable_backlog_items") + raw_first_items = agent_todo_summary.get("first_executable_items") advancement_items = ( - agent_todo_summary.get("executable_backlog_items") - if isinstance(agent_todo_summary.get("executable_backlog_items"), list) - else agent_todo_summary.get("first_executable_items") - if isinstance(agent_todo_summary.get("first_executable_items"), list) + raw_backlog_items + if isinstance(raw_backlog_items, list) + else raw_first_items + if isinstance(raw_first_items, list) else [] ) executable_items = [ @@ -518,11 +510,8 @@ def _scoped_user_gate_fallback( def _first_executable_todo_text(agent_todo_summary: dict[str, Any] | None) -> str | None: if not isinstance(agent_todo_summary, dict): return None - items = ( - agent_todo_summary.get("first_executable_items") - if isinstance(agent_todo_summary.get("first_executable_items"), list) - else [] - ) + raw_items = agent_todo_summary.get("first_executable_items") + items = raw_items if isinstance(raw_items, list) else [] for item in items: if not isinstance(item, dict): continue @@ -540,7 +529,8 @@ def _first_monitor_todo_text(agent_todo_summary: dict[str, Any] | None) -> str | if not isinstance(agent_todo_summary, dict): return None for key in ("monitor_due_items", "monitor_open_items"): - items = agent_todo_summary.get(key) if isinstance(agent_todo_summary.get(key), list) else [] + raw_items = agent_todo_summary.get(key) + items = raw_items if isinstance(raw_items, list) else [] for item in items: if not isinstance(item, dict): continue @@ -946,7 +936,8 @@ def _agent_scope_monitor_blocked_resume_candidates( continue if item.get("resume_ready") is not False: continue - condition = item.get("resume_condition") if isinstance(item.get("resume_condition"), dict) else {} + raw_condition = item.get("resume_condition") + condition = raw_condition if isinstance(raw_condition, dict) else {} if normalize_todo_status(condition.get("target_status")) != TODO_STATUS_OPEN: continue target_todo_id = normalize_todo_id( diff --git a/loopx/control_plane/agents/capability_gate.py b/loopx/control_plane/agents/capability_gate.py index 5e9786f188..eaac3df43e 100644 --- a/loopx/control_plane/agents/capability_gate.py +++ b/loopx/control_plane/agents/capability_gate.py @@ -315,16 +315,16 @@ def _collect_capability_gate_candidates( agent_todo_summary: dict[str, Any], ) -> tuple[list[dict[str, Any]], str]: raw_items, source = _select_advancement_candidate_source(agent_todo_summary) + raw_due_monitor_items = agent_todo_summary.get("monitor_due_items") due_monitor_items = ( - agent_todo_summary.get("monitor_due_items") - if isinstance(agent_todo_summary.get("monitor_due_items"), list) - else [] + raw_due_monitor_items if isinstance(raw_due_monitor_items, list) else [] + ) + raw_blocked_due_monitor_items = agent_todo_summary.get( + "monitor_capability_blocked_due_items" ) blocked_due_monitor_items = ( - agent_todo_summary.get("monitor_capability_blocked_due_items") - if isinstance( - agent_todo_summary.get("monitor_capability_blocked_due_items"), list - ) + raw_blocked_due_monitor_items + if isinstance(raw_blocked_due_monitor_items, list) else [] ) monitor_sources: list[str] = [] diff --git a/loopx/control_plane/scheduler/state_transition_rules.py b/loopx/control_plane/scheduler/state_transition_rules.py index a615a574d6..516e5230f7 100644 --- a/loopx/control_plane/scheduler/state_transition_rules.py +++ b/loopx/control_plane/scheduler/state_transition_rules.py @@ -276,15 +276,10 @@ def decide_scheduler_backoff_state( state_acknowledges_rrule = result.get( "scheduler_state_acknowledges_current_rrule" ) - string_fields = { - key: result.get(key) - for key in ( - "current_rrule", - "last_applied_rrule", - "observed_host_rrule", - "effective_host_rrule", - ) - } + current_rrule = result.get("current_rrule") + last_applied_rrule = result.get("last_applied_rrule") + observed_result_rrule = result.get("observed_host_rrule") + effective_host_rrule = result.get("effective_host_rrule") recorded_failure = result.get("recorded_host_failure") if ( result.get("operation") != "backoff" @@ -300,7 +295,10 @@ def decide_scheduler_backoff_state( or not isinstance(repeated_failed_pair, bool) or not isinstance(current_rrule_applied, bool) or not isinstance(state_acknowledges_rrule, bool) - or any(not isinstance(value, str) for value in string_fields.values()) + or not isinstance(current_rrule, str) + or not isinstance(last_applied_rrule, str) + or not isinstance(observed_result_rrule, str) + or not isinstance(effective_host_rrule, str) or (recorded_failure is not None and not isinstance(recorded_failure, dict)) ): raise RuntimeError("TypeScript scheduler backoff result shape mismatch") @@ -319,10 +317,10 @@ def decide_scheduler_backoff_state( cadence=cadence, host=host, current_interval_minutes=current_interval, - current_rrule=string_fields["current_rrule"], - last_applied_rrule=string_fields["last_applied_rrule"], - observed_host_rrule=string_fields["observed_host_rrule"], - effective_host_rrule=string_fields["effective_host_rrule"], + current_rrule=current_rrule, + last_applied_rrule=last_applied_rrule, + observed_host_rrule=observed_result_rrule, + effective_host_rrule=effective_host_rrule, all_host_update_failures=_object_tuple( result.get("all_host_update_failures"), label="scheduler all-host failure cache", diff --git a/loopx/control_plane/todos/contract.py b/loopx/control_plane/todos/contract.py index 5ad391b761..0e6759c52b 100644 --- a/loopx/control_plane/todos/contract.py +++ b/loopx/control_plane/todos/contract.py @@ -11,7 +11,7 @@ from .completion_state import ( normalize_todo_completion_continuation, normalize_todo_completion_recovery, - normalize_todo_no_followup, + normalize_todo_no_followup as normalize_todo_no_followup, require_todo_completion_metadata, ) from .resume_condition import ( diff --git a/loopx/control_plane/todos/decision_scope.py b/loopx/control_plane/todos/decision_scope.py index 8490d78648..c12a604d78 100644 --- a/loopx/control_plane/todos/decision_scope.py +++ b/loopx/control_plane/todos/decision_scope.py @@ -380,7 +380,8 @@ def build_required_decision_scope_repair_hint( ) -> dict[str, Any] | None: if consistency.get("ok") is not False: return None - errors = consistency.get("errors") if isinstance(consistency.get("errors"), list) else [] + raw_errors = consistency.get("errors") + errors = raw_errors if isinstance(raw_errors, list) else [] missing_gate_scope = any( isinstance(error, dict) and error.get("reason_code") == "multi_agent_user_gate_missing_scope" diff --git a/loopx/control_plane/todos/deferred_resume.py b/loopx/control_plane/todos/deferred_resume.py index ae43a3a46d..10572a4e90 100644 --- a/loopx/control_plane/todos/deferred_resume.py +++ b/loopx/control_plane/todos/deferred_resume.py @@ -170,11 +170,13 @@ def todo_summary_deferred_items( ) -> list[dict[str, Any]]: if not isinstance(value, dict): return [] - source_items = value.get(key) if isinstance(value.get(key), list) else [] + raw_source_items = value.get(key) + source_items = raw_source_items if isinstance(raw_source_items, list) else [] if not source_items and key == "deferred_items": + raw_items = value.get("items") source_items = [ item - for item in value.get("items", []) + for item in (raw_items if isinstance(raw_items, list) else []) if isinstance(item, dict) and todo_item_is_deferred(item) ] items: list[dict[str, Any]] = [] @@ -214,14 +216,14 @@ def _dedupe_todo_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]: def todo_summary_resume_blocked_items(value: dict[str, Any]) -> list[dict[str, Any]]: if not isinstance(value, dict): return [] + raw_source_items = value.get("resume_blocked_items") source_items = ( - value.get("resume_blocked_items") - if isinstance(value.get("resume_blocked_items"), list) - else [] + list(raw_source_items) if isinstance(raw_source_items, list) else [] ) if not source_items: for key in ("items", "backlog_items", "first_open_items"): - raw_items = value.get(key) if isinstance(value.get(key), list) else [] + raw_value_items = value.get(key) + raw_items = raw_value_items if isinstance(raw_value_items, list) else [] source_items.extend(item for item in raw_items if isinstance(item, dict)) items: list[dict[str, Any]] = [] for item in source_items: @@ -250,7 +252,8 @@ def _monitor_target_todo_ids(value: dict[str, Any]) -> set[str]: "backlog_items", "first_open_items", ): - source_items = value.get(key) if isinstance(value.get(key), list) else [] + raw_items = value.get(key) + source_items = raw_items if isinstance(raw_items, list) else [] for item in source_items: if not isinstance(item, dict): continue @@ -270,11 +273,8 @@ def todo_summary_monitor_blocked_resume_items( for item in todo_summary_resume_blocked_items(value): if _todo_task_class(item) != TODO_TASK_CLASS_ADVANCEMENT: continue - condition = ( - item.get("resume_condition") - if isinstance(item.get("resume_condition"), dict) - else {} - ) + raw_condition = item.get("resume_condition") + condition = raw_condition if isinstance(raw_condition, dict) else {} if condition.get("kind") == "monitor_changed": # This is the intentional typed wait lifecycle. The monitor stays # independently schedulable and its generation fence owns resume; @@ -334,11 +334,8 @@ def todo_summary_blocked_successor_items( if claimed_by and claimed_by != agent_id: continue resume_when = normalize_todo_resume_when(item.get("resume_when")) - condition = ( - item.get("resume_condition") - if isinstance(item.get("resume_condition"), dict) - else {} - ) + raw_condition = item.get("resume_condition") + condition = raw_condition if isinstance(raw_condition, dict) else {} if not resume_when or condition.get("satisfied") is not False: continue target_task_class = normalize_todo_task_class( diff --git a/loopx/control_plane/todos/handoff_gate.py b/loopx/control_plane/todos/handoff_gate.py index 97465b3446..2d10511e8d 100644 --- a/loopx/control_plane/todos/handoff_gate.py +++ b/loopx/control_plane/todos/handoff_gate.py @@ -217,7 +217,8 @@ def todo_summary_handoff_gates(value: dict[str, Any]) -> list[dict[str, Any]]: projected = value.get("handoff_gates") if isinstance(projected, list): return [item for item in projected if isinstance(item, dict)] - source_items = value.get("items") if isinstance(value.get("items"), list) else [] + raw_items = value.get("items") + source_items = raw_items if isinstance(raw_items, list) else [] return build_todo_handoff_gate_states(source_items) diff --git a/loopx/control_plane/todos/projection.py b/loopx/control_plane/todos/projection.py index c16f0c4428..c6b570ed26 100644 --- a/loopx/control_plane/todos/projection.py +++ b/loopx/control_plane/todos/projection.py @@ -88,8 +88,9 @@ def todo_priority_rank(value: Any, *, text_mode: str = "label") -> int: def todo_index_rank(item: dict[str, Any]) -> int: + raw_index = item.get("index") try: - return int(item.get("index")) + return int(raw_index) if raw_index is not None else TODO_MISSING_INDEX except (TypeError, ValueError): return TODO_MISSING_INDEX @@ -722,11 +723,8 @@ def todo_summary_first_executable_item( ) -> dict[str, Any] | None: if not isinstance(summary, dict): return None - items = ( - summary.get("first_executable_items") - if isinstance(summary.get("first_executable_items"), list) - else [] - ) + raw_items = summary.get("first_executable_items") + items = raw_items if isinstance(raw_items, list) else [] for item in items: if not isinstance(item, dict): continue diff --git a/loopx/control_plane/todos/quota_summary.py b/loopx/control_plane/todos/quota_summary.py index 2184cdb5c1..96b2c957b4 100644 --- a/loopx/control_plane/todos/quota_summary.py +++ b/loopx/control_plane/todos/quota_summary.py @@ -287,10 +287,16 @@ def validate_todo_source_contract( key: _strict_non_negative_int(value.get(key)) for key in ("total_count", "open_count", "done_count", "deferred_count") } + total_count = counts["total_count"] + open_count = counts["open_count"] + done_count = counts["done_count"] + deferred_count = counts["deferred_count"] valid_counts = ( - all(count is not None for count in counts.values()) - and counts["total_count"] - == counts["open_count"] + counts["done_count"] + counts["deferred_count"] + total_count is not None + and open_count is not None + and done_count is not None + and deferred_count is not None + and total_count == open_count + done_count + deferred_count ) valid_proof = bool( isinstance(proof, dict) @@ -299,11 +305,12 @@ def validate_todo_source_contract( and proof.get("derived") is True and bool(str(value.get("source_section") or "").strip()) and type(proof.get("item_count")) is int - and proof.get("item_count") == counts["total_count"] + and proof.get("item_count") == total_count ) valid_terminal_closure = bool( valid_counts and valid_proof + and isinstance(proof, dict) and _terminal_closure_proof_is_valid( value, counts=counts, @@ -320,18 +327,25 @@ def validate_todo_source_contract( intent = value.get("closure_intent") terminal_proof = value.get("terminal_closure_proof") + intent_count = intent.get("count") if isinstance(intent, dict) else None valid_intent = bool( valid_terminal_closure and isinstance(intent, dict) and intent.get("schema_version") == "todo_closure_intent_v0" and intent.get("kind") == "no_followup" and intent.get("derived") is True - and type(intent.get("count")) is int - and 0 < intent.get("count") <= counts["done_count"] + and type(intent_count) is int + and done_count is not None + and 0 < intent_count <= done_count and isinstance(terminal_proof, dict) - and intent.get("count") == terminal_proof.get("no_followup_count") + and intent_count == terminal_proof.get("no_followup_count") + ) + closure_intent = ( + {**intent, "source": "todo_no_followup"} + if valid_intent and isinstance(intent, dict) + else None ) - return completeness, {**intent, "source": "todo_no_followup"} if valid_intent else None + return completeness, closure_intent def _build_quota_todo_lanes( diff --git a/loopx/control_plane/todos/route_continuation.py b/loopx/control_plane/todos/route_continuation.py index c5c07a201b..111e8e42ba 100644 --- a/loopx/control_plane/todos/route_continuation.py +++ b/loopx/control_plane/todos/route_continuation.py @@ -124,7 +124,8 @@ def todo_summary_route_continuation_candidates( "route_continuation_replan_candidates", "route_continuation_candidates", ): - raw_items = value.get(key) if isinstance(value.get(key), list) else [] + source = value.get(key) + raw_items = source if isinstance(source, list) else [] source_items.extend(item for item in raw_items if isinstance(item, dict)) source_items.extend( diff --git a/loopx/control_plane/todos/succession_warning.py b/loopx/control_plane/todos/succession_warning.py index 6adb3385a4..985630c365 100644 --- a/loopx/control_plane/todos/succession_warning.py +++ b/loopx/control_plane/todos/succession_warning.py @@ -152,7 +152,7 @@ def build_todo_succession_warning_lanes( ][:item_limit] count = warning.get("count", summary.get("completed_without_successor_count")) try: - count = max(0, int(count)) + count = max(0, int(count)) if count is not None else len(items) except (TypeError, ValueError): count = len(items) if count <= 0 and not items: @@ -193,18 +193,17 @@ def todo_succession_gap_items( if not isinstance(summary, dict): return [] - warning = ( - summary.get("todo_succession_warning") - if isinstance(summary.get("todo_succession_warning"), dict) - else {} - ) + raw_warning = summary.get("todo_succession_warning") + warning = raw_warning if isinstance(raw_warning, dict) else {} if warning and warning.get("reason_code") != TODO_SUCCESSION_WARNING_REASON_CODE: return [] + warning_items = warning.get("items") + completed_items = summary.get("completed_without_successor_items") source_items = ( - warning.get("items") - if isinstance(warning.get("items"), list) - else summary.get("completed_without_successor_items") - if isinstance(summary.get("completed_without_successor_items"), list) + warning_items + if isinstance(warning_items, list) + else completed_items + if isinstance(completed_items, list) else [] ) items = [item for item in source_items if isinstance(item, dict)] diff --git a/loopx/control_plane/todos/summary_item.py b/loopx/control_plane/todos/summary_item.py index 621bff566d..d180d6b20d 100644 --- a/loopx/control_plane/todos/summary_item.py +++ b/loopx/control_plane/todos/summary_item.py @@ -200,7 +200,8 @@ def todo_summary_source_items(value: dict[str, Any]) -> list[dict[str, Any]]: ready_successor_todo_ids = handoff_ready_successor_todo_ids(value) open_items: list[dict[str, Any]] = [] for key in TODO_SUMMARY_SOURCE_KEYS: - source_items = value.get(key) if isinstance(value.get(key), list) else [] + raw_items = value.get(key) + source_items = raw_items if isinstance(raw_items, list) else [] for item in source_items: if not isinstance(item, dict) or item.get("done") is True: continue @@ -251,7 +252,8 @@ def todo_planning_source_items( planning_items: list[dict[str, Any]] = [] seen: set[str] = set() for key in TODO_PLANNING_SOURCE_KEYS: - source_items = value.get(key) if isinstance(value.get(key), list) else [] + raw_items = value.get(key) + source_items = raw_items if isinstance(raw_items, list) else [] for item in source_items: if not isinstance(item, dict) or ( not include_terminal and item.get("status") == "done" diff --git a/loopx/control_plane/todos/todo_summary.py b/loopx/control_plane/todos/todo_summary.py index 61a7f9ab82..ca4ad85d45 100644 --- a/loopx/control_plane/todos/todo_summary.py +++ b/loopx/control_plane/todos/todo_summary.py @@ -1,6 +1,7 @@ from __future__ import annotations from dataclasses import dataclass +from datetime import datetime import re from typing import Any, Callable, Optional @@ -628,19 +629,23 @@ def todo_item_is_actionable_open(item: dict[str, Any]) -> bool: return projection_todo_item_is_actionable_open(item) -def todo_item_next_due_at(item: dict[str, Any]): +def todo_item_next_due_at(item: dict[str, Any]) -> datetime | None: return projection_todo_item_next_due_at(item) -def todo_item_expires_at(item: dict[str, Any]): +def todo_item_expires_at(item: dict[str, Any]) -> datetime | None: return projection_todo_item_expires_at(item) -def todo_item_is_due_monitor(item: dict[str, Any], *, now=None) -> bool: +def todo_item_is_due_monitor( + item: dict[str, Any], *, now: datetime | None = None +) -> bool: return projection_todo_item_is_due_monitor(item, now=now, task_text_keys=("text",)) -def todo_item_missing_monitor_schedule(item: dict[str, Any], *, now=None) -> bool: +def todo_item_missing_monitor_schedule( + item: dict[str, Any], *, now: datetime | None = None +) -> bool: return projection_todo_item_missing_monitor_schedule(item, now=now, task_text_keys=("text",)) @@ -778,7 +783,8 @@ def dependency_blocker_summary( goal_id = str(item.get("goal_id") or "") if not goal_id or goal_id == current_goal_id: continue - user_todos = item.get("user_todos") if isinstance(item.get("user_todos"), dict) else {} + raw_user_todos = item.get("user_todos") + user_todos = raw_user_todos if isinstance(raw_user_todos, dict) else {} for todo in user_todos.get("items") or []: if not isinstance(todo, dict) or todo.get("done"): continue @@ -940,8 +946,9 @@ def todo_item_is_succession_tracked_completion(item: dict[str, Any]) -> bool: def _completed_succession_sort_key(item: dict[str, Any]) -> tuple[str, int]: + raw_index = item.get("index") try: - index = int(item.get("index")) + index = int(raw_index) if raw_index is not None else 0 except (TypeError, ValueError): index = 0 timestamp = str(item.get("updated_at") or item.get("completed_at") or "") @@ -1193,7 +1200,7 @@ def compact_todo_group( for item in lanes.open_items if normalize_todo_id(item.get("todo_id")) not in watch_only_ids ] - summary = { + summary: dict[str, Any] = { "schema_version": "todo_summary_v0", "source_section": source_section, "total_count": len(items), diff --git a/loopx/control_plane/todos/user_gate.py b/loopx/control_plane/todos/user_gate.py index 290bb7a90d..bcbf10eab3 100644 --- a/loopx/control_plane/todos/user_gate.py +++ b/loopx/control_plane/todos/user_gate.py @@ -162,9 +162,10 @@ def apply_scoped_user_gate_fallback_projection( if projected.get("effective_action") in {"skip", "monitor_quiet_skip", None}: projected["effective_action"] = "scoped_user_gate_fallback" + raw_execution_obligation = projected.get("execution_obligation") execution_obligation = ( - dict(projected.get("execution_obligation")) - if isinstance(projected.get("execution_obligation"), dict) + dict(raw_execution_obligation) + if isinstance(raw_execution_obligation, dict) else {} ) execution_obligation.update( @@ -199,9 +200,10 @@ def build_gate_prompt( question = str(item.get("operator_question") or "").strip() recommended_action = str(item.get("recommended_action") or "").strip() next_handoff_condition = str(item.get("next_handoff_condition") or "").strip() + raw_missing_gates = item.get("missing_gates") missing_gates = [ str(gate).strip() - for gate in (item.get("missing_gates") if isinstance(item.get("missing_gates"), list) else []) + for gate in (raw_missing_gates if isinstance(raw_missing_gates, list) else []) if str(gate).strip() ] if user_todo_summary is None: diff --git a/loopx/control_plane/work_items/autonomous_replan_obligation.py b/loopx/control_plane/work_items/autonomous_replan_obligation.py index 9cefced291..d9684708db 100644 --- a/loopx/control_plane/work_items/autonomous_replan_obligation.py +++ b/loopx/control_plane/work_items/autonomous_replan_obligation.py @@ -59,7 +59,7 @@ def with_replan_novelty_guidance(action: str) -> str: def replan_obligation_id_from_packet(value: Any) -> str | None: - packet = value if isinstance(value, Mapping) else {} + packet: Mapping[str, Any] = value if isinstance(value, Mapping) else {} return normalize_todo_replan_obligation_id(packet.get("obligation_id")) @@ -68,10 +68,9 @@ def todo_lifecycle_settlement_obligation( ) -> Mapping[str, Any] | None: """Return the lifecycle-settlement subtype from a status payload or obligation.""" - obligation = ( - value.get("autonomous_replan_obligation") - if isinstance(value.get("autonomous_replan_obligation"), Mapping) - else value + raw_obligation = value.get("autonomous_replan_obligation") + obligation: Mapping[str, Any] = ( + raw_obligation if isinstance(raw_obligation, Mapping) else value ) if obligation.get("resolution_mode") != TODO_LIFECYCLE_SETTLEMENT_RESOLUTION_MODE: return None @@ -91,13 +90,14 @@ def project_todo_lifecycle_settlement_reentry( obligation = todo_lifecycle_settlement_obligation(payload) if obligation is None: return None + raw_triggers = obligation.get("triggers") triggers = [ { key: item.get(key) for key in ("kind", "todo_id", "completion_turn_key") if item.get(key) is not None } - for item in obligation.get("triggers") or [] + for item in (raw_triggers if isinstance(raw_triggers, list) else []) if isinstance(item, Mapping) and item.get("kind") == "completed_advancement_without_successor" and normalize_todo_id(item.get("todo_id")) @@ -132,14 +132,14 @@ def build_autonomous_replan_cli_actions( ) if lifecycle_reentry is not None: return list(lifecycle_reentry["next_cli_actions"]) - packet = ( - payload.get("replan_action_packet") - if isinstance(payload.get("replan_action_packet"), Mapping) - else {} + raw_packet = payload.get("replan_action_packet") + packet: Mapping[str, Any] = ( + raw_packet if isinstance(raw_packet, Mapping) else {} ) - writeback_contract = ( - packet.get("writeback_contract") - if isinstance(packet.get("writeback_contract"), Mapping) + raw_writeback_contract = packet.get("writeback_contract") + writeback_contract: Mapping[str, Any] = ( + raw_writeback_contract + if isinstance(raw_writeback_contract, Mapping) else {} ) successor_command = str( @@ -391,7 +391,11 @@ def _monitor_no_change_evidence( if not isinstance(agent_todos, dict): return None raw_monitors = agent_todos.get("monitor_open_items") - monitors = [item for item in raw_monitors or [] if isinstance(item, dict)] + monitors = [ + item + for item in (raw_monitors if isinstance(raw_monitors, list) else []) + if isinstance(item, dict) + ] stalled: list[tuple[int, dict[str, Any]]] = [] for item in monitors: try: @@ -454,8 +458,9 @@ def _future_due_blocking_monitor( if not accountable_agent_id: return None raw_monitors = agent_todos.get("monitor_open_items") + monitor_items = raw_monitors if isinstance(raw_monitors, list) else [] monitors_by_id: dict[str, dict[str, Any]] = {} - for monitor in raw_monitors or []: + for monitor in monitor_items: if not isinstance(monitor, dict): continue monitor_todo_id = normalize_todo_id(monitor.get("todo_id")) @@ -475,11 +480,8 @@ def _future_due_blocking_monitor( claimed_by = normalize_todo_claimed_by(item.get("claimed_by")) if accountable_agent_id and claimed_by and claimed_by != accountable_agent_id: continue - condition = ( - item.get("resume_condition") - if isinstance(item.get("resume_condition"), dict) - else {} - ) + raw_condition = item.get("resume_condition") + condition = raw_condition if isinstance(raw_condition, dict) else {} monitor_todo_id = normalize_todo_id( item.get("blocking_monitor_todo_id") or condition.get("target_todo_id") diff --git a/loopx/control_plane/work_items/progress_observation.py b/loopx/control_plane/work_items/progress_observation.py index c8be9405ae..512521688e 100644 --- a/loopx/control_plane/work_items/progress_observation.py +++ b/loopx/control_plane/work_items/progress_observation.py @@ -449,10 +449,9 @@ def semantic_delta_from_writeback( path_outcome = "" if isinstance(agent_vision, Mapping): vision_state = normalize_goal_vision_state(agent_vision.get("state")) - path_delta = ( - agent_vision.get("path_delta") - if isinstance(agent_vision.get("path_delta"), Mapping) - else {} + raw_path_delta = agent_vision.get("path_delta") + path_delta: Mapping[str, Any] = ( + raw_path_delta if isinstance(raw_path_delta, Mapping) else {} ) path_outcome = str(path_delta.get("outcome") or "").strip() if vision_state != "no_followup" or path_outcome != "stop": @@ -463,15 +462,13 @@ def semantic_delta_from_writeback( ) if isinstance(agent_vision, Mapping): - patch = ( - agent_vision.get("vision_patch") - if isinstance(agent_vision.get("vision_patch"), Mapping) - else {} + raw_patch = agent_vision.get("vision_patch") + patch: Mapping[str, Any] = ( + raw_patch if isinstance(raw_patch, Mapping) else {} ) + raw_path_delta = agent_vision.get("path_delta") path_delta = ( - agent_vision.get("path_delta") - if isinstance(agent_vision.get("path_delta"), Mapping) - else {} + raw_path_delta if isinstance(raw_path_delta, Mapping) else {} ) path_outcome = str(path_delta.get("outcome") or "").strip() evidence_refs = [ @@ -629,17 +626,18 @@ def build_replan_action_packet( successor_priority = str(todo_action.get("priority") or "P1").strip() if successor_priority not in {"P0", "P1", "P2", "P3", "P4"}: successor_priority = "P1" - selected_gap = ( + raw_selected_gap = ( bounded_research_frontier.get("selected_gap") if isinstance(bounded_research_frontier, Mapping) - and isinstance(bounded_research_frontier.get("selected_gap"), Mapping) else None ) - raw_successor_binding = ( - selected_gap.get("successor_binding") - if isinstance(selected_gap, Mapping) - and isinstance(selected_gap.get("successor_binding"), Mapping) - else {} + selected_gap: Mapping[str, Any] | None = ( + raw_selected_gap if isinstance(raw_selected_gap, Mapping) else None + ) + selected_gap_values: Mapping[str, Any] = selected_gap or {} + raw_binding = selected_gap_values.get("successor_binding") + raw_successor_binding: Mapping[str, Any] = ( + raw_binding if isinstance(raw_binding, Mapping) else {} ) successor_binding = replan_successor_semantic_binding( action_kind=raw_successor_binding.get("action_kind"), @@ -650,9 +648,9 @@ def build_replan_action_packet( ) writeback_contract: dict[str, Any] = {} successor_summary = str( - (selected_gap or {}).get("successor_summary") or "" + selected_gap_values.get("successor_summary") or "" ).strip()[:240] - if isinstance(selected_gap, Mapping) and successor_summary and successor_binding: + if selected_gap is not None and successor_summary and successor_binding: safe_goal_id = _stable_id(goal_id, field="goal_id") or "" safe_agent_id = ( _stable_id(agent_id or obligation.get("agent_id"), field="agent_id") diff --git a/loopx/control_plane/work_items/project_asset.py b/loopx/control_plane/work_items/project_asset.py index 60264fbfdb..6b5d250f08 100644 --- a/loopx/control_plane/work_items/project_asset.py +++ b/loopx/control_plane/work_items/project_asset.py @@ -351,7 +351,8 @@ def project_asset_handoff_check_projection(item: dict[str, Any]) -> dict[str, An if not isinstance(project_asset, dict): return None - quota = project_asset.get("quota") if isinstance(project_asset.get("quota"), dict) else {} + raw_quota = project_asset.get("quota") + quota = raw_quota if isinstance(raw_quota, dict) else {} if not quota and isinstance(item.get("quota"), dict): quota = item["quota"] From edc168277163297f6666b18e47453ae08d7e6254 Mon Sep 17 00:00:00 2001 From: huangruiteng Date: Sat, 5 Sep 2026 01:54:39 +0800 Subject: [PATCH 11/11] fix(todos): honor monitor writeback runtime root Signed-off-by: huangruiteng --- loopx/control_plane/quota/monitor_poll.py | 14 ++++++++++++++ .../scheduler/monitor_poll_writeback.py | 17 ++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/loopx/control_plane/quota/monitor_poll.py b/loopx/control_plane/quota/monitor_poll.py index 1f85463c54..cdd43afee9 100644 --- a/loopx/control_plane/quota/monitor_poll.py +++ b/loopx/control_plane/quota/monitor_poll.py @@ -112,6 +112,7 @@ def _vision_wait_state(before: dict[str, Any]) -> dict[str, Any]: def _registry_due_monitor( *, registry_path: Path | None, + runtime_root: Path | None, goal_id: str, todo_id: str | None, target_key: str | None, @@ -122,6 +123,7 @@ def _registry_due_monitor( item = resolve_monitor_todo_item( registry_path=registry_path, goal_id=goal_id, + runtime_root=runtime_root, todo_id=todo_id, target_key=target_key, ) @@ -145,12 +147,14 @@ def _decision_packet( todo_id: str | None, target_key: str | None, registry_path: Path | None = None, + runtime_root: Path | None = None, authorized_due_monitor_poll: bool | None = None, ) -> dict[str, Any]: lane = _mapping(before.get("work_lane_contract")) due_candidates = _due_monitor_candidates(before) registry_due = _registry_due_monitor( registry_path=registry_path, + runtime_root=runtime_root, goal_id=goal_id, todo_id=todo_id, target_key=target_key, @@ -636,12 +640,22 @@ def record_quota_monitor_poll_for_decision( if normalized_turn_id else f"quota-monitor-poll:{goal_id}:{uuid.uuid4().hex}" ) + if execute and (safe_todo_id or safe_target_key): + from ..coordination.legacy_writer_fence import ( + require_legacy_coordination_write_allowed, + ) + + require_legacy_coordination_write_allowed( + runtime_root=runtime_root, + goal_id=goal_id, + ) decision = _decision_packet( before, goal_id=goal_id, todo_id=safe_todo_id, target_key=safe_target_key, registry_path=registry_path, + runtime_root=runtime_root, ) observation = _observation_packet( before=before, diff --git a/loopx/control_plane/scheduler/monitor_poll_writeback.py b/loopx/control_plane/scheduler/monitor_poll_writeback.py index f3a2d47117..f060c67209 100644 --- a/loopx/control_plane/scheduler/monitor_poll_writeback.py +++ b/loopx/control_plane/scheduler/monitor_poll_writeback.py @@ -115,6 +115,7 @@ def resolve_monitor_todo_item( *, registry_path: Path, goal_id: str, + runtime_root: Path | None = None, todo_id: str | None = None, target_key: str | None = None, ) -> dict[str, Any]: @@ -124,7 +125,12 @@ def resolve_monitor_todo_item( safe_target_key = str(target_key or "").strip() if not normalized_todo_id and not safe_target_key: raise ValueError("monitor todo writeback requires --todo-id or --target-key") - payload = list_goal_todos(registry_path=registry_path, goal_id=goal_id, role="agent") + payload = list_goal_todos( + registry_path=registry_path, + goal_id=goal_id, + role="agent", + runtime_root_arg=str(runtime_root) if runtime_root is not None else None, + ) items = payload.get("todos") if isinstance(payload.get("todos"), list) else [] if normalized_todo_id: matches = [ @@ -202,9 +208,17 @@ def write_monitor_poll_todo_state( """ from ...todos import add_goal_todo, update_goal_todo + from ..coordination.legacy_writer_fence import ( + require_legacy_coordination_write_allowed, + ) if not todo_id and not target_key: return None + if execute: + require_legacy_coordination_write_allowed( + runtime_root=runtime_root, + goal_id=goal_id, + ) safe_result_hash = str(result_hash or "").strip() if not safe_result_hash: raise ValueError("monitor todo writeback requires --result-hash") @@ -223,6 +237,7 @@ def write_monitor_poll_todo_state( item = resolve_monitor_todo_item( registry_path=registry_path, goal_id=goal_id, + runtime_root=runtime_root, todo_id=todo_id, target_key=target_key, )