From a5243f6d005a5cf55d50c6db9bffe88f8be10460 Mon Sep 17 00:00:00 2001 From: Will Eaton Date: Fri, 4 Sep 2026 09:56:56 -0400 Subject: [PATCH 01/11] Fold the session log once, in the contract Every consumer of state/session.jsonl scanned it its own way: ResumeFold for counters, the recovery tail scan for crash classification, and the controller's ingest enum. Add crucible_contract::LoopState with one exhaustive apply, a classify() that reproduces the recovery precedence, and a resume_view() that matches ResumeFold, so a new event kind is taught to one place. Additive wire changes, WIRE_VERSION unchanged: an Unknown catch-all so a newer writer never makes an older reader refuse a log, an optional envelope ts, task, source and park on approval_wait, trace_id, by and source on approval_resolved, a suspended shutdown outcome, and the run-workspace and approval-waits artifact kinds. CONTRACT_VERSION 1.3.0. The old folds stay in place for now. A parity module folds every recovery and resume test log, and the run6 fixture, through both and asserts they agree. Assisted-by: Claude --- crucible-contract/src/artifact.rs | 23 +- crucible-contract/src/lib.rs | 10 +- crucible-contract/src/session.rs | 134 +++- crucible-contract/src/session/fold.rs | 962 ++++++++++++++++++++++++++ crucible/src/loop_driver.rs | 11 +- crucible/src/recovery.rs | 156 +++-- crucible/src/stream.rs | 6 + crucible/tests/contract_version.rs | 2 +- docs/crucible-contract.md | 44 +- 9 files changed, 1278 insertions(+), 70 deletions(-) create mode 100644 crucible-contract/src/session/fold.rs diff --git a/crucible-contract/src/artifact.rs b/crucible-contract/src/artifact.rs index fcc04abf..0ad8b2cf 100644 --- a/crucible-contract/src/artifact.rs +++ b/crucible-contract/src/artifact.rs @@ -11,7 +11,8 @@ use std::fmt; use std::fmt::Write as _; use std::str::FromStr; -const MIB: u64 = 1024 * 1024; +const KIB: u64 = 1024; +const MIB: u64 = 1024 * KIB; /// The content digest of an artifact's (compressed) bytes, `sha256:`. Both the /// engine and the controller drop-box call this, so the two digests are byte-for-byte comparable @@ -28,7 +29,8 @@ pub fn content_digest(bytes: &[u8]) -> String { } /// The Tier 2 artifact kinds. The serde spelling is the `{kind}` path segment of the ingest route -/// (`scope-pack`, `scope-transcript`, `run-session`, `run-files`, `otel-log`). +/// (`scope-pack`, `scope-transcript`, `run-session`, `run-files`, `run-workspace`, +/// `approval-waits`, `otel-log`). #[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] #[serde(rename_all = "kebab-case")] pub enum ArtifactKind { @@ -40,6 +42,11 @@ pub enum ArtifactKind { RunSession, /// The gzipped tar of a loop run's `state/files`, one directory per task that captured. RunFiles, + /// The gzipped tar a suspended run leaves for its resume: `state/` minus the session log, + /// a bundle of the workspace repo, and `state/resume.json`. + RunWorkspace, + /// The JSON list of approval gates a parked run is waiting on, replaced on every change. + ApprovalWaits, /// The raw OTLP jsonl the in-process collector captured next to the agent. OtelLog, } @@ -52,6 +59,8 @@ impl ArtifactKind { ArtifactKind::ScopeTranscript => "scope-transcript", ArtifactKind::RunSession => "run-session", ArtifactKind::RunFiles => "run-files", + ArtifactKind::RunWorkspace => "run-workspace", + ArtifactKind::ApprovalWaits => "approval-waits", ArtifactKind::OtelLog => "otel-log", } } @@ -63,16 +72,20 @@ impl ArtifactKind { ArtifactKind::ScopeTranscript => 32 * MIB, ArtifactKind::RunSession => 128 * MIB, ArtifactKind::RunFiles => 256 * MIB, + ArtifactKind::RunWorkspace => 32 * MIB, + ArtifactKind::ApprovalWaits => 64 * KIB, ArtifactKind::OtelLog => 32 * MIB, } } /// Every kind, for exhaustive iteration in tests and route registration. - const ALL: [ArtifactKind; 5] = [ + const ALL: [ArtifactKind; 7] = [ ArtifactKind::ScopePack, ArtifactKind::ScopeTranscript, ArtifactKind::RunSession, ArtifactKind::RunFiles, + ArtifactKind::RunWorkspace, + ArtifactKind::ApprovalWaits, ArtifactKind::OtelLog, ]; } @@ -172,6 +185,8 @@ mod tests { assert_eq!(ArtifactKind::ScopePack.as_str(), "scope-pack"); assert_eq!(ArtifactKind::ScopeTranscript.as_str(), "scope-transcript"); assert_eq!(ArtifactKind::RunSession.as_str(), "run-session"); + assert_eq!(ArtifactKind::RunWorkspace.as_str(), "run-workspace"); + assert_eq!(ArtifactKind::ApprovalWaits.as_str(), "approval-waits"); assert_eq!(ArtifactKind::OtelLog.as_str(), "otel-log"); } @@ -180,6 +195,8 @@ mod tests { assert_eq!(ArtifactKind::ScopePack.max_bytes(), 16 * MIB); assert_eq!(ArtifactKind::ScopeTranscript.max_bytes(), 32 * MIB); assert_eq!(ArtifactKind::RunSession.max_bytes(), 128 * MIB); + assert_eq!(ArtifactKind::RunWorkspace.max_bytes(), 32 * MIB); + assert_eq!(ArtifactKind::ApprovalWaits.max_bytes(), 64 * KIB); assert_eq!(ArtifactKind::OtelLog.max_bytes(), 32 * MIB); } diff --git a/crucible-contract/src/lib.rs b/crucible-contract/src/lib.rs index db2d7d98..e8ff2176 100644 --- a/crucible-contract/src/lib.rs +++ b/crucible-contract/src/lib.rs @@ -27,7 +27,7 @@ pub mod verdict; /// `crucible --contract-version` and the runtime image carries it as the /// `io.crucible.contract-version` OCI label, so a deployed image can be matched against the /// controller it talks to without a probe. -pub const CONTRACT_VERSION: &str = "1.2.0"; +pub const CONTRACT_VERSION: &str = "1.3.0"; pub use admission::{ ADMISSION_WIRE_VERSION, AdmissionEvent, AdmissionKey, AdmissionOutcome, AdmittedInput, @@ -58,6 +58,12 @@ pub use refine::{ }; pub use report::{REPORT_FILE, ReportResult, RunReport, TaskReport}; pub use scope::{ScopeReport, StageName, StageResult}; -pub use session::{PrLinkWire, RowWire, SessionEvent, WIRE_VERSION, decode, encode}; +pub use session::fold::{ + Classification, LoopState, OpenApproval, OpenPlan, ResumeView, ShutdownOutcome, TaskResultWire, + TurnEvidence, WaitMode, +}; +pub use session::{ + Line, PrLinkWire, RowWire, SessionEvent, WIRE_VERSION, decode, decode_line, encode, encode_at, +}; pub use tier::{Disposition, Tier, TierParseError}; pub use verdict::{GroundedErrorKind, GroundedVerdict}; diff --git a/crucible-contract/src/session.rs b/crucible-contract/src/session.rs index cd36c091..13f5e2e0 100644 --- a/crucible-contract/src/session.rs +++ b/crucible-contract/src/session.rs @@ -5,11 +5,15 @@ //! This is the wire-only half of the format: plain-data types and the codec, with no dependency //! on the CLI's in-process `Row`/`Phase` state (that bridge lives in `crucible::session`). +pub mod fold; + use crate::event::AgentEvent; use crate::identity::RunIdentity; use serde::{Deserialize, Serialize}; -/// Bump only on a breaking change to the on-disk shape. +/// Bump only on a breaking change to the on-disk shape. Adding a variant or a defaulted field is +/// not one: [`SessionEvent::Unknown`] absorbs a kind this reader predates, and every field added +/// since v1 defaults on decode. pub const WIRE_VERSION: u8 = 1; /// How one declared grade-evidence task ended up. A lossy grade (`join = "passed"`) @@ -58,7 +62,7 @@ impl std::fmt::Display for EvidenceEntry { /// A plain-serde mirror of `Row`, so `Row`'s fields can churn without touching the /// on-disk contract. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct RowWire { pub iter: u32, pub decision: String, @@ -99,7 +103,7 @@ pub struct RowWire { /// onto the kept candidate rows' `pr_url`. A single-repo run emits one link with `name` empty; a /// composite run emits one per touched component. Mirrors `crucible::publish::PrLink` but adds /// `Deserialize`, for the same reason [`RowWire`] mirrors `Row`. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct PrLinkWire { pub url: String, pub repo: String, @@ -113,7 +117,7 @@ pub struct PrLinkWire { } /// A validated plan task emitted for visualization. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct PlanTaskWire { pub name: String, /// Task kind label: `agent`, `command`, or a reducer name like `top_k`. @@ -333,21 +337,35 @@ pub enum SessionEvent { span_id: String, }, /// The loop is exiting, emitted exactly once as the LAST line of the session log. `outcome` is - /// one of `finished`/`solved`/`budget`/`stopped`/`escalated`/`stalled`/`error`. The viewer keys - /// its terminal state off this line: a dead stream with no `Shutdown` line means the pod died - /// mid-run rather than exiting cleanly. + /// one of `finished`/`solved`/`budget`/`stopped`/`escalated`/`stalled`/`error`/`suspended`. + /// The viewer keys its terminal state off this line: a dead stream with no `Shutdown` line + /// means the pod died mid-run rather than exiting cleanly. `suspended` is a clean exit that + /// left an approval open on purpose; a resume with the resolution continues past it. Shutdown { outcome: String, reason: String, }, - /// The loop began waiting on a mediated-provisioning approval. A dangling ApprovalWait - /// means the run died or stopped with the approval outstanding; resume re-parks a - /// block-mode one. + /// The loop began waiting on an approval: a mediated-provisioning ask, a distress marker, or a + /// plan `approve` task. A dangling ApprovalWait means the run died, stopped, or suspended with + /// the approval outstanding; resume re-parks a block-mode one unless it is handed the + /// resolution. ApprovalWait { handle: String, #[serde(default)] trace_id: String, mode: String, + /// The plan task that opened the gate; absent for provisioning and distress waits. + #[serde(default, skip_serializing_if = "Option::is_none")] + task: Option, + /// Where the resolution comes from, as the engine resolved it at the gate: + /// `{"kind":"native"}`, `{"kind":"github_pr","url":..,"until":..}`, or + /// `{"kind":"jira","key":..,"until":{..}}`. Absent on logs written before gates existed. + #[serde(default, skip_serializing_if = "Option::is_none")] + source: Option, + /// `park` (the pod idles) or `suspend` (the pod snapshots and exits). Absent on older + /// logs, which always parked. + #[serde(default, skip_serializing_if = "Option::is_none")] + park: Option, }, /// The wait above reached a terminal outcome: `granted`/`denied`/`timeout`. Deliberately /// NOT emitted on a stop-while-parked (a stop doesn't resolve the ask), so a resumed run @@ -356,6 +374,16 @@ pub enum SessionEvent { outcome: String, #[serde(default)] reason: String, + /// The `trace_id` of the wait this resolves; empty on older logs, which resolved the one + /// open wait. + #[serde(default, skip_serializing_if = "String::is_empty")] + trace_id: String, + /// Who resolved it, when the source knows. + #[serde(default, skip_serializing_if = "Option::is_none")] + by: Option, + /// Which source resolved it (`native`, `github_pr`, `jira`, `timeout`, ...). + #[serde(default, skip_serializing_if = "Option::is_none")] + source: Option, }, /// A mediated write refused by the run's declared output bounds. The refusal fails the /// requesting tool call; it never terminates the run. @@ -382,12 +410,19 @@ pub enum SessionEvent { #[serde(default)] detail: String, }, + /// A `kind` this reader does not know: written by a newer engine. Folds count it and + /// classify from the events they do know; a resume refuses when one precedes no terminal. + #[serde(other)] + Unknown, } /// Borrowing envelope so encoding doesn't clone the event. #[derive(Serialize)] struct EnvelopeRef<'a> { v: u8, + /// Unix seconds when the line was written. Absent on logs written before the field existed. + #[serde(skip_serializing_if = "Option::is_none")] + ts: Option, #[serde(flatten)] event: &'a SessionEvent, } @@ -398,6 +433,8 @@ struct Envelope { #[serde(default = "default_version")] #[allow(dead_code)] // read for forward-compat; we accept any v for now v: u8, + #[serde(default)] + ts: Option, #[serde(flatten)] event: SessionEvent, } @@ -406,12 +443,25 @@ fn default_version() -> u8 { WIRE_VERSION } +/// One decoded line: the event and the moment it was written, when the writer stamped one. +#[derive(Debug, Clone)] +pub struct Line { + pub ts: Option, + pub event: SessionEvent, +} + /// Encode one event as a single NDJSON line (no trailing newline). Infallible for our /// own types; on the impossible serde error we emit a valid `note` line rather than /// corrupt the log. pub fn encode(ev: &SessionEvent) -> String { + encode_at(ev, None) +} + +/// [`encode`] with a write timestamp (unix seconds) on the envelope. +pub fn encode_at(ev: &SessionEvent, ts: Option) -> String { serde_json::to_string(&EnvelopeRef { v: WIRE_VERSION, + ts, event: ev, }) .unwrap_or_else(|e| { @@ -422,11 +472,19 @@ pub fn encode(ev: &SessionEvent) -> String { /// Decode one NDJSON line. Returns `None` for blank or torn/partial lines so a tailing /// reader can skip them. pub fn decode(line: &str) -> Option { + decode_line(line).map(|l| l.event) +} + +/// [`decode`] keeping the envelope's timestamp. +pub fn decode_line(line: &str) -> Option { let t = line.trim(); if t.is_empty() { return None; } - crate::json::from_str::(t).ok().map(|e| e.event) + crate::json::from_str::(t).ok().map(|e| Line { + ts: e.ts, + event: e.event, + }) } #[cfg(test)] @@ -589,10 +647,35 @@ mod tests { handle: "https://github.com/wseaton/llm-d-router/pull/7".into(), trace_id: "model=Qwen/Qwen3-0.6B;c=48".into(), mode: "block".into(), + task: None, + source: None, + park: None, + }, + SessionEvent::ApprovalWait { + handle: "https://github.com/wseaton/llm-d-router/pull/7".into(), + trace_id: "approve:run-1:review".into(), + mode: "block".into(), + task: Some("review".into()), + source: Some(serde_json::json!({ + "kind": "github_pr", + "url": "https://github.com/wseaton/llm-d-router/pull/7", + "until": "approved" + })), + park: Some("suspend".into()), }, SessionEvent::ApprovalResolved { outcome: "granted".into(), reason: "concurrency=48".into(), + trace_id: String::new(), + by: None, + source: None, + }, + SessionEvent::ApprovalResolved { + outcome: "denied".into(), + reason: "changes requested".into(), + trace_id: "approve:run-1:review".into(), + by: Some("alice".into()), + source: Some("github_pr".into()), }, SessionEvent::Recovery { class: RecoveryClass::DiedMidTurn, @@ -641,15 +724,44 @@ mod tests { handle, trace_id, mode, + task, + source, + park, } => { assert_eq!(handle, "h"); assert_eq!(trace_id, ""); assert_eq!(mode, "continue"); + assert_eq!(task, None); + assert_eq!(source, None); + assert_eq!(park, None); } other => panic!("wrong variant: {other:?}"), } } + /// A kind this reader predates decodes as `Unknown` instead of failing the whole line, so + /// a rolling deploy never turns a newer log into "resume refused" or "ingest refused". + #[test] + fn unknown_kind_decodes_as_unknown_and_round_trips() { + let ev = decode(r#"{"v":1,"kind":"teleport","where":"elsewhere"}"#) + .expect("unknown kind decodes"); + assert!(matches!(ev, SessionEvent::Unknown), "{ev:?}"); + assert_round_trips(SessionEvent::Unknown); + } + + #[test] + fn envelope_timestamp_is_optional_and_survives_the_trip() { + let bare = encode(&SessionEvent::Finished); + assert!(!bare.contains("\"ts\""), "{bare}"); + let stamped = encode_at(&SessionEvent::Finished, Some(1_725_000_000.5)); + assert!(stamped.contains(r#""ts":1725000000.5"#), "{stamped}"); + let line = decode_line(&stamped).expect("stamped line decodes"); + assert_eq!(line.ts, Some(1_725_000_000.5)); + assert!(matches!(line.event, SessionEvent::Finished)); + let old = decode_line(r#"{"v":1,"kind":"finished"}"#).expect("old line decodes"); + assert_eq!(old.ts, None); + } + #[test] fn agent_events_round_trip_including_raw() { for event in [ diff --git a/crucible-contract/src/session/fold.rs b/crucible-contract/src/session/fold.rs new file mode 100644 index 00000000..5abfc0df --- /dev/null +++ b/crucible-contract/src/session/fold.rs @@ -0,0 +1,962 @@ +//! The session log folded into one state: every consumer of `state/session.jsonl` (the +//! engine's resume, its crash classification, the controller's ingest and rebuild) reads the +//! same [`LoopState`] by applying the same [`LoopState::apply`] to the same events. +//! +//! Only what the log can reproduce lives here. Process-local state (control bridge slots, +//! marker files, handles) is the engine's, reconstructed from the admission ledger. + +use crate::event::AgentEvent; +use crate::identity::RunIdentity; +use crate::session::{PlanTaskWire, PrLinkWire, RecoveryClass, RowWire, SessionEvent}; +use std::collections::BTreeMap; + +/// `Shutdown.outcome` tokens. Unknown maps to `Other` so a newer writer never breaks an +/// older reader. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ShutdownOutcome { + Finished, + Solved, + Budget, + Stopped, + Escalated, + Stalled, + Error, + Suspended, + Other(String), +} + +impl ShutdownOutcome { + pub fn parse(token: &str) -> Self { + match token { + "finished" => ShutdownOutcome::Finished, + "solved" => ShutdownOutcome::Solved, + "budget" => ShutdownOutcome::Budget, + "stopped" => ShutdownOutcome::Stopped, + "escalated" => ShutdownOutcome::Escalated, + "stalled" => ShutdownOutcome::Stalled, + "error" => ShutdownOutcome::Error, + "suspended" => ShutdownOutcome::Suspended, + other => ShutdownOutcome::Other(other.to_string()), + } + } + + pub fn as_str(&self) -> &str { + match self { + ShutdownOutcome::Finished => "finished", + ShutdownOutcome::Solved => "solved", + ShutdownOutcome::Budget => "budget", + ShutdownOutcome::Stopped => "stopped", + ShutdownOutcome::Escalated => "escalated", + ShutdownOutcome::Stalled => "stalled", + ShutdownOutcome::Error => "error", + ShutdownOutcome::Suspended => "suspended", + ShutdownOutcome::Other(t) => t, + } + } +} + +/// `ApprovalWait.mode` tokens. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WaitMode { + /// The run parks until the approval resolves. + Block, + /// The run keeps iterating; the grant lands whenever it arrives. + Continue, +} + +impl WaitMode { + /// Unknown tokens degrade to `Continue`: a reader never re-parks on a mode it cannot honor. + pub fn parse(token: &str) -> Self { + if token == "block" { + WaitMode::Block + } else { + WaitMode::Continue + } + } + + pub fn as_str(self) -> &'static str { + match self { + WaitMode::Block => "block", + WaitMode::Continue => "continue", + } + } +} + +/// Evidence scraped from a dangling turn's agent events. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct TurnEvidence { + /// Events inside the dangling AgentStart bracket. + pub agent_events: u32, + /// Last per-turn cost seen (`Tokens.cost_usd` or `OtelSummary.cost_usd`). + pub last_cost_usd: Option, + /// Last error text (`Error.message` or an is_error `Result.error`), verbatim. + pub last_error: Option, + /// From the preceding AgentSession line. + pub session: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DanglingSession { + pub name: String, + /// 1-based, from the AgentSession line. + pub turn: u32, +} + +/// An approval the log left open (ApprovalWait with no ApprovalResolved after it). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OpenApproval { + pub handle: String, + pub trace_id: String, + pub mode: WaitMode, + /// The plan task that opened the gate; `None` for a provisioning or distress wait. + pub task: Option, + /// The resolution source, verbatim from the wire; `None` on logs written before it existed. + pub source: Option, +} + +/// A PlanAdmitted whose iteration never accounted (no Row after it). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OpenPlan { + pub plan_version: u32, + /// Declared task names, in order. + pub declared: Vec, + /// TaskResult names seen after this PlanAdmitted. + pub resulted: Vec, + /// Admitted before the first iteration Phase. + pub wide: bool, +} + +/// What the tail of the log says happened. +#[derive(Debug, Clone, PartialEq)] +pub enum Classification { + /// A trailing Shutdown line: the previous process exited on purpose. + CleanExit { + outcome: ShutdownOutcome, + reason: String, + }, + /// No decided rows: nothing to resume from. + DiedInBaseline, + /// Died before the first iteration Phase. + DiedInWideRound { + plan: Option, + turn: Option<(u32, TurnEvidence)>, + }, + /// AgentStart with no AgentDone: the turn was in flight. + DiedMidTurn { + iter: u32, + evidence: TurnEvidence, + approval: Option, + }, + /// Turn finished but no Row for that iteration: died in measure/decide/keep. + DiedDeciding { iter: u32, evidence: TurnEvidence }, + /// PlanAdmitted for an iteration with no Row and no dangling turn. + DiedInPlanTask { iter: u32, plan: OpenPlan }, + /// Parked on a block-mode approval with no turn in flight. + DiedAwaitingApproval { approval: OpenApproval }, + /// No dangling turn, plan, or approval; died between a Row and the next AgentStart. + DiedBetweenIterations { last_iter: u32 }, +} + +impl Classification { + pub fn class(&self) -> RecoveryClass { + match self { + Classification::CleanExit { .. } => RecoveryClass::CleanExit, + Classification::DiedInBaseline => RecoveryClass::DiedInBaseline, + Classification::DiedInWideRound { .. } => RecoveryClass::DiedInWideRound, + Classification::DiedMidTurn { .. } => RecoveryClass::DiedMidTurn, + Classification::DiedDeciding { .. } => RecoveryClass::DiedDeciding, + Classification::DiedInPlanTask { .. } => RecoveryClass::DiedInPlanTask, + Classification::DiedAwaitingApproval { .. } => RecoveryClass::DiedAwaitingApproval, + Classification::DiedBetweenIterations { .. } => RecoveryClass::DiedBetweenIterations, + } + } + + /// The iteration the interruption touched (0 if none). + pub fn iter(&self) -> u32 { + match self { + Classification::DiedMidTurn { iter, .. } + | Classification::DiedDeciding { iter, .. } + | Classification::DiedInPlanTask { iter, .. } => *iter, + Classification::DiedBetweenIterations { last_iter } => *last_iter, + _ => 0, + } + } + + /// One-line evidence summary for the Recovery event and the resume note. + pub fn detail(&self) -> String { + match self { + Classification::CleanExit { outcome, reason } => { + format!("previous run exited {}: {reason}", outcome.as_str()) + } + Classification::DiedInBaseline => "no decided rows in the log".to_string(), + Classification::DiedInWideRound { plan, turn } => { + let mut s = "died in the wide tournament".to_string(); + if let Some(p) = plan { + s.push_str(&format!( + ", plan v{} open ({}/{} tasks resulted)", + p.plan_version, + p.resulted.len(), + p.declared.len() + )); + } + if turn.is_some() { + s.push_str(", a turn was in flight"); + } + s + } + Classification::DiedMidTurn { + iter, + evidence, + approval, + } => { + let mut s = format!( + "turn in flight at iter {iter}, {} agent events", + evidence.agent_events + ); + if let Some(c) = evidence.last_cost_usd { + s.push_str(&format!(", last cost ${c:.2}")); + } + if let Some(sess) = &evidence.session { + s.push_str(&format!(", session {} turn {}", sess.name, sess.turn)); + } + if let Some(e) = &evidence.last_error { + s.push_str(&format!(", last error: {}", trunc(e, 200))); + } + if let Some(a) = approval { + s.push_str(&format!(", approval {} outstanding", a.handle)); + } + s + } + Classification::DiedDeciding { iter, evidence } => { + let mut s = format!("turn at iter {iter} completed but was never decided"); + if let Some(sess) = &evidence.session { + s.push_str(&format!( + " (session {} turn {} cursor advanced ungraded)", + sess.name, sess.turn + )); + } + s + } + Classification::DiedInPlanTask { iter, plan } => format!( + "{}plan v{} at iter {iter} never accounted ({}/{} tasks resulted)", + if plan.wide { "wide " } else { "" }, + plan.plan_version, + plan.resulted.len(), + plan.declared.len() + ), + Classification::DiedAwaitingApproval { approval } => { + format!( + "parked on approval {} ({})", + approval.handle, approval.trace_id + ) + } + Classification::DiedBetweenIterations { last_iter } => { + format!("died between iterations, last decided iter {last_iter}") + } + } + } +} + +fn trunc(s: &str, max: usize) -> String { + if s.chars().count() <= max { + return s.to_string(); + } + let cut: String = s.chars().take(max).collect(); + format!("{cut}…") +} + +/// One settled plan task, keyed by name (fan-out instances carry `node[key]`). +#[derive(Debug, Clone, PartialEq)] +pub struct TaskResultWire { + pub status: String, + pub task_kind: String, + pub iter: u32, + pub attempts: u32, + pub cost_usd: f64, + pub secs: f64, + pub note: String, + pub output: Option, +} + +/// The latest admitted work graph. +#[derive(Debug, Clone, PartialEq)] +pub struct PlanWire { + pub plan_version: u32, + pub budget_usd: f64, + pub tasks: Vec, +} + +/// The counters `--resume` restores, as the log carries them. +#[derive(Debug, Clone, PartialEq)] +pub struct ResumeView { + /// Decided deep-loop rows (no `wide`, no `infra`), in log order. + pub rows: Vec, + pub best_score: f64, + pub best_tiebreak: Option, + pub baseline_score: f64, + pub baseline_total: u64, + pub spent: f64, + /// First iteration to run (last logged iteration + 1). + pub next_iter: u32, + pub solved_any: bool, + pub identity: Option, + /// Head branches of every draft PR prior segments already opened. + pub published_branches: Vec, +} + +/// The log folded into one state. +#[derive(Debug, Clone, Default)] +pub struct LoopState { + /// Every Row verbatim, in log order. + pub rows: Vec, + pub solved_any: bool, + /// Last Budget line. + pub spent: f64, + pub elapsed_secs: Option, + /// `Summary.best_score`, when the run got that far. + pub summary_best: Option, + /// Last Identity line: a run resumed more than once re-emits one each time. + pub identity: Option, + pub pr_links: Vec, + pub escalation: Option<(String, String, String)>, + /// The latest admitted graph and the results settled under it. A PlanAdmitted with a + /// different `plan_version` clears the results. + pub plan: Option, + pub plan_results: BTreeMap, + /// Every Recovery line, in order. + pub recoveries: Vec<(RecoveryClass, u32, String)>, + /// The trailing Shutdown. A resumed process appends past its predecessor's Shutdown, so any + /// later event clears it. + pub terminal: Option<(ShutdownOutcome, String)>, + /// Lines whose `kind` this reader does not know. + pub unknown: u32, + // Open brackets, exactly the facts the tail classifier needs. + saw_iteration_phase: bool, + saw_any_row: bool, + last_row_iter: u32, + last_phase_iter: u32, + pending_session: Option, + open_turn: Option<(u32, TurnEvidence)>, + done_unrowed: Option<(u32, TurnEvidence)>, + open_plan: Option, + open_approval: Option, +} + +impl LoopState { + /// Fold every decodable line of a session log. Blank and torn lines are skipped, exactly + /// as [`crate::session::decode`] skips them. + pub fn from_lines<'a>(lines: impl IntoIterator) -> Self { + let mut state = LoopState::default(); + for line in lines { + if let Some(ev) = crate::session::decode(line) { + state.apply(&ev); + } + } + state + } + + /// Apply one event. Exhaustive: adding a variant to [`SessionEvent`] is a compile error + /// here until the fold says what it means. + pub fn apply(&mut self, ev: &SessionEvent) { + if self.terminal.is_some() && !matches!(ev, SessionEvent::Shutdown { .. }) { + self.terminal = None; + } + match ev { + SessionEvent::Start { .. } => {} + SessionEvent::Phase { phase, iter } => { + if phase == "iteration" { + self.saw_iteration_phase = true; + self.last_phase_iter = *iter; + self.done_unrowed = None; + } + } + SessionEvent::Note { .. } => {} + SessionEvent::Row { row, solved } => { + if !matches!(row.phase.as_deref(), Some("wide") | Some("infra")) { + self.saw_any_row = true; + self.last_row_iter = row.iter; + self.solved_any |= *solved; + } + if self + .done_unrowed + .as_ref() + .is_some_and(|(iter, _)| *iter == row.iter) + { + self.done_unrowed = None; + } + self.open_plan = None; + self.rows.push(row.clone()); + } + SessionEvent::AgentStart { iter } => { + self.open_turn = Some(( + *iter, + TurnEvidence { + session: self.pending_session.take(), + ..TurnEvidence::default() + }, + )); + } + SessionEvent::AgentSession { session, turn, .. } => { + self.pending_session = Some(DanglingSession { + name: session.clone(), + turn: *turn, + }); + } + SessionEvent::Agent { event } => { + if let Some((_, evidence)) = &mut self.open_turn { + evidence.agent_events += 1; + match event { + AgentEvent::Tokens(t) => { + if let Some(c) = t.cost_usd { + evidence.last_cost_usd = Some(c); + } + } + AgentEvent::OtelSummary { cost_usd, .. } => { + evidence.last_cost_usd = Some(*cost_usd); + } + AgentEvent::Error { message, .. } => { + evidence.last_error = Some(message.clone()); + } + AgentEvent::Result { + is_error: true, + error: Some(e), + .. + } => { + evidence.last_error = Some(e.clone()); + } + _ => {} + } + } + } + SessionEvent::AgentDone => { + self.done_unrowed = self.open_turn.take(); + } + SessionEvent::Budget { + spent, + elapsed_secs, + } => { + self.spent = *spent; + self.elapsed_secs = Some(*elapsed_secs); + } + SessionEvent::Summary { best_score, .. } => self.summary_best = *best_score, + SessionEvent::Escalation { + category, + reason, + evidence, + } => { + self.escalation = Some((category.clone(), reason.clone(), evidence.clone())); + } + SessionEvent::Segment { .. } => {} + SessionEvent::Identity { identity } => self.identity = Some(identity.clone()), + SessionEvent::PrLinks { links } => self.pr_links.extend(links.iter().cloned()), + SessionEvent::Finished => {} + SessionEvent::PlanAdmitted { + plan_version, + budget_usd, + tasks, + .. + } => { + if self.plan.as_ref().map(|p| p.plan_version) != Some(*plan_version) { + self.plan_results.clear(); + } + self.plan = Some(PlanWire { + plan_version: *plan_version, + budget_usd: *budget_usd, + tasks: tasks.clone(), + }); + self.open_plan = Some(OpenPlan { + plan_version: *plan_version, + declared: tasks.iter().map(|t| t.name.clone()).collect(), + resulted: Vec::new(), + wide: !self.saw_iteration_phase, + }); + } + SessionEvent::AsksEmitted { .. } => {} + SessionEvent::TaskResult { + task, + status, + task_kind, + iter, + attempts, + cost_usd, + output, + note, + secs, + .. + } => { + if let Some(plan) = &mut self.open_plan { + plan.resulted.push(task.clone()); + } + self.plan_results.insert( + task.clone(), + TaskResultWire { + status: status.clone(), + task_kind: task_kind.clone(), + iter: *iter, + attempts: *attempts, + cost_usd: *cost_usd, + secs: *secs, + note: note.clone(), + output: output.clone(), + }, + ); + } + SessionEvent::Shutdown { outcome, reason } => { + self.terminal = Some((ShutdownOutcome::parse(outcome), reason.clone())); + } + SessionEvent::ApprovalWait { + handle, + trace_id, + mode, + task, + source, + .. + } => { + self.open_approval = Some(OpenApproval { + handle: handle.clone(), + trace_id: trace_id.clone(), + mode: WaitMode::parse(mode), + task: task.clone(), + source: source.clone(), + }); + } + SessionEvent::ApprovalResolved { .. } => self.open_approval = None, + SessionEvent::OutputRefused { .. } => {} + SessionEvent::Recovery { + class, + iter, + detail, + } => self.recoveries.push((*class, *iter, detail.clone())), + SessionEvent::Unknown => self.unknown += 1, + } + } + + /// Whether any deep-loop row was decided. A log without one is not resumable. + pub fn has_rows(&self) -> bool { + self.saw_any_row + } + + /// The approval the log left open, whatever the classification. + pub fn open_approval(&self) -> Option<&OpenApproval> { + self.open_approval.as_ref() + } + + /// Declared plan tasks with no result under the current plan. + pub fn plan_open(&self) -> Vec { + let Some(plan) = &self.plan else { + return Vec::new(); + }; + plan.tasks + .iter() + .map(|t| t.name.clone()) + .filter(|name| !self.plan_results.contains_key(name)) + .collect() + } + + /// What the tail says happened. Each earlier case subsumes the later ones. + pub fn classify(&self) -> Classification { + if let Some((outcome, reason)) = &self.terminal { + return Classification::CleanExit { + outcome: outcome.clone(), + reason: reason.clone(), + }; + } + if !self.saw_any_row { + return Classification::DiedInBaseline; + } + if !self.saw_iteration_phase { + return Classification::DiedInWideRound { + plan: self.open_plan.clone(), + turn: self.open_turn.clone(), + }; + } + if let Some((iter, evidence)) = &self.open_turn { + return Classification::DiedMidTurn { + iter: *iter, + evidence: evidence.clone(), + approval: self.open_approval.clone(), + }; + } + if let Some(plan) = &self.open_plan { + return Classification::DiedInPlanTask { + iter: self.last_phase_iter, + plan: plan.clone(), + }; + } + if let Some(approval) = self + .open_approval + .as_ref() + .filter(|a| a.mode == WaitMode::Block) + { + return Classification::DiedAwaitingApproval { + approval: approval.clone(), + }; + } + if let Some((iter, evidence)) = &self.done_unrowed { + return Classification::DiedDeciding { + iter: *iter, + evidence: evidence.clone(), + }; + } + Classification::DiedBetweenIterations { + last_iter: self.last_row_iter, + } + } + + /// Decided deep-loop rows: neither a wide-round lane nor a never-started turn record. + pub fn deep_rows(&self) -> impl DoubleEndedIterator { + self.rows + .iter() + .filter(|r| !matches!(r.phase.as_deref(), Some("wide") | Some("infra"))) + } + + /// The counters a resume restores. Decided rows carry `score`/`total`, so baseline and + /// best restore exactly; keeps are monotone within a segment, so the last kept row is the + /// best and its tiebreak travels with the best score. + pub fn resume_view(&self) -> ResumeView { + let rows: Vec = self.deep_rows().cloned().collect(); + let baseline_score = rows.first().and_then(|r| r.score).unwrap_or(f64::INFINITY); + let baseline_total = rows.first().and_then(|r| r.total).unwrap_or(0); + let best_score = self.summary_best.unwrap_or_else(|| { + rows.iter() + .filter(|r| r.decision == "keep") + .filter_map(|r| r.score) + .fold(baseline_score, f64::min) + }); + let best_tiebreak = rows + .iter() + .rev() + .find(|r| r.decision == "keep") + .or_else(|| rows.first()) + .and_then(|r| r.tiebreak); + let next_iter = rows.iter().map(|r| r.iter).max().unwrap_or(0) + 1; + ResumeView { + rows, + best_score, + best_tiebreak, + baseline_score, + baseline_total, + spent: self.spent, + next_iter, + solved_any: self.solved_any, + identity: self.identity.clone(), + published_branches: self.pr_links.iter().map(|l| l.branch.clone()).collect(), + } + } + + /// The run's cost when no Budget line landed: the sum of settled plan-task costs. + pub fn cost_usd(&self) -> f64 { + if self.spent > 0.0 { + return self.spent; + } + self.plan_results.values().map(|r| r.cost_usd).sum() + } + + /// The run's best score: the Summary's, else the last kept deep row's. + pub fn best_score(&self) -> Option { + self.summary_best.or_else(|| { + self.deep_rows() + .rev() + .find(|r| r.decision == "keep") + .and_then(|r| r.score) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::session::encode; + + fn row(iter: u32, decision: &str, score: f64, phase: Option<&str>) -> SessionEvent { + SessionEvent::Row { + row: RowWire { + iter, + decision: decision.into(), + note: String::new(), + detail: String::new(), + diff: String::new(), + diffstat: String::new(), + score: Some(score), + tiebreak: None, + total: None, + phase: phase.map(str::to_string), + kept_snap: None, + evidence: Vec::new(), + candidate_md: String::new(), + }, + solved: false, + } + } + + fn iteration(iter: u32) -> SessionEvent { + SessionEvent::Phase { + phase: "iteration".into(), + iter, + } + } + + fn task(name: &str, cost: f64) -> SessionEvent { + SessionEvent::TaskResult { + task: name.into(), + status: "pass".into(), + plan_version: 1, + task_kind: "command".into(), + iter: 1, + digest: String::new(), + job: String::new(), + attempts: 1, + cost_usd: cost, + metric: None, + output: Some(serde_json::json!({"n": 1})), + note: String::new(), + secs: 0.5, + trace_id: String::new(), + span_id: String::new(), + } + } + + fn plan(version: u32, names: &[&str]) -> SessionEvent { + SessionEvent::PlanAdmitted { + plan_version: version, + reason: String::new(), + budget_usd: 1.0, + tasks: names + .iter() + .map(|n| PlanTaskWire { + name: (*n).into(), + kind: "command".into(), + depends_on: Vec::new(), + session: String::new(), + needs: String::new(), + required: true, + join: String::new(), + stage: String::new(), + over: String::new(), + max_fanout: 0, + }) + .collect(), + } + } + + fn fold(events: &[SessionEvent]) -> LoopState { + let mut s = LoopState::default(); + for ev in events { + s.apply(ev); + } + s + } + + #[test] + fn from_lines_skips_blank_and_torn_lines_and_counts_unknown_kinds() { + let body = [ + encode(&row(0, "baseline", 240.0, None)), + String::new(), + r#"{"v":1,"kind":"teleport"}"#.to_string(), + r#"{"v":1,"kind":"row","row":{"iter":1,"deci"#.to_string(), + encode(&iteration(1)), + ] + .join("\n"); + let s = LoopState::from_lines(body.lines()); + assert_eq!(s.rows.len(), 1); + assert_eq!(s.unknown, 1); + assert!(s.has_rows()); + } + + #[test] + fn a_trailing_shutdown_is_terminal_and_a_later_event_clears_it() { + let mut events = vec![ + row(0, "baseline", 240.0, None), + iteration(1), + SessionEvent::Shutdown { + outcome: "suspended".into(), + reason: "approve:r:gate".into(), + }, + ]; + let s = fold(&events); + assert_eq!( + s.classify(), + Classification::CleanExit { + outcome: ShutdownOutcome::Suspended, + reason: "approve:r:gate".into() + } + ); + events.push(SessionEvent::Note { + msg: "resumed".into(), + }); + let s = fold(&events); + assert!(s.terminal.is_none()); + assert_eq!( + s.classify(), + Classification::DiedBetweenIterations { last_iter: 0 } + ); + } + + #[test] + fn shutdown_tokens_round_trip_and_unknown_is_other() { + for token in [ + "finished", + "solved", + "budget", + "stopped", + "escalated", + "stalled", + "error", + "suspended", + ] { + assert_eq!(ShutdownOutcome::parse(token).as_str(), token); + } + assert_eq!( + ShutdownOutcome::parse("whatever"), + ShutdownOutcome::Other("whatever".into()) + ); + } + + #[test] + fn plan_results_key_by_task_and_reset_when_the_plan_version_changes() { + let s = fold(&[ + row(0, "baseline", 240.0, None), + iteration(1), + plan(1, &["a", "b[x]", "b[y]"]), + task("a", 0.2), + task("b[x]", 0.3), + ]); + assert_eq!(s.plan_open(), vec!["b[y]".to_string()]); + assert_eq!(s.cost_usd(), 0.5, "no budget line: task costs sum"); + assert!(matches!( + s.classify(), + Classification::DiedInPlanTask { iter: 1, .. } + )); + + let mut events = vec![ + row(0, "baseline", 240.0, None), + iteration(1), + plan(1, &["a"]), + task("a", 0.2), + row(1, "keep", 200.0, None), + iteration(2), + plan(2, &["a", "c"]), + ]; + let s = fold(&events); + assert!(s.plan_results.is_empty(), "a new plan version starts empty"); + assert_eq!(s.plan_open(), vec!["a".to_string(), "c".to_string()]); + events.push(task("a", 0.1)); + let s = fold(&events); + assert_eq!(s.plan_open(), vec!["c".to_string()]); + } + + #[test] + fn a_budget_line_wins_over_the_task_cost_sum() { + let s = fold(&[ + row(0, "baseline", 240.0, None), + plan(1, &["a"]), + task("a", 0.2), + SessionEvent::Budget { + spent: 1.5, + elapsed_secs: 10, + }, + ]); + assert_eq!(s.cost_usd(), 1.5); + assert_eq!(s.elapsed_secs, Some(10)); + } + + #[test] + fn best_score_prefers_the_summary_then_the_last_kept_deep_row() { + let base = vec![ + row(0, "baseline", 240.0, None), + iteration(1), + row(1, "keep", 220.0, None), + row(2, "wide-keep-0", 100.0, Some("wide")), + row(3, "discard", 260.0, None), + ]; + let s = fold(&base); + assert_eq!(s.best_score(), Some(220.0)); + assert_eq!(s.deep_rows().count(), 3, "the wide row is not a deep row"); + let mut with_summary = base.clone(); + with_summary.push(SessionEvent::Summary { + rows: Vec::new(), + gate: "bench".into(), + best_score: Some(210.0), + }); + assert_eq!(fold(&with_summary).best_score(), Some(210.0)); + } + + #[test] + fn the_resume_view_restores_baseline_best_and_next_iter_from_deep_rows() { + let s = fold(&[ + row(0, "baseline", 240.0, None), + iteration(1), + row(1, "keep", 220.0, None), + row(1, "infra", 0.0, Some("infra")), + row(2, "discard", 260.0, None), + SessionEvent::Budget { + spent: 2.5, + elapsed_secs: 40, + }, + ]); + let v = s.resume_view(); + assert_eq!(v.rows.len(), 3); + assert_eq!(v.baseline_score, 240.0); + assert_eq!(v.best_score, 220.0); + assert_eq!(v.next_iter, 3); + assert_eq!(v.spent, 2.5); + } + + #[test] + fn an_open_gate_carries_its_task_and_source_and_a_resolution_closes_it() { + let source = + serde_json::json!({"kind": "jira", "key": "PROJ-1", "until": {"status": "Ready"}}); + let mut events = vec![ + row(0, "baseline", 240.0, None), + iteration(1), + SessionEvent::ApprovalWait { + handle: "PROJ-1".into(), + trace_id: "approve:r:gate".into(), + mode: "block".into(), + task: Some("gate".into()), + source: Some(source.clone()), + park: Some("suspend".into()), + }, + ]; + let s = fold(&events); + let open = s.open_approval().expect("gate is open"); + assert_eq!(open.task.as_deref(), Some("gate")); + assert_eq!(open.source.as_ref(), Some(&source)); + assert_eq!(open.mode, WaitMode::Block); + assert!(matches!( + s.classify(), + Classification::DiedAwaitingApproval { .. } + )); + events.push(SessionEvent::ApprovalResolved { + outcome: "granted".into(), + reason: String::new(), + trace_id: "approve:r:gate".into(), + by: Some("alice".into()), + source: Some("jira".into()), + }); + assert!(fold(&events).open_approval().is_none()); + } + + #[test] + fn an_unknown_wait_mode_never_reparks() { + assert_eq!(WaitMode::parse("block"), WaitMode::Block); + assert_eq!(WaitMode::parse("continue"), WaitMode::Continue); + assert_eq!(WaitMode::parse("sideways"), WaitMode::Continue); + let s = fold(&[ + row(0, "baseline", 240.0, None), + iteration(1), + SessionEvent::ApprovalWait { + handle: "h".into(), + trace_id: "t".into(), + mode: "sideways".into(), + task: None, + source: None, + park: None, + }, + ]); + assert_eq!( + s.classify(), + Classification::DiedBetweenIterations { last_iter: 0 } + ); + } +} diff --git a/crucible/src/loop_driver.rs b/crucible/src/loop_driver.rs index 37cc3deb..654bc589 100644 --- a/crucible/src/loop_driver.rs +++ b/crucible/src/loop_driver.rs @@ -56,6 +56,13 @@ pub(crate) struct ResumeState { pub published_branches: Vec, } +#[cfg(test)] +impl ResumeState { + pub(crate) fn has_rows_for_parity(&self) -> bool { + !self.rows.is_empty() + } +} + /// Optional runtime state for special loop starts (remote control and resume). Kept in /// one argument so the core loop boundary stays small as front-ends evolve. #[derive(Default)] @@ -2185,7 +2192,9 @@ mod tests { /// The counter fold as `--resume` consumes it (through the classifier), so these /// replay tests exercise the same path `run.rs` takes. fn load_resume_state(session_log: &std::path::Path) -> Result { - crate::recovery::classify_session(session_log).map(|s| s.resume) + let got = crate::recovery::classify_session(session_log)?; + crate::recovery::parity::assert_matches(session_log, &got); + Ok(got.resume) } /// The 401 that killed a 5h turn, plus the other transport signatures, classify as retryable; diff --git a/crucible/src/recovery.rs b/crucible/src/recovery.rs index 2c6df2b9..e60cc8b9 100644 --- a/crucible/src/recovery.rs +++ b/crucible/src/recovery.rs @@ -90,47 +90,7 @@ pub(crate) enum Classification { DiedBetweenIterations { last_iter: u32 }, } -/// `Shutdown.outcome` tokens. Unknown maps to `Other` so a newer writer never breaks an -/// older resumer. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) enum ShutdownOutcome { - Finished, - Solved, - Budget, - Stopped, - Escalated, - Stalled, - Error, - Other(String), -} - -impl ShutdownOutcome { - pub(crate) fn parse(token: &str) -> Self { - match token { - "finished" => ShutdownOutcome::Finished, - "solved" => ShutdownOutcome::Solved, - "budget" => ShutdownOutcome::Budget, - "stopped" => ShutdownOutcome::Stopped, - "escalated" => ShutdownOutcome::Escalated, - "stalled" => ShutdownOutcome::Stalled, - "error" => ShutdownOutcome::Error, - other => ShutdownOutcome::Other(other.to_string()), - } - } - - fn as_str(&self) -> &str { - match self { - ShutdownOutcome::Finished => "finished", - ShutdownOutcome::Solved => "solved", - ShutdownOutcome::Budget => "budget", - ShutdownOutcome::Stopped => "stopped", - ShutdownOutcome::Escalated => "escalated", - ShutdownOutcome::Stalled => "stalled", - ShutdownOutcome::Error => "error", - ShutdownOutcome::Other(t) => t, - } - } -} +pub(crate) use crucible_contract::ShutdownOutcome; fn trunc(s: &str, max: usize) -> String { if s.chars().count() <= max { @@ -359,6 +319,7 @@ impl TailScan { handle, trace_id, mode, + .. } => { self.open_approval = Some(PendingApproval { handle: handle.clone(), @@ -510,9 +471,11 @@ pub(crate) fn plan_recovery(s: &SessionRecovery, iterations: u32, max_cost: f64) ), }; } - // Budget falls through: the operator may resume with a raised cap. + // Budget falls through: the operator may resume with a raised cap. Suspended + // falls through with its approval still open, so the wait re-arms below. ShutdownOutcome::Budget | ShutdownOutcome::Stopped + | ShutdownOutcome::Suspended | ShutdownOutcome::Stalled | ShutdownOutcome::Error | ShutdownOutcome::Other(_) => {} @@ -594,6 +557,86 @@ pub(crate) fn resume_approval( } } +/// The old folds against the contract's [`crucible_contract::LoopState`]: every test that +/// classifies a log runs both and proves they agree, so the contract fold can replace the +/// scanners here without a behavior change. +#[cfg(test)] +pub(crate) mod parity { + use super::{Classification, SessionRecovery}; + use crate::provisioning::WaitMode; + use crucible_contract::{LoopState, WaitMode as WireWaitMode}; + use std::path::Path; + + pub(crate) fn fold(session_log: &Path) -> LoopState { + let body = std::fs::read_to_string(session_log).expect("session log readable"); + LoopState::from_lines(body.lines()) + } + + /// Both folds over the same file agree on the classification, the resume counters, and the + /// open approval. + pub(crate) fn assert_matches(session_log: &Path, got: &SessionRecovery) { + let state = fold(session_log); + let theirs = state.classify(); + assert_eq!(theirs.class(), got.classification.class(), "class"); + assert_eq!(theirs.iter(), got.classification.iter(), "iter"); + assert_eq!(theirs.detail(), got.classification.detail(), "detail"); + if let Classification::CleanExit { outcome, reason } = &got.classification { + match &theirs { + crucible_contract::Classification::CleanExit { + outcome: o, + reason: r, + } => { + assert_eq!(o.as_str(), outcome.as_str(), "outcome token"); + assert_eq!(r, reason, "reason"); + } + other => panic!("contract fold classified {other:?}"), + } + } + assert_eq!( + state.has_rows(), + got.resume.has_rows_for_parity(), + "has_rows" + ); + let view = state.resume_view(); + let rs = &got.resume; + assert_eq!(view.rows.len(), rs.rows.len(), "row count"); + for (w, r) in view.rows.iter().zip(rs.rows.iter()) { + assert_eq!(w.iter, r.iter, "row iter"); + assert_eq!(w.decision, r.decision, "row decision"); + assert_eq!(w.score, r.score, "row score"); + assert_eq!(w.tiebreak, r.tiebreak, "row tiebreak"); + assert_eq!(w.total, r.total, "row total"); + assert_eq!(w.phase, r.phase, "row phase"); + assert_eq!(w.kept_snap, r.kept_snap, "row kept_snap"); + } + assert_eq!(view.best_score, rs.best_score, "best_score"); + assert_eq!(view.best_tiebreak, rs.best_tiebreak, "best_tiebreak"); + assert_eq!(view.baseline_score, rs.baseline_score, "baseline_score"); + assert_eq!(view.baseline_total, rs.baseline_total, "baseline_total"); + assert_eq!(view.spent, rs.spent, "spent"); + assert_eq!(view.next_iter, rs.next_iter, "next_iter"); + assert_eq!(view.solved_any, rs.solved_any, "solved_any"); + assert_eq!(view.identity, rs.identity, "identity"); + assert_eq!( + view.published_branches, rs.published_branches, + "published_branches" + ); + match (state.open_approval(), &got.pending_approval) { + (None, None) => {} + (Some(a), Some(b)) => { + assert_eq!(a.handle, b.handle, "approval handle"); + assert_eq!(a.trace_id, b.trace_id, "approval trace"); + let mode = match b.mode { + WaitMode::Block => WireWaitMode::Block, + WaitMode::Continue => WireWaitMode::Continue, + }; + assert_eq!(a.mode, mode, "approval mode"); + } + (a, b) => panic!("open approval disagrees: contract {a:?}, scanner {b:?}"), + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -645,10 +688,21 @@ mod tests { fn classify(name: &str, events: &[SessionEvent]) -> SessionRecovery { let path = write_log(name, events); let got = classify_session(&path).unwrap(); + super::parity::assert_matches(&path, &got); let _ = std::fs::remove_file(&path); got } + /// A rowless log is refused by the scanner and folds to `has_rows() == false` in the + /// contract, the same verdict. + #[test] + fn rowless_log_folds_without_rows_in_the_contract_too() { + let path = write_log("rowless-parity", &[iteration_phase(1)]); + assert!(classify_session(&path).is_err()); + assert!(!super::parity::fold(&path).has_rows()); + let _ = std::fs::remove_file(&path); + } + #[test] fn trailing_shutdown_classifies_clean_exit_per_outcome() { for (token, want) in [ @@ -881,6 +935,9 @@ mod tests { handle: "https://example.com/pr/7".into(), trace_id: "c=48".into(), mode: "block".into(), + task: None, + source: None, + park: None, }); let got = classify("awaiting", &events); match &got.classification { @@ -913,10 +970,16 @@ mod tests { handle: "h".into(), trace_id: "t".into(), mode: "block".into(), + task: None, + source: None, + park: None, }); events.push(SessionEvent::ApprovalResolved { outcome: "denied".into(), reason: "policy".into(), + trace_id: String::new(), + by: None, + source: None, }); let got = classify("resolved", &events); assert!(got.pending_approval.is_none()); @@ -937,6 +1000,9 @@ mod tests { handle: "h".into(), trace_id: "regime-x".into(), mode: "continue".into(), + task: None, + source: None, + park: None, }); events.push(row(1, "discard", 250.0)); let got = classify("continue-wait", &events); @@ -971,6 +1037,9 @@ mod tests { handle: "h".into(), trace_id: "t".into(), mode: "block".into(), + task: None, + source: None, + park: None, }); events.push(SessionEvent::Shutdown { outcome: "stopped".into(), @@ -1011,6 +1080,9 @@ mod tests { handle: "h".into(), trace_id: "t".into(), mode: "continue".into(), + task: None, + source: None, + park: None, }); events.push(SessionEvent::AgentStart { iter: 2 }); let got = classify("precedence", &events); diff --git a/crucible/src/stream.rs b/crucible/src/stream.rs index 5a6ec0a9..d1c80449 100644 --- a/crucible/src/stream.rs +++ b/crucible/src/stream.rs @@ -300,6 +300,9 @@ impl Reporter for SessionReporter { handle: handle.to_string(), trace_id: trace_id.to_string(), mode: mode.as_str().to_string(), + task: None, + source: None, + park: None, }); } @@ -307,6 +310,9 @@ impl Reporter for SessionReporter { self.emit(&SessionEvent::ApprovalResolved { outcome: outcome.to_string(), reason: reason.to_string(), + trace_id: String::new(), + by: None, + source: None, }); } diff --git a/crucible/tests/contract_version.rs b/crucible/tests/contract_version.rs index 29286f6d..ed38a6e9 100644 --- a/crucible/tests/contract_version.rs +++ b/crucible/tests/contract_version.rs @@ -43,7 +43,7 @@ fn wire_versions_are_pinned_to_contract_version() { ); assert_eq!( (CONTRACT_VERSION, wire), - ("1.2.0", (1, 1, 1, "v2")), + ("1.3.0", (1, 1, 1, "v2")), "a wire version changed without bumping CONTRACT_VERSION" ); } diff --git a/docs/crucible-contract.md b/docs/crucible-contract.md index e4a993f8..20c947af 100644 --- a/docs/crucible-contract.md +++ b/docs/crucible-contract.md @@ -712,8 +712,10 @@ Additive event kinds beyond the compat set include: a hard-warning `note` event, never an abort. - **`shutdown`**: `{ outcome, reason }`, emitted **exactly once**, as the **last** line of every run (after `finished`/`summary`). `outcome` is one of `finished`/`solved`/`budget`/`stopped`/ - `escalated`/`stalled`/`error`. Session-log consumers key a run's terminal state off this line; a - dead stream with **no** `shutdown` line means the pod likely died mid-run, not a clean exit. + `escalated`/`stalled`/`error`/`suspended`. Session-log consumers key a run's terminal state off + this line; a dead stream with **no** `shutdown` line means the pod likely died mid-run, not a + clean exit. `suspended` is a clean exit that left an `approval_wait` open on purpose: the run + snapshotted itself and expects a `--resume` carrying the resolution. `--resume` consumes this invariant, not just documents it: a resumed run classifies the log tail (see `recovery` below) and a trailing `shutdown` is the "exited on purpose" signal. In a resumed (appended) log, only the **trailing** `shutdown` counts; one followed by more events @@ -721,14 +723,21 @@ Additive event kinds beyond the compat set include: - **`agent_session`**: `{ session, action, turn }`, emitted before a persistent agent turn so a viewer can draw continuation lanes and distinguish `started` from `resumed`. It deliberately contains neither the provider cursor nor native transcript content. -- **`approval_wait`**: `{ handle, trace_id, mode }`, emitted when the loop reads the agent's - pending-provisioning marker. `mode` is `block` (the loop parks idle) or `continue` (it keeps - iterating in the frozen regime). Bracket invariant: every `approval_wait` is closed by an - `approval_resolved` **except** on stop-while-parked and process death, so a dangling wait in - the log tail means the run ended with the approval outstanding, and a resume re-parks a - block-mode one and re-registers the approval key so an operator `approve` still resolves it. -- **`approval_resolved`**: `{ outcome, reason }` with `outcome` one of `granted`/`denied`/ - `timeout`. A grant is emitted at the iteration-head rescope drain (the single re-baseline +- **`approval_wait`**: `{ handle, trace_id, mode, task?, source?, park? }`, emitted when the + loop reads the agent's pending-provisioning marker or reaches a plan `approve` task. `mode` is + `block` (the loop parks idle) or `continue` (it keeps iterating in the frozen regime). `task` + names the plan task that opened the gate; `source` is the resolution source the engine + resolved at the gate (`{"kind":"native"}`, `{"kind":"github_pr","url","until"}`, + `{"kind":"jira","key","until"}`); `park` is `park` or `suspend`. All three are absent on + provisioning and distress waits and on older logs. Bracket invariant: every `approval_wait` + is closed by an `approval_resolved` **except** on stop-while-parked, suspend, and process + death, so a dangling wait in the log tail means the run ended with the approval outstanding, + and a resume re-parks a block-mode one (or continues past it when handed the resolution) and + re-registers the approval key so an operator `approve` still resolves it. +- **`approval_resolved`**: `{ outcome, reason, trace_id?, by?, source? }` with `outcome` one of + `granted`/`denied`/`timeout`. `trace_id` names the wait it resolves (empty on older logs, + which resolved the one open wait); `by` and `source` record who and which source resolved it + when known. A grant is emitted at the iteration-head rescope drain (the single re-baseline site); a stop deliberately emits nothing (a stop doesn't resolve the ask). - **`recovery`**: `{ class, iter, detail }`, emitted once per `--resume` right after the resume note: how the resumed process classified its predecessor's end. `class` is one of @@ -738,6 +747,21 @@ Additive event kinds beyond the compat set include: evidence summary. Purely a record: the loop acts on the in-process classification, never by re-reading this line. +Two reader rules make the format additive rather than frozen: + +- **Unknown kinds.** A line whose `kind` this reader does not know decodes as `unknown` instead + of failing. Folds count it and classify from the events they do know; a `--resume` refuses + when an unknown line precedes no terminal `shutdown`, because it cannot know whether that + line opened a bracket. +- **Envelope timestamp.** The envelope may carry `ts` (unix seconds, `{"v":1,"ts":…,"kind":…}`). + Readers never require it; a writer that stamps it lets a fold account wall-clock spans such + as parked time without a process-side clock. + +**One fold, every consumer.** `crucible-contract::LoopState` is the session log folded into +one state: `apply` is exhaustive over every event kind, `classify` names how a dead run ended, +and `resume_view` is what `--resume` restores. The engine's resume and crash classification and +the controller's ingest read the same fold, so a new event kind is taught to one place. + **`RunIdentity`** (`crucible/src/identity.rs`) is the comparability key: two runs' scores are comparable only if it matches. It's a hash-of-hashes (`v1:`) over, per component (one unnamed entry for a single-domain run, one per `[[component]]` for a composite): `repo` From 509c98db793a6ca7f5eb2f829f0836806970e4a8 Mon Sep 17 00:00:00 2001 From: Will Eaton Date: Fri, 4 Sep 2026 10:06:46 -0400 Subject: [PATCH 02/11] Retire the wide tournament No domain pack sets [search], and every piece the wide round adds is a second copy of something the deep loop already has: a second runner over Row, a second phase to classify, a second filter in every fold, and a third TaskKind dispatch site. The event-sourced loop refactor would have to carry all of it. Delete WideConfig, WideOutcome, run_wide_tournament, wide_template, WideRunner, wide_propose, render_wide_prompt, the [search] manifest table and its validation, --wide and --wide-keep, and the engine.measure_diff operation. The died_in_wide_round classification arm goes with them; a plan admitted before any iteration phase is an open plan like any other. The wire stays readable: phase "wide" and died_in_wide_round remain legal tokens because old logs carry them, and readers keep such rows out of the deep loop's baseline and best as the resume fold always did. ADR-0024 records the decision and supersedes ADR-0010; RFC-0001 C-SEARCH is deprecated and the [search] rule leaves C-MANIFEST. Assisted-by: Claude --- README.md | 5 +- crucible-contract/src/session/fold.rs | 38 +- crucible/src/engine.rs | 2 +- crucible/src/flow.rs | 2 +- crucible/src/loop_driver.rs | 73 +- crucible/src/loop_graph.rs | 729 +----------------- crucible/src/main.rs | 17 +- crucible/src/manifest/mod.rs | 20 +- crucible/src/manifest/search.rs | 186 ----- crucible/src/manifest/workflow.rs | 1 - crucible/src/openshell/run.rs | 2 +- crucible/src/openshell/sandbox.rs | 2 +- crucible/src/plan/exec.rs | 2 +- crucible/src/plan/harness.rs | 2 +- crucible/src/plan/ir.rs | 9 +- crucible/src/plan/worktree.rs | 3 +- crucible/src/recovery.rs | 79 +- crucible/src/reporter.rs | 4 +- crucible/src/run.rs | 2 - docs/SUMMARY.md | 1 + .../0010-candidate-portfolios-and-search.md | 2 +- docs/adr/0028-retire-the-wide-tournament.md | 43 ++ docs/adr/index.md | 3 +- docs/crucible-contract.md | 26 +- docs/crucible.md | 15 +- docs/getting-started.md | 1 - docs/how-it-works.md | 4 +- docs/rfc/RFC-0001.md | 10 +- docs/task-lane.md | 2 +- docs/work-graphs.md | 9 +- examples/adversarial-review/README.md | 3 +- ...oit-search-over-reviewable-candidates.toml | 3 +- .../ADR-0024-retire-the-wide-tournament.toml | 47 ++ gov/rfc/RFC-0001/clauses/C-MANIFEST.toml | 2 - gov/rfc/RFC-0001/clauses/C-SEARCH.toml | 2 +- gov/rfc/RFC-0001/rfc.toml | 4 +- 36 files changed, 185 insertions(+), 1170 deletions(-) delete mode 100644 crucible/src/manifest/search.rs create mode 100644 docs/adr/0028-retire-the-wide-tournament.md create mode 100644 gov/adr/ADR-0024-retire-the-wide-tournament.toml diff --git a/README.md b/README.md index f3a8442d..a97ea987 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,6 @@ The main manifest sections are: | `[agent]` | yes | Configures the backend, harness, model, goal, prompt, environment, and sandbox. | | `[judge]` | no | Defines `measure_cmd`, score `direction`, and optional gate self-tests. Omitted entirely, the run is a task: every completed turn is kept, unscored (see [docs/task-lane.md](docs/task-lane.md)). | | `[world]` | no | Adds apply, snapshot, and restore commands for state outside Git. | -| `[search]` | no | Configures a parallel wide round before the iterative deep loop. | | `[workflow]` | no | Defines the task graph used by an iteration. | | `[deploy]` | no | Defines build and deployment values used by rendered cluster runs. | | `[build.]` | no | Defines a named image build target. | @@ -332,8 +331,8 @@ Running `crucible` without a subcommand starts an optimization loop and requires | `crucible rank-grounded` | Performs one read-only, code-grounded ranking turn over an existing checkout. | | `crucible build` | Executes a named build configuration and prints the resulting digest-pinned image reference. | -Common loop controls include `--iterations`, `--wide`, `--wide-keep`, `--max-cost`, -`--max-time`, `--ui`, `--resume`, and `--no-early-stop`. +Common loop controls include `--iterations`, `--max-cost`, `--max-time`, `--ui`, `--resume`, +and `--no-early-stop`. ## Repository layout diff --git a/crucible-contract/src/session/fold.rs b/crucible-contract/src/session/fold.rs index 5abfc0df..047a93f6 100644 --- a/crucible-contract/src/session/fold.rs +++ b/crucible-contract/src/session/fold.rs @@ -122,8 +122,6 @@ pub struct OpenPlan { pub declared: Vec, /// TaskResult names seen after this PlanAdmitted. pub resulted: Vec, - /// Admitted before the first iteration Phase. - pub wide: bool, } /// What the tail of the log says happened. @@ -136,11 +134,6 @@ pub enum Classification { }, /// No decided rows: nothing to resume from. DiedInBaseline, - /// Died before the first iteration Phase. - DiedInWideRound { - plan: Option, - turn: Option<(u32, TurnEvidence)>, - }, /// AgentStart with no AgentDone: the turn was in flight. DiedMidTurn { iter: u32, @@ -162,7 +155,6 @@ impl Classification { match self { Classification::CleanExit { .. } => RecoveryClass::CleanExit, Classification::DiedInBaseline => RecoveryClass::DiedInBaseline, - Classification::DiedInWideRound { .. } => RecoveryClass::DiedInWideRound, Classification::DiedMidTurn { .. } => RecoveryClass::DiedMidTurn, Classification::DiedDeciding { .. } => RecoveryClass::DiedDeciding, Classification::DiedInPlanTask { .. } => RecoveryClass::DiedInPlanTask, @@ -189,21 +181,6 @@ impl Classification { format!("previous run exited {}: {reason}", outcome.as_str()) } Classification::DiedInBaseline => "no decided rows in the log".to_string(), - Classification::DiedInWideRound { plan, turn } => { - let mut s = "died in the wide tournament".to_string(); - if let Some(p) = plan { - s.push_str(&format!( - ", plan v{} open ({}/{} tasks resulted)", - p.plan_version, - p.resulted.len(), - p.declared.len() - )); - } - if turn.is_some() { - s.push_str(", a turn was in flight"); - } - s - } Classification::DiedMidTurn { iter, evidence, @@ -238,8 +215,7 @@ impl Classification { s } Classification::DiedInPlanTask { iter, plan } => format!( - "{}plan v{} at iter {iter} never accounted ({}/{} tasks resulted)", - if plan.wide { "wide " } else { "" }, + "plan v{} at iter {iter} never accounted ({}/{} tasks resulted)", plan.plan_version, plan.resulted.len(), plan.declared.len() @@ -331,7 +307,6 @@ pub struct LoopState { /// Lines whose `kind` this reader does not know. pub unknown: u32, // Open brackets, exactly the facts the tail classifier needs. - saw_iteration_phase: bool, saw_any_row: bool, last_row_iter: u32, last_phase_iter: u32, @@ -365,7 +340,6 @@ impl LoopState { SessionEvent::Start { .. } => {} SessionEvent::Phase { phase, iter } => { if phase == "iteration" { - self.saw_iteration_phase = true; self.last_phase_iter = *iter; self.done_unrowed = None; } @@ -468,7 +442,6 @@ impl LoopState { plan_version: *plan_version, declared: tasks.iter().map(|t| t.name.clone()).collect(), resulted: Vec::new(), - wide: !self.saw_iteration_phase, }); } SessionEvent::AsksEmitted { .. } => {} @@ -564,12 +537,6 @@ impl LoopState { if !self.saw_any_row { return Classification::DiedInBaseline; } - if !self.saw_iteration_phase { - return Classification::DiedInWideRound { - plan: self.open_plan.clone(), - turn: self.open_turn.clone(), - }; - } if let Some((iter, evidence)) = &self.open_turn { return Classification::DiedMidTurn { iter: *iter, @@ -603,7 +570,8 @@ impl LoopState { } } - /// Decided deep-loop rows: neither a wide-round lane nor a never-started turn record. + /// Decided deep-loop rows: not a never-started turn record, and not a `wide` lane row from a + /// log written before the wide tournament was removed. pub fn deep_rows(&self) -> impl DoubleEndedIterator { self.rows .iter() diff --git a/crucible/src/engine.rs b/crucible/src/engine.rs index ad67d8d1..70865ec5 100644 --- a/crucible/src/engine.rs +++ b/crucible/src/engine.rs @@ -242,7 +242,7 @@ fn dispatch_parent() -> Option { /// standalone run (no controller, so no `TRACEPARENT`) gets the same span self-rooted. Returns /// `None` only when the engine's OTLP exporter is not installed. Enter the returned span for the /// life of the loop: the `openshell_turn` spans, created on the same thread, then nest under it -/// (wide-round turns run on their own threads and root themselves, no thread-local to inherit). +/// (isolated plan-task turns run on their own threads and root themselves, no thread-local to inherit). /// /// CONSUMER kind is recorded only in the dispatched case, where it pairs with the controller's /// PRODUCER span — the async producer/consumer edge a service-graph processor draws the diff --git a/crucible/src/flow.rs b/crucible/src/flow.rs index 522151df..3325c964 100644 --- a/crucible/src/flow.rs +++ b/crucible/src/flow.rs @@ -8,7 +8,7 @@ //! `flow.json`, so extraction and emission never mix. The session log alone covers //! decisions, scores, edits, rungs, budget and publish; the span export adds wall-clock //! (iteration windows, turn durations, the agent's per-tool-call timeline, rung -//! durations). Wide-round and infra rows are out of scope: this is the deep-loop +//! durations). Infra rows are out of scope: this is the deep-loop //! overview a human is walked through. use anyhow::Result; diff --git a/crucible/src/loop_driver.rs b/crucible/src/loop_driver.rs index 654bc589..00ba81ac 100644 --- a/crucible/src/loop_driver.rs +++ b/crucible/src/loop_driver.rs @@ -15,10 +15,6 @@ use crucible_contract::admission::AdmissionOutcome; use std::sync::atomic::Ordering; use std::time::{Duration, Instant}; -#[derive(Debug, thiserror::Error)] -#[error("winner produced no diff")] -struct WinnerProducedNoDiff; - #[derive(Debug, thiserror::Error)] #[error("baseline measurement invalid: {note}")] struct BaselineInvalid { @@ -582,7 +578,6 @@ fn run_loop_body( let heartbeat = heartbeat_handle.map(std::sync::Arc::as_ref); let started = Instant::now(); let start_iter: u32; - let is_resume = runtime.resume.is_some(); // Held across the whole body: an approved re-scope re-baselines, and re-measuring a rescoped // baseline without an agent turn is impossible for a codegen domain, so it reuses this same // preflight measurement. A resumed run whose baseline was never measured re-runs preflight @@ -815,65 +810,6 @@ fn run_loop_body( &run.segment.regime, ); - // Wide round: fan out N candidates before the deep loop, if configured. The winner's diff - // seeds the deep loop's workspace. Skipped on resume (the wide rows already live in the - // session log). - if !is_resume - && let Some(wide_cfg) = crate::loop_graph::WideConfig::resolve(args, args.search.as_ref()) - { - // The tournament runs as a work-graph template (parallel isolated proposes, - // serial diff scoring, engine top_k) on both loop paths. The winner diff travels - // as text: the candidate worktrees are removed before seed time, so re-deriving a - // diff from one silently yields nothing. - let result = crate::loop_graph::run_wide_tournament( - &wide_cfg, - args, - p, - prep, - r, - world, - judge, - run.segment.baseline_score, - )?; - let winner = result.winners.first().copied(); - let winner_diff = winner.and_then(|id| result.diffs.get(&id).cloned()); - let rows = result.rows; - - for row in &rows { - r.row(row, false); - run.rows.push(row.clone()); - } - write_results(p, &prep.goal, &prep.prior, &run.rows)?; - - if let Some(winner_id) = winner { - r.note(&format!( - "wide round complete: seeding deep loop with candidate {winner_id}" - )); - let applied = winner_diff - .filter(|d| !d.trim().is_empty()) - .ok_or_else(|| WinnerProducedNoDiff.into()) - .and_then(|d| crate::plan::worktree::apply(&p.workspace, &d)); - if let Err(e) = applied { - r.note(&format!( - "failed to apply winner diff: {e:#} — deep loop starts from baseline" - )); - } else { - // Snapshot the seeded state so the deep loop has a base to work from. - match world.snapshot("wide: winner applied") { - Ok(snap) => { - if let Some(sha) = world.commit_sha(&snap) { - run.kept_shas.push(sha); - } - run.segment.best_snap = Snapshot(snap); - } - Err(e) => r.note(&format!("snapshot after wide winner failed: {e:#}")), - } - } - } else { - r.note("wide round produced no winners — deep loop starts from baseline"); - } - } - // How this run ends. Each early exit sets it before breaking; a loop that runs out of // iterations leaves it `Finished`. One match below folds it into the `Outcome`. let mut exit = LoopExit::Finished; @@ -1593,10 +1529,10 @@ impl ResumeFold { use session::{IntoRow, SessionEvent}; match ev { SessionEvent::Row { row, solved } => { - // Wide-round rows (phase:"wide") are historical context only on resume; they - // must not count toward next_iter or influence the deep loop's baseline/best. // Infra-dead rows (phase:"infra") record turns that never started — their // iteration was never consumed, so counting them would skip it on resume. + // Wide-round rows (phase:"wide") come from logs written before the wide + // tournament was removed; they never counted toward the deep loop. if matches!(row.phase.as_deref(), Some("wide") | Some("infra")) { return; } @@ -2445,7 +2381,7 @@ mod tests { } #[test] - fn resume_filters_out_wide_phase_rows() { + fn resume_ignores_legacy_wide_phase_rows() { use session::{RowWire, SessionEvent, encode}; let mk_deep = |iter, decision: &str, score: f64| SessionEvent::Row { row: RowWire { @@ -3473,8 +3409,6 @@ mod tests { disclosure: None, output_bounds: None, iterations, - wide: 0, - wide_keep: 1, graph_loop: false, no_early_stop, ui: crate::Ui::Headless, @@ -3503,7 +3437,6 @@ mod tests { results_bucket: String::new(), pr_repo: String::new(), component_pr_repos: Vec::new(), - search: None, workflow: None, workflow_frozen_injects: Vec::new(), workflow_toolbox_exclude: Vec::new(), diff --git a/crucible/src/loop_graph.rs b/crucible/src/loop_graph.rs index d93e9478..e574eb15 100644 --- a/crucible/src/loop_graph.rs +++ b/crucible/src/loop_graph.rs @@ -7,9 +7,7 @@ use std::cell::RefCell; use std::collections::BTreeMap; -use std::path::{Path, PathBuf}; use std::rc::Rc; -use std::sync::atomic::Ordering; use std::time::Instant; use anyhow::{Context, Result}; @@ -21,12 +19,11 @@ use crate::plan::exec::{ Attempt, AttemptOutcome, BatchItem, ExecCfg, Substrate, TaskRunner, TaskStatus, execute, }; use crate::plan::ir::{ - Direction, EngineOp, Isolation, Join, Plan, PlanBudget, Stage, Task, TaskKind, TaskName, - ValidPlan, + EngineOp, Join, Plan, PlanBudget, Stage, Task, TaskKind, TaskName, ValidPlan, }; use crate::reporter::{Reporter, Row, TurnBudget}; use crate::session::{EvidenceDisposition, EvidenceEntry}; -use crate::{Args, Paths, Prepared, STOP, agent, control}; +use crate::{Args, Paths, agent, control}; use crucible::crucible::{Judge, MeasureCtx, Reading, World}; #[derive(Debug, thiserror::Error)] @@ -734,11 +731,7 @@ impl TaskRunner for LoopTaskRunner<'_, R> { source, .. } => self.decide(task, source.as_ref()), - TaskKind::Engine { - op: EngineOp::MeasureDiff, - .. - } - | TaskKind::TopK { .. } => fail( + TaskKind::TopK { .. } => fail( 0.0, format!( "unexpected task kind in the loop template: {}", @@ -785,549 +778,6 @@ fn fail(cost_usd: f64, note: String) -> Attempt { } } -// --------------------------------------------------------------------------- -// The wide tournament as a template: N isolated propose tasks fan out in -// parallel worktrees, their diffs are scored serially on the shared deployment -// by MeasureDiff tasks, and an engine top_k with a lossy join ranks whatever -// survived. The driver seeds the deep loop from the winner's diff text. -// --------------------------------------------------------------------------- - -/// The resolved wide-round config, merged from CLI flags + manifest `[search]`. CLI wins. -pub struct WideConfig { - pub n: u32, - pub k: u32, - pub approaches: Vec, -} - -impl WideConfig { - /// Merge CLI flags (`--wide`, `--wide-keep`) with the manifest's `[search]`. CLI wins. - pub fn resolve(args: &Args, search: Option<&crate::manifest::SearchCfg>) -> Option { - let n = if args.wide > 0 { - args.wide - } else { - search.map(|s| s.wide).unwrap_or(0) - }; - if n == 0 { - return None; - } - let k = if args.wide_keep > 0 && args.wide > 0 { - args.wide_keep - } else { - search.map(|s| s.policy_k).unwrap_or(1) - }; - let approaches = search.map(|s| s.approaches.clone()).unwrap_or_default(); - if approaches.len() < n as usize { - return None; - } - Some(WideConfig { n, k, approaches }) - } -} - -/// What the wide tournament left behind for the driver. -pub(crate) struct WideOutcome { - /// Winning candidate ids, best first. - pub winners: Vec, - /// Every candidate row (skip/fail/measured), for the session log. - pub rows: Vec, - /// Candidate id → captured diff text, for seeding the deep loop. Only winners are - /// resolved here. - pub diffs: BTreeMap, -} - -#[allow(clippy::too_many_arguments)] -pub(crate) fn run_wide_tournament( - cfg: &WideConfig, - args: &Args, - p: &Paths, - prep: &Prepared, - r: &mut R, - world: &dyn World, - judge: &dyn Judge, - baseline_score: f64, -) -> Result { - r.note(&format!( - "wide round: fanning out {} candidates (top-{} advance)", - cfg.n, cfg.k - )); - let wide_dir = p.state.join("wide").join(&prep.run_id); - std::fs::create_dir_all(&wide_dir) - .with_context(|| format!("creating wide-round dir {}", wide_dir.display()))?; - - let direction = match judge.direction() { - crate::command_judge::Direction::Lower => Direction::Lower, - crate::command_judge::Direction::Higher => Direction::Higher, - }; - let plan = wide_template(cfg, prep, direction)?; - r.plan_event(&crate::plan::cli::plan_admitted_event(&plan)); - r.note("wide: starting parallel PROPOSE turns"); - let snap = world - .snapshot("wide-pre-measure") - .context("wide pre-measure snapshot")?; - - let mut runner = WideRunner { - args, - p, - world, - judge, - r, - wide_dir, - baseline_score, - approaches: (0..cfg.n) - .map(|id| (id, cfg.approaches[id as usize].clone())) - .collect(), - snap, - measure_note_emitted: false, - rows: Vec::new(), - fatal: None, - }; - let mut task_events = Vec::new(); - let outcome = execute( - &plan, - &Substrate::default(), - ExecCfg::default(), - &mut runner, - |task, result| { - task_events.push(crate::plan::cli::task_result_event( - plan.plan().version, - 0, - task, - result, - )); - }, - ); - for ev in &task_events { - runner.r.plan_event(ev); - } - if let Some(e) = runner.fatal.take() { - return Err(e); - } - - let mut winners: Vec = Vec::new(); - if let Some(kept) = outcome - .results - .get(&"pick".into()) - .and_then(|r| r.output.as_ref()) - .and_then(|o| o.get("kept")) - .and_then(Value::as_array) - { - for (rank, entry) in kept.iter().enumerate() { - let id = entry - .get("task") - .and_then(Value::as_str) - .and_then(|n| n.rsplit('-').next()) - .and_then(|s| s.parse::().ok()); - let score = entry.get("score").and_then(Value::as_f64); - if let (Some(id), Some(score)) = (id, score) { - runner.r.note(&format!( - "wide round: rank {} = candidate {id} (score: {score:.2})", - rank + 1 - )); - winners.push(id); - } - } - } - if winners.is_empty() { - runner - .r - .note("wide round: no candidates scored, deep loop starts from baseline"); - } - - let diffs: BTreeMap = winners - .iter() - .filter_map(|id| { - outcome - .results - .get(&TaskName(format!("propose-{id}"))) - .and_then(|r| r.output.as_ref()) - .and_then(|o| o.get("diff")) - .and_then(Value::as_str) - .map(|d| (*id, d.to_string())) - }) - .collect(); - Ok(WideOutcome { - winners, - rows: runner.rows, - diffs, - }) -} - -/// The wide tournament template: `propose-{i} (isolated, advisory) → measure-{i} -/// (advisory) → pick (top_k over whatever passed)`. Everything is advisory: a dead -/// candidate never gates the tournament, matching the loop's skip/fail rows. The budget -/// stays driver-owned, like the iteration template's. -fn wide_template(cfg: &WideConfig, prep: &Prepared, direction: Direction) -> Result { - let mut tasks: Vec = Vec::new(); - for id in 0..cfg.n { - tasks.push(Task { - name: TaskName(format!("propose-{id}")), - task: TaskKind::Agent { - prompt: render_wide_prompt( - &prep.template, - &prep.goal, - &cfg.approaches[id as usize], - ), - harness: None, - model: None, - effort: None, - }, - depends_on: vec![], - session: None, - needs: "any".to_string(), - required: false, - isolation: Some(Isolation::Worktree), - join: Join::All, - stage: Stage::Iteration, - emits: Vec::new(), - emits_files: Vec::new(), - over: None, - max_fanout: None, - }); - } - for id in 0..cfg.n { - tasks.push(Task { - name: TaskName(format!("measure-{id}")), - task: TaskKind::Engine { - op: EngineOp::MeasureDiff, - source: None, - tiebreak: None, - }, - depends_on: vec![TaskName(format!("propose-{id}"))], - session: None, - needs: "any".to_string(), - required: false, - isolation: None, - join: Join::All, - stage: Stage::Iteration, - emits: Vec::new(), - emits_files: Vec::new(), - over: None, - max_fanout: None, - }); - } - tasks.push(Task { - name: "pick".into(), - task: TaskKind::TopK { - k: cfg.k, - direction, - }, - depends_on: (0..cfg.n) - .map(|id| TaskName(format!("measure-{id}"))) - .collect(), - session: None, - needs: "any".to_string(), - required: false, - isolation: None, - join: Join::Passed, - stage: Stage::Iteration, - emits: Vec::new(), - emits_files: Vec::new(), - over: None, - max_fanout: None, - }); - Plan { - version: 1, - reason: None, - budget: PlanBudget { usd: f64::MAX }, - tasks, - } - .validate() - .context("building the wide tournament template") -} - -/// [`TaskRunner`] for the wide template. Propose batches run threaded (one worktree per -/// candidate); MeasureDiff serializes on the main workspace. -struct WideRunner<'a, R: Reporter> { - args: &'a Args, - p: &'a Paths, - world: &'a dyn World, - judge: &'a dyn Judge, - r: &'a mut R, - wide_dir: PathBuf, - baseline_score: f64, - approaches: BTreeMap, - /// The pre-measure rollback token every scoring pass restores to. - snap: String, - measure_note_emitted: bool, - rows: Vec, - /// A failed world restore leaves the workspace in an unknown state; abort the run. - fatal: Option, -} - -impl WideRunner<'_, R> { - fn restore_or_fatal(&mut self) -> bool { - match self.world.restore(&self.snap) { - Ok(()) => true, - Err(e) => { - self.fatal = Some(e.context("restoring after a wide candidate measure")); - false - } - } - } - - fn measure_diff(&mut self, task: &Task, inputs: &BTreeMap) -> Attempt { - if !self.measure_note_emitted { - self.r.note("wide: measuring candidates serially"); - self.measure_note_emitted = true; - } - if STOP.load(Ordering::SeqCst) { - return fail(0.0, "stop requested".to_string()); - } - let Some(id) = candidate_id(&task.name) else { - return fail(0.0, format!("task {} has no candidate id", task.name)); - }; - let approach = self.approaches.get(&id).cloned().unwrap_or_default(); - let diff = inputs - .values() - .next() - .and_then(|v| v.get("diff")) - .and_then(Value::as_str) - .unwrap_or_default() - .to_string(); - if diff.trim().is_empty() { - self.r - .note(&format!("wide candidate {id}: no diff produced, skipping")); - self.rows.push(Row { - iter: 0, - decision: "wide-skip".into(), - note: format!("candidate {id} ({approach}) produced no diff"), - phase: Some("wide".into()), - ..Default::default() - }); - return fail(0.0, "no diff produced".to_string()); - } - self.r.note(&format!( - "wide candidate {id}: measuring (approach: {approach})" - )); - if let Err(e) = crate::plan::worktree::apply(&self.p.workspace, &diff) { - self.r - .note(&format!("wide candidate {id}: apply failed: {e:#}")); - if !self.restore_or_fatal() { - return fail(0.0, "restore failed".to_string()); - } - self.rows.push(Row { - iter: 0, - decision: "wide-fail".into(), - note: format!("candidate {id} apply failed: {e:#}"), - phase: Some("wide".into()), - ..Default::default() - }); - return fail(0.0, format!("apply failed: {e:#}")); - } - if let Err(e) = self.world.apply() { - self.r - .note(&format!("wide candidate {id}: world apply failed: {e:#}")); - if !self.restore_or_fatal() { - return fail(0.0, "restore failed".to_string()); - } - self.rows.push(Row { - iter: 0, - decision: "wide-fail".into(), - note: format!("candidate {id} world apply failed: {e:#}"), - phase: Some("wide".into()), - ..Default::default() - }); - return fail(0.0, format!("world apply failed: {e:#}")); - } - let ctx = MeasureCtx { - baseline_score: Some(self.baseline_score), - baseline_total: None, - best_score: Some(self.baseline_score), - }; - match self.judge.measure(&ctx) { - Ok(reading) => { - let (diff_text, diffstat) = self.world.staged_diff(); - let decision = if self.judge.decide(&reading, self.baseline_score, None).keep { - "wide-keep" - } else { - "wide-discard" - }; - let row = Row { - iter: 0, - decision: decision.into(), - note: format!("[wide candidate {id}] {approach} — {}", reading.note), - detail: self.judge.detail(&reading), - diff: diff_text, - diffstat, - score: reading.score, - tiebreak: reading.tiebreak, - total: reading.detail.get("total").and_then(Value::as_u64), - phase: Some("wide".into()), - kept_snap: None, - evidence: Vec::new(), - candidate_md: String::new(), - }; - self.r.row(&row, false); - self.rows.push(row); - if !self.restore_or_fatal() { - return fail(0.0, "restore failed".to_string()); - } - match reading.score { - Some(s) if s.is_finite() => pass(serde_json::json!({ "score": s })), - // The reducer needs a finite number, so a scoreless candidate - // does not rank. - _ => fail(0.0, "no finite score to rank".to_string()), - } - } - Err(e) => { - self.r - .note(&format!("wide candidate {id}: measure failed: {e:#}")); - self.rows.push(Row { - iter: 0, - decision: "wide-fail".into(), - note: format!("candidate {id} measure failed: {e:#}"), - phase: Some("wide".into()), - ..Default::default() - }); - if !self.restore_or_fatal() { - return fail(0.0, "restore failed".to_string()); - } - fail(0.0, format!("measure failed: {e:#}")) - } - } - } -} - -impl TaskRunner for WideRunner<'_, R> { - fn run(&mut self, task: &Task, _attempt: u32, inputs: &BTreeMap) -> Attempt { - match &task.task { - TaskKind::Agent { prompt, .. } => { - // A batch of one (wide n=1) lands here instead of run_many. - let Some(id) = candidate_id(&task.name) else { - return fail(0.0, format!("task {} has no candidate id", task.name)); - }; - let pending = match crate::plan::worktree::capture_diff(&self.p.workspace) { - Ok(p) => p, - Err(e) => { - return fail(0.0, format!("capturing the workspace state failed: {e:#}")); - } - }; - wide_propose( - self.args, - &self.p.workspace, - &self.wide_dir, - self.p.skills.clone(), - id, - prompt, - &pending, - ) - } - TaskKind::Engine { - op: EngineOp::MeasureDiff, - .. - } => self.measure_diff(task, inputs), - other => fail( - 0.0, - format!( - "unexpected task kind in the wide template: {}", - other.label() - ), - ), - } - } - - fn run_many(&mut self, batch: &[BatchItem<'_>]) -> Vec { - let workspace = self.p.workspace.clone(); - let wide_dir = self.wide_dir.clone(); - let skills = self.p.skills.clone(); - let args = self.args; - // Every candidate clones the same workspace, so its pending state is captured once - // here: concurrent `git add -A` in one repo races on `.git/index.lock`. - let pending = match crate::plan::worktree::capture_diff(&workspace) { - Ok(p) => p, - Err(e) => { - let note = format!("capturing the workspace state failed: {e:#}"); - return batch.iter().map(|_| fail(0.0, note.clone())).collect(); - } - }; - std::thread::scope(|s| { - let handles: Vec<_> = batch - .iter() - .map(|b| { - let workspace = workspace.clone(); - let wide_dir = wide_dir.clone(); - let skills = skills.clone(); - let args = args.clone(); - let pending = pending.as_str(); - let parsed = match (&b.task.task, candidate_id(&b.task.name)) { - (TaskKind::Agent { prompt, .. }, Some(id)) => Some((id, prompt.clone())), - _ => None, - }; - s.spawn(move || match parsed { - Some((id, prompt)) => { - wide_propose(&args, &workspace, &wide_dir, skills, id, &prompt, pending) - } - None => fail(0.0, "non-agent task in a propose batch".to_string()), - }) - }) - .collect(); - handles - .into_iter() - .map(|h| { - h.join() - .unwrap_or_else(|_| fail(0.0, "propose thread panicked".to_string())) - }) - .collect() - }) - } -} - -/// One candidate's PROPOSE turn in a private worktree: clone, run the turn, capture the -/// staged diff as the task output, clean up. The diff text is the only thing that leaves -/// the worktree. Wide turns are not streamed and their cost is not booked against the run -/// budget. -fn wide_propose( - args: &Args, - workspace: &Path, - wide_dir: &Path, - skills: Option, - id: u32, - prompt: &str, - pending: &str, -) -> Attempt { - if STOP.load(Ordering::SeqCst) { - return pass(serde_json::json!({ "diff": "" })); - } - let worktree = wide_dir.join(format!("candidate-{id}")); - if let Err(e) = crate::plan::worktree::setup(workspace, &worktree, pending) { - return fail(0.0, format!("worktree setup failed: {e:#}")); - } - let cand_paths = Paths::for_worktree(worktree.clone(), skills); - let _ = std::fs::create_dir_all(&cand_paths.state); - let turn = agent::run_turn(args, &cand_paths, prompt, false, |_line, _stream, _ev| {}); - if let Some(failure) = turn.failure() { - let _ = std::fs::remove_dir_all(&worktree); - return fail(turn.cost_usd, failure.to_string()); - } - let diff = crate::plan::worktree::capture_diff(&worktree); - let _ = std::fs::remove_dir_all(&worktree); - match diff { - Ok(diff) => pass(serde_json::json!({ "diff": diff })), - Err(e) => fail(0.0, format!("capturing the candidate diff failed: {e:#}")), - } -} - -/// The numeric suffix of `propose-{id}` / `measure-{id}` task names. -fn candidate_id(name: &TaskName) -> Option { - name.0.rsplit('-').next().and_then(|s| s.parse().ok()) -} - -/// Build the wide-round prompt. The approach biases the candidate; the template provides -/// the domain's structure. The status is omitted (no prior score in the wide round). -fn render_wide_prompt(template: &str, goal: &str, approach: &str) -> String { - let status = "no prior score (wide round, first attempt)"; - let out = template - .replace("{{GOAL}}", goal.trim()) - .replace("{{STATUS}}", status) - .replace("{{BEST_SCORE}}", status) - .replace("{{STEER}}", ""); - format!( - "## Approach constraint (MANDATORY)\n\ - You MUST use this specific approach: {approach}\n\ - Implement a minimal, working version of this approach. Do not deviate.\n\n\ - {out}" - ) -} - #[cfg(test)] mod tests { use super::*; @@ -1421,7 +871,6 @@ mod tests { true, 2, BUMP, - false, Some( "\n[[workflow.task]]\nname = \"review\"\nkind = \"command\"\ncommand = \"exit 1\"\n", ), @@ -1548,7 +997,7 @@ mod tests { source = "grade" depends_on = ["grade"] "#; - let trace = run_counter_cfg(true, 2, BUMP, false, Some(workflow)); + let trace = run_counter_cfg(true, 2, BUMP, Some(workflow)); assert_eq!(trace.best, 3.0, "{}", describe(&trace)); assert_eq!( trace @@ -1623,7 +1072,7 @@ mod tests { source = "grade" depends_on = ["grade"] "#; - let trace = run_counter_cfg(true, 1, BUMP, false, Some(workflow)); + let trace = run_counter_cfg(true, 1, BUMP, Some(workflow)); assert_eq!( trace .rows @@ -1712,7 +1161,7 @@ mod tests { source = "grade" depends_on = ["grade"] "#; - let trace = run_counter_cfg(true, 1, BUMP, false, Some(workflow)); + let trace = run_counter_cfg(true, 1, BUMP, Some(workflow)); assert_eq!( trace .rows @@ -1774,14 +1223,13 @@ mod tests { /// path. sh stands in for nu so the fixture is self-contained. The measure declares /// solved at value >= 3, so a multi-iteration run exercises the early stop. fn run_counter(graph_loop: bool, iterations: u32, bump: &str) -> RunTrace { - run_counter_cfg(graph_loop, iterations, bump, false, None) + run_counter_cfg(graph_loop, iterations, bump, None) } fn run_counter_cfg( graph_loop: bool, iterations: u32, bump: &str, - wide: bool, workflow: Option<&str>, ) -> RunTrace { // A counter, not a timestamp: two tests starting in the same microsecond would get @@ -1813,11 +1261,6 @@ mod tests { } let manifest_path = dir.join("crucible.toml"); let workflow_block = workflow.unwrap_or(""); - let search_block = if wide { - "\n[search]\nwide = 3\napproaches = [\"plus one\", \"add a unit\", \"increment\"]\npolicy_k = 1\n" - } else { - "" - }; std::fs::write( &manifest_path, format!( @@ -1835,7 +1278,7 @@ mod tests { measure_cmd = "./measure.sh" direction = "higher" objective = "value" - {search_block}{workflow_block}"# + {workflow_block}"# ), ) .unwrap(); @@ -1857,7 +1300,6 @@ mod tests { args.agent_cmd = m.agent.agent_cmd.clone(); args.iterations = iterations; args.graph_loop = graph_loop; - args.search = m.search.clone(); args.workflow = m.workflow.clone(); let prep = Prepared { @@ -1997,8 +1439,8 @@ mod tests { legacy.shutdown, graph.shutdown, "shutdown outcome must match" ); - // Both traces may carry additive plan lines (the wide tournament is graph-shaped - // on both loop paths); the parity claim is about everything else. + // Both traces may carry additive plan lines; the parity claim is about everything + // else. let strip = |t: &RunTrace| -> Vec { t.kinds .iter() @@ -2032,65 +1474,6 @@ mod tests { ); } - /// The wide tournament as a template: parallel isolated proposes, serial - /// diff scoring, top_k, winner seed: produces the same rows, notes order, seed - /// state, and event sequence under both paths, then the deep loop runs on - /// top of the seeded workspace under both paths. - #[test] - fn counter_parity_wide_tournament() { - let legacy = run_counter_cfg(false, 2, BUMP, true, None); - let graph = run_counter_cfg(true, 2, BUMP, true, None); - - // Shape pinned on the legacy path first: 3 measured candidates at value 2 (each - // row appears twice on the wire: once at measure time, once in the driver's - // fold), then the seeded deep loop solves at 3 on its first iteration. - let decisions: Vec<&str> = legacy.rows.iter().map(|(_, d, _)| d.as_str()).collect(); - assert_eq!( - decisions, - [ - "baseline", - "wide-keep", - "wide-keep", - "wide-keep", // measure-time emissions - "wide-keep", - "wide-keep", - "wide-keep", // driver re-emissions - "keep", // deep iteration on the seeded workspace - ] - ); - let scores: Vec> = legacy.rows.iter().map(|(_, _, s)| *s).collect(); - assert_eq!( - scores, - [ - Some(1.0), - Some(2.0), - Some(2.0), - Some(2.0), - Some(2.0), - Some(2.0), - Some(2.0), - Some(3.0) - ], - "the winner seed must actually land: the deep iteration starts from 2" - ); - assert_eq!(legacy.shutdown, "solved"); - - assert_parity(&legacy, &graph); - assert_eq!( - graph.kinds.iter().filter(|k| *k == "plan_admitted").count(), - 2, - "one admitted plan for the tournament, one for the deep round" - ); - assert_eq!( - graph.kinds.iter().filter(|k| *k == "task_result").count(), - 7 + 4, - "3 proposes + 3 measures + pick, then the 4 deep-round tasks" - ); - } - - /// The full round loop under the graph path: keep, a regression that - /// exercises discard/restore, then the solve that stops the run early (3 of 5 - /// budgeted iterations): identical to the typestate path. #[test] fn counter_parity_multi_iteration_with_discard_restore_and_early_stop() { let legacy = run_counter(false, 5, ZIGZAG); @@ -2125,96 +1508,4 @@ mod tests { "task_result lines carry their loop round" ); } - - fn default_args() -> crate::Args { - crate::Cli::parse_from(["crucible"]).run - } - - fn search_cfg(wide: u32, approaches: &[&str], k: u32) -> crate::manifest::SearchCfg { - crate::manifest::SearchCfg { - wide, - approaches: approaches.iter().map(|s| s.to_string()).collect(), - policy: "top-k".into(), - policy_k: k, - } - } - - #[test] - fn wide_config_resolve_none_when_no_wide() { - assert!(WideConfig::resolve(&default_args(), None).is_none()); - } - - #[test] - fn wide_config_resolve_from_cli() { - let search = search_cfg(3, &["a", "b", "c"], 1); - let args = crate::Args { - wide: 3, - wide_keep: 2, - ..default_args() - }; - let cfg = WideConfig::resolve(&args, Some(&search)).unwrap(); - assert_eq!(cfg.n, 3); - assert_eq!(cfg.k, 2); - } - - #[test] - fn wide_config_resolve_from_manifest() { - let search = search_cfg(4, &["a", "b", "c", "d"], 2); - let cfg = WideConfig::resolve(&default_args(), Some(&search)).unwrap(); - assert_eq!(cfg.n, 4); - assert_eq!(cfg.k, 2); - } - - #[test] - fn wide_config_cli_wide_wins_over_manifest_wide() { - let search = search_cfg(5, &["a", "b", "c"], 1); - let args = crate::Args { - wide: 3, - wide_keep: 1, - ..default_args() - }; - assert_eq!(WideConfig::resolve(&args, Some(&search)).unwrap().n, 3); - } - - #[test] - fn wide_config_none_when_too_few_approaches() { - let search = search_cfg(3, &["a"], 1); - let args = crate::Args { - wide: 3, - wide_keep: 1, - ..default_args() - }; - assert!(WideConfig::resolve(&args, Some(&search)).is_none()); - } - - #[test] - fn wide_prompt_includes_approach_constraint() { - let prompt = render_wide_prompt( - "goal={{GOAL}} status={{BEST_SCORE}}", - "lower p99", - "metrics-scrape approach", - ); - assert!(prompt.contains("metrics-scrape approach")); - assert!(prompt.contains("MUST use this specific approach")); - assert!(prompt.contains("goal=lower p99")); - } - - #[test] - fn wide_prompt_replaces_steer_placeholder() { - let prompt = render_wide_prompt( - "{{GOAL}} {{STATUS}} steer:{{STEER}}", - "improve throughput", - "batch-all", - ); - assert!(prompt.contains("improve throughput")); - assert!(prompt.contains("steer:")); - assert!(!prompt.contains("{{STEER}}")); - } - - #[test] - fn wide_prompt_without_steer_placeholder() { - let prompt = render_wide_prompt("goal={{GOAL}}", "my goal", "approach-x"); - assert!(prompt.contains("goal=my goal")); - assert!(!prompt.contains("{{STEER}}")); - } } diff --git a/crucible/src/main.rs b/crucible/src/main.rs index 935dd211..e196085f 100644 --- a/crucible/src/main.rs +++ b/crucible/src/main.rs @@ -104,8 +104,8 @@ pub(crate) fn kill_pid(pid: i32) { } } -/// Registry of live agent-child PIDs so Ctrl+C can kill ALL concurrent children (wide-round -/// parallel agents and the serial deep-loop agent alike). +/// Registry of live agent-child PIDs so Ctrl+C can kill ALL concurrent children (parallel +/// isolated plan tasks and the serial deep-loop agent alike). pub(crate) mod pid_registry { use std::sync::Mutex; @@ -641,16 +641,6 @@ pub(crate) struct Args { /// Max agent iterations. #[arg(long, default_value_t = 3)] pub iterations: u32, - /// Wide-round breadth: fan out N independent candidates in parallel before the deep loop. - /// Each candidate gets one PROPOSE turn biased to a distinct `[search].approaches` entry, - /// measured serially, ranked by the gate. The winner seeds the deep loop. 0 = no wide round - /// (pure deep, the default). Overrides `[search].wide`. - #[arg(long, default_value_t = 0)] - pub wide: u32, - /// How many wide-round winners seed a deep loop (top-K by score). Default 1. Only - /// meaningful when `--wide > 0`. Overrides `[search].policy_k`. - #[arg(long, default_value_t = 1)] - pub wide_keep: u32, /// Run each iteration as a canonical work-graph plan (propose → apply → measure → decide) /// through the shared plan executor instead of the hand-sequenced stages. Same events, /// same decisions (parity-gated), plus additive plan lines on the session log. @@ -780,9 +770,6 @@ pub(crate) struct Args { /// each `embed` match lands in the PR body and the S3 run record. No CLI flag. #[arg(skip)] pub artifacts: Vec, - /// Wide-round search config (from `[search]`). No CLI flag, set by `run_from_manifest`. - #[arg(skip)] - pub search: Option, /// Manifest-only authored workflow. #[arg(skip)] pub workflow: Option, diff --git a/crucible/src/manifest/mod.rs b/crucible/src/manifest/mod.rs index ff3feacc..cdca0dfa 100644 --- a/crucible/src/manifest/mod.rs +++ b/crucible/src/manifest/mod.rs @@ -12,7 +12,6 @@ mod openshell; pub mod outputs; mod preflight; mod relay; -mod search; mod secret; mod selftest; mod wiring; @@ -29,7 +28,6 @@ pub use openshell::OpenshellCfg; pub use outputs::OutputsCfg; pub use preflight::{MODE_PLACEHOLDER, PreflightCfg}; pub use relay::RelayFile; -pub use search::SearchCfg; pub use secret::{SecretDecl, SecretError, SecretKind}; pub use selftest::SelftestCfg; pub use workflow::{KEPT_INPUT, WorkflowCaps, WorkflowCfg, WorkflowError, WorkflowType}; @@ -258,9 +256,6 @@ pub struct Manifest { /// rebuilds an image leaves it unset. #[serde(default)] pub deploy: Option, - /// Wide-round search config. Optional, most domains run pure-deep. - #[serde(default)] - pub search: Option, /// Authored iteration graph; absent uses the default workflow. #[serde(default)] pub workflow: Option, @@ -359,8 +354,8 @@ pub struct Workspace { /// the discard's `reset --hard` reverts tracked content regardless. Carry pipeline output the /// repo doesn't track. /// - A path that doesn't exist yet is fine; the exclude is prospective. - /// - Wide-tournament worktrees are fresh checkouts with no untracked carried dirs, so the wide - /// round doesn't benefit. + /// - Isolated plan-task worktrees are fresh checkouts with no untracked carried dirs, so + /// they don't benefit. /// - Composite manifests reject the key: per-component carry-forward is a non-goal. #[serde(default)] pub carry_forward: Vec, @@ -745,7 +740,6 @@ struct CommonCfg<'a> { workspace: &'a Workspace, agent: &'a AgentCfg, judge: Option<&'a JudgeCfg>, - search: &'a Option, workflow: &'a Option, build: &'a BTreeMap, preflight: &'a Option, @@ -762,7 +756,6 @@ fn validate_common(c: CommonCfg<'_>) -> Result<()> { validate_carry_forward(&c.workspace.carry_forward)?; validate_artifacts(&c.workspace.artifact)?; validate_codex_api_key(c.agent.codex.api_key.as_deref())?; - search::validate_search(c.search)?; if let Some(w) = c.workflow { w.validate()?; } @@ -785,7 +778,6 @@ fn validate_common(c: CommonCfg<'_>) -> Result<()> { .as_ref() .is_some_and(|w| w.workflow_type == WorkflowType::Playbook); for (present, table) in [ - (c.search.is_some(), "[search]"), (c.workflow.is_some() && !playbook, "[workflow]"), (c.preflight.is_some(), "[preflight]"), ] { @@ -910,7 +902,6 @@ impl Manifest { workspace: &self.workspace, agent: &self.agent, judge: self.judge.as_ref(), - search: &self.search, workflow: &self.workflow, build: &self.build, preflight: &self.preflight, @@ -1051,9 +1042,6 @@ pub struct CompositeManifest { /// without forking the base domain manifest. #[serde(default)] pub deploy: BTreeMap, - /// Wide-round search config. Optional. - #[serde(default)] - pub search: Option, /// Authored iteration graph; absent uses the default workflow. #[serde(default)] pub workflow: Option, @@ -1182,7 +1170,6 @@ impl CompositeManifest { workspace: &self.workspace, agent: &self.agent, judge: Some(&self.judge), - search: &self.search, workflow: &self.workflow, build: &self.build, preflight: &self.preflight, @@ -1363,9 +1350,8 @@ mod tests { } #[test] - fn task_lane_rejects_search_workflow_and_preflight() { + fn task_lane_rejects_workflow_and_preflight() { for table in [ - "[search]\nwide = 2\napproaches = [\"a\", \"b\"]", "[workflow]\ntype = \"custom\"\nresult = \"t\"\n[[workflow.task]]\nname = \"t\"\nkind = \"evaluate\"\ncommand = \"true\"", "[preflight]\ncommands = [\"true\"]", ] { diff --git a/crucible/src/manifest/search.rs b/crucible/src/manifest/search.rs deleted file mode 100644 index a8d05db3..00000000 --- a/crucible/src/manifest/search.rs +++ /dev/null @@ -1,186 +0,0 @@ -use serde::Deserialize; - -/// Why a `[search]` block is unusable. Checked at manifest load, before any wide round spends. -#[derive(Debug, thiserror::Error, PartialEq)] -pub enum SearchError { - #[error( - "[search].approaches needs at least {wide} entries (one per wide candidate), got {got} \ - — diversity must be engineered, not random" - )] - TooFewApproaches { wide: u32, got: usize }, - #[error("[search].policy must be \"top-k\" (the only v1 policy), got {got:?}")] - UnknownPolicy { got: String }, - #[error("[search].policy_k must be in 1..={wide} (wide), got {got}")] - PolicyKOutOfRange { wide: u32, got: u32 }, -} - -/// Wide-round search config: how many candidates, which approaches, which tournament policy. -/// Required `approaches` when `wide > 0`, no auto-generated fallback. -#[derive(Deserialize, Clone)] -#[serde(deny_unknown_fields)] -pub struct SearchCfg { - /// Fan-out breadth: N independent candidates per wide round. 0 or absent = no wide round. - #[serde(default)] - pub wide: u32, - /// Distinct approach descriptions, one per candidate slot. REQUIRED when `wide > 0`: diversity - /// must be engineered, not random. - #[serde(default)] - pub approaches: Vec, - /// Tournament policy name. v1 ships only `"top-k"`. - #[serde(default = "default_search_policy")] - pub policy: String, - /// K for the `top-k` policy (how many wide-round winners seed a deep loop). - #[serde(default = "default_policy_k")] - pub policy_k: u32, -} - -pub fn validate_search(search: &Option) -> Result<(), SearchError> { - let Some(s) = search else { return Ok(()) }; - if s.wide == 0 { - return Ok(()); - } - if s.approaches.len() < s.wide as usize { - return Err(SearchError::TooFewApproaches { - wide: s.wide, - got: s.approaches.len(), - }); - } - if s.policy != "top-k" { - return Err(SearchError::UnknownPolicy { - got: s.policy.clone(), - }); - } - if s.policy_k == 0 || s.policy_k > s.wide { - return Err(SearchError::PolicyKOutOfRange { - wide: s.wide, - got: s.policy_k, - }); - } - Ok(()) -} - -fn default_search_policy() -> String { - "top-k".to_string() -} -fn default_policy_k() -> u32 { - 1 -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::manifest::Manifest; - - #[test] - fn search_cfg_valid() { - let s = Some(SearchCfg { - wide: 3, - approaches: vec!["a".into(), "b".into(), "c".into()], - policy: "top-k".into(), - policy_k: 1, - }); - assert!(validate_search(&s).is_ok()); - } - - #[test] - fn search_cfg_zero_wide_always_valid() { - let s = Some(SearchCfg { - wide: 0, - approaches: vec![], - policy: "anything".into(), - policy_k: 0, - }); - assert!(validate_search(&s).is_ok()); - } - - #[test] - fn search_cfg_none_is_valid() { - assert!(validate_search(&None).is_ok()); - } - - #[test] - fn search_cfg_too_few_approaches() { - let s = Some(SearchCfg { - wide: 3, - approaches: vec!["a".into(), "b".into()], - policy: "top-k".into(), - policy_k: 1, - }); - assert_eq!( - validate_search(&s).unwrap_err(), - SearchError::TooFewApproaches { wide: 3, got: 2 } - ); - } - - #[test] - fn search_cfg_bad_policy() { - let s = Some(SearchCfg { - wide: 2, - approaches: vec!["a".into(), "b".into()], - policy: "round-robin".into(), - policy_k: 1, - }); - let err = validate_search(&s).unwrap_err(); - assert_eq!( - err, - SearchError::UnknownPolicy { - got: "round-robin".to_owned() - } - ); - } - - #[test] - fn search_cfg_k_zero() { - let s = Some(SearchCfg { - wide: 2, - approaches: vec!["a".into(), "b".into()], - policy: "top-k".into(), - policy_k: 0, - }); - assert!(validate_search(&s).is_err()); - } - - #[test] - fn search_cfg_k_exceeds_wide() { - let s = Some(SearchCfg { - wide: 2, - approaches: vec!["a".into(), "b".into()], - policy: "top-k".into(), - policy_k: 3, - }); - assert!(validate_search(&s).is_err()); - } - - #[test] - fn search_cfg_parses_from_toml() { - let toml = r#" - [repo] - path = "." - [workspace] - dir = "ws" - [agent] - backend = "command" - agent_cmd = "true" - [judge] - measure_cmd = "echo 42" - direction = "lower" - - [search] - wide = 3 - approaches = ["cache optimization", "algorithm swap", "parallelism"] - policy = "top-k" - policy_k = 2 - "#; - let dir = std::env::temp_dir().join("crucible-test-search-toml"); - let _ = std::fs::create_dir_all(&dir); - let path = dir.join("crucible.toml"); - std::fs::write(&path, toml).unwrap(); - let m = Manifest::load_frozen(&path).unwrap(); - let s = m.search.unwrap(); - assert_eq!(s.wide, 3); - assert_eq!(s.approaches.len(), 3); - assert_eq!(s.policy, "top-k"); - assert_eq!(s.policy_k, 2); - let _ = std::fs::remove_dir_all(&dir); - } -} diff --git a/crucible/src/manifest/workflow.rs b/crucible/src/manifest/workflow.rs index 2f205ff1..259de06c 100644 --- a/crucible/src/manifest/workflow.rs +++ b/crucible/src/manifest/workflow.rs @@ -56,7 +56,6 @@ impl EngineOp { EngineOp::Measure => "engine.measure", EngineOp::Grade => "engine.grade", EngineOp::Decide => "engine.decide", - EngineOp::MeasureDiff => "engine.measure_diff", } } } diff --git a/crucible/src/openshell/run.rs b/crucible/src/openshell/run.rs index 570420f4..cfe41f8e 100644 --- a/crucible/src/openshell/run.rs +++ b/crucible/src/openshell/run.rs @@ -9,7 +9,7 @@ //! (blocked-connection attempts are turn telemetry) → download the workspace and private session back → delete //! the sandbox (per-turn-fresh, so a discarded iteration leaves no residue). //! -//! The sandbox name is derived per (process, workspace), so parallel wide-round candidates and parallel +//! The sandbox name is derived per (process, workspace), so parallel isolated plan tasks and parallel //! crucible processes sharing one gateway never collide on a fixed name. use crate::agent::{self, TurnFailure, TurnOutcome}; diff --git a/crucible/src/openshell/sandbox.rs b/crucible/src/openshell/sandbox.rs index 65df1903..1a9c7cd7 100644 --- a/crucible/src/openshell/sandbox.rs +++ b/crucible/src/openshell/sandbox.rs @@ -79,7 +79,7 @@ mod tests { use std::path::Path; let a = name_for(Path::new("/state/worktrees/lane-a")); let b = name_for(Path::new("/state/worktrees/lane-b")); - assert_ne!(a, b, "wide-round lanes get distinct sandboxes"); + assert_ne!(a, b, "isolated lanes get distinct sandboxes"); assert_eq!( a, name_for(Path::new("/state/worktrees/lane-a")), diff --git a/crucible/src/plan/exec.rs b/crucible/src/plan/exec.rs index 09a1d734..254a3a87 100644 --- a/crucible/src/plan/exec.rs +++ b/crucible/src/plan/exec.rs @@ -2101,7 +2101,7 @@ mod tests { #[test] fn passed_join_folds_only_passing_dependencies() { - // Wide's reducer shape: one candidate fails, the reducer still ranks the rest. + // A lossy reducer: one candidate fails, the reducer still ranks the rest. let mut tasks = vec![ task("m-ok", &[], "any", false), task("m-bad", &[], "any", false), diff --git a/crucible/src/plan/harness.rs b/crucible/src/plan/harness.rs index 339f3a04..a00c5129 100644 --- a/crucible/src/plan/harness.rs +++ b/crucible/src/plan/harness.rs @@ -387,7 +387,7 @@ fn run_task( }; // A private clone of the workspace. Its edits are discarded on cleanup: what leaves an // isolated task is its declared output, so this is for review/analysis work, not for - // coding tasks whose diff has to survive (the wide tournament carries those out itself). + // coding tasks whose diff has to survive. let root = paths.state.join("plan-iso"); if let Err(e) = std::fs::create_dir_all(&root) { return transport(format!("creating the isolation root failed: {e}")); diff --git a/crucible/src/plan/ir.rs b/crucible/src/plan/ir.rs index 2faf6124..cbf83879 100644 --- a/crucible/src/plan/ir.rs +++ b/crucible/src/plan/ir.rs @@ -74,10 +74,6 @@ pub enum EngineOp { Grade, /// `Judge::decide`: rule keep/discard against the run's best. Decide, - /// The wide tournament's scoring stage: apply an upstream candidate diff to the main - /// workspace, `World::apply`, measure with the frozen judge, restore. Serialized by - /// construction (never isolation-marked), because candidates share one deployment. - MeasureDiff, } /// Where a task executes. Authorable (`isolation = "worktree"`); a runner that cannot @@ -129,8 +125,8 @@ pub enum Join { #[default] All, /// Dispatch once every dependency is terminal, folding only the passing outputs: a - /// reducer over a lossy fan-out (the wide `top_k`: skipped/failed candidates just - /// don't rank), or a join over reviewers where one being advisory must not stop the run. + /// reducer over a lossy fan-out (a `top_k` where skipped/failed candidates just don't + /// rank), or a join over reviewers where one being advisory must not stop the run. Passed, /// Dispatch once every dependency is terminal whatever it settled as, unless the run has /// halted, forwarding each one as an entry carrying its status, note, output, and whether @@ -214,7 +210,6 @@ impl TaskKind { EngineOp::Measure => "engine_measure", EngineOp::Grade => "engine_grade", EngineOp::Decide => "engine_decide", - EngineOp::MeasureDiff => "engine_measure_diff", }, } } diff --git a/crucible/src/plan/worktree.rs b/crucible/src/plan/worktree.rs index 9e9e66a0..281ec313 100644 --- a/crucible/src/plan/worktree.rs +++ b/crucible/src/plan/worktree.rs @@ -2,8 +2,7 @@ //! carry work out of one. An isolated task's edits never touch the shared workspace: what //! leaves is its structured output (and, where the runner asks for it, a captured diff). //! -//! Used by the wide tournament's parallel proposers and by any plan task marked -//! `isolation = "worktree"`. +//! Used by any plan task marked `isolation = "worktree"`. use std::path::Path; diff --git a/crucible/src/recovery.rs b/crucible/src/recovery.rs index e60cc8b9..8566406d 100644 --- a/crucible/src/recovery.rs +++ b/crucible/src/recovery.rs @@ -55,8 +55,6 @@ pub(crate) struct OpenPlan { pub declared: Vec, /// TaskResult names seen after this PlanAdmitted. pub resulted: Vec, - /// Admitted before the first iteration Phase (the wide tournament). - pub wide: bool, } /// What the tail of the log says happened. @@ -69,11 +67,6 @@ pub(crate) enum Classification { }, /// Start (maybe Phase baseline) but no decided rows: nothing to resume from. DiedInBaseline, - /// Died before the deep loop's first iteration, inside the wide tournament. - DiedInWideRound { - plan: Option, - turn: Option<(u32, TurnEvidence)>, - }, /// AgentStart with no AgentDone: the turn was in flight. DiedMidTurn { iter: u32, @@ -105,7 +98,6 @@ impl Classification { match self { Classification::CleanExit { .. } => RecoveryClass::CleanExit, Classification::DiedInBaseline => RecoveryClass::DiedInBaseline, - Classification::DiedInWideRound { .. } => RecoveryClass::DiedInWideRound, Classification::DiedMidTurn { .. } => RecoveryClass::DiedMidTurn, Classification::DiedDeciding { .. } => RecoveryClass::DiedDeciding, Classification::DiedInPlanTask { .. } => RecoveryClass::DiedInPlanTask, @@ -132,21 +124,6 @@ impl Classification { format!("previous run exited {}: {reason}", outcome.as_str()) } Classification::DiedInBaseline => "no decided rows in the log".to_string(), - Classification::DiedInWideRound { plan, turn } => { - let mut s = "died in the wide tournament".to_string(); - if let Some(p) = plan { - s.push_str(&format!( - ", plan v{} open ({}/{} tasks resulted)", - p.plan_version, - p.resulted.len(), - p.declared.len() - )); - } - if turn.is_some() { - s.push_str(", a turn was in flight"); - } - s - } Classification::DiedMidTurn { iter, evidence, @@ -181,8 +158,7 @@ impl Classification { s } Classification::DiedInPlanTask { iter, plan } => format!( - "{}plan v{} at iter {iter} never accounted ({}/{} tasks resulted)", - if plan.wide { "wide " } else { "" }, + "plan v{} at iter {iter} never accounted ({}/{} tasks resulted)", plan.plan_version, plan.resulted.len(), plan.declared.len() @@ -205,8 +181,7 @@ struct TailScan { /// Only a TRAILING Shutdown counts: a resumed process appends past its /// predecessor's Shutdown, so any later event clears it. shutdown: Option<(String, String)>, - saw_iteration_phase: bool, - /// Any decided (non-wide, non-infra) row. + /// Any decided (non-infra) row. saw_any_row: bool, last_row_iter: u32, last_phase_iter: u32, @@ -233,7 +208,6 @@ impl TailScan { } SessionEvent::Phase { phase, iter } => { if phase == "iteration" { - self.saw_iteration_phase = true; self.last_phase_iter = *iter; // Safety net for paths that account an iteration without a Row. self.done_unrowed = None; @@ -307,7 +281,6 @@ impl TailScan { plan_version: *plan_version, declared: tasks.iter().map(|t| t.name.clone()).collect(), resulted: Vec::new(), - wide: !self.saw_iteration_phase, }); } SessionEvent::TaskResult { task, .. } => { @@ -348,11 +321,6 @@ impl TailScan { } } else if !self.saw_any_row { Classification::DiedInBaseline - } else if !self.saw_iteration_phase { - Classification::DiedInWideRound { - plan: self.open_plan, - turn: self.open_turn, - } } else if let Some((iter, evidence)) = self.open_turn { Classification::DiedMidTurn { iter, @@ -878,7 +846,6 @@ mod tests { assert_eq!(plan.plan_version, 7); assert_eq!(plan.declared, vec!["propose", "measure"]); assert!(plan.resulted.is_empty()); - assert!(!plan.wide); } other => panic!("expected DiedInPlanTask, got {other:?}"), } @@ -905,25 +872,35 @@ mod tests { ); } + /// A plan admitted before any iteration phase (a log from before the wide tournament was + /// removed) is an open plan like any other. #[test] - fn pre_iteration_plan_classifies_died_in_wide_round() { - // A wide tournament plan admitted before any iteration Phase. - let events = vec![ - row(0, "baseline", 240.0), - SessionEvent::PlanAdmitted { - plan_version: 1, - reason: String::new(), - budget_usd: 5.0, - tasks: vec![], - }, - ]; - let got = classify("wide", &events); + fn pre_iteration_plan_classifies_died_in_plan_task() { + let mut events = vec![row(0, "baseline", 240.0)]; + events.push(SessionEvent::PlanAdmitted { + plan_version: 1, + reason: String::new(), + budget_usd: 1.0, + tasks: vec![PlanTaskWire { + name: "propose-0".into(), + kind: "agent".into(), + depends_on: vec![], + session: String::new(), + needs: "any".into(), + required: true, + join: "all".into(), + stage: "iteration".into(), + over: String::new(), + max_fanout: 0, + }], + }); + let got = classify("pre-iteration-plan", &events); match &got.classification { - Classification::DiedInWideRound { plan, turn } => { - assert!(plan.as_ref().is_some_and(|p| p.wide)); - assert!(turn.is_none()); + Classification::DiedInPlanTask { iter, plan } => { + assert_eq!(*iter, 0, "no iteration phase was seen"); + assert_eq!(plan.declared, vec!["propose-0".to_string()]); } - other => panic!("expected DiedInWideRound, got {other:?}"), + other => panic!("expected DiedInPlanTask, got {other:?}"), } } diff --git a/crucible/src/reporter.rs b/crucible/src/reporter.rs index 75ad934b..7b644cb6 100644 --- a/crucible/src/reporter.rs +++ b/crucible/src/reporter.rs @@ -64,8 +64,8 @@ pub struct Row { pub tiebreak: Option, /// Total test count for this row (test gate), for the same reason. pub total: Option, - /// `Some("wide")` for wide-round rows, `Some("infra")` for never-started turn - /// records; `None` for the deep (default) loop. + /// `Some("infra")` for never-started turn records; `None` for the deep (default) loop. + /// Logs written before the wide tournament was removed carry `Some("wide")`. pub phase: Option, /// The World snapshot token committed when this row was kept (a git world packs the /// commit sha). Carried on the wire so a resume can restore the kept-best tree instead diff --git a/crucible/src/run.rs b/crucible/src/run.rs index 1ae06cdb..f9e85495 100644 --- a/crucible/src/run.rs +++ b/crucible/src/run.rs @@ -678,7 +678,6 @@ fn run_from_manifest(args: Args) -> Result<()> { .filter(|(_, _, frozen)| *frozen) .map(|(src, dst, _)| (src, dst)) .collect(); - args.search = m.search.clone(); args.workflow = m.workflow.clone(); args.workflow_frozen_injects = m.frozen_inject_pairs(&manifest_dir)?; args.workflow_toolbox_exclude = m.agent.toolbox_exclude.clone(); @@ -793,7 +792,6 @@ fn run_composite(args: Args, manifest_path: PathBuf) -> Result<()> { seed_diff: read_seed_diff(&manifest_dir, m.agent.seed_diff.as_deref())?, }; - args.search = m.search.clone(); args.workflow = m.workflow.clone(); args.workflow_frozen_injects = Vec::new(); args.workflow_toolbox_exclude = m.agent.toolbox_exclude.clone(); diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 6a247bd7..e4697413 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -53,3 +53,4 @@ - [ADR 0024: Admission ledger for external inputs](./adr/0024-admission-ledger.md) - [ADR 0025: Durable tool steps for broker builds and measures](./adr/0025-durable-tool-steps.md) - [ADR 0026: The no-judge task lane](./adr/0026-no-judge-task-lane.md) + - [ADR 0028: Retire the wide tournament](./adr/0028-retire-the-wide-tournament.md) diff --git a/docs/adr/0010-candidate-portfolios-and-search.md b/docs/adr/0010-candidate-portfolios-and-search.md index 04c1b085..420d7b95 100644 --- a/docs/adr/0010-candidate-portfolios-and-search.md +++ b/docs/adr/0010-candidate-portfolios-and-search.md @@ -1,6 +1,6 @@ # ADR 0010: Candidate portfolios — explore/exploit search over reviewable candidates -**Status:** Accepted; implemented (v1, 2026-07-02: parallel propose turns + serialized measurement + top-k ranking behind `--wide N` / `[search]`; since re-implemented as a [work-graph](../work-graphs.md) template compiled from `[search]`, with the same `[search]` surface; round-robin and successive-halving policies remain future impls) +**Status:** Superseded by [ADR 0028](./0028-retire-the-wide-tournament.md) (2026-09-04). Was implemented (v1, 2026-07-02: parallel propose turns + serialized measurement + top-k ranking behind `--wide N` / `[search]`; later re-implemented as a [work-graph](../work-graphs.md) template compiled from `[search]`). The wide round, `[search]`, and `--wide` no longer exist; breadth is authored as a fan-out in the work graph. **Date:** 2026-06-28 **Related:** [ADR-0004](./0004-core-loop-state-model.md) (the core loop is a *sequential* refinement of one line; this generalizes it to a population), [ADR-0009](./0009-composite-domains.md) (a composite candidate diff --git a/docs/adr/0028-retire-the-wide-tournament.md b/docs/adr/0028-retire-the-wide-tournament.md new file mode 100644 index 00000000..6d421215 --- /dev/null +++ b/docs/adr/0028-retire-the-wide-tournament.md @@ -0,0 +1,43 @@ +# ADR 0028: Retire the wide tournament + +**Status:** Accepted; implemented (2026-09-04). Governance source: `gov/adr/ADR-0024`. +**Date:** 2026-09-04 +**Related:** [ADR-0010](./0010-candidate-portfolios-and-search.md) (superseded), [ADR-0004](./0004-core-loop-state-model.md) +(the deep loop that remains), [work graphs](../work-graphs.md) (where breadth now lives) + +## Context + +ADR-0010 added a wide round before the deep loop: N parallel propose turns in per-candidate +worktrees, serial diff scoring on the shared deployment, a top-k fold, and the winner's diff +seeding the deep loop. It shipped as a work-graph template with its own manifest table +(`[search]`), two CLI flags (`--wide`, `--wide-keep`), a row phase (`"wide"`), a crash class +(`died_in_wide_round`), and a second row-accounting filter in the resume fold. + +No domain pack sets `[search]`, and the loop is being refactored into one event-sourced state +machine. Every piece the wide round adds is a second copy of something the deep loop already +has: a second runner over `Row`, a second phase to classify, a second filter in every fold, and +a third `TaskKind` dispatch site. Each would have to be carried through the refactor and taught +to the approval-gate work that follows it. + +## Decision + +Delete the wide tournament: the runner, the template, the `[search]` table and its validation, +the `--wide` and `--wide-keep` flags, and the `engine.measure_diff` operation. The +`died_in_wide_round` arm goes with them; a plan admitted before any iteration phase now +classifies as an open plan like any other. + +The wire stays readable. `phase: "wide"` remains a legal row value and `died_in_wide_round` +remains a decodable recovery-class token, because logs written before this decision carry +them; readers keep such rows out of the deep loop's baseline and best, exactly as the resume +fold did. RFC-0001 C-SEARCH is deprecated and the `[search]` rule leaves C-MANIFEST. + +Breadth, when a run wants it, is authored inside the work graph: isolated plan tasks fanned out +over a list and a `top_k` reducer express the same search without a separate pre-loop stage. + +## Consequences + +- One runner over `Row`, one phase vocabulary, one row filter. The event-sourced refactor and + the approval gates have one fewer shape to carry. +- A run that wants breadth authors it in `workflow.star`; no shipped template does that yet. +- Old session logs with `"wide"` rows render those rows as legacy context rather than as a + named stage. diff --git a/docs/adr/index.md b/docs/adr/index.md index 33c5e593..45c6e605 100644 --- a/docs/adr/index.md +++ b/docs/adr/index.md @@ -17,7 +17,7 @@ Expand this section in the sidebar to browse the full list. | [0007](./0007-isolation-preflight.md) | Isolation pre-flight (the metric that misframed #1109) | Accepted (process) | | [0008](./0008-domains-as-immutable-composes.md) | Domains as immutable composes (the rpm-ostree model) | Partially implemented | | [0009](./0009-composite-domains.md) | Composite domains (combined multi-component autoresearch) | Implemented | -| [0010](./0010-candidate-portfolios-and-search.md) | Candidate portfolios — explore/exploit search | Implemented (v1) | +| [0010](./0010-candidate-portfolios-and-search.md) | Candidate portfolios — explore/exploit search | Superseded by 0028 | | [0012](./0012-rendered-deployments.md) | Crucible-rendered deployments, generating the loop/broker/deployment manifests | Implemented | | [0014](./0014-scoping-pipeline.md) | Scoping as a governed pipeline, `crucible scope ` | Partially implemented | | [0017](./0017-turn-result-contract.md) | Turn result contract, structured state back from turn pods | Implemented | @@ -30,3 +30,4 @@ Expand this section in the sidebar to browse the full list. | [0025](./0025-durable-tool-steps.md) | Durable tool steps for broker builds and measures | Implemented | | [0026](./0026-no-judge-task-lane.md) | The no-judge task lane | Implemented | | [0027](./0027-measurement-sessions.md) | Measurement sessions, one warm engine and many observations | Proposed | +| [0028](./0028-retire-the-wide-tournament.md) | Retire the wide tournament | Implemented | diff --git a/docs/crucible-contract.md b/docs/crucible-contract.md index 20c947af..26d0d9f1 100644 --- a/docs/crucible-contract.md +++ b/docs/crucible-contract.md @@ -76,11 +76,6 @@ apply_cmd = "..." # optional. §3. snapshot_cmd = "..." # optional. §3. restore_cmd = "..." # optional. §3. -[search] # optional (ADR-0010). Absent or wide=0 → pure-deep, the default. -wide = 0 # u32. N parallel propose turns before the deep loop. default 0 -approaches = ["...", "..."] # REQUIRED when wide > 0: one distinct approach string per candidate slot -policy = "top-k" # only v1 policy. default "top-k" -policy_k = 1 # how many wide-round winners seed the deep loop. default 1, must be in 1..=wide ``` Rules: @@ -89,8 +84,6 @@ Rules: which still owns git memory and layers the given commands on top. - `[judge.selftest]`, if present, requires both `good_cmd` and `bad_cmd` (a self-test that only stages one side isn't a control); `runs` must be `>= 1`. -- `[search]`, if present with `wide > 0`, requires `approaches.len() >= wide` (hard error, - diversity is engineered, not auto-generated) and `policy_k` in `1..=wide`. - Unknown keys are an error (typo protection), not silently ignored. - **Frozen loading (`load_frozen`).** When the manifest file lives *inside* the workspace it targets (the BYO on-ramp: `crucible init` scaffolds `[repo] path = "."`), the engine parses it @@ -116,18 +109,7 @@ The workspace is restored to pristine on every exit path (pass, fail, or error). no `[judge.selftest]` isn't an error, `crucible check` warns instead, since the gate hasn't been proven to discriminate. -### 1.2 Wide-round search (`[search]`, ADR-0010) - -`[search].wide > 0` (or `--wide N` on the CLI, which overrides the manifest) fans out `N` -independent PROPOSE turns in per-candidate git worktrees under the state dir before the deep -loop starts, one turn per `approaches` entry biased into its prompt. Each candidate's diff is -applied (cherry-picked) into the shared main workspace and measured serially there (measurement -never runs concurrently, only proposal does). The scored set is ranked by `[search].policy` (v1: -`"top-k"`); the `policy_k` (or `--wide-keep K`) winner(s) seed the deep loop, which then runs as -normal. Session rows from the wide round carry an additive `phase: "wide"` field (§7) so a -consumer can tell a wide-round row from a deep-loop row without a wire-shape change. - -### 1.3 Scope-authored workflows (`workflow.star`) +### 1.2 Scope-authored workflows (`workflow.star`) A scoped pack may include `workflow.star` beside `crucible.toml`. It is authoring syntax, not a runtime interpreter: scope compiles it to the existing `[[workflow.task]]` manifest IR before @@ -701,9 +683,9 @@ free-text label like `"score"`/`"bench"`, not the deleted enum). This keeps `--r remote viewer, and already-published S3 runs loading. Do **not** rename the wire key. A `row` event's wire record (`RowWire`) carries an additive, optional `phase` field -(`"wide"` for a wide-round candidate row, absent for a deep-loop row). It's `skip_serializing_if -= "Option::is_none"`, so a deep-only run's wire bytes are unchanged from before wide rounds -existed. +(`"infra"` for a never-started turn record, absent for a deep-loop row). It's `skip_serializing_if += "Option::is_none"`. Logs written before the wide tournament was removed carry `"wide"` rows; +readers keep them out of the deep loop's baseline and best. Additive event kinds beyond the compat set include: diff --git a/docs/crucible.md b/docs/crucible.md index 4c7db743..8155b7b9 100644 --- a/docs/crucible.md +++ b/docs/crucible.md @@ -47,11 +47,9 @@ flowchart LR - **accept?**: keep if it's strictly better; otherwise restore the last good state. - **remember**: kept states are git commits; every step is an NDJSON event. -This single-candidate line of descent is the default, and it's not the only shape: a -**wide round** ([ADR 0010](./adr/0010-candidate-portfolios-and-search.md)) can fan `N` -independent propose turns out in parallel first, biased to distinct `[search].approaches`, -rank them by the same Judge, and seed the loop above with the winner. `--wide 0` (the -default) skips it entirely, no engine change, just a pre-loop step. +This single-candidate line of descent is the loop. Breadth, when a run wants it, is an +authored fan-out inside the work graph (isolated plan tasks and a `top_k` reducer), not a +separate pre-loop stage. ## The contract: framework owns it, the repo owns the implementations @@ -93,8 +91,8 @@ Limits worth knowing before you use it: - A path that doesn't exist yet is fine; the exclude is prospective. - Nested paths (`target/codegen-out`) work: the clean descends into the parent instead of deleting it. Entries must be plain relative paths, so no `.`, `..`, or leading `/`. -- Wide-tournament rounds run in fresh worktrees with no untracked carried dirs, so they don't - benefit. +- Isolated plan-task worktrees are fresh checkouts with no untracked carried dirs, so they + don't benefit. - Composite manifests reject the key; per-component carry-forward is a non-goal. Omitting it is the default and keeps today's fresh-start behavior exactly, which is what a @@ -161,7 +159,6 @@ Any language works the same way, because the engine reads only the JSON verdict. | **Profiler** | generic profile-over-MCP: pprof for a Go service, GPU traces for a model server | `crucible-broker::profile` (ADR-0006) | | **Build + deploy** | engine-side build, and `crucible deploy render` projects the loop/deployment manifests, digest-pinned | `forge` + `crucible/src/deploy/` (ADR-0005 / 0012) | | **Publish** | publish-on-keep to S3 + a draft PR per fork; authorized review comments re-steer the run | `publish.rs` / `crucible-broker::draft_pr` | -| **Search (wide round)** | optional fan-out of N parallel propose turns, ranked by the same Judge, before the deep loop ([ADR 0010](./adr/0010-candidate-portfolios-and-search.md)) | `crucible/src/wide/` | | **Self-test** | `crucible check` proves the Judge can tell a known-good config from a known-bad one before a run trusts it | `[judge.selftest]` + `selftest.rs` | | **On-ramp** | `crucible init` scaffolds a manifest + measure stub onto an existing repo; `crucible check` validates it with no agent turn; `crucible scope` ingests a goal and freezes a `SCOPE.md` (ADR-0014 S0) | `init.rs` / `check.rs` / `scope.rs` | | **Preflight** | runs the domain's rung ladder against the unmodified tree before iteration 1; a failure refuses the run, and the optional baseline rung seeds `segment.baseline_score` | `[preflight]` + `preflight.rs` | @@ -302,7 +299,7 @@ On the wire the run is the normal shape with `gate: "task"` as the discriminator no-op). - The trust boundary is unchanged: the agent still holds no credentials. Output lands as a draft PR; privileged write actions (merging, closing issues) stay broker-tool material. -- Composites still require `[judge]`; so do `[search]`, `[workflow]`, and `[preflight]`, +- Composites still require `[judge]`; so do `[workflow]` and `[preflight]`, which a task manifest rejects at load. The scope pipeline still rejects judge-less proposed packs, and a scored judge may not claim `objective = "task"`. diff --git a/docs/getting-started.md b/docs/getting-started.md index 1684af59..2b32940b 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -121,7 +121,6 @@ parser rejects unknown keys, so typos fail loudly): | `[judge]` | The frozen objective. | `measure_cmd`, `direction` (`lower`/`higher`), `[judge.selftest]` (`good_cmd`/`bad_cmd`) | | `[world]` | Reversibility beyond git. | `apply_cmd`, `snapshot_cmd`, `restore_cmd` (omit all three → pure `GitWorld`) | | `[deploy]` | Image rebuild targets (deploy domains). | `[deploy.buildah]` `registry`/`dockerfile`, `[deploy.env]` | -| `[search]` | Wide-round fan-out before the deep loop. | `wide`, `approaches` (required when `wide > 0`), `policy_k` | | `[composite]` | Assemble N domains into one run. | `[composite].name`, `[[component]]` `domain`/`pr_repo` | The single most important rule: **omit `[world]` entirely and you get `GitWorld`** (git is the diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 0cbf81f3..789de5b4 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -8,8 +8,7 @@ privileged operations live host-side behind a mediated broker the agent can only ```mermaid flowchart TD issue["GitHub issue / Jira ticket"] -->|scope --issue| goal["Run goal
(frozen objective)"] - goal --> wide["wide round (optional)
N parallel propose → rank → winner seeds"] - wide --> propose + goal --> propose subgraph control["Control plane (operator, human-in-the-loop)"] direction LR @@ -61,7 +60,6 @@ flowchart TD | Stage | What happens | Deeper | | --- | --- | --- | | **Goal** | A GitHub issue or Jira ticket becomes a frozen run objective. The objective never moves once the run starts. | [ADR 0001](./adr/0001-adaptive-harness.md) | -| **wide round** | Optional: `--wide N` fans out N independent propose turns (one per `[search].approaches` entry) in parallel, ranks them by the gate, and the winner seeds the deep loop below. 0 (default) skips straight to propose. | [ADR 0010](./adr/0010-candidate-portfolios-and-search.md) | | **propose** | The agent reads the history and edits the world toward the goal. The proposal policy is a pluggable backend (`local` / `openshell` / `command`), not the engine. | [What crucible is](./crucible.md) | | **apply** | Make the candidate live. For a code repo the edits *are* the apply; for a deploy domain it builds + sets the image. | [ADR 0005](./adr/0005-engine-side-builds.md), [ADR 0012](./adr/0012-rendered-deployments.md) | | **measure** | The frozen judge scores the candidate once. The agent is handed a `World`, never the `Judge`, so it can't tune the test it's graded on. | [ADR 0001](./adr/0001-adaptive-harness.md) | diff --git a/docs/rfc/RFC-0001.md b/docs/rfc/RFC-0001.md index 5ac6e5fc..f286738d 100644 --- a/docs/rfc/RFC-0001.md +++ b/docs/rfc/RFC-0001.md @@ -1,5 +1,5 @@ - + # RFC-0001: Crucible implementation contract @@ -28,8 +28,6 @@ Required fields: [repo], when present, must contain exactly one of url or path. [judge.selftest], if present, must require both good_cmd and bad_cmd; runs must be >= 1. -[search], if present with wide > 0, must require approaches.len() >= wide and policy_k in 1..=wide. - Unknown manifest keys must be rejected as errors. Frozen loading: when the manifest lives inside the workspace it targets, the engine must parse it from the workspace's pristine base commit, not the current working tree. Before any base commit exists (the very first run), the engine must hard-warn and trust the working tree for that run only. @@ -52,7 +50,9 @@ A manifest with no [judge.selftest] must not be an error; crucible check must wa *Since: v0.1.0* -### [RFC-0001:C-SEARCH] Wide-round search protocol (Normative) +### [RFC-0001:C-SEARCH] Wide-round search protocol (Normative) + +> **Status:** deprecated [search].wide > 0 (or --wide N on the CLI, which overrides the manifest) must fan out N independent PROPOSE turns in per-candidate git worktrees before the deep loop starts, one turn per approaches entry biased into its prompt. @@ -303,6 +303,8 @@ Admit the playbook lane - the declared lane scopes the constructor namespace, so a playbook cannot name or be offered a scored constructor - session log gains asks_emitted, so what a run proposed is auditable apart from what an orchestrator admitted - skill joins the enumerated constructors: an agent task whose prompt is assembled from a shipped SKILL.md and the invocation's arguments +- the wide tournament is retired (ADR-0024): C-SEARCH is deprecated, [search] leaves C-MANIFEST, and phase: "wide" is a legacy row value readers keep out of the deep loop +- engine.measure_diff leaves the engine operations, since only the wide template produced it ### v0.1.0 (2026-08-19) diff --git a/docs/task-lane.md b/docs/task-lane.md index e4ab297e..099d57f9 100644 --- a/docs/task-lane.md +++ b/docs/task-lane.md @@ -58,7 +58,7 @@ there is no baseline, no score, no discard. The run exits 0 when the iterations | What does exit 0 mean? | The run completed. It does not certify the chore succeeded — read the rows or the PR. | | What's on the wire? | `Start.gate == "task"` is the discriminator; an iter-0 `baseline-skipped` row, then `keep` rows with `score: null`. | | Can a composite be a task? | No. Composites exist to combine scored components; `[judge]` stays required there. | -| What else is off the table? | `[search]`, `[workflow]`, and `[preflight]` — all three need scores, so a task manifest rejects them at load. `objective = "task"` is reserved on scored judges. | +| What else is off the table? | `[workflow]` and `[preflight]` — both need scores, so a task manifest rejects them at load. `objective = "task"` is reserved on scored judges. | | Scheduled runs? | Render the pod with `crucible deploy render` (a playbook launch adds `--playbook --max-time `) and drive it from any scheduler (a Kubernetes CronJob works today); native controller scheduling is planned. | ## The runnable reference diff --git a/docs/work-graphs.md b/docs/work-graphs.md index 12192c60..ab19ad91 100644 --- a/docs/work-graphs.md +++ b/docs/work-graphs.md @@ -4,8 +4,8 @@ A **plan** is a versioned DAG of **tasks** that a deterministic executor runs. T turns, plan-authored commands, or engine-owned reducers. The executor owns advancement: a task never decides what runs next. -Today, the engine supplies a default loop graph and a wide-tournament template, and a human or -pack can supply TOML or JSON through the `plan` CLI. Workflow admission separates authorable +Today, the engine supplies a default loop graph, and a human or pack can supply TOML or JSON +through the `plan` CLI. Workflow admission separates authorable topology from authority: an orchestrator must advertise the workflow type and engine operations it can safely execute. @@ -76,7 +76,7 @@ depends_on = ["measure"] | `command` | A plan-authored command returning JSON on its last stdout line. | | `evaluate` | A measurement command. `pass = false` vetoes; paired `threshold` + `direction` grade numeric `score`. | | `top_k` | Engine-owned reducer: keep the `k` best inputs by their `score` field. Needs at least one dependency. | -| `engine` | Capability-owned operation (`propose`, `apply`, `measure`, `grade`, `decide`, or `measure_diff`). Only an admitting orchestrator can execute it. | +| `engine` | Capability-owned operation (`propose`, `apply`, `measure`, `grade`, or `decide`). Only an admitting orchestrator can execute it. | Serialization is not authority. A workflow may author and sequence engine nodes, but admission requires matching capabilities such as `workflow.autoresearch`, `engine.apply`, and @@ -224,8 +224,7 @@ task can run; workflow capabilities control what the orchestrator is authorized Same decisions and same session events as the default path, plus additive `plan_admitted` and `task_result` lines. Cross-round state, keep/discard, and every between-round control (parking, -steering, re-scoping, budget) stay with the driver. The wide round runs as a template compiled -from `[search]` on both paths. +steering, re-scoping, budget) stay with the driver. The templates carry no budget of their own: the run budget is the driver's, checked between rounds, so a turn that overruns the cap is still measured and decided. diff --git a/examples/adversarial-review/README.md b/examples/adversarial-review/README.md index 8f0f914f..8b712b87 100644 --- a/examples/adversarial-review/README.md +++ b/examples/adversarial-review/README.md @@ -122,8 +122,7 @@ Two reviewers concurrently: 25.5s wall, against 36s for a single review. ## Limits -- Isolated plan tasks discard workspace edits. Carrying edits out of an isolated task is - the wide tournament's path, not this one. +- Isolated plan tasks discard workspace edits; what leaves one is its declared output. - `plan run` reports cost per task but no token counts (`TaskResult.metric`/`secs` are unset). - `harness` and `model` are per-task, but the shipped harnesses (`claude`, `hermes`) both serve Anthropic models over Vertex. Cross-vendor panels need a harness that does not exist. diff --git a/gov/adr/ADR-0010-candidate-portfolios-explore-exploit-search-over-reviewable-candidates.toml b/gov/adr/ADR-0010-candidate-portfolios-explore-exploit-search-over-reviewable-candidates.toml index 537f997e..8bb340b2 100644 --- a/gov/adr/ADR-0010-candidate-portfolios-explore-exploit-search-over-reviewable-candidates.toml +++ b/gov/adr/ADR-0010-candidate-portfolios-explore-exploit-search-over-reviewable-candidates.toml @@ -3,8 +3,9 @@ [govctl] id = "ADR-0010" title = "Candidate portfolios — explore/exploit search over reviewable candidates" -status = "accepted" +status = "superseded" date = "2026-06-28" +superseded_by = "ADR-0024" refs = [ "ADR-0001", "ADR-0004", diff --git a/gov/adr/ADR-0024-retire-the-wide-tournament.toml b/gov/adr/ADR-0024-retire-the-wide-tournament.toml new file mode 100644 index 00000000..69f56730 --- /dev/null +++ b/gov/adr/ADR-0024-retire-the-wide-tournament.toml @@ -0,0 +1,47 @@ +#:schema ../schema/adr.schema.json + +[govctl] +id = "ADR-0024" +title = "Retire the wide tournament" +status = "accepted" +date = "2026-09-04" +refs = [ + "ADR-0010", + "ADR-0004", + "RFC-0001", + "RFC-0002", +] + +[content] +context = """ +[[ADR-0010]] added a wide round before the deep loop: N parallel propose turns in per-candidate worktrees, serial diff scoring on the shared deployment, a top-k fold, and the winner's diff seeding the deep loop. It shipped as a work-graph template (`WideRunner`, `wide_template`, `MeasureDiff`) with its own manifest table (`[search]`), two CLI flags (`--wide`, `--wide-keep`), a row phase (`"wide"`), a crash class (`died_in_wide_round`), and a second row-accounting filter in the resume fold. + +No domain pack in crucible-domains sets `[search]`, and the loop is being refactored into one event-sourced state machine (a contract-level fold of the session log, a pure decision step, and a host that runs commands). Every piece the wide round adds is a second copy of something the deep loop already has: a second runner over `Row`, a second phase to classify, a second filter in every fold, and a third `TaskKind` dispatch site. Each would have to be carried through the refactor and taught to the approval-gate work that follows it. + +The breadth the wide round bought is available inside the work graph: an authored fan-out of isolated plan tasks and a `top_k` reducer express the same search without a separate pre-loop stage.""" +decision = """ +Delete the wide tournament. `WideConfig`, `WideOutcome`, `run_wide_tournament`, `wide_template`, `WideRunner`, `wide_propose`, `render_wide_prompt`, the `[search]` manifest table and its validation, the `--wide` and `--wide-keep` flags, and the `engine.measure_diff` operation are removed. The `died_in_wide_round` classification arm goes with them; a plan admitted before any iteration phase now classifies as an open plan like any other. + +The wire stays readable. `phase: "wide"` remains a legal row value and `died_in_wide_round` remains a decodable recovery-class token, because logs written before this decision carry them; readers keep such rows out of the deep loop's baseline and best, exactly as the resume fold did. [[RFC-0001:C-SEARCH]] is deprecated and the `[search]` rule leaves [[RFC-0001:C-MANIFEST]]. + +This supersedes [[ADR-0010]]. The candidate-portfolio idea it recorded is not rejected; its home moves from a dedicated stage to the authored graph.""" +consequences = """ +Positive: the loop has one runner over `Row`, one phase vocabulary, and one row filter, so the event-sourced refactor and the approval gates have one fewer shape to carry. `loop_graph.rs` loses its second `TaskRunner`; `recovery.rs` and the contract fold lose a classification arm; the manifest loses a table. + +Negative: a run that wants breadth has to author it as a fan-out in `workflow.star`, and no shipped template does that yet. Old session logs with `"wide"` rows render those rows as legacy context rather than as a named stage. + +The removal lands as its own behavior-preserving change with the parity tests green, before the state-machine refactor touches the same files.""" + +[[content.alternatives]] +text = "Delete the wide tournament; breadth lives in the authored work graph" +status = "accepted" + +[[content.alternatives]] +text = "Keep the wide round and carry it through the state-machine refactor" +status = "rejected" +rejection_reason = "Doubles the shapes the reducer, the classifier, and the gate work must handle, for a stage no deployed pack uses." + +[[content.alternatives]] +text = "Keep [search] as an authored-graph sugar that expands to a fan-out template" +status = "rejected" +rejection_reason = "A template no one runs is code no one tests; the DSL can express the same graph explicitly when a pack wants it." diff --git a/gov/rfc/RFC-0001/clauses/C-MANIFEST.toml b/gov/rfc/RFC-0001/clauses/C-MANIFEST.toml index c18b5302..0b88c3e6 100644 --- a/gov/rfc/RFC-0001/clauses/C-MANIFEST.toml +++ b/gov/rfc/RFC-0001/clauses/C-MANIFEST.toml @@ -21,8 +21,6 @@ Required fields: [repo], when present, must contain exactly one of url or path. [judge.selftest], if present, must require both good_cmd and bad_cmd; runs must be >= 1. -[search], if present with wide > 0, must require approaches.len() >= wide and policy_k in 1..=wide. - Unknown manifest keys must be rejected as errors. Frozen loading: when the manifest lives inside the workspace it targets, the engine must parse it from the workspace's pristine base commit, not the current working tree. Before any base commit exists (the very first run), the engine must hard-warn and trust the working tree for that run only.""" diff --git a/gov/rfc/RFC-0001/clauses/C-SEARCH.toml b/gov/rfc/RFC-0001/clauses/C-SEARCH.toml index 13bf33bf..37c38dbc 100644 --- a/gov/rfc/RFC-0001/clauses/C-SEARCH.toml +++ b/gov/rfc/RFC-0001/clauses/C-SEARCH.toml @@ -4,7 +4,7 @@ id = "C-SEARCH" title = "Wide-round search protocol" kind = "normative" -status = "active" +status = "deprecated" since = "0.1.0" [content] diff --git a/gov/rfc/RFC-0001/rfc.toml b/gov/rfc/RFC-0001/rfc.toml index 8017558b..f5759b4b 100644 --- a/gov/rfc/RFC-0001/rfc.toml +++ b/gov/rfc/RFC-0001/rfc.toml @@ -8,7 +8,7 @@ status = "normative" phase = "spec" owners = ["@Will Eaton"] created = "2026-08-19" -updated = "2026-08-22" +updated = "2026-09-04" signature = "ff8072ad4fecc176564e2ffbbf52c069b261c784901d2028972e8a983ac8b969" [[sections]] @@ -53,6 +53,8 @@ changed = [ "the declared lane scopes the constructor namespace, so a playbook cannot name or be offered a scored constructor", "session log gains asks_emitted, so what a run proposed is auditable apart from what an orchestrator admitted", "skill joins the enumerated constructors: an agent task whose prompt is assembled from a shipped SKILL.md and the invocation's arguments", + "the wide tournament is retired (ADR-0024): C-SEARCH is deprecated, [search] leaves C-MANIFEST, and phase: \"wide\" is a legacy row value readers keep out of the deep loop", + "engine.measure_diff leaves the engine operations, since only the wide template produced it", ] [[changelog]] From 166b7c4d56dfa8833ae7c9028ceff1c2f7c72768 Mon Sep 17 00:00:00 2001 From: Will Eaton Date: Fri, 4 Sep 2026 10:21:19 -0400 Subject: [PATCH 03/11] Split the loop into a decision machine and an effect host The loop's decisions were threaded through one 800-line function together with every effect they cause, and a resume rebuilt state through two more scanners of the same log. Add crucible::machine: LoopCfg, RunState, Segment, LoopExit and a Machine whose methods answer the head-of-iteration checks, fold an iteration's outcome into a row, and do the keep/discard and budget bookkeeping from plain inputs, with no I/O. loop_driver keeps the host: it performs every effect in the order it always has and hands the results to the machine, so the session log a run writes is unchanged. Resume and crash classification now read the contract's LoopState fold; ResumeFold and the recovery tail scan are gone, and the parity oracle that proved the fold matched them goes with them. The plan lane gains its resume fold: execute_from seeds a graph with the results an earlier process settled, and a run that died inside a plan hands the resumed iteration the passing pack tasks under the same plan, so they are reported rather than re-dispatched (RFC-0002 C-PLAYBOOK-RESUME). Engine operations always run again, since the world they produced did not survive. Assisted-by: Claude --- crucible/src/loop_driver.rs | 1065 +++++++++++++---------------------- crucible/src/loop_graph.rs | 121 +++- crucible/src/machine.rs | 791 ++++++++++++++++++++++++++ crucible/src/main.rs | 1 + crucible/src/plan/exec.rs | 91 +++ crucible/src/recovery.rs | 549 ++++-------------- crucible/src/run.rs | 1 + docs/crucible.md | 2 +- 8 files changed, 1507 insertions(+), 1114 deletions(-) create mode 100644 crucible/src/machine.rs diff --git a/crucible/src/loop_driver.rs b/crucible/src/loop_driver.rs index 00ba81ac..6d78f2dc 100644 --- a/crucible/src/loop_driver.rs +++ b/crucible/src/loop_driver.rs @@ -6,6 +6,11 @@ //! workspace prep, front-end choice) lives in [`crate::run`]; this module is just the loop and //! its helpers. +pub(crate) use crate::machine::IterStep; +use crate::machine::{ + BudgetHit, DistressOutcome, LoopCfg, LoopExit, MAX_DEAD_TURN_ATTEMPTS, Machine, ParkOutcome, + RunState, Segment, Settle, +}; use crate::reporter::{AgentTurn, Outcome, Phase, Reporter, Row, Stop, TurnBudget}; use crate::{Args, Paths, Prepared, STOP}; use crate::{control, escalation, provisioning, publish, session}; @@ -52,13 +57,6 @@ pub(crate) struct ResumeState { pub published_branches: Vec, } -#[cfg(test)] -impl ResumeState { - pub(crate) fn has_rows_for_parity(&self) -> bool { - !self.rows.is_empty() - } -} - /// Optional runtime state for special loop starts (remote control and resume). Kept in /// one argument so the core loop boundary stays small as front-ends evolve. #[derive(Default)] @@ -73,95 +71,33 @@ pub(crate) struct LoopRuntime<'a> { /// The liveness beat's view of the loop, refreshed wherever the control status is. `None` /// when the beat is disabled (`CRUCIBLE_HEARTBEAT_SECS=0`). pub heartbeat: Option>, + /// The plan tasks the previous process settled in the iteration it died in, so a resumed + /// graph iteration does not re-dispatch them; present only with `resume`. + pub prior_plan: Option, } -/// An opaque rollback token from [`World::snapshot`]. The engine never inspects it (a git -/// world packs a sha, a command world packs `"\t"`); the newtype just keeps it -/// from being confused with the run's other strings (regime, fingerprint, note). -struct Snapshot(String); - -impl Snapshot { - fn as_str(&self) -> &str { - &self.0 - } -} - -/// Everything an approved judge-changing re-scope replaces *atomically*: scores across a segment -/// boundary are not comparable, so the regime, its fingerprint, the re-baselined scores, and the -/// rollback snapshot move as one set. Swapping the whole `Segment` in a single assignment means -/// a re-scope can't half-update the goalpost. -struct Segment { +/// Measure a fresh baseline and open a new comparable segment for `regime`. Used both for the +/// initial segment 0 and for an approved re-scope; the returned [`Row`] is the baseline row +/// (the initial path logs it, a re-scope discards it). +fn baseline_segment( + world: &dyn World, + judge: &dyn Judge, + goal: &str, regime: String, - fingerprint: String, - baseline_score: f64, - /// Mutable within the segment: a kept improvement lowers it. A re-scope resets it to the new - /// baseline. - best_score: f64, - /// The kept best's secondary tiebreak scalar, tracked with `best_score` so a - /// primary-score tie can be ruled on the secondary axis. `None` when the kept best - /// (or the baseline) declared none. - best_tiebreak: Option, - baseline_total: u64, - best_snap: Snapshot, -} - -impl Segment { - /// Measure a fresh baseline and open a new comparable segment for `regime`. Used both for the - /// initial segment 0 and for an approved re-scope; the returned [`Row`] is the baseline row - /// (the initial path logs it, a re-scope discards it). - fn baseline( - world: &dyn World, - judge: &dyn Judge, - goal: &str, - regime: String, - source: BaselineSource, - ) -> Result<(Self, Row)> { - let (baseline_score, baseline_total, snap, row) = run_baseline(world, judge, source)?; - let fingerprint = fingerprint(goal, &judge.objective(), ®ime); - let segment = Segment { - regime, - fingerprint, - baseline_score, - best_score: baseline_score, - best_tiebreak: row.tiebreak, - baseline_total, - best_snap: Snapshot(snap), - }; - Ok((segment, row)) - } -} - -/// The per-run state [`run_loop`] threads across iterations. Bundled so a new gate plugs into -/// a named context instead of adding a 14th mutable binding, and so the segment-scoped fields -/// can be swapped atomically (see [`Segment`]). -struct Run { - rows: Vec, - spent: f64, - /// SHAs of commits this session kept, for the publish summary. A resumed run rebuilds the - /// pre-resume keeps from the log's kept rows (see [`restore_kept_best`]). - kept_shas: Vec, - /// The pristine upstream SHA the workspace was checked out at, captured from segment 0's baseline - /// snapshot BEFORE any agent commit (a later re-baseline would see kept commits, so this is taken - /// once). It's the true PR base (the diff base for publish-on-keep) replacing the pod wrapper's - /// `/tmp/base-shas` hack. `None` on resume (the pristine base lived in the original run). - base_sha: Option, - /// The pristine baseline snapshot TOKEN, captured once at segment 0 (same moment as `base_sha`). - /// A composite world reads its per-component base shas out of this (the multi-fork publish path); - /// a single-repo world ignores it. `None` on resume (the original run held it). - base_snap: Option, - solved_any: bool, - /// Idle time spent parked on a human approval, excluded from the time cap. - parked_total: Duration, - /// Set when the agent blocked on a pending approval with no frozen-regime fallback; the loop - /// parks at the next iteration head until the re-scope lands. - pending_block: Option, - /// Head branches prior segments already opened PRs from (restored from the log's `pr_links` - /// events; empty on a fresh run). Publish skips candidates whose branch is in here. - published_branches: Vec, - /// The run's spend against its declared output bounds, for the writes the engine performs - /// itself (RFC-0001:C-OUTPUTS). The broker tallies its own tools separately. - outputs: crate::outputs::OutputTally, - segment: Segment, + source: BaselineSource, +) -> Result<(Segment, Row)> { + let (baseline_score, baseline_total, snap, row) = run_baseline(world, judge, source)?; + let fingerprint = fingerprint(goal, &judge.objective(), ®ime); + let segment = Segment { + regime, + fingerprint, + baseline_score, + best_score: baseline_score, + best_tiebreak: row.tiebreak, + baseline_total, + best_snap: snap, + }; + Ok((segment, row)) } /// The run's spend against its declared output bounds: the frozen pack's when one was projected, @@ -175,55 +111,6 @@ fn output_tally(args: &Args, p: &Paths) -> crate::outputs::OutputTally { ) } -/// Consecutive never-started turns (transport/sandbox death before the agent produced -/// anything) after which the run halts as [`LoopExit::Stalled`]. Such a turn re-runs its -/// iteration instead of consuming it, so without this bound one dead node could spin the -/// run forever (run 6 burned 7 of 9 iterations on a single sandbox that never came up). -const MAX_DEAD_TURN_ATTEMPTS: u32 = 3; - -/// How a run ended, the single enumeration of every way the loop exits, replacing the old -/// `escalated: Option` flag plus the scattered `break`s. Mapped to an [`Outcome`] once -/// at the end of [`run_loop`]. -enum LoopExit { - /// The `for` ran every iteration without an early exit. - Finished, - /// A kept candidate satisfied the win condition. - Solved, - /// A cost or time cap was reached. - Budget, - /// Ctrl+C / a stop signal (at an interrupt checkpoint or while parked). - Stopped, - /// The agent declared the harness inadequate, or a `block` approval was denied with no - /// fallback: halt for human review. The escalation itself is reported and the world rolled - /// back eagerly at the break site (differently per site) so the variant only needs to - /// mark the run as "needs human" for the exit code. - Escalated, - /// [`MAX_DEAD_TURN_ATTEMPTS`] consecutive turns died on transport before starting: the - /// run is stalled on infrastructure, not out of iterations. - Stalled, -} - -impl LoopExit { - /// The wire token + human-readable reason for [`Reporter::shutdown`]. `error` (a bail from - /// inside the loop, never reaching this variant) is reported separately by [`run_loop`]. - fn shutdown_reason(&self) -> (&'static str, &'static str) { - match self { - LoopExit::Finished => ("finished", "all iterations completed"), - LoopExit::Solved => ("solved", "a kept candidate satisfied the win condition"), - LoopExit::Budget => ("budget", "a cost or time cap was reached"), - LoopExit::Stopped => ("stopped", "stop signal received"), - LoopExit::Escalated => ( - "escalated", - "the agent declared the harness inadequate — halted for human review", - ), - LoopExit::Stalled => ( - "stalled", - "the run stalled on consecutive transport failures — no turn could start", - ), - } - } -} - /// One turn's linear protocol, typed so its illegal orderings stop compiling: the candidate /// moves `Proposed → Applied → Measured`, and only a `Measured` can be decided. You cannot /// `measure` before `apply` (no method), nor `decide` without a [`crucible::crucible::Reading`] (the @@ -260,32 +147,6 @@ pub(crate) struct Decided { pub(crate) reading: crucible::crucible::Reading, } -/// One iteration's outcome in driver vocabulary, produced by either path (the typestate -/// chain or the graph template) and folded by the shared keep/discard tail in -/// [`run_loop_body`]. -pub(crate) enum IterStep { - Decided(Box), - /// Discard and move on (failed turn, failed apply, gate rejection). The reason lands in the - /// iteration's Row, so the run summary counts every iteration honestly — a run that lost all - /// its iterations must not read as a clean "finished" with an empty scoreboard. - Discarded { - reason: String, - }, - /// The turn never started: a transport-class death (sandbox setup, auth, connection) - /// before the agent produced anything. There is no candidate to discard, so the driver - /// re-runs the SAME iteration instead of consuming it, bounded by - /// [`MAX_DEAD_TURN_ATTEMPTS`] consecutive attempts. - NeverStarted { - reason: String, - }, - /// Halt for human review (the escalation is already reported). - Escalated, - /// Park at the next iteration head on a blocking approval. - Parked(provisioning::PendingProvisioning), - /// Stop signal at the post-turn checkpoint. - Stopped, -} - /// What the post-turn sentinel drains decided, in the exact order the loop checks them. /// Notes and the structured escalation event are emitted in here; the caller owns the /// world rollback and the loop control that follows. Shared by the typestate path and the @@ -563,6 +424,9 @@ pub(crate) fn run_loop( } } +/// The host: performs every effect the loop needs and hands the results to the +/// [`crate::machine::Machine`], which owns the decisions. Effects happen in the order they +/// always have, so the session log a run writes is unchanged. fn run_loop_body( args: &Args, p: &Paths, @@ -572,19 +436,26 @@ fn run_loop_body( judge: &dyn Judge, runtime: LoopRuntime<'_>, ) -> Result { + let mut runtime = runtime; let control = runtime.control; let ledger = runtime.ledger.as_deref(); let heartbeat_handle = runtime.heartbeat.as_ref(); let heartbeat = heartbeat_handle.map(std::sync::Arc::as_ref); let started = Instant::now(); - let start_iter: u32; + let cfg = LoopCfg { + iterations: args.iterations, + max_time: args.max_time(), + max_park: args.max_park(), + no_early_stop: args.no_early_stop, + }; + let mut outputs = output_tally(args, p); // Held across the whole body: an approved re-scope re-baselines, and re-measuring a rescoped // baseline without an agent turn is impossible for a codegen domain, so it reuses this same // preflight measurement. A resumed run whose baseline was never measured re-runs preflight // and sets this; a resume with a finite baseline skips preflight entirely. let mut preflight_baseline: Option = None; - let mut run = if let Some(rs) = runtime.resume { + let mut m = if let Some(rs) = runtime.resume.take() { // Resume restores state in-memory only: the log already holds the prior // `start` + rows, so re-emitting them would double-count on replay. We append // just the continuation (a resume note, then the new iterations). Segment 0 opens @@ -608,30 +479,33 @@ fn run_loop_body( baseline_total: rs.baseline_total, best_snap: resumed_best.best_snap, }; - start_iter = rs.next_iter; - let mut run = Run { - rows: rs.rows, - spent: rs.spent, - kept_shas: resumed_best.kept_shas, - base_sha: None, - base_snap: None, - solved_any: rs.solved_any, - parked_total: Duration::ZERO, - pending_block: None, - published_branches: rs.published_branches, - outputs: output_tally(args, p), - segment, - }; + let start_iter = rs.next_iter; + let mut m = Machine::new( + cfg, + RunState { + rows: rs.rows, + spent: rs.spent, + kept_shas: resumed_best.kept_shas, + base_sha: None, + base_snap: None, + solved_any: rs.solved_any, + parked_total: Duration::ZERO, + pending_block: None, + published_branches: rs.published_branches, + segment, + }, + start_iter, + ); update_control_status( control, "resume", start_iter.saturating_sub(1), - run.segment.best_score, - run.spent, + m.run.segment.best_score, + m.run.spent, ); r.note(&format!( "resumed: {} prior rows restored, continuing at iter {start_iter}", - run.rows.len() + m.run.rows.len() )); if let Some(why) = &resumed_best.degraded { r.note(&format!("resume: {why}")); @@ -639,7 +513,7 @@ fn run_loop_body( // The ledger is read FIRST: a grant recorded before the death settles what the // dangling approval bracket means. let replay = ledger.map(crate::admission::AdmissionLedger::replay_for_resume); - if let Some(rec) = runtime.recovery { + if let Some(rec) = runtime.recovery.take() { r.recovery(rec.class, rec.iter, &rec.detail); let approval = crate::recovery::resume_approval(&rec, replay.as_ref()); if let Some(why) = &approval.note { @@ -648,7 +522,7 @@ fn run_loop_body( if let (Some(control), Some(regime)) = (control, approval.pending_regime) { control.set_pending_regime(regime); } - run.pending_block = approval.repark; + m.run.pending_block = approval.repark; } if let (Some(ledger), Some(replay)) = (ledger, replay) { replay_admissions(ledger, control, replay, r); @@ -672,15 +546,15 @@ fn run_loop_body( // baseline comes from the judge, not from preflight. if prep.skip_baseline && let Some(cfg) = &prep.preflight - && !run.segment.baseline_score.is_finite() + && !m.run.segment.baseline_score.is_finite() { r.phase(Phase::Preflight); update_control_status( control, "preflight", start_iter.saturating_sub(1), - run.segment.best_score, - run.spent, + m.run.segment.best_score, + m.run.spent, ); match crate::preflight::run(cfg, &prep.preflight_modes, &p.workspace, r) { Ok(seeded) => preflight_baseline = seeded, @@ -700,8 +574,8 @@ fn run_loop_body( ..Default::default() }; r.row(&row, false); - run.rows.push(row); - write_results(p, &prep.goal, &prep.prior, &run.rows)?; + m.record(row); + write_results(p, &prep.goal, &prep.prior, &m.run.rows)?; return Err(e); } } @@ -710,12 +584,12 @@ fn run_loop_body( // (pre-preflight) can carry a finite kept best with a sentinel baseline, and // clobbering that best would discard real progress. if let Some(pb) = &preflight_baseline { - run.segment.baseline_score = pb.score; - if !run.segment.best_score.is_finite() { + m.run.segment.baseline_score = pb.score; + if !m.run.segment.best_score.is_finite() { let snap = world.snapshot("preflight baseline")?; - run.segment.best_score = pb.score; - run.segment.best_tiebreak = pb.tiebreak; - run.segment.best_snap = Snapshot(snap); + m.run.segment.best_score = pb.score; + m.run.segment.best_tiebreak = pb.tiebreak; + m.run.segment.best_snap = snap; } let base_row = Row { iter: 0, @@ -726,16 +600,15 @@ fn run_loop_body( ..Default::default() }; r.row(&base_row, false); - run.rows.push(base_row); + m.record(base_row); } } - write_results(p, &prep.goal, &prep.prior, &run.rows)?; - run + write_results(p, &prep.goal, &prep.prior, &m.run.rows)?; + m } else { r.start(&prep.goal, &judge.objective()); r.identity(&prep.identity); - start_iter = 1; if let Some(cfg) = &prep.preflight { r.phase(Phase::Preflight); @@ -768,7 +641,7 @@ fn run_loop_body( r.phase(Phase::Baseline); update_control_status(control, "baseline", 0, f64::INFINITY, 0.0); - let (segment, base_row) = Segment::baseline( + let (segment, base_row) = baseline_segment( world, judge, &prep.goal, @@ -778,61 +651,61 @@ fn run_loop_body( // The pristine base: segment 0's baseline snapshot is the upstream checkout, before any // agent commit. Captured here (not at publish time) because a kept iteration advances HEAD, // so by end-of-run `git rev-parse HEAD` is the candidate, not the base. - let base_sha = world.commit_sha(segment.best_snap.as_str()); + let base_sha = world.commit_sha(&segment.best_snap); // The composite multi-fork publish path needs the full baseline token (per-component base // shas), not just the single `commit_sha`; capture it once here, same moment as `base_sha`. - let base_snap = Some(segment.best_snap.as_str().to_string()); - let mut run = Run { - rows: Vec::new(), - spent: 0.0_f64, - kept_shas: Vec::new(), - base_sha, - base_snap, - solved_any: false, - parked_total: Duration::ZERO, - pending_block: None, - published_branches: Vec::new(), - outputs: output_tally(args, p), - segment, - }; + let base_snap = Some(segment.best_snap.clone()); + let mut m = Machine::new( + cfg, + RunState { + rows: Vec::new(), + spent: 0.0_f64, + kept_shas: Vec::new(), + base_sha, + base_snap, + solved_any: false, + parked_total: Duration::ZERO, + pending_block: None, + published_branches: Vec::new(), + segment, + }, + 1, + ); r.row(&base_row, false); - run.rows.push(base_row); - write_results(p, &prep.goal, &prep.prior, &run.rows)?; - update_control_status(control, "baseline", 0, run.segment.best_score, run.spent); - run + m.record(base_row); + write_results(p, &prep.goal, &prep.prior, &m.run.rows)?; + update_control_status( + control, + "baseline", + 0, + m.run.segment.best_score, + m.run.spent, + ); + m }; - // Announce segment 0. A re-scope (below) swaps `run.segment` for a fresh one, marking a new + // Announce segment 0. A re-scope (below) swaps the segment for a fresh one, marking a new // comparable segment: scores across a boundary are NOT comparable. r.segment( - &run.segment.fingerprint, - run.segment.baseline_score, - &run.segment.regime, + &m.run.segment.fingerprint, + m.run.segment.baseline_score, + &m.run.segment.regime, ); - // How this run ends. Each early exit sets it before breaking; a loop that runs out of - // iterations leaves it `Finished`. One match below folds it into the `Outcome`. - let mut exit = LoopExit::Finished; - // `it` advances only when a turn actually started: a never-started attempt re-runs the - // same iteration (hence a `while`, not a `for`), and `dead_turns` counts the consecutive - // never-started attempts that bound the re-runs. - let mut dead_turns: u32 = 0; - // The marker this run already parked on (by its `ts_ms`), so a marker the operator left in - // place can't re-park the loop, and a fresh distress still can. - let mut parked_distress_ts: Option = None; // The mtime of the malformed marker already complained about; a rewrite re-notes. let mut bad_marker_seen: Option> = None; - let mut it = start_iter; - while it <= args.iterations { + while m.exit().is_none() && m.has_iterations() { + let it = m.it; wait_if_paused(control, r); // The agent blocked on a pending approval last turn (it had no frozen-regime fallback). // Park here (idle, budget-paused) until the approval lands as a re-scope (the broker // fires it over the control bridge) or we're told to stop. The drain below then // re-baselines into the granted regime. - if let Some(pp) = run.pending_block.take() { - match park_for_approval(control, ledger, r, &mut run.parked_total, args.max_park()) { - ParkOutcome::Resumed => {} // the re-scope drain below re-baselines - ParkOutcome::Denied(why) => { + if let Some(pp) = m.take_pending_block() { + let (outcome, parked) = park_for_approval(control, ledger, r, m.cfg.max_park); + match (m.on_park(parked, &outcome), &outcome) { + (None, _) => {} // the re-scope drain below re-baselines + (Some(LoopExit::Escalated), ParkOutcome::Denied(why)) => { // `block` means the agent had no frozen-regime fallback, a denial leaves // nothing to do, so escalate-halt for a human. r.note(&format!( @@ -843,17 +716,13 @@ fn run_loop_body( control, "escalated", it, - run.segment.best_score, - run.spent, + m.run.segment.best_score, + m.run.spent, ); - world.restore(run.segment.best_snap.as_str())?; - exit = LoopExit::Escalated; - break; - } - ParkOutcome::Stopped => { - exit = LoopExit::Stopped; + world.restore(&m.run.segment.best_snap)?; break; } + (Some(_), _) => break, } } // An approved judge-changing grant arrived (via the control channel / MCP): re-baseline @@ -868,14 +737,14 @@ fn run_loop_body( // One atomic swap of the goalpost: the new regime, its fingerprint, the re-baselined // scores, and the fresh rollback snapshot all land together. The admission // settles only after the swap: a baseline error leaves it for the resume. - let (segment, _row) = Segment::baseline( + let (segment, _row) = baseline_segment( world, judge, &prep.goal, new_regime.clone(), baseline_source(prep.skip_baseline, preflight_baseline.as_ref()), )?; - run.segment = segment; + m.rescope(segment); if let Some(ledger) = ledger { let _ = ledger.settle( &rescope_key, @@ -884,11 +753,11 @@ fn run_loop_body( ); } r.segment( - &run.segment.fingerprint, - run.segment.baseline_score, - &run.segment.regime, + &m.run.segment.fingerprint, + m.run.segment.baseline_score, + &m.run.segment.regime, ); - write_results(p, &prep.goal, &prep.prior, &run.rows)?; + write_results(p, &prep.goal, &prep.prior, &m.run.rows)?; } // A denial that arrived while *continuing* (the agent had a fallback, so the loop never // parked) just means the regime change won't happen; note it and stay in the frozen regime. @@ -910,55 +779,50 @@ fn run_loop_body( // Modeled as an approval wait (same bracket, same parked-time accounting); the operator's // `rm` of the marker is the grant. match crate::distress::read_marker() { - Some(Ok(marker)) if parked_distress_ts != Some(marker.ts_ms) => { - parked_distress_ts = Some(marker.ts_ms); - // The turn that raised distress is already numbered; this row annotates it. - let row = Row { - iter: it.saturating_sub(1), - decision: "distressed".to_string(), - note: marker.reason.clone(), - ..Default::default() - }; - // An in-place restart re-reads a marker the operator never cleared and re-parks - // (correct: no grant was given), but the row for it is already in the resumed log. - if !run.rows.iter().any(|prior| { - prior.decision == row.decision - && prior.iter == row.iter - && prior.note == row.note - }) { - r.row(&row, false); - run.rows.push(row); - write_results(p, &prep.goal, &prep.prior, &run.rows)?; - } - r.note(&format!( - "distress: {}, suspended awaiting the operator (clear {})", - marker.reason, - forge::storage_root().join("distress").display() - )); - for item in &marker.evidence { - r.note(&format!("distress evidence: {item}")); - } - update_control_status(control, "distressed", it, run.segment.best_score, run.spent); - r.approval_wait( - crate::distress::HANDLE, - crate::distress::HANDLE, - provisioning::WaitMode::Block, - ); - match park_for_distress(p, &run.rows, r, &mut run.parked_total, args.max_park()) { - DistressOutcome::Cleared => { - r.approval_resolved("granted", "distress cleared by operator"); - r.note("distress cleared, resuming"); - // The head re-checks budget/interrupts before the next turn runs. - continue; + Some(Ok(marker)) => { + if let Some(row) = m.on_distress(marker.ts_ms, &marker.reason) { + // An in-place restart re-reads a marker the operator never cleared and + // re-parks (correct: no grant was given), but the row for it is already in + // the resumed log. + if let Some(row) = row { + r.row(&row, false); + m.record(row); + write_results(p, &prep.goal, &prep.prior, &m.run.rows)?; } - DistressOutcome::Stopped => { - exit = LoopExit::Stopped; - break; + r.note(&format!( + "distress: {}, suspended awaiting the operator (clear {})", + marker.reason, + forge::storage_root().join("distress").display() + )); + for item in &marker.evidence { + r.note(&format!("distress evidence: {item}")); } - DistressOutcome::TimedOut => { - r.note("distress park timed out, stopping with state preserved"); - exit = LoopExit::Stopped; - break; + update_control_status( + control, + "distressed", + it, + m.run.segment.best_score, + m.run.spent, + ); + r.approval_wait( + crate::distress::HANDLE, + crate::distress::HANDLE, + provisioning::WaitMode::Block, + ); + let (outcome, parked) = park_for_distress(p, &m.run.rows, r, m.cfg.max_park); + match m.on_distress_park(parked, outcome) { + None => { + r.approval_resolved("granted", "distress cleared by operator"); + r.note("distress cleared, resuming"); + // The head re-checks budget/interrupts before the next turn runs. + continue; + } + Some(_) => { + if outcome == DistressOutcome::TimedOut { + r.note("distress park timed out, stopping with state preserved"); + } + break; + } } } } @@ -971,30 +835,37 @@ fn run_loop_body( r.note(&format!("distress marker unreadable ({why}), not parking")); } } - _ => {} + None => {} } - if matches!(r.check_interrupt(p, &run.rows), Stop::Quit) { - exit = LoopExit::Stopped; + if matches!(r.check_interrupt(p, &m.run.rows), Stop::Quit) { + m.end(LoopExit::Stopped); break; } - if over_budget(args, control, run.spent, started, run.parked_total, r) { - exit = LoopExit::Budget; + if let Some(hit) = m.over_budget(live_max_cost(args, control), started.elapsed()) { + note_budget_hit(r, args, hit); + m.end(LoopExit::Budget); break; } r.phase(Phase::Iteration(it)); // One span per loop round, entered for the iteration's whole body on this thread: the // turn span, gate evaluations, and broker traceparent files all nest under it, so a trace // groups by iteration and `iter` is queryable directly. - let iter_span = tracing::info_span!("iteration", iter = it, spent_usd = run.spent); + let iter_span = tracing::info_span!("iteration", iter = it, spent_usd = m.run.spent); let _iter_span = iter_span.enter(); // The broker's distress page prints the iteration it fired on; best-effort by design, a // missing stamp only costs the page a "?". - write_turn_meta(it, run.spent); - update_control_status(control, "iteration", it, run.segment.best_score, run.spent); - beat_position(heartbeat, it, run.spent); - write_results(p, &prep.goal, &prep.prior, &run.rows)?; + write_turn_meta(it, m.run.spent); + update_control_status( + control, + "iteration", + it, + m.run.segment.best_score, + m.run.spent, + ); + beat_position(heartbeat, it, m.run.spent); + write_results(p, &prep.goal, &prep.prior, &m.run.rows)?; - let status = judge.status(run.segment.best_score); + let status = judge.status(m.run.segment.best_score); // Un-carried steers, in admission order. The keys settle after the turn ran, so a // turn that never started re-delivers the same batch. let steer_batch = crate::admission::drain_steer(ledger, &p.steer); @@ -1003,7 +874,7 @@ fn run_loop_body( } // The pack-declared seed diff goes to iteration 1 only: it's starting material, not // standing guidance, and later iterations already stand on whatever iter 1 kept. - let seed = if it == 1 { + let seed = if m.wants_seed() { prep.seed_diff.as_deref() } else { None @@ -1015,7 +886,7 @@ fn run_loop_body( )); } let resume_prompt = - render_resume_prompt(&status, &run.segment.regime, steer_batch.text.as_deref()); + render_resume_prompt(&status, &m.run.segment.regime, steer_batch.text.as_deref()); let prompt = render_prompt( &prep.template, &prep.goal, @@ -1038,20 +909,21 @@ fn run_loop_body( it, prompt: &prompt, resume_prompt: &resume_prompt, - rows: &run.rows, - baseline_score: run.segment.baseline_score, - baseline_total: run.segment.baseline_total, - best_score: run.segment.best_score, - best_tiebreak: run.segment.best_tiebreak, - spent_before: run.spent, + rows: &m.run.rows, + baseline_score: m.run.segment.baseline_score, + baseline_total: m.run.segment.baseline_total, + best_score: m.run.segment.best_score, + best_tiebreak: m.run.segment.best_tiebreak, + spent_before: m.run.spent, started, heartbeat: heartbeat_handle.cloned(), workflow: args.workflow.as_ref(), + prior: runtime.prior_plan.take_if(|prior| prior.iter == it), }, r, )?; - run.spent += cost; - beat_position(heartbeat, it, run.spent); + m.add_cost(cost); + beat_position(heartbeat, it, m.run.spent); step } else { let turn = r.run_agent( @@ -1062,20 +934,20 @@ fn run_loop_body( None, None, TurnBudget { - spent_before: run.spent, + spent_before: m.run.spent, started, max_cost: live_max_cost(args, control), heartbeat: heartbeat_handle.cloned(), }, ); - run.spent += turn.cost; + m.add_cost(turn.cost); if let Some(control) = control { - control.set_spend(run.spent); + control.set_spend(m.run.spent); } - beat_position(heartbeat, it, run.spent); - r.budget(run.spent, started.elapsed()); + beat_position(heartbeat, it, m.run.spent); + r.budget(m.run.spent, started.elapsed()); - match drain_turn_markers(r, p, control, it, &turn, &run.rows) { + match drain_turn_markers(r, p, control, it, &turn, &m.run.rows) { TurnVerdict::Proceed => { // Make the candidate live (deploy domains build+push+set-image); a no-op // for the agent-edit/git worlds where the edit IS the candidate. A failed @@ -1083,15 +955,15 @@ fn run_loop_body( match Iteration::proposed(it).apply(world) { Ok(applied) => { let ctx = crucible::crucible::MeasureCtx { - baseline_score: Some(run.segment.baseline_score), - baseline_total: Some(run.segment.baseline_total), - best_score: Some(run.segment.best_score), + baseline_score: Some(m.run.segment.baseline_score), + baseline_total: Some(m.run.segment.baseline_total), + best_score: Some(m.run.segment.best_score), }; IterStep::Decided(Box::new( applied.measure(judge, &ctx, p, world)?.decide( judge, - run.segment.best_score, - run.segment.best_tiebreak, + m.run.segment.best_score, + m.run.segment.best_tiebreak, ), )) } @@ -1116,118 +988,106 @@ fn run_loop_body( } }; - // Any step other than NeverStarted proves a turn started: reset the stall streak. // A started turn carried the steer batch in its prompt, which settles an admitted // steer ("delivered", not "heeded"); a never-started turn leaves the batch owed. - if !matches!(&step, IterStep::NeverStarted { .. }) { - dead_turns = 0; - if let Some(ledger) = ledger { - ledger.settle_all( - &steer_batch.keys, - AdmissionOutcome::Applied, - &format!("delivered in iter {it}"), - ); - } + if !matches!(&step, IterStep::NeverStarted { .. }) + && let Some(ledger) = ledger + { + ledger.settle_all( + &steer_batch.keys, + AdmissionOutcome::Applied, + &format!("delivered in iter {it}"), + ); } let Decided { mut row, verdict, reading, - } = match step { - IterStep::Decided(d) => *d, - IterStep::Discarded { reason } => { - let mut row = Row { - iter: it, - decision: "discarded".to_string(), - note: reason, - ..Default::default() - }; + } = match m.settle(step) { + Settle::Decide(d) => *d, + Settle::Discard { mut row } => { fold_distress_notes(r, &mut row.note); r.row(&row, false); - run.rows.push(row); - world.restore(run.segment.best_snap.as_str())?; - it += 1; + m.record(row); + world.restore(&m.run.segment.best_snap)?; + m.advance(); continue; } // A never-started turn produced no candidate, so there is nothing to charge the // iteration for: log the dead attempt faithfully (row + note), then re-run the // same `it`. Bounded so a dead node stalls the run instead of burning it to the // iteration cap as a fake "finished". - IterStep::NeverStarted { reason } => { - dead_turns += 1; - let row = Row { - iter: it, - decision: "infra-dead".to_string(), - note: reason, - phase: Some("infra".to_string()), - ..Default::default() - }; + Settle::Rerun { + row, + attempt, + stalled, + } => { r.row(&row, false); - run.rows.push(row); - write_results(p, &prep.goal, &prep.prior, &run.rows)?; - world.restore(run.segment.best_snap.as_str())?; - if dead_turns >= MAX_DEAD_TURN_ATTEMPTS { + m.record(row); + write_results(p, &prep.goal, &prep.prior, &m.run.rows)?; + world.restore(&m.run.segment.best_snap)?; + if stalled { r.note(&format!( - "{dead_turns} consecutive turns died before starting — the run is stalled" + "{attempt} consecutive turns died before starting — the run is stalled" )); - exit = LoopExit::Stalled; break; } r.note(&format!( - "turn never started (attempt {dead_turns}/{MAX_DEAD_TURN_ATTEMPTS}) — re-running iter {it} without consuming it" + "turn never started (attempt {attempt}/{MAX_DEAD_TURN_ATTEMPTS}) — re-running iter {it} without consuming it" )); continue; } - IterStep::Escalated => { - update_control_status(control, "escalated", it, run.segment.best_score, run.spent); - world.restore(run.segment.best_snap.as_str())?; - exit = LoopExit::Escalated; + Settle::Escalate => { + update_control_status( + control, + "escalated", + it, + m.run.segment.best_score, + m.run.spent, + ); + world.restore(&m.run.segment.best_snap)?; break; } - IterStep::Parked(pp) => { - update_control_status(control, "parked", it, run.segment.best_score, run.spent); - run.pending_block = Some(pp); - write_results(p, &prep.goal, &prep.prior, &run.rows)?; - it += 1; + Settle::Park => { + update_control_status(control, "parked", it, m.run.segment.best_score, m.run.spent); + write_results(p, &prep.goal, &prep.prior, &m.run.rows)?; + m.advance(); continue; } - IterStep::Stopped => { - exit = LoopExit::Stopped; - break; - } + Settle::Stop => break, }; fold_distress_notes(r, &mut row.note); if verdict.keep { - if let Some(s) = reading.score { - run.segment.best_score = s; - // The kept candidate defines BOTH axes, even when its tiebreak is absent: - // carrying a stale tiebreak forward would compare the next tie against a - // scalar the current best never earned. - run.segment.best_tiebreak = reading.tiebreak; - update_control_status(control, "iteration", it, run.segment.best_score, run.spent); - } // The World owns reversibility now: snapshot commits the kept state (git memory) // and captures any external state; the engine never touches git directly. - match world.snapshot(&format!("iter {it}: keep ({})", reading.note)) { - Ok(snap) => { - if let Some(sha) = world.commit_sha(&snap) { - run.kept_shas.push(sha); - } - // The row carries the token so a resume can restore this kept tree. - row.kept_snap = Some(snap.clone()); - run.segment.best_snap = Snapshot(snap); - } - Err(e) => r.note(&format!("snapshot failed (change still live): {e:#}")), + let snapshot = world + .snapshot(&format!("iter {it}: keep ({})", reading.note)) + .map(|snap| { + let sha = world.commit_sha(&snap); + (snap, sha) + }) + .map_err(|e| format!("{e:#}")); + m.keep(&mut row, &reading, verdict.solved, snapshot.clone()); + if reading.score.is_some() { + update_control_status( + control, + "iteration", + it, + m.run.segment.best_score, + m.run.spent, + ); + } + if let Err(why) = snapshot { + r.note(&format!("snapshot failed (change still live): {why}")); } - run.solved_any |= verdict.solved; } else { - world.restore(run.segment.best_snap.as_str())?; + world.restore(&m.run.segment.best_snap)?; } r.row(&row, verdict.solved); - run.rows.push(row); - write_results(p, &prep.goal, &prep.prior, &run.rows)?; + m.record(row); + write_results(p, &prep.goal, &prep.prior, &m.run.rows)?; // Snapshot durable state per decided iteration so cross-run memory survives a killed pod // (the end-of-run publish below only fires on a clean exit). No branch push mid-run. @@ -1240,80 +1100,61 @@ fn run_loop_body( goal: &prep.goal, model: args.model(), gate: judge.objective(), - rows: &run.rows, - baseline_score: run.segment.baseline_score, - best_score: run.segment.best_score, + rows: &m.run.rows, + baseline_score: m.run.segment.baseline_score, + best_score: m.run.segment.best_score, improved: judge.improved( - run.segment.best_score, - run.segment.baseline_score, - run.solved_any, + m.run.segment.best_score, + m.run.segment.baseline_score, + m.run.solved_any, ), - kept_shas: &run.kept_shas, - base_sha: run.base_sha.as_deref(), + kept_shas: &m.run.kept_shas, + base_sha: m.run.base_sha.as_deref(), // Progress publish never pushes branches (S3 only), so no composite targets here. components: &[], - published_branches: &run.published_branches, - cost_usd: run.spent, + published_branches: &m.run.published_branches, + cost_usd: m.run.spent, elapsed: started.elapsed(), identity_digest: &prep.identity.digest, seed_hash: &prep.identity.seed_hash, }, ); - if over_budget(args, control, run.spent, started, run.parked_total, r) { - exit = LoopExit::Budget; - break; - } - if verdict.keep && verdict.solved && !args.no_early_stop { - exit = LoopExit::Solved; - break; + if let Some(hit) = m.after_decide(&verdict, live_max_cost(args, control), started.elapsed()) + { + note_budget_hit(r, args, hit); } - it += 1; } + let exit = m.exit().unwrap_or(LoopExit::Finished); // Run-scoped epilogue: expensive one-shot checks (a 90-minute racecheck, a slow perf // rung) that cannot ride the per-iteration graph run once here, against the final kept // candidate. Advisory by contract: rows land in the log, RESULTS.md, the summary, and // the PR body, but nothing here can un-keep the candidate, and a concluded run stays // concluded even if the epilogue itself cannot run. - if matches!( - exit, - LoopExit::Finished | LoopExit::Budget | LoopExit::Solved - ) && let Some(workflow) = args.workflow.as_ref().filter(|w| w.has_epilogue()) + if exit.concluded() + && let Some(workflow) = args.workflow.as_ref().filter(|w| w.has_epilogue()) { - let kept = run - .rows - .iter() - .rev() - .find(|row| row.decision == "keep") - .map(|row| crate::loop_graph::KeptContext { - iter: row.iter, - score: row.score, - tiebreak: row.tiebreak, - sha: run.kept_shas.last().cloned(), - snapshot: row.kept_snap.clone(), - note: row.note.clone(), - }); - match kept { + match m.kept_context() { None => r.note("epilogue skipped: the run kept nothing"), Some(kept) => { // The epilogue measures the kept tree, not whatever the last discard left // behind; skip loudly rather than score the wrong tree. - if let Err(e) = world.restore(run.segment.best_snap.as_str()) { + if let Err(e) = world.restore(&m.run.segment.best_snap) { r.note(&format!( "epilogue skipped: restoring the kept best failed: {e:#}" )); } else { match crate::loop_graph::run_epilogue(args, p, workflow, &kept, r) { Ok((rows, cost)) => { - run.spent += cost; - beat_position(heartbeat, args.iterations, run.spent); - r.budget(run.spent, started.elapsed()); + m.add_cost(cost); + beat_position(heartbeat, args.iterations, m.run.spent); + r.budget(m.run.spent, started.elapsed()); for row in rows { r.row(&row, false); - run.rows.push(row); + m.record(row); } - write_results(p, &prep.goal, &prep.prior, &run.rows)?; + write_results(p, &prep.goal, &prep.prior, &m.run.rows)?; } Err(e) => r.note(&format!("epilogue failed to run (advisory): {e:#}")), } @@ -1322,31 +1163,32 @@ fn run_loop_body( } } - r.summary(&run.rows, &judge.objective(), run.segment.best_score); + r.summary(&m.run.rows, &judge.objective(), m.run.segment.best_score); update_control_status( control, "finished", args.iterations, - run.segment.best_score, - run.spent, + m.run.segment.best_score, + m.run.spent, ); - beat_position(heartbeat, args.iterations, run.spent); + beat_position(heartbeat, args.iterations, m.run.spent); // Publish-on-keep: durable artifacts off the (possibly ephemeral) pod. Runs here // so it fires on every exit path (clean finish, Ctrl+C, or budget-stop) and is // best-effort: a publish failure logs but never masks the loop's real outcome. let improved = judge.improved( - run.segment.best_score, - run.segment.baseline_score, - run.solved_any, + m.run.segment.best_score, + m.run.segment.baseline_score, + m.run.solved_any, ); // Composite multi-fork targets: a composite world resolves its touched components from the // baseline + best tokens; the per-component fork comes from the manifest map on `args`. // Empty for a single-repo world (it publishes via `base_sha`/`kept_shas` instead). - let components = run + let components = m + .run .base_snap .as_deref() - .and_then(|base| world.publish_components(base, run.segment.best_snap.as_str())) + .and_then(|base| world.publish_components(base, &m.run.segment.best_snap)) .map(|pc| publish::composite_targets(pc, &args.component_pr_repos)) .unwrap_or_default(); let prs = publish::publish( @@ -1358,20 +1200,20 @@ fn run_loop_body( goal: &prep.goal, model: args.model(), gate: judge.objective(), - rows: &run.rows, - baseline_score: run.segment.baseline_score, - best_score: run.segment.best_score, + rows: &m.run.rows, + baseline_score: m.run.segment.baseline_score, + best_score: m.run.segment.best_score, improved, - kept_shas: &run.kept_shas, - base_sha: run.base_sha.as_deref(), + kept_shas: &m.run.kept_shas, + base_sha: m.run.base_sha.as_deref(), components: &components, - published_branches: &run.published_branches, - cost_usd: run.spent, + published_branches: &m.run.published_branches, + cost_usd: m.run.spent, elapsed: started.elapsed(), identity_digest: &prep.identity.digest, seed_hash: &prep.identity.seed_hash, }, - &mut run.outputs, + &mut outputs, ); // Record the opened PR(s) on the session log so the controller's pull-ingest can fold them onto // the kept candidates' `pr_url` (the P1 fix). Best-effort by construction, a single-repo run @@ -1395,8 +1237,8 @@ fn run_loop_body( Ok(Outcome { improved, - solved: run.solved_any, - escalated: matches!(exit, LoopExit::Escalated), + solved: m.run.solved_any, + escalated: exit == LoopExit::Escalated, }) } @@ -1443,7 +1285,7 @@ fn spawn_feedback_watcher(r: &mut R, prs: &[publish::PrLink], p: &P /// The resumed segment's best tree, score, and keeps, resolved together by /// [`restore_kept_best`] so the score can never be paired with a tree it did not measure. struct ResumedBest { - best_snap: Snapshot, + best_snap: String, best_score: f64, best_tiebreak: Option, kept_shas: Vec, @@ -1467,7 +1309,7 @@ fn restore_kept_best( let Some(last_kept) = rows.iter().rev().find(|row| row.decision == "keep") else { // No keeps: the re-prepared checkout IS the baseline the logged scores measured. return Ok(ResumedBest { - best_snap: Snapshot(world.snapshot("resume").context("resume snapshot")?), + best_snap: world.snapshot("resume").context("resume snapshot")?, best_score: logged_best, best_tiebreak: logged_tiebreak, kept_shas: Vec::new(), @@ -1491,13 +1333,13 @@ fn restore_kept_best( .filter_map(|row| row.kept_snap.as_deref()) .filter_map(|snap| world.commit_sha(snap)) .collect(), - best_snap: Snapshot(snap), + best_snap: snap, best_score: logged_best, best_tiebreak: logged_tiebreak, degraded: None, }), Err(why) => Ok(ResumedBest { - best_snap: Snapshot(world.snapshot("resume").context("resume snapshot")?), + best_snap: world.snapshot("resume").context("resume snapshot")?, best_score: worst_score(direction), // The score's artifact is gone, so its tiebreak goes with it. best_tiebreak: None, @@ -1511,88 +1353,21 @@ fn restore_kept_best( } } -/// The counter fold `--resume` replays from the session log. Fed one event at a time so -/// [`crate::recovery::classify_session`] can drive it and the tail scanner in one pass. -/// Decided rows carry `score`/`total`, so baseline + best restore exactly. -#[derive(Default)] -pub(crate) struct ResumeFold { - rows: Vec, - spent: f64, - summary_best: Option, - solved_any: bool, - identity: Option, - published_branches: Vec, -} - -impl ResumeFold { - pub(crate) fn feed(&mut self, ev: &session::SessionEvent) { - use session::{IntoRow, SessionEvent}; - match ev { - SessionEvent::Row { row, solved } => { - // Infra-dead rows (phase:"infra") record turns that never started — their - // iteration was never consumed, so counting them would skip it on resume. - // Wide-round rows (phase:"wide") come from logs written before the wide - // tournament was removed; they never counted toward the deep loop. - if matches!(row.phase.as_deref(), Some("wide") | Some("infra")) { - return; - } - self.solved_any |= *solved; - self.rows.push(row.clone().into_row()); - } - SessionEvent::Budget { spent, .. } => self.spent = *spent, - SessionEvent::Summary { best_score, .. } => self.summary_best = *best_score, - // Last one wins: a run resumed more than once re-emits a fresh identity each time. - SessionEvent::Identity { identity } => self.identity = Some(identity.clone()), - // Accumulated across segments: every branch any prior publish opened a PR from, - // so a replayed finish can recognize an already-published kept commit. - SessionEvent::PrLinks { links } => { - self.published_branches - .extend(links.iter().map(|l| l.branch.clone())); - } - _ => {} - } - } - - /// A rowless log is unresumable; the caller refuses before [`finish`](Self::finish). - pub(crate) fn has_rows(&self) -> bool { - !self.rows.is_empty() - } - - pub(crate) fn finish(self) -> ResumeState { - let baseline_score = self - .rows - .first() - .and_then(|r| r.score) - .unwrap_or(f64::INFINITY); - let baseline_total = self.rows.first().and_then(|r| r.total).unwrap_or(0); - let best_score = self.summary_best.unwrap_or_else(|| { - self.rows - .iter() - .filter(|r| r.decision == "keep") - .filter_map(|r| r.score) - .fold(baseline_score, f64::min) - }); - // Keeps are monotone within a segment, so the last kept row IS the best; its tiebreak - // travels with the best score. No keeps = the baseline's (usually absent) tiebreak. - let best_tiebreak = self - .rows - .iter() - .rev() - .find(|r| r.decision == "keep") - .or_else(|| self.rows.first()) - .and_then(|r| r.tiebreak); - let next_iter = self.rows.iter().map(|r| r.iter).max().unwrap_or(0) + 1; +impl ResumeState { + /// The engine's typed view of what the contract fold restored. + pub(crate) fn from_view(view: crucible_contract::ResumeView) -> Self { + use session::IntoRow; ResumeState { - rows: self.rows, - best_score, - best_tiebreak, - baseline_score, - baseline_total, - spent: self.spent, - next_iter, - solved_any: self.solved_any, - identity: self.identity, - published_branches: self.published_branches, + rows: view.rows.into_iter().map(IntoRow::into_row).collect(), + best_score: view.best_score, + best_tiebreak: view.best_tiebreak, + baseline_score: view.baseline_score, + baseline_total: view.baseline_total, + spent: view.spent, + next_iter: view.next_iter, + solved_any: view.solved_any, + identity: view.identity, + published_branches: view.published_branches, } } } @@ -1646,8 +1421,6 @@ fn update_control_status( } } -/// True when a cost/time cap is set and reached; notes it on `r`. `parked_total` is idle time -/// spent waiting on a human approval, excluded from the wall-clock the time cap measures. /// The effective cost cap: a live control override wins over the CLI arg. pub(crate) fn live_max_cost(args: &Args, control: Option<&control::ControlState>) -> f64 { control @@ -1655,46 +1428,16 @@ pub(crate) fn live_max_cost(args: &Args, control: Option<&control::ControlState> .unwrap_or(args.max_cost) } -fn over_budget( - args: &Args, - control: Option<&control::ControlState>, - spent: f64, - started: Instant, - parked_total: Duration, - r: &mut R, -) -> bool { - let max_cost = live_max_cost(args, control); - if max_cost > 0.0 && spent >= max_cost { - r.note(&format!( - "budget: cost ${spent:.4} reached cap ${:.2} — stopping", - max_cost - )); - return true; - } - if let Some(cap) = args.max_time() { - // Subtract parked (approval-wait) time: idling on a human must not burn the time budget. - let active = started.elapsed().saturating_sub(parked_total); - if active >= cap { - r.note(&format!( - "budget: time cap {} reached — stopping", - args.max_time - )); - return true; - } +fn note_budget_hit(r: &mut R, args: &Args, hit: BudgetHit) { + match hit { + BudgetHit::Cost { spent, cap } => r.note(&format!( + "budget: cost ${spent:.4} reached cap ${cap:.2} — stopping" + )), + BudgetHit::Time => r.note(&format!( + "budget: time cap {} reached — stopping", + args.max_time + )), } - false -} - -/// Why a [`park_for_approval`] ended, the terminal provisioning outcome the loop waited on. -enum ParkOutcome { - /// A grant landed as a re-scope (the watcher sends this once provisioning is ready). The - /// iteration-head drain re-baselines into the new regime. - Resumed, - /// Not granted: a denial (operator / forge / policy) or a park timeout. The caller decides - /// whether to resume frozen or escalate, per the agent's mode. - Denied(String), - /// Ctrl+C / stop while parked. - Stopped, } /// Park the loop until a pending approval reaches a terminal outcome: a re-scope (the grant is @@ -1706,12 +1449,11 @@ fn park_for_approval( control: Option<&control::ControlState>, ledger: Option<&crate::admission::AdmissionLedger>, r: &mut R, - parked_total: &mut Duration, timeout: Option, -) -> ParkOutcome { +) -> (ParkOutcome, Duration) { let Some(control) = control else { r.note("block requested but no control bridge to receive an approval — continuing"); - return ParkOutcome::Resumed; + return (ParkOutcome::Resumed, Duration::ZERO); }; r.note("parked: idle, awaiting approval (budget paused)"); let start = Instant::now(); @@ -1739,13 +1481,12 @@ fn park_for_approval( } std::thread::sleep(Duration::from_millis(250)); }; - *parked_total += start.elapsed(); match &outcome { ParkOutcome::Resumed => r.note("approval received — resuming with a re-scope"), ParkOutcome::Denied(why) => r.note(&format!("approval not granted: {why}")), ParkOutcome::Stopped => {} } - outcome + (outcome, start.elapsed()) } /// Fold the turn's info/warn distress notes onto the row being written: they are per-turn signals, @@ -1758,17 +1499,6 @@ fn fold_distress_notes(r: &mut R, note: &mut String) { } } -/// Why a [`park_for_distress`] ended. -#[derive(Debug, PartialEq, Eq)] -enum DistressOutcome { - /// The operator removed the marker: that IS the grant, resume at the next iteration head. - Cleared, - /// Ctrl+C / stop / a control interrupt while suspended. - Stopped, - /// `--max-park` elapsed with the marker still in place. - TimedOut, -} - /// Suspend the loop while the broker's distress marker exists. Idle and budget-paused: the caller /// folds the elapsed time into `parked_total`, so suspended wall-clock burns no time budget. The /// engine never deletes the marker: the operator's `rm` is the resume grant. @@ -1776,9 +1506,8 @@ fn park_for_distress( p: &Paths, rows: &[Row], r: &mut R, - parked_total: &mut Duration, timeout: Option, -) -> DistressOutcome { +) -> (DistressOutcome, Duration) { let start = Instant::now(); let outcome = loop { if crate::distress::read_marker().is_none() { @@ -1792,8 +1521,7 @@ fn park_for_distress( } std::thread::sleep(Duration::from_millis(250)); }; - *parked_total += start.elapsed(); - outcome + (outcome, start.elapsed()) } /// Stamp the current iteration where the broker's distress page reads it (`/turn-meta.json`, @@ -2128,9 +1856,7 @@ mod tests { /// The counter fold as `--resume` consumes it (through the classifier), so these /// replay tests exercise the same path `run.rs` takes. fn load_resume_state(session_log: &std::path::Path) -> Result { - let got = crate::recovery::classify_session(session_log)?; - crate::recovery::parity::assert_matches(session_log, &got); - Ok(got.resume) + crate::recovery::classify_session(session_log).map(|s| s.resume) } /// The 401 that killed a 5h turn, plus the other transport signatures, classify as retryable; @@ -2716,8 +2442,7 @@ mod tests { }); let mut r = NoteCapture::default(); - let mut parked = Duration::ZERO; - let outcome = park_for_approval(Some(&control), None, &mut r, &mut parked, None); + let (outcome, parked) = park_for_approval(Some(&control), None, &mut r, None); h.join().unwrap(); assert!(matches!(outcome, ParkOutcome::Resumed)); @@ -2757,8 +2482,7 @@ mod tests { deliver.set_deny(AdmissionKey::new("d1"), "over budget".into()); }); let mut r = NoteCapture::default(); - let mut parked = Duration::ZERO; - let outcome = park_for_approval(Some(&control), None, &mut r, &mut parked, None); + let (outcome, _parked) = park_for_approval(Some(&control), None, &mut r, None); h.join().unwrap(); match outcome { ParkOutcome::Denied(why) => { @@ -2779,12 +2503,10 @@ mod tests { // No signal ever arrives; a short --max-park bounds the wait and resolves to Denied. let control = std::sync::Arc::new(control::ControlState::default()); let mut r = NoteCapture::default(); - let mut parked = Duration::ZERO; - let outcome = park_for_approval( + let (outcome, parked) = park_for_approval( Some(&control), None, &mut r, - &mut parked, Some(Duration::from_millis(120)), ); match outcome { @@ -2803,8 +2525,7 @@ mod tests { fn park_without_control_bridge_does_not_block() { // No bridge => nothing could deliver an approval; park notes and returns rather than hang. let mut r = NoteCapture::default(); - let mut parked = Duration::ZERO; - let outcome = park_for_approval(None, None, &mut r, &mut parked, None); + let (outcome, parked) = park_for_approval(None, None, &mut r, None); assert!(matches!(outcome, ParkOutcome::Resumed)); assert_eq!(parked, Duration::ZERO); assert!(r.notes.iter().any(|n| n.contains("no control bridge"))); @@ -2842,8 +2563,7 @@ mod tests { }); let mut r = NoteCapture::default(); - let mut parked = Duration::ZERO; - let outcome = park_for_distress(&f.paths, &[], &mut r, &mut parked, None); + let (outcome, parked) = park_for_distress(&f.paths, &[], &mut r, None); h.join().unwrap(); assert_eq!(outcome, DistressOutcome::Cleared); @@ -2861,9 +2581,8 @@ mod tests { quit_after: Some(1), ..Default::default() }; - let mut parked = Duration::ZERO; assert_eq!( - park_for_distress(&f.paths, &[], &mut r, &mut parked, None), + park_for_distress(&f.paths, &[], &mut r, None).0, DistressOutcome::Stopped ); assert!( @@ -2877,17 +2596,9 @@ mod tests { let (f, root) = distress_fixture("loop-timeout", 1); write_marker(&root.dir, "nobody came", 1); let mut r = NoteCapture::default(); - let mut parked = Duration::ZERO; - assert_eq!( - park_for_distress( - &f.paths, - &[], - &mut r, - &mut parked, - Some(Duration::from_millis(120)) - ), - DistressOutcome::TimedOut - ); + let (outcome, parked) = + park_for_distress(&f.paths, &[], &mut r, Some(Duration::from_millis(120))); + assert_eq!(outcome, DistressOutcome::TimedOut); assert!(parked >= Duration::from_millis(100), "{parked:?}"); assert!(root.dir.join("distress").exists()); } @@ -2898,22 +2609,22 @@ mod tests { // longer than its time cap is not over budget, because suspended time buys nothing. let mut f = fixture(1, 0.0, false); f.args.max_time = "0.1s".into(); - let started = Instant::now(); - std::thread::sleep(Duration::from_millis(150)); - let mut r = NoteCapture::default(); + let cfg = LoopCfg { + iterations: f.args.iterations, + max_time: f.args.max_time(), + max_park: f.args.max_park(), + no_early_stop: f.args.no_early_stop, + }; + let mut m = Machine::new(cfg, resume_run_state(resume_state_for_test(1)), 1); + m.run.parked_total = Duration::from_millis(150); assert!( - !over_budget( - &f.args, - None, - 0.0, - started, - Duration::from_millis(150), - &mut r - ), + m.over_budget(0.0, Duration::from_millis(150)).is_none(), "suspended wall-clock must not burn the time cap" ); - assert!( - over_budget(&f.args, None, 0.0, started, Duration::ZERO, &mut r), + m.run.parked_total = Duration::ZERO; + assert_eq!( + m.over_budget(0.0, Duration::from_millis(150)), + Some(BudgetHit::Time), "the same elapsed time WITHOUT a park is over budget" ); } @@ -2991,14 +2702,14 @@ mod tests { fn a_distressed_row_folds_as_inert_on_resume() { // The distressed row is an annotation, not a measurement: it carries no score and no // kept snapshot, so a resume must not treat it as the baseline or the best. - let mut fold = ResumeFold::default(); + let mut fold = crucible_contract::LoopState::default(); for ev in [ logged_row(1, "keep", "", Some(10.0), Some("snap-1")), logged_row(1, "distressed", "torch skew", None, None), ] { - fold.feed(&ev); + fold.apply(&ev); } - let state = fold.finish(); + let state = ResumeState::from_view(fold.resume_view()); assert_eq!(state.best_score, 10.0, "the kept row still defines best"); assert_eq!( state.next_iter, 2, @@ -3139,14 +2850,14 @@ mod tests { let marker = root.dir.join("distress"); write_marker(&root.dir, "torch skew", 42); // The pre-restart log: iteration 1 ran, then this same marker parked the head. - let mut fold = ResumeFold::default(); + let mut fold = crucible_contract::LoopState::default(); for ev in [ logged_row(1, "discard", "", Some(100.0), None), logged_row(1, "distressed", "torch skew", None, None), ] { - fold.feed(&ev); + fold.apply(&ev); } - let prior_run = fold.finish(); + let prior_run = ResumeState::from_view(fold.resume_view()); let mut r = RecordingReporter::default(); let clear = marker.clone(); let operator = std::thread::spawn(move || { @@ -4257,6 +3968,30 @@ mod tests { assert_eq!(r.shutdowns[0].0, "error"); } + /// The machine's state for a resumed run, as the host builds it minus the world calls. + fn resume_run_state(rs: ResumeState) -> RunState { + RunState { + rows: rs.rows, + spent: rs.spent, + kept_shas: Vec::new(), + base_sha: None, + base_snap: None, + solved_any: rs.solved_any, + parked_total: Duration::ZERO, + pending_block: None, + published_branches: rs.published_branches, + segment: Segment { + regime: "default".into(), + fingerprint: String::new(), + baseline_score: rs.baseline_score, + best_score: rs.best_score, + best_tiebreak: rs.best_tiebreak, + baseline_total: rs.baseline_total, + best_snap: String::new(), + }, + } + } + fn resume_state_for_test(next_iter: u32) -> ResumeState { ResumeState { rows: vec![ @@ -4318,6 +4053,7 @@ mod tests { recovery: Some(recovery), ledger: None, heartbeat: None, + prior_plan: None, }, ) .expect("resumed run finishes"); @@ -4369,6 +4105,7 @@ mod tests { recovery: Some(recovery), ledger: None, heartbeat: None, + prior_plan: None, }, ) .expect("resumed run finishes"); @@ -4846,8 +4583,8 @@ mod tests { &[&touch_cmd], Some(r#"echo '{"score":42.0,"tiebreak":0.1,"note":"seeded on resume"}'"#), )); - // The prior session: exactly one preflight-failed row, no score. This is what - // ResumeFold::finish produces when the only row is a preflight refusal. + // The prior session: exactly one preflight-failed row, no score. This is what the + // contract fold produces when the only row is a preflight refusal. let rs = ResumeState { rows: vec![Row { iter: 0, diff --git a/crucible/src/loop_graph.rs b/crucible/src/loop_graph.rs index e574eb15..d9a4f0b5 100644 --- a/crucible/src/loop_graph.rs +++ b/crucible/src/loop_graph.rs @@ -16,7 +16,8 @@ use serde_json::Value; use crate::loop_driver::{self, Decided, IterStep, Measured, TurnVerdict}; use crate::manifest::{WorkflowCaps, WorkflowCfg, WorkflowType}; use crate::plan::exec::{ - Attempt, AttemptOutcome, BatchItem, ExecCfg, Substrate, TaskRunner, TaskStatus, execute, + Attempt, AttemptOutcome, BatchItem, ExecCfg, Substrate, TaskResult, TaskRunner, TaskStatus, + execute, execute_from, }; use crate::plan::ir::{ EngineOp, Join, Plan, PlanBudget, Stage, Task, TaskKind, TaskName, ValidPlan, @@ -62,6 +63,54 @@ pub(crate) struct IterCtx<'a> { pub started: Instant, pub heartbeat: Option>, pub workflow: Option<&'a WorkflowCfg>, + /// Results a previous process settled in this same iteration, from the session log. + pub prior: Option, +} + +/// The plan tasks a dead process settled in the iteration it died in, as the contract fold +/// restored them. A resumed iteration whose freshly built plan declares the same tasks under +/// the same version starts from these instead of re-dispatching them. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct PriorPlan { + pub iter: u32, + pub plan_version: u32, + pub declared: Vec, + pub results: BTreeMap, +} + +impl PriorPlan { + /// The results to seed `plan` with: only when the plan is the one the log admitted (same + /// version, same declared tasks), and only the tasks that passed. Engine operations are + /// never seeded: the world a passing `propose` or `apply` produced did not survive the + /// process, so they run again. + fn seed(&self, plan: &ValidPlan) -> BTreeMap { + let declared: Vec = plan.plan().tasks.iter().map(|t| t.name.0.clone()).collect(); + if plan.plan().version != self.plan_version || declared != self.declared { + return BTreeMap::new(); + } + plan.plan() + .tasks + .iter() + .filter(|t| !matches!(t.task, TaskKind::Engine { .. })) + .filter_map(|t| { + let prior = self.results.get(&t.name.0)?; + let status = TaskStatus::parse(&prior.status)?; + (status == TaskStatus::Pass).then(|| { + ( + t.name.clone(), + TaskResult { + status, + attempts: prior.attempts, + cost_usd: prior.cost_usd, + output: prior.output.clone(), + note: (!prior.note.is_empty()).then(|| prior.note.clone()), + fanout: None, + }, + ) + }) + }) + .collect() + } } /// Run one admitted autoresearch iteration and return its step and cost. @@ -75,6 +124,17 @@ pub(crate) fn run_iteration(cx: IterCtx<'_>, r: &mut R) -> Result<( caps = caps.with_persistent_sessions(); } let plan = iteration_template(cx.workflow, &caps)?; + let prior = cx + .prior + .as_ref() + .map(|prior| prior.seed(&plan)) + .unwrap_or_default(); + if !prior.is_empty() { + r.note(&format!( + "resume: {} task(s) settled before the run died carry over; the rest run", + prior.len() + )); + } let result_task = cx .workflow .filter(|workflow| !workflow.is_legacy_splice()) @@ -120,11 +180,12 @@ pub(crate) fn run_iteration(cx: IterCtx<'_>, r: &mut R) -> Result<( // The runner and the on_result hook both need the reporter; collect the wire lines // here and append them after the executor returns (they're additive either way). let mut task_events = Vec::new(); - let outcome = execute( + let outcome = execute_from( &plan, &Substrate::default(), ExecCfg::default(), &mut runner, + prior, |task, result| { task_states .borrow_mut() @@ -821,6 +882,62 @@ mod tests { assert_eq!(dep("decide"), ["measure"]); } + /// A resumed iteration re-uses what the dead process settled only when the plan is the + /// one it admitted, and only the passing pack tasks: engine operations run again because + /// the world they produced did not survive the process. + #[test] + fn a_prior_plan_seeds_only_passing_pack_tasks_of_the_same_plan() { + let w: WorkflowCfg = toml::from_str( + "[[task]]\nname = \"review\"\nkind = \"command\"\ncommand = \"true\"\n [[task]]\nname = \"lint\"\nkind = \"command\"\ncommand = \"true\"\ndepends_on = [\"review\"]\n", + ) + .unwrap(); + w.validate().unwrap(); + let plan = iteration_template(Some(&w), &WorkflowCaps::autoresearch_engine()).unwrap(); + let declared: Vec = plan.plan().tasks.iter().map(|t| t.name.0.clone()).collect(); + let wire = |status: &str| crucible_contract::TaskResultWire { + status: status.into(), + task_kind: "command".into(), + iter: 2, + attempts: 1, + cost_usd: 0.1, + secs: 0.2, + note: String::new(), + output: Some(serde_json::json!({"ok": true})), + }; + let prior = PriorPlan { + iter: 2, + plan_version: 1, + declared: declared.clone(), + results: BTreeMap::from([ + ("propose".to_string(), wire("pass")), + ("review".to_string(), wire("pass")), + ("lint".to_string(), wire("fail")), + ]), + }; + let seeded = prior.seed(&plan); + let names: Vec<&str> = seeded.keys().map(|n| n.0.as_str()).collect(); + assert_eq!(names, ["review"], "passing pack task only: {names:?}"); + assert_eq!(seeded[&TaskName::from("review")].status, TaskStatus::Pass); + assert_eq!(seeded[&TaskName::from("review")].cost_usd, 0.1); + + let other_version = PriorPlan { + plan_version: 2, + ..prior.clone() + }; + assert!( + other_version.seed(&plan).is_empty(), + "a different plan version seeds nothing" + ); + let other_shape = PriorPlan { + declared: vec!["review".into()], + ..prior + }; + assert!( + other_shape.seed(&plan).is_empty(), + "a different task set seeds nothing" + ); + } + /// Epilogue tasks stay out of the per-iteration plan entirely (legacy splice and /// fully-authored form) and land in their own post-loop template. #[test] diff --git a/crucible/src/machine.rs b/crucible/src/machine.rs new file mode 100644 index 00000000..2a5e20aa --- /dev/null +++ b/crucible/src/machine.rs @@ -0,0 +1,791 @@ +//! The loop's decisions, with no I/O in them. +//! +//! [`Machine`] holds the run's state and answers "what happens next" from plain inputs: the +//! head-of-iteration checks in the order the loop has always made them, the fold of a turn's +//! outcome into a row, the keep/discard bookkeeping, and the exit. The host in +//! [`crate::loop_driver`] performs every effect (agent turns, world snapshots, the reporter, the +//! control bridge, publish) and hands the results back. A test drives the machine with literal +//! inputs and asserts the decisions; nothing here can touch a file, a socket, or a clock. +//! +//! ```text +//! host machine +//! ─────────────────────────────────────── ─────────────────────────────── +//! poll control/markers/clock ──inputs──▶ head() ──Head──▶ act +//! run turn / plan ──IterStep──▶ settle() ──Settle─▶ act +//! snapshot / restore world ──result───▶ keep() / discard() +//! poll budget ──inputs──▶ over_budget() ──exit───▶ shutdown +//! ``` + +use std::time::Duration; + +use crate::provisioning::PendingProvisioning; +use crate::reporter::Row; + +/// The loop's ceilings and switches, read once from `Args`. +#[derive(Debug, Clone)] +pub(crate) struct LoopCfg { + pub iterations: u32, + pub max_time: Option, + pub max_park: Option, + pub no_early_stop: bool, +} + +/// The comparable segment: a baseline and everything measured against it. A re-scope swaps +/// the whole thing, so a goalpost can never be half-moved. +#[derive(Debug, Clone)] +pub(crate) struct Segment { + pub regime: String, + pub fingerprint: String, + pub baseline_score: f64, + pub best_score: f64, + pub best_tiebreak: Option, + pub baseline_total: u64, + /// The world snapshot token of the kept best, the rollback target. + pub best_snap: String, +} + +/// The run's state across iterations. Everything the log can reproduce is here in the same +/// shape the contract fold restores it; the snapshot tokens and the pending park are the +/// process-local additions. +#[derive(Debug, Clone)] +pub(crate) struct RunState { + pub rows: Vec, + pub spent: f64, + /// SHAs of commits this run kept, for the publish summary. + pub kept_shas: Vec, + /// The pristine upstream SHA segment 0 was measured on; `None` on resume. + pub base_sha: Option, + /// The pristine baseline snapshot token; `None` on resume. + pub base_snap: Option, + pub solved_any: bool, + /// Idle time spent parked on a human, excluded from the time cap. + pub parked_total: Duration, + /// The block-mode approval the next head parks on. + pub pending_block: Option, + /// Branches prior publishes opened PRs from; publish skips them. + pub published_branches: Vec, + pub segment: Segment, +} + +/// How a run ended: the single enumeration of every way the loop exits. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum LoopExit { + /// The loop ran every iteration without an early exit. + Finished, + /// A kept candidate satisfied the win condition. + Solved, + /// A cost or time cap was reached. + Budget, + /// A stop signal, at a checkpoint or while parked. + Stopped, + /// The agent declared the harness inadequate, or a `block` approval was denied with no + /// fallback: halt for human review. + Escalated, + /// [`MAX_DEAD_TURN_ATTEMPTS`] consecutive turns died before starting. + Stalled, +} + +impl LoopExit { + /// The wire token and reason for the shutdown line. + pub(crate) fn shutdown_reason(self) -> (&'static str, &'static str) { + match self { + LoopExit::Finished => ("finished", "all iterations completed"), + LoopExit::Solved => ("solved", "a kept candidate satisfied the win condition"), + LoopExit::Budget => ("budget", "a cost or time cap was reached"), + LoopExit::Stopped => ("stopped", "stop signal received"), + LoopExit::Escalated => ( + "escalated", + "the agent declared the harness inadequate — halted for human review", + ), + LoopExit::Stalled => ( + "stalled", + "the run stalled on consecutive transport failures — no turn could start", + ), + } + } + + /// Whether the run concluded on its own terms, so the epilogue and the final publish run. + pub(crate) fn concluded(self) -> bool { + matches!( + self, + LoopExit::Finished | LoopExit::Budget | LoopExit::Solved + ) + } +} + +/// Consecutive never-started turns after which the run halts as [`LoopExit::Stalled`]. Such a +/// turn re-runs its iteration instead of consuming it, so without this bound one dead node could +/// spin the run forever. +pub(crate) const MAX_DEAD_TURN_ATTEMPTS: u32 = 3; + +/// Why a park ended, as the host observed it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ParkOutcome { + /// A grant landed as a re-scope; the head's rescope drain re-baselines. + Resumed, + /// A denial or a park timeout. + Denied(String), + /// A stop while parked. + Stopped, +} + +/// Why a distress park ended. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum DistressOutcome { + /// The operator removed the marker: that is the grant. + Cleared, + Stopped, + TimedOut, +} + +/// A cap the head found reached. +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) enum BudgetHit { + Cost { spent: f64, cap: f64 }, + Time, +} + +/// One iteration's outcome, produced by either loop path and folded by [`Machine::settle`]. +pub(crate) enum IterStep { + Decided(Box), + /// Discard and move on; the reason lands in the row. + Discarded { + reason: String, + }, + /// The turn never started: re-run the same iteration, bounded. + NeverStarted { + reason: String, + }, + /// Halt for human review (the escalation is already reported). + Escalated, + /// Park at the next head on a blocking approval. + Parked(PendingProvisioning), + /// Stop signal at the post-turn checkpoint. + Stopped, +} + +/// What the host does with a settled iteration. +pub(crate) enum Settle { + /// Record `row`, restore the best tree, advance. + Discard { + row: Row, + }, + /// Record `row`, restore the best tree, re-run the same iteration; `stalled` ends the run. + Rerun { + row: Row, + attempt: u32, + stalled: bool, + }, + /// Restore the best tree and end the run for a human. + Escalate, + /// The park is armed for the next head; advance. + Park, + Stop, + /// A measured candidate: keep or discard per `verdict`. + Decide(Box), +} + +/// The loop's decisions over its state. +pub(crate) struct Machine { + pub cfg: LoopCfg, + pub run: RunState, + /// The iteration about to run. Advances only when a turn actually started. + pub it: u32, + dead_turns: u32, + /// The distress marker this run already parked on, by its `ts_ms`. + parked_distress_ts: Option, + exit: Option, +} + +impl Machine { + pub(crate) fn new(cfg: LoopCfg, run: RunState, start_iter: u32) -> Self { + Machine { + cfg, + run, + it: start_iter, + dead_turns: 0, + parked_distress_ts: None, + exit: None, + } + } + + /// The exit, once one is set. The host's loop condition. + pub(crate) fn exit(&self) -> Option { + self.exit + } + + /// Whether another iteration may run. + pub(crate) fn has_iterations(&self) -> bool { + self.it <= self.cfg.iterations + } + + pub(crate) fn end(&mut self, exit: LoopExit) { + self.exit = Some(exit); + } + + /// The block-mode approval the head parks on, taken so it parks once. + pub(crate) fn take_pending_block(&mut self) -> Option { + self.run.pending_block.take() + } + + /// The park is over. A denial on a `block` wait leaves nothing to try, so it ends the run + /// for a human; a stop ends it as stopped; a grant lets the head's rescope drain proceed. + pub(crate) fn on_park(&mut self, parked: Duration, outcome: &ParkOutcome) -> Option { + self.run.parked_total += parked; + let exit = match outcome { + ParkOutcome::Resumed => return None, + ParkOutcome::Denied(_) => LoopExit::Escalated, + ParkOutcome::Stopped => LoopExit::Stopped, + }; + self.exit = Some(exit); + Some(exit) + } + + /// An approved re-scope re-baselined: the new segment replaces the old one whole. + pub(crate) fn rescope(&mut self, segment: Segment) { + self.run.segment = segment; + } + + /// A distress marker was read at the head. `Some(row)` when this marker has not parked the + /// run before; the row annotates the turn that raised it, and is omitted when an identical + /// row is already on the record (an in-place restart re-reads a marker the operator never + /// cleared). + pub(crate) fn on_distress(&mut self, ts_ms: u64, reason: &str) -> Option> { + if self.parked_distress_ts == Some(ts_ms) { + return None; + } + self.parked_distress_ts = Some(ts_ms); + let row = Row { + iter: self.it.saturating_sub(1), + decision: "distressed".to_string(), + note: reason.to_string(), + ..Default::default() + }; + let seen = self.run.rows.iter().any(|prior| { + prior.decision == row.decision && prior.iter == row.iter && prior.note == row.note + }); + Some((!seen).then_some(row)) + } + + /// The distress park is over. + pub(crate) fn on_distress_park( + &mut self, + parked: Duration, + outcome: DistressOutcome, + ) -> Option { + self.run.parked_total += parked; + match outcome { + DistressOutcome::Cleared => None, + DistressOutcome::Stopped | DistressOutcome::TimedOut => { + self.exit = Some(LoopExit::Stopped); + self.exit + } + } + } + + /// A cap reached, if any. `elapsed` is the run's wall clock; parked time is excluded. + pub(crate) fn over_budget(&self, live_max_cost: f64, elapsed: Duration) -> Option { + if live_max_cost > 0.0 && self.run.spent >= live_max_cost { + return Some(BudgetHit::Cost { + spent: self.run.spent, + cap: live_max_cost, + }); + } + if let Some(cap) = self.cfg.max_time + && elapsed.saturating_sub(self.run.parked_total) >= cap + { + return Some(BudgetHit::Time); + } + None + } + + /// The pack seed diff goes to iteration 1 only. + pub(crate) fn wants_seed(&self) -> bool { + self.it == 1 + } + + pub(crate) fn add_cost(&mut self, usd: f64) { + self.run.spent += usd; + } + + /// Fold one iteration's outcome. Any step other than `NeverStarted` proves a turn started + /// and resets the stall streak; a never-started turn keeps the iteration and counts toward + /// [`MAX_DEAD_TURN_ATTEMPTS`]. + pub(crate) fn settle(&mut self, step: IterStep) -> Settle { + if !matches!(step, IterStep::NeverStarted { .. }) { + self.dead_turns = 0; + } + let it = self.it; + match step { + IterStep::Decided(d) => Settle::Decide(d), + IterStep::Discarded { reason } => { + let row = Row { + iter: it, + decision: "discarded".to_string(), + note: reason, + ..Default::default() + }; + Settle::Discard { row } + } + IterStep::NeverStarted { reason } => { + self.dead_turns += 1; + let row = Row { + iter: it, + decision: "infra-dead".to_string(), + note: reason, + phase: Some("infra".to_string()), + ..Default::default() + }; + let stalled = self.dead_turns >= MAX_DEAD_TURN_ATTEMPTS; + if stalled { + self.exit = Some(LoopExit::Stalled); + } + Settle::Rerun { + row, + attempt: self.dead_turns, + stalled, + } + } + IterStep::Escalated => { + self.exit = Some(LoopExit::Escalated); + Settle::Escalate + } + IterStep::Parked(pp) => { + self.run.pending_block = Some(pp); + Settle::Park + } + IterStep::Stopped => { + self.exit = Some(LoopExit::Stopped); + Settle::Stop + } + } + } + + /// A settled row goes on the record. + pub(crate) fn record(&mut self, row: Row) { + self.run.rows.push(row); + } + + /// The candidate was kept: the segment's best moves to its reading, and the snapshot the + /// host took becomes the rollback target. `snapshot` is `Err(why)` when the world could not + /// snapshot; the change stays live and the best keeps its old rollback target. + pub(crate) fn keep( + &mut self, + row: &mut Row, + reading: &crucible::crucible::Reading, + solved: bool, + snapshot: Result<(String, Option), String>, + ) { + if let Some(s) = reading.score { + self.run.segment.best_score = s; + // The kept candidate defines both axes: a stale tiebreak would compare the next tie + // against a scalar the current best never earned. + self.run.segment.best_tiebreak = reading.tiebreak; + } + if let Ok((snap, sha)) = snapshot { + if let Some(sha) = sha { + self.run.kept_shas.push(sha); + } + row.kept_snap = Some(snap.clone()); + self.run.segment.best_snap = snap; + } + self.run.solved_any |= solved; + } + + /// After a decided iteration: a cap or an early solve ends the run, else the next + /// iteration is up. + pub(crate) fn after_decide( + &mut self, + verdict: &crucible::crucible::Decision, + live_max_cost: f64, + elapsed: Duration, + ) -> Option { + if let Some(hit) = self.over_budget(live_max_cost, elapsed) { + self.exit = Some(LoopExit::Budget); + return Some(hit); + } + if verdict.keep && verdict.solved && !self.cfg.no_early_stop { + self.exit = Some(LoopExit::Solved); + return None; + } + self.it += 1; + None + } + + /// The iteration was consumed without a decision (a discard, a park). + pub(crate) fn advance(&mut self) { + self.it += 1; + } + + /// The kept context the epilogue runs against, from the last kept row. + pub(crate) fn kept_context(&self) -> Option { + self.run + .rows + .iter() + .rev() + .find(|row| row.decision == "keep") + .map(|row| crate::loop_graph::KeptContext { + iter: row.iter, + score: row.score, + tiebreak: row.tiebreak, + sha: self.run.kept_shas.last().cloned(), + snapshot: row.kept_snap.clone(), + note: row.note.clone(), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crucible::crucible::{Decision, Reading}; + + fn cfg() -> LoopCfg { + LoopCfg { + iterations: 3, + max_time: Some(Duration::from_secs(100)), + max_park: None, + no_early_stop: false, + } + } + + fn run() -> RunState { + RunState { + rows: vec![Row { + iter: 0, + decision: "baseline".into(), + score: Some(240.0), + ..Default::default() + }], + spent: 0.0, + kept_shas: Vec::new(), + base_sha: None, + base_snap: None, + solved_any: false, + parked_total: Duration::ZERO, + pending_block: None, + published_branches: Vec::new(), + segment: Segment { + regime: "default".into(), + fingerprint: "f".into(), + baseline_score: 240.0, + best_score: 240.0, + best_tiebreak: None, + baseline_total: 0, + best_snap: "base".into(), + }, + } + } + + fn reading(score: f64) -> Reading { + Reading { + valid: true, + score: Some(score), + tiebreak: None, + solved: false, + note: String::new(), + detail: serde_json::Value::Null, + } + } + + fn decided(it: u32, keep: bool, solved: bool, score: f64) -> IterStep { + IterStep::Decided(Box::new(crate::loop_driver::Decided { + row: Row { + iter: it, + decision: if keep { "keep" } else { "discard" }.into(), + score: Some(score), + ..Default::default() + }, + verdict: Decision { keep, solved }, + reading: reading(score), + })) + } + + #[test] + fn a_kept_candidate_moves_the_best_and_the_rollback_target() { + let mut m = Machine::new(cfg(), run(), 1); + let Settle::Decide(d) = m.settle(decided(1, true, false, 220.0)) else { + panic!("a decided step settles as Decide"); + }; + let mut row = d.row; + m.keep( + &mut row, + &d.reading, + false, + Ok(("snap-1".into(), Some("sha-1".into()))), + ); + assert_eq!(m.run.segment.best_score, 220.0); + assert_eq!(m.run.segment.best_snap, "snap-1"); + assert_eq!(row.kept_snap.as_deref(), Some("snap-1")); + assert_eq!(m.run.kept_shas, vec!["sha-1".to_string()]); + assert!(!m.run.solved_any); + assert!( + m.after_decide(&d.verdict, 5.0, Duration::ZERO).is_none(), + "no cap hit" + ); + assert_eq!(m.it, 2, "a decided iteration advances"); + assert!(m.exit().is_none()); + } + + #[test] + fn a_failed_snapshot_keeps_the_score_but_not_the_rollback_target() { + let mut m = Machine::new(cfg(), run(), 1); + let mut row = Row::default(); + m.keep(&mut row, &reading(200.0), false, Err("git said no".into())); + assert_eq!(m.run.segment.best_score, 200.0); + assert_eq!(m.run.segment.best_snap, "base"); + assert!(row.kept_snap.is_none()); + assert!(m.run.kept_shas.is_empty()); + } + + #[test] + fn a_solved_keep_ends_the_run_unless_early_stop_is_off() { + let mut m = Machine::new(cfg(), run(), 1); + let verdict = Decision { + keep: true, + solved: true, + }; + assert!(m.after_decide(&verdict, 5.0, Duration::ZERO).is_none()); + assert_eq!(m.exit(), Some(LoopExit::Solved)); + + let mut c = cfg(); + c.no_early_stop = true; + let mut m = Machine::new(c, run(), 1); + assert!(m.after_decide(&verdict, 5.0, Duration::ZERO).is_none()); + assert_eq!(m.exit(), None); + assert_eq!(m.it, 2); + } + + #[test] + fn caps_read_the_live_override_and_exclude_parked_time() { + let mut m = Machine::new(cfg(), run(), 1); + m.add_cost(4.0); + assert!(m.over_budget(5.0, Duration::ZERO).is_none()); + assert_eq!( + m.over_budget(3.0, Duration::ZERO), + Some(BudgetHit::Cost { + spent: 4.0, + cap: 3.0 + }), + "a live cap below the spend hits" + ); + assert_eq!( + m.over_budget(0.0, Duration::from_secs(150)), + Some(BudgetHit::Time) + ); + m.run.parked_total = Duration::from_secs(60); + assert!( + m.over_budget(0.0, Duration::from_secs(150)).is_none(), + "parked time is not active time" + ); + assert!( + m.after_decide( + &Decision { + keep: false, + solved: false + }, + 3.0, + Duration::ZERO + ) + .is_some() + ); + assert_eq!(m.exit(), Some(LoopExit::Budget)); + } + + #[test] + fn never_started_turns_rerun_the_same_iteration_then_stall() { + let mut m = Machine::new(cfg(), run(), 2); + for attempt in 1..MAX_DEAD_TURN_ATTEMPTS { + let Settle::Rerun { + row, + attempt: n, + stalled, + } = m.settle(IterStep::NeverStarted { + reason: "401".into(), + }) + else { + panic!("a never-started turn reruns"); + }; + assert_eq!(row.iter, 2); + assert_eq!(row.phase.as_deref(), Some("infra")); + assert_eq!(n, attempt); + assert!(!stalled); + assert_eq!(m.it, 2, "the iteration is not consumed"); + } + let Settle::Rerun { stalled, .. } = m.settle(IterStep::NeverStarted { + reason: "401".into(), + }) else { + panic!("rerun"); + }; + assert!(stalled); + assert_eq!(m.exit(), Some(LoopExit::Stalled)); + } + + #[test] + fn a_started_turn_resets_the_stall_streak() { + let mut m = Machine::new(cfg(), run(), 1); + let _ = m.settle(IterStep::NeverStarted { reason: "x".into() }); + let _ = m.settle(IterStep::NeverStarted { reason: "x".into() }); + let Settle::Discard { row } = m.settle(IterStep::Discarded { + reason: "turn failed".into(), + }) else { + panic!("discard"); + }; + assert_eq!(row.decision, "discarded"); + m.advance(); + let Settle::Rerun { attempt, .. } = m.settle(IterStep::NeverStarted { reason: "x".into() }) + else { + panic!("rerun"); + }; + assert_eq!(attempt, 1, "the streak restarted after a started turn"); + } + + #[test] + fn escalate_park_and_stop_set_the_exit_or_arm_the_park() { + let mut m = Machine::new(cfg(), run(), 1); + assert!(matches!(m.settle(IterStep::Escalated), Settle::Escalate)); + assert_eq!(m.exit(), Some(LoopExit::Escalated)); + + let mut m = Machine::new(cfg(), run(), 1); + let pp = PendingProvisioning { + mode: crate::provisioning::WaitMode::Block, + trace_id: "t".into(), + handle: "h".into(), + }; + assert!(matches!(m.settle(IterStep::Parked(pp)), Settle::Park)); + assert!(m.exit().is_none()); + assert_eq!( + m.take_pending_block().map(|p| p.trace_id), + Some("t".to_string()) + ); + assert!(m.take_pending_block().is_none(), "taken once"); + + let mut m = Machine::new(cfg(), run(), 1); + assert!(matches!(m.settle(IterStep::Stopped), Settle::Stop)); + assert_eq!(m.exit(), Some(LoopExit::Stopped)); + } + + #[test] + fn a_park_denial_on_a_block_wait_escalates_and_the_wait_is_not_active_time() { + let mut m = Machine::new(cfg(), run(), 1); + assert_eq!( + m.on_park(Duration::from_secs(30), &ParkOutcome::Resumed), + None + ); + assert_eq!(m.run.parked_total, Duration::from_secs(30)); + assert_eq!( + m.on_park(Duration::from_secs(5), &ParkOutcome::Denied("no".into())), + Some(LoopExit::Escalated) + ); + assert_eq!(m.run.parked_total, Duration::from_secs(35)); + let mut m = Machine::new(cfg(), run(), 1); + assert_eq!( + m.on_park(Duration::ZERO, &ParkOutcome::Stopped), + Some(LoopExit::Stopped) + ); + } + + #[test] + fn a_distress_marker_parks_once_per_timestamp_and_never_duplicates_its_row() { + let mut m = Machine::new(cfg(), run(), 3); + let first = m.on_distress(7, "torch skew").expect("a new marker parks"); + let row = first.expect("first sighting writes a row"); + assert_eq!(row.iter, 2, "annotates the turn that raised it"); + assert_eq!(row.decision, "distressed"); + m.record(row); + assert!( + m.on_distress(7, "torch skew").is_none(), + "the same marker never re-parks" + ); + let again = m + .on_distress(8, "torch skew") + .expect("a fresh marker parks"); + assert!(again.is_none(), "an identical row is already on the record"); + assert_eq!( + m.on_distress_park(Duration::from_secs(9), DistressOutcome::Cleared), + None + ); + assert_eq!(m.run.parked_total, Duration::from_secs(9)); + assert_eq!( + m.on_distress_park(Duration::ZERO, DistressOutcome::TimedOut), + Some(LoopExit::Stopped) + ); + } + + #[test] + fn a_rescope_swaps_the_whole_segment() { + let mut m = Machine::new(cfg(), run(), 1); + m.rescope(Segment { + regime: "c=48".into(), + fingerprint: "g".into(), + baseline_score: 300.0, + best_score: 300.0, + best_tiebreak: Some(1.0), + baseline_total: 9, + best_snap: "s2".into(), + }); + assert_eq!(m.run.segment.regime, "c=48"); + assert_eq!(m.run.segment.best_snap, "s2"); + assert_eq!(m.run.segment.baseline_total, 9); + } + + #[test] + fn the_seed_rides_iteration_one_only_and_iterations_are_bounded() { + let mut m = Machine::new(cfg(), run(), 1); + assert!(m.wants_seed()); + m.advance(); + assert!(!m.wants_seed()); + m.advance(); + assert!(m.has_iterations()); + m.advance(); + assert!(!m.has_iterations()); + } + + #[test] + fn the_kept_context_is_the_last_kept_row() { + let mut m = Machine::new(cfg(), run(), 1); + assert!(m.kept_context().is_none()); + m.record(Row { + iter: 1, + decision: "keep".into(), + score: Some(220.0), + kept_snap: Some("s1".into()), + note: "first".into(), + ..Default::default() + }); + m.run.kept_shas.push("sha1".into()); + m.record(Row { + iter: 2, + decision: "discard".into(), + ..Default::default() + }); + let kept = m.kept_context().expect("a keep exists"); + assert_eq!(kept.iter, 1); + assert_eq!(kept.snapshot.as_deref(), Some("s1")); + assert_eq!(kept.sha.as_deref(), Some("sha1")); + } + + #[test] + fn shutdown_tokens_match_the_wire_vocabulary() { + for (exit, token) in [ + (LoopExit::Finished, "finished"), + (LoopExit::Solved, "solved"), + (LoopExit::Budget, "budget"), + (LoopExit::Stopped, "stopped"), + (LoopExit::Escalated, "escalated"), + (LoopExit::Stalled, "stalled"), + ] { + assert_eq!(exit.shutdown_reason().0, token); + assert_eq!( + crucible_contract::ShutdownOutcome::parse(token).as_str(), + token + ); + } + assert!(LoopExit::Finished.concluded()); + assert!(LoopExit::Budget.concluded()); + assert!(LoopExit::Solved.concluded()); + assert!(!LoopExit::Stopped.concluded()); + assert!(!LoopExit::Escalated.concluded()); + assert!(!LoopExit::Stalled.concluded()); + } +} diff --git a/crucible/src/main.rs b/crucible/src/main.rs index e196085f..8fedc5c6 100644 --- a/crucible/src/main.rs +++ b/crucible/src/main.rs @@ -46,6 +46,7 @@ mod init; mod issue; mod loop_driver; mod loop_graph; +mod machine; mod pr_watch; mod preflight; mod provisioning; diff --git a/crucible/src/plan/exec.rs b/crucible/src/plan/exec.rs index 254a3a87..76a13b99 100644 --- a/crucible/src/plan/exec.rs +++ b/crucible/src/plan/exec.rs @@ -173,6 +173,20 @@ pub enum TaskStatus { } impl TaskStatus { + /// The status a wire token names; `None` for a token this reader does not know. + pub fn parse(token: &str) -> Option { + [ + TaskStatus::Pass, + TaskStatus::Fail, + TaskStatus::Transport, + TaskStatus::Skipped, + TaskStatus::Blocked, + TaskStatus::Truncated, + ] + .into_iter() + .find(|s| s.as_str() == token) + } + /// The stable wire token (`SessionEvent::TaskResult.status`). pub fn as_str(self) -> &'static str { match self { @@ -343,6 +357,21 @@ pub fn execute( substrate: &Substrate, cfg: ExecCfg, runner: &mut dyn TaskRunner, + on_result: impl FnMut(&Task, &TaskResult), +) -> PlanOutcome { + execute_from(plan, substrate, cfg, runner, BTreeMap::new(), on_result) +} + +/// [`execute`] with results already settled by an earlier process. A task named in `prior` is +/// never dispatched: its result stands, its output feeds its dependents, and it is reported +/// through `on_result` like any other terminal result so the log accounts for it. This is how a +/// resumed run continues a graph instead of re-running the tasks that already passed. +pub fn execute_from( + plan: &ValidPlan, + substrate: &Substrate, + cfg: ExecCfg, + runner: &mut dyn TaskRunner, + prior: BTreeMap, mut on_result: impl FnMut(&Task, &TaskResult), ) -> PlanOutcome { let runnable = runnable_set(plan, substrate); @@ -373,6 +402,12 @@ pub fn execute( let started = Instant::now(); let mut results: BTreeMap = BTreeMap::new(); + for task in plan.tasks_topo() { + if let Some(result) = prior.get(&task.name) { + on_result(task, result); + results.insert(task.name.clone(), result.clone()); + } + } let mut spent = 0.0f64; let budget = plan.plan().budget.usd; let mut halted: Option = None; @@ -1487,6 +1522,62 @@ mod tests { assert_eq!(r.seen_inputs["b"], vec!["a".to_string()]); } + /// A resumed run hands the executor what its predecessor settled: those tasks are reported + /// and never dispatched, and their outputs reach their dependents like a fresh pass would. + #[test] + fn prior_results_are_reported_not_dispatched_and_feed_dependents() { + let plan = valid( + vec![task("a", &[], "any", true), task("b", &["a"], "any", true)], + 10.0, + ); + let mut r = ScriptRunner::new(); + let prior = BTreeMap::from([( + TaskName::from("a"), + TaskResult { + status: TaskStatus::Pass, + attempts: 2, + cost_usd: 0.4, + output: Some(serde_json::json!({"from": "before"})), + note: None, + fanout: None, + }, + )]); + let mut reported = Vec::new(); + let out = execute_from( + &plan, + &any_substrate(), + ExecCfg::default(), + &mut r, + prior, + |task, result| reported.push((task.name.0.clone(), result.status)), + ); + assert!(out.valid); + assert_eq!(out.exit, PlanExit::Completed); + assert_eq!( + reported, + vec![ + ("a".to_string(), TaskStatus::Pass), + ("b".to_string(), TaskStatus::Pass) + ], + "the prior result is reported first, then the dispatched one" + ); + assert!( + !r.seen_inputs.contains_key("a"), + "a settled task is never dispatched again" + ); + assert_eq!(r.seen_inputs["b"], vec!["a".to_string()]); + assert_eq!( + out.results[&"a".into()].attempts, + 2, + "the prior result stands" + ); + assert!( + out.spent_usd < 0.4, + "the prior task's cost was booked by the run that spent it, not again: {}", + out.spent_usd + ); + } + /// The wall-clock ceiling blocks every task not yet dispatched and invalidates the run. /// Unlike the cost ceiling it is checked before dispatch, because elapsed time is known /// continuously while a cost total is only known once an attempt finishes. diff --git a/crucible/src/recovery.rs b/crucible/src/recovery.rs index 8566406d..cd25fd80 100644 --- a/crucible/src/recovery.rs +++ b/crucible/src/recovery.rs @@ -1,14 +1,14 @@ -//! Recovery classification: one pass over a dead run's session log names how it died. -//! Reads only the durable log (marker files are consume-on-read and gone by park time). -//! Facts only; policy lives in [`plan_recovery`]. +//! Recovery classification: the session log, folded once by the contract, names how a dead run +//! ended. Reads only the durable log (marker files are consume-on-read and gone by park time). +//! Facts come from [`crucible_contract::LoopState`]; policy lives in [`plan_recovery`]. -use crate::event::AgentEvent; -use crate::loop_driver::{ResumeFold, ResumeState}; +use crate::loop_driver::ResumeState; use crate::provisioning::{self, WaitMode}; -use crate::session::{self, RecoveryClass, SessionEvent}; +use crate::session::RecoveryClass; use anyhow::{Context, Result}; +use crucible_contract::LoopState; use crucible_contract::admission::AdmissionKey; -use std::io::BufRead; +pub(crate) use crucible_contract::{Classification, OpenApproval, ShutdownOutcome}; use std::path::Path; #[derive(Debug, thiserror::Error)] @@ -18,374 +18,51 @@ struct EmptySessionLog { class: String, } -/// Evidence scraped from a dangling turn's Agent events; retry policy lives elsewhere. -#[derive(Debug, Clone, Default)] -pub(crate) struct TurnEvidence { - /// Events inside the dangling AgentStart bracket. - pub agent_events: u32, - /// Last per-turn cost seen (`Tokens.cost_usd` or `OtelSummary.cost_usd`). - pub last_cost_usd: Option, - /// Last error text (`Error.message` or an is_error `Result.error`), verbatim. - pub last_error: Option, - /// From the preceding AgentSession line. - pub session: Option, -} - -#[derive(Debug, Clone)] -pub(crate) struct DanglingSession { - pub name: String, - /// 1-based, from the AgentSession line. - pub turn: u32, -} - -/// An approval the dead run was still waiting on (dangling ApprovalWait). -#[derive(Debug, Clone)] -pub(crate) struct PendingApproval { - pub handle: String, - pub trace_id: String, - pub mode: WaitMode, -} - -/// PlanAdmitted with its iteration's Row absent. The graph runner batches TaskResult -/// emission until the executor returns, so a mid-plan death leaves `resulted` empty. -#[derive(Debug, Clone)] -pub(crate) struct OpenPlan { - pub plan_version: u32, - /// PlanTaskWire names, in order. - pub declared: Vec, - /// TaskResult names seen after this PlanAdmitted. - pub resulted: Vec, -} - -/// What the tail of the log says happened. -#[derive(Debug, Clone)] -pub(crate) enum Classification { - /// Shutdown line present: the previous process exited on purpose. - CleanExit { - outcome: ShutdownOutcome, - reason: String, - }, - /// Start (maybe Phase baseline) but no decided rows: nothing to resume from. - DiedInBaseline, - /// AgentStart with no AgentDone: the turn was in flight. - DiedMidTurn { - iter: u32, - evidence: TurnEvidence, - approval: Option, - }, - /// Turn finished (AgentDone) but no Row for that iter: died in measure/decide/keep. - DiedDeciding { iter: u32, evidence: TurnEvidence }, - /// Graph loop: PlanAdmitted for an iteration with no Row and no dangling turn. - DiedInPlanTask { iter: u32, plan: OpenPlan }, - /// Parked on a block-mode approval (dangling ApprovalWait, no turn in flight). - DiedAwaitingApproval { approval: PendingApproval }, - /// No dangling turn/plan/approval; died between a Row and the next AgentStart. - DiedBetweenIterations { last_iter: u32 }, -} - -pub(crate) use crucible_contract::ShutdownOutcome; - -fn trunc(s: &str, max: usize) -> String { - if s.chars().count() <= max { - return s.to_string(); - } - let cut: String = s.chars().take(max).collect(); - format!("{cut}…") -} - -impl Classification { - pub(crate) fn class(&self) -> RecoveryClass { - match self { - Classification::CleanExit { .. } => RecoveryClass::CleanExit, - Classification::DiedInBaseline => RecoveryClass::DiedInBaseline, - Classification::DiedMidTurn { .. } => RecoveryClass::DiedMidTurn, - Classification::DiedDeciding { .. } => RecoveryClass::DiedDeciding, - Classification::DiedInPlanTask { .. } => RecoveryClass::DiedInPlanTask, - Classification::DiedAwaitingApproval { .. } => RecoveryClass::DiedAwaitingApproval, - Classification::DiedBetweenIterations { .. } => RecoveryClass::DiedBetweenIterations, - } - } - - /// The iteration the interruption touched (0 if none). - pub(crate) fn iter(&self) -> u32 { - match self { - Classification::DiedMidTurn { iter, .. } - | Classification::DiedDeciding { iter, .. } - | Classification::DiedInPlanTask { iter, .. } => *iter, - Classification::DiedBetweenIterations { last_iter } => *last_iter, - _ => 0, - } - } - - /// One-line evidence summary for the Recovery event and the resume note. - pub(crate) fn detail(&self) -> String { - match self { - Classification::CleanExit { outcome, reason } => { - format!("previous run exited {}: {reason}", outcome.as_str()) - } - Classification::DiedInBaseline => "no decided rows in the log".to_string(), - Classification::DiedMidTurn { - iter, - evidence, - approval, - } => { - let mut s = format!( - "turn in flight at iter {iter}, {} agent events", - evidence.agent_events - ); - if let Some(c) = evidence.last_cost_usd { - s.push_str(&format!(", last cost ${c:.2}")); - } - if let Some(sess) = &evidence.session { - s.push_str(&format!(", session {} turn {}", sess.name, sess.turn)); - } - if let Some(e) = &evidence.last_error { - s.push_str(&format!(", last error: {}", trunc(e, 200))); - } - if let Some(a) = approval { - s.push_str(&format!(", approval {} outstanding", a.handle)); - } - s - } - Classification::DiedDeciding { iter, evidence } => { - let mut s = format!("turn at iter {iter} completed but was never decided"); - if let Some(sess) = &evidence.session { - s.push_str(&format!( - " (session {} turn {} cursor advanced ungraded)", - sess.name, sess.turn - )); - } - s - } - Classification::DiedInPlanTask { iter, plan } => format!( - "plan v{} at iter {iter} never accounted ({}/{} tasks resulted)", - plan.plan_version, - plan.resulted.len(), - plan.declared.len() - ), - Classification::DiedAwaitingApproval { approval } => { - format!( - "parked on approval {} ({})", - approval.handle, approval.trace_id - ) - } - Classification::DiedBetweenIterations { last_iter } => { - format!("died between iterations, last decided iter {last_iter}") - } - } - } -} - -#[derive(Default)] -struct TailScan { - /// Only a TRAILING Shutdown counts: a resumed process appends past its - /// predecessor's Shutdown, so any later event clears it. - shutdown: Option<(String, String)>, - /// Any decided (non-infra) row. - saw_any_row: bool, - last_row_iter: u32, - last_phase_iter: u32, - /// AgentSession seen; the next AgentStart claims it. - pending_session: Option, - /// AgentStart .. missing AgentDone. - open_turn: Option<(u32, TurnEvidence)>, - /// AgentDone landed, the Row for that iter has not. - done_unrowed: Option<(u32, TurnEvidence)>, - /// PlanAdmitted .. missing its Row. - open_plan: Option, - /// ApprovalWait .. missing ApprovalResolved. - open_approval: Option, -} - -impl TailScan { - fn feed(&mut self, ev: &SessionEvent) { - if self.shutdown.is_some() && !matches!(ev, SessionEvent::Shutdown { .. }) { - self.shutdown = None; - } - match ev { - SessionEvent::Shutdown { outcome, reason } => { - self.shutdown = Some((outcome.clone(), reason.clone())); - } - SessionEvent::Phase { phase, iter } => { - if phase == "iteration" { - self.last_phase_iter = *iter; - // Safety net for paths that account an iteration without a Row. - self.done_unrowed = None; - } - } - SessionEvent::AgentSession { session, turn, .. } => { - self.pending_session = Some(DanglingSession { - name: session.clone(), - turn: *turn, - }); - } - SessionEvent::AgentStart { iter } => { - self.open_turn = Some(( - *iter, - TurnEvidence { - session: self.pending_session.take(), - ..TurnEvidence::default() - }, - )); - } - SessionEvent::Agent { event } => { - if let Some((_, evidence)) = &mut self.open_turn { - evidence.agent_events += 1; - match event { - AgentEvent::Tokens(t) => { - if let Some(c) = t.cost_usd { - evidence.last_cost_usd = Some(c); - } - } - AgentEvent::OtelSummary { cost_usd, .. } => { - evidence.last_cost_usd = Some(*cost_usd); - } - AgentEvent::Error { message, .. } => { - evidence.last_error = Some(message.clone()); - } - AgentEvent::Result { - is_error: true, - error: Some(e), - .. - } => { - evidence.last_error = Some(e.clone()); - } - _ => {} - } - } - } - SessionEvent::AgentDone => { - self.done_unrowed = self.open_turn.take(); - } - SessionEvent::Row { row, .. } => { - if !matches!(row.phase.as_deref(), Some("wide") | Some("infra")) { - self.saw_any_row = true; - self.last_row_iter = row.iter; - } - if self - .done_unrowed - .as_ref() - .is_some_and(|(iter, _)| *iter == row.iter) - { - self.done_unrowed = None; - } - // Any row after an admission accounts the plan's iteration. - self.open_plan = None; - } - SessionEvent::PlanAdmitted { - plan_version, - tasks, - .. - } => { - self.open_plan = Some(OpenPlan { - plan_version: *plan_version, - declared: tasks.iter().map(|t| t.name.clone()).collect(), - resulted: Vec::new(), - }); - } - SessionEvent::TaskResult { task, .. } => { - if let Some(plan) = &mut self.open_plan { - plan.resulted.push(task.clone()); - } - } - SessionEvent::ApprovalWait { - handle, - trace_id, - mode, - .. - } => { - self.open_approval = Some(PendingApproval { - handle: handle.clone(), - trace_id: trace_id.clone(), - // Unknown tokens degrade to Continue: never re-park on a mode - // this binary can't honor. - mode: if mode == "block" { - WaitMode::Block - } else { - WaitMode::Continue - }, - }); - } - SessionEvent::ApprovalResolved { .. } => self.open_approval = None, - _ => {} - } - } - - /// Precedence: each earlier case subsumes the later ones. - fn finish(mut self) -> (Classification, Option) { - let pending = self.open_approval.clone(); - let classification = if let Some((outcome, reason)) = self.shutdown { - Classification::CleanExit { - outcome: ShutdownOutcome::parse(&outcome), - reason, - } - } else if !self.saw_any_row { - Classification::DiedInBaseline - } else if let Some((iter, evidence)) = self.open_turn { - Classification::DiedMidTurn { - iter, - evidence, - approval: self.open_approval.take(), - } - } else if let Some(plan) = self.open_plan { - Classification::DiedInPlanTask { - iter: self.last_phase_iter, - plan, - } - } else if let Some(approval) = self.open_approval.take_if(|a| a.mode == WaitMode::Block) { - Classification::DiedAwaitingApproval { approval } - } else if let Some((iter, evidence)) = self.done_unrowed { - Classification::DiedDeciding { iter, evidence } - } else { - Classification::DiedBetweenIterations { - last_iter: self.last_row_iter, - } - }; - (classification, pending) - } -} - -/// Everything `--resume` needs from the log, produced in one pass. +/// Everything `--resume` needs from the log, produced in one fold. #[derive(Debug)] pub(crate) struct SessionRecovery { pub resume: ResumeState, pub classification: Classification, /// Dangling approval regardless of class, for re-registration. - pub pending_approval: Option, + pub pending_approval: Option, + /// The plan tasks settled in the iteration the run died in, when it died inside a plan. + pub prior_plan: Option, } /// Replay the session log into resume counters and a tail classification. Torn final /// lines are skipped; a rowless log is a refusal (`--resume` means continue, not restart). pub(crate) fn classify_session(session_log: &Path) -> Result { - let file = std::fs::File::open(session_log).with_context(|| { + let body = std::fs::read_to_string(session_log).with_context(|| { format!( "reading session log {} to resume (run with --ui stream first?)", session_log.display() ) })?; - let mut fold = ResumeFold::default(); - let mut scan = TailScan::default(); - for line in std::io::BufReader::new(file).lines() { - let line = - line.with_context(|| format!("reading session log {}", session_log.display()))?; - let Some(ev) = session::decode(&line) else { - continue; - }; - fold.feed(&ev); - scan.feed(&ev); - } - let (classification, pending_approval) = scan.finish(); - if !fold.has_rows() { + let state = LoopState::from_lines(body.lines()); + let classification = state.classify(); + if !state.has_rows() { return Err(EmptySessionLog { path: session_log.to_path_buf(), class: classification.class().to_string(), } .into()); } + let prior_plan = match (&classification, &state.plan) { + (Classification::DiedInPlanTask { iter, .. }, Some(plan)) => { + Some(crate::loop_graph::PriorPlan { + iter: *iter, + plan_version: plan.plan_version, + declared: plan.tasks.iter().map(|t| t.name.clone()).collect(), + results: state.plan_results.clone(), + }) + } + _ => None, + }; Ok(SessionRecovery { - resume: fold.finish(), + resume: ResumeState::from_view(state.resume_view()), + pending_approval: state.open_approval().cloned(), classification, - pending_approval, + prior_plan, }) } @@ -456,7 +133,7 @@ pub(crate) fn plan_recovery(s: &SessionRecovery, iterations: u32, max_cost: f64) repark: s .pending_approval .as_ref() - .filter(|a| a.mode == WaitMode::Block) + .filter(|a| a.mode == crucible_contract::WaitMode::Block) .map(|a| provisioning::PendingProvisioning { mode: WaitMode::Block, trace_id: a.trace_id.clone(), @@ -525,90 +202,12 @@ pub(crate) fn resume_approval( } } -/// The old folds against the contract's [`crucible_contract::LoopState`]: every test that -/// classifies a log runs both and proves they agree, so the contract fold can replace the -/// scanners here without a behavior change. -#[cfg(test)] -pub(crate) mod parity { - use super::{Classification, SessionRecovery}; - use crate::provisioning::WaitMode; - use crucible_contract::{LoopState, WaitMode as WireWaitMode}; - use std::path::Path; - - pub(crate) fn fold(session_log: &Path) -> LoopState { - let body = std::fs::read_to_string(session_log).expect("session log readable"); - LoopState::from_lines(body.lines()) - } - - /// Both folds over the same file agree on the classification, the resume counters, and the - /// open approval. - pub(crate) fn assert_matches(session_log: &Path, got: &SessionRecovery) { - let state = fold(session_log); - let theirs = state.classify(); - assert_eq!(theirs.class(), got.classification.class(), "class"); - assert_eq!(theirs.iter(), got.classification.iter(), "iter"); - assert_eq!(theirs.detail(), got.classification.detail(), "detail"); - if let Classification::CleanExit { outcome, reason } = &got.classification { - match &theirs { - crucible_contract::Classification::CleanExit { - outcome: o, - reason: r, - } => { - assert_eq!(o.as_str(), outcome.as_str(), "outcome token"); - assert_eq!(r, reason, "reason"); - } - other => panic!("contract fold classified {other:?}"), - } - } - assert_eq!( - state.has_rows(), - got.resume.has_rows_for_parity(), - "has_rows" - ); - let view = state.resume_view(); - let rs = &got.resume; - assert_eq!(view.rows.len(), rs.rows.len(), "row count"); - for (w, r) in view.rows.iter().zip(rs.rows.iter()) { - assert_eq!(w.iter, r.iter, "row iter"); - assert_eq!(w.decision, r.decision, "row decision"); - assert_eq!(w.score, r.score, "row score"); - assert_eq!(w.tiebreak, r.tiebreak, "row tiebreak"); - assert_eq!(w.total, r.total, "row total"); - assert_eq!(w.phase, r.phase, "row phase"); - assert_eq!(w.kept_snap, r.kept_snap, "row kept_snap"); - } - assert_eq!(view.best_score, rs.best_score, "best_score"); - assert_eq!(view.best_tiebreak, rs.best_tiebreak, "best_tiebreak"); - assert_eq!(view.baseline_score, rs.baseline_score, "baseline_score"); - assert_eq!(view.baseline_total, rs.baseline_total, "baseline_total"); - assert_eq!(view.spent, rs.spent, "spent"); - assert_eq!(view.next_iter, rs.next_iter, "next_iter"); - assert_eq!(view.solved_any, rs.solved_any, "solved_any"); - assert_eq!(view.identity, rs.identity, "identity"); - assert_eq!( - view.published_branches, rs.published_branches, - "published_branches" - ); - match (state.open_approval(), &got.pending_approval) { - (None, None) => {} - (Some(a), Some(b)) => { - assert_eq!(a.handle, b.handle, "approval handle"); - assert_eq!(a.trace_id, b.trace_id, "approval trace"); - let mode = match b.mode { - WaitMode::Block => WireWaitMode::Block, - WaitMode::Continue => WireWaitMode::Continue, - }; - assert_eq!(a.mode, mode, "approval mode"); - } - (a, b) => panic!("open approval disagrees: contract {a:?}, scanner {b:?}"), - } - } -} - #[cfg(test)] mod tests { use super::*; - use crate::session::{PlanTaskWire, RowWire, encode}; + use crate::event::AgentEvent; + use crate::session::{PlanTaskWire, RowWire, SessionEvent, encode}; + use crucible_contract::WaitMode as WireMode; fn write_log(name: &str, events: &[SessionEvent]) -> std::path::PathBuf { let path = std::env::temp_dir().join(format!( @@ -656,21 +255,10 @@ mod tests { fn classify(name: &str, events: &[SessionEvent]) -> SessionRecovery { let path = write_log(name, events); let got = classify_session(&path).unwrap(); - super::parity::assert_matches(&path, &got); let _ = std::fs::remove_file(&path); got } - /// A rowless log is refused by the scanner and folds to `has_rows() == false` in the - /// contract, the same verdict. - #[test] - fn rowless_log_folds_without_rows_in_the_contract_too() { - let path = write_log("rowless-parity", &[iteration_phase(1)]); - assert!(classify_session(&path).is_err()); - assert!(!super::parity::fold(&path).has_rows()); - let _ = std::fs::remove_file(&path); - } - #[test] fn trailing_shutdown_classifies_clean_exit_per_outcome() { for (token, want) in [ @@ -851,6 +439,71 @@ mod tests { } } + /// A death inside a plan hands the resume the tasks that settled under it, so the resumed + /// iteration can start from them. + #[test] + fn a_death_inside_a_plan_carries_the_settled_tasks_for_the_resume() { + let mut events = prefix(); + events.push(SessionEvent::PlanAdmitted { + plan_version: 1, + reason: String::new(), + budget_usd: 1.0, + tasks: ["review", "lint"] + .iter() + .map(|n| PlanTaskWire { + name: (*n).into(), + kind: "command".into(), + depends_on: vec![], + session: String::new(), + needs: "any".into(), + required: true, + join: "all".into(), + stage: "iteration".into(), + over: String::new(), + max_fanout: 0, + }) + .collect(), + }); + events.push(SessionEvent::TaskResult { + task: "review".into(), + status: "pass".into(), + plan_version: 1, + task_kind: "command".into(), + iter: 1, + digest: String::new(), + job: String::new(), + attempts: 1, + cost_usd: 0.3, + metric: None, + output: Some(serde_json::json!({"n": 1})), + note: String::new(), + secs: 1.0, + trace_id: String::new(), + span_id: String::new(), + }); + let got = classify("plan-prior", &events); + assert!(matches!( + got.classification, + Classification::DiedInPlanTask { iter: 1, .. } + )); + let prior = got + .prior_plan + .expect("a plan death carries its settled tasks"); + assert_eq!(prior.iter, 1); + assert_eq!(prior.plan_version, 1); + assert_eq!( + prior.declared, + vec!["review".to_string(), "lint".to_string()] + ); + assert_eq!(prior.results.len(), 1); + assert_eq!(prior.results["review"].status, "pass"); + + let mut events = prefix(); + events.push(row(1, "keep", 210.0)); + let got = classify("no-plan", &events); + assert!(got.prior_plan.is_none(), "no plan death, nothing to carry"); + } + #[test] fn a_row_closes_the_open_plan() { let mut events = prefix(); @@ -920,7 +573,7 @@ mod tests { match &got.classification { Classification::DiedAwaitingApproval { approval } => { assert_eq!(approval.handle, "https://example.com/pr/7"); - assert_eq!(approval.mode, WaitMode::Block); + assert_eq!(approval.mode, WireMode::Block); } other => panic!("expected DiedAwaitingApproval, got {other:?}"), } @@ -992,7 +645,7 @@ mod tests { got.classification ); let pending = got.pending_approval.as_ref().expect("wait still open"); - assert_eq!(pending.mode, WaitMode::Continue); + assert_eq!(pending.mode, WireMode::Continue); match plan_recovery(&got, 5, 0.0) { RecoveryPlan::Continue { repark, @@ -1168,6 +821,7 @@ mod tests { reason: "r".into(), }, pending_approval: None, + prior_plan: None, } } @@ -1243,6 +897,7 @@ mod tests { }, classification: Classification::DiedBetweenIterations { last_iter: 5 }, pending_approval: None, + prior_plan: None, }; match plan_recovery(&s, 5, 10.0) { RecoveryPlan::NoOp { message } => { diff --git a/crucible/src/run.rs b/crucible/src/run.rs index f9e85495..839c312f 100644 --- a/crucible/src/run.rs +++ b/crucible/src/run.rs @@ -1067,6 +1067,7 @@ fn drive_loop( recovery: Some(recovery), ledger: Some(ledger), heartbeat: beat.clone(), + prior_plan: recovered.prior_plan, }, )? } diff --git a/docs/crucible.md b/docs/crucible.md index 8155b7b9..af20dbbe 100644 --- a/docs/crucible.md +++ b/docs/crucible.md @@ -360,7 +360,7 @@ real wall-clock: time-proportional columns, check durations, the agent's tool-ca ## Concept → code (where to look) -- loop / budget / keep-discard: `crucible/src/run.rs` + `crucible/src/loop_driver.rs` +- loop / budget / keep-discard: `crucible/src/run.rs` + `crucible/src/loop_driver.rs` (the host: every effect) + `crucible/src/machine.rs` (the decisions, no I/O) - the contract traits: `crucible/src/crucible.rs` - World/Judge batteries: `crucible/src/command_world.rs` (`GitWorld`/`CommandWorld`/`CompositeWorld`), `crucible/src/command_judge.rs` (`CommandJudge`) - composite domains: `crucible/src/manifest.rs` (`CompositeManifest`) + `crucible/src/run.rs` (`run_composite`) From 2575038b1e6d4a4c37f75e224a755a31dc98088e Mon Sep 17 00:00:00 2001 From: Will Eaton Date: Fri, 4 Sep 2026 13:22:59 -0400 Subject: [PATCH 04/11] Restore the dead-turn streak across a resume MAX_DEAD_TURN_ATTEMPTS bounds the attempts an iteration gets when a turn never starts, but the counter lived in Machine and began at zero on every construction, so it bounded attempts per process rather than per iteration. A pod that burned two attempts and died handed its successor three more, and a harness that cannot start never reaches the stall. LoopState::dead_turns reads the streak off the log: trailing rows in the infra phase, skipping a distressed row, which annotates the turn before it rather than reporting one, so it neither ends the streak nor counts toward it. The count travels on ResumeView into RunState, beside the rest of the state a resume restores. No wire change; those rows already carry phase "infra". Assisted-by: Claude --- crucible-contract/src/session/fold.rs | 53 +++++++++++++++++++++++++++ crucible/src/loop_driver.rs | 11 ++++++ crucible/src/machine.rs | 37 +++++++++++++++---- crucible/src/recovery.rs | 2 + 4 files changed, 96 insertions(+), 7 deletions(-) diff --git a/crucible-contract/src/session/fold.rs b/crucible-contract/src/session/fold.rs index 047a93f6..871cb3f4 100644 --- a/crucible-contract/src/session/fold.rs +++ b/crucible-contract/src/session/fold.rs @@ -274,6 +274,8 @@ pub struct ResumeView { pub spent: f64, /// First iteration to run (last logged iteration + 1). pub next_iter: u32, + /// Never-started turns already spent on `next_iter`, so the attempt bound survives a resume. + pub dead_turns: u32, pub solved_any: bool, pub identity: Option, /// Head branches of every draft PR prior segments already opened. @@ -578,6 +580,18 @@ impl LoopState { .filter(|r| !matches!(r.phase.as_deref(), Some("wide") | Some("infra"))) } + /// Never-started turns at the tail of the log: the streak an in-flight iteration has + /// already spent. Any row from a turn that started ends the streak; a `distressed` row + /// annotates the turn before it rather than reporting one, so it does not. + pub fn dead_turns(&self) -> u32 { + self.rows + .iter() + .rev() + .filter(|r| r.decision != "distressed") + .take_while(|r| r.phase.as_deref() == Some("infra")) + .count() as u32 + } + /// The counters a resume restores. Decided rows carry `score`/`total`, so baseline and /// best restore exactly; keeps are monotone within a segment, so the last kept row is the /// best and its tiebreak travels with the best score. @@ -600,6 +614,7 @@ impl LoopState { let next_iter = rows.iter().map(|r| r.iter).max().unwrap_or(0) + 1; ResumeView { rows, + dead_turns: self.dead_turns(), best_score, best_tiebreak, baseline_score, @@ -868,6 +883,44 @@ mod tests { assert_eq!(v.best_score, 220.0); assert_eq!(v.next_iter, 3); assert_eq!(v.spent, 2.5); + assert_eq!( + v.dead_turns, 0, + "a decided row after the dead turn ends the streak" + ); + } + + #[test] + fn the_resume_view_carries_the_dead_turn_streak_the_log_ends_on() { + // A pod that died mid-streak must not hand its successor a fresh attempt budget. + let s = fold(&[ + row(0, "baseline", 240.0, None), + iteration(1), + row(1, "keep", 220.0, None), + iteration(2), + row(2, "infra-dead", 0.0, Some("infra")), + row(2, "infra-dead", 0.0, Some("infra")), + ]); + let v = s.resume_view(); + assert_eq!(v.next_iter, 2, "the never-started iteration re-runs"); + assert_eq!(v.dead_turns, 2); + + // A distress row annotates the turn before it; it does not report one, so it neither + // ends the streak nor counts toward it. + let mut with_distress = vec![ + row(0, "baseline", 240.0, None), + iteration(2), + row(2, "infra-dead", 0.0, Some("infra")), + ]; + with_distress.push(row(1, "distressed", 0.0, None)); + assert_eq!(fold(&with_distress).resume_view().dead_turns, 1); + + // No dead turn at the tail, no streak. + assert_eq!( + fold(&[row(0, "baseline", 240.0, None)]) + .resume_view() + .dead_turns, + 0 + ); } #[test] diff --git a/crucible/src/loop_driver.rs b/crucible/src/loop_driver.rs index 6d78f2dc..b4dee4e6 100644 --- a/crucible/src/loop_driver.rs +++ b/crucible/src/loop_driver.rs @@ -45,6 +45,8 @@ pub(crate) struct ResumeState { pub spent: f64, /// First iteration to run (last logged iter + 1). pub next_iter: u32, + /// Never-started turns the log already spent on `next_iter`. + pub dead_turns: u32, pub solved_any: bool, /// The last [`crate::identity::RunIdentity`] the original run recorded, if any (older logs /// predate this event). The resume path recomputes the identity fresh and hard-warns, never @@ -490,6 +492,7 @@ fn run_loop_body( base_snap: None, solved_any: rs.solved_any, parked_total: Duration::ZERO, + dead_turns: rs.dead_turns, pending_block: None, published_branches: rs.published_branches, segment, @@ -665,6 +668,7 @@ fn run_loop_body( base_snap, solved_any: false, parked_total: Duration::ZERO, + dead_turns: 0, pending_block: None, published_branches: Vec::new(), segment, @@ -1365,6 +1369,7 @@ impl ResumeState { baseline_total: view.baseline_total, spent: view.spent, next_iter: view.next_iter, + dead_turns: view.dead_turns, solved_any: view.solved_any, identity: view.identity, published_branches: view.published_branches, @@ -3978,6 +3983,7 @@ mod tests { base_snap: None, solved_any: rs.solved_any, parked_total: Duration::ZERO, + dead_turns: rs.dead_turns, pending_block: None, published_branches: rs.published_branches, segment: Segment { @@ -4013,6 +4019,7 @@ mod tests { baseline_total: 0, spent: 0.0, next_iter, + dead_turns: 0, solved_any: false, identity: None, best_tiebreak: None, @@ -4598,6 +4605,7 @@ mod tests { baseline_total: 0, spent: 0.0, next_iter: 1, + dead_turns: 0, solved_any: false, identity: None, published_branches: Vec::new(), @@ -4693,6 +4701,7 @@ mod tests { baseline_total: 0, spent: 0.0, next_iter: 2, + dead_turns: 0, solved_any: false, identity: None, published_branches: Vec::new(), @@ -4761,6 +4770,7 @@ mod tests { baseline_total: 0, spent: 0.0, next_iter: 2, + dead_turns: 0, solved_any: false, identity: None, published_branches: Vec::new(), @@ -4838,6 +4848,7 @@ mod tests { baseline_total: 0, spent: 0.0, next_iter: 1, + dead_turns: 0, solved_any: false, identity: None, published_branches: Vec::new(), diff --git a/crucible/src/machine.rs b/crucible/src/machine.rs index 2a5e20aa..34e8614d 100644 --- a/crucible/src/machine.rs +++ b/crucible/src/machine.rs @@ -58,8 +58,12 @@ pub(crate) struct RunState { /// The pristine baseline snapshot token; `None` on resume. pub base_snap: Option, pub solved_any: bool, - /// Idle time spent parked on a human, excluded from the time cap. + /// Idle time spent parked on a human, excluded from the time cap. A resumed process + /// measures its own wall clock, so this starts at zero there. pub parked_total: Duration, + /// Never-started turns already spent on the current iteration. Restored on resume so the + /// attempt bound counts the iteration's attempts, not one process's. + pub dead_turns: u32, /// The block-mode approval the next head parks on. pub pending_block: Option, /// Branches prior publishes opened PRs from; publish skips them. @@ -191,7 +195,6 @@ pub(crate) struct Machine { pub run: RunState, /// The iteration about to run. Advances only when a turn actually started. pub it: u32, - dead_turns: u32, /// The distress marker this run already parked on, by its `ts_ms`. parked_distress_ts: Option, exit: Option, @@ -203,7 +206,6 @@ impl Machine { cfg, run, it: start_iter, - dead_turns: 0, parked_distress_ts: None, exit: None, } @@ -313,7 +315,7 @@ impl Machine { /// [`MAX_DEAD_TURN_ATTEMPTS`]. pub(crate) fn settle(&mut self, step: IterStep) -> Settle { if !matches!(step, IterStep::NeverStarted { .. }) { - self.dead_turns = 0; + self.run.dead_turns = 0; } let it = self.it; match step { @@ -328,7 +330,7 @@ impl Machine { Settle::Discard { row } } IterStep::NeverStarted { reason } => { - self.dead_turns += 1; + self.run.dead_turns += 1; let row = Row { iter: it, decision: "infra-dead".to_string(), @@ -336,13 +338,13 @@ impl Machine { phase: Some("infra".to_string()), ..Default::default() }; - let stalled = self.dead_turns >= MAX_DEAD_TURN_ATTEMPTS; + let stalled = self.run.dead_turns >= MAX_DEAD_TURN_ATTEMPTS; if stalled { self.exit = Some(LoopExit::Stalled); } Settle::Rerun { row, - attempt: self.dead_turns, + attempt: self.run.dead_turns, stalled, } } @@ -458,6 +460,7 @@ mod tests { ..Default::default() }], spent: 0.0, + dead_turns: 0, kept_shas: Vec::new(), base_sha: None, base_snap: None, @@ -621,6 +624,26 @@ mod tests { assert_eq!(m.exit(), Some(LoopExit::Stalled)); } + #[test] + fn a_resumed_run_keeps_the_streak_the_log_already_spent() { + // The bound is per iteration, not per process: a pod that died after two dead turns + // gets one more attempt on resume, not three. + let mut resumed = run(); + resumed.dead_turns = MAX_DEAD_TURN_ATTEMPTS - 1; + let mut m = Machine::new(cfg(), resumed, 2); + let Settle::Rerun { + attempt, stalled, .. + } = m.settle(IterStep::NeverStarted { + reason: "401".into(), + }) + else { + panic!("rerun"); + }; + assert_eq!(attempt, MAX_DEAD_TURN_ATTEMPTS); + assert!(stalled, "the attempt the resume inherited is the last one"); + assert_eq!(m.exit(), Some(LoopExit::Stalled)); + } + #[test] fn a_started_turn_resets_the_stall_streak() { let mut m = Machine::new(cfg(), run(), 1); diff --git a/crucible/src/recovery.rs b/crucible/src/recovery.rs index cd25fd80..fbb8cfea 100644 --- a/crucible/src/recovery.rs +++ b/crucible/src/recovery.rs @@ -811,6 +811,7 @@ mod tests { baseline_total: 0, spent, next_iter, + dead_turns: 0, solved_any: false, identity: None, best_tiebreak: None, @@ -890,6 +891,7 @@ mod tests { baseline_total: 0, spent: 3.5, next_iter: 6, + dead_turns: 0, solved_any: false, identity: None, best_tiebreak: None, From dba1c74964ce3cb778af890ac39d7a20c1d9b0d4 Mon Sep 17 00:00:00 2001 From: Will Eaton Date: Fri, 4 Sep 2026 15:52:34 -0400 Subject: [PATCH 05/11] Gate a playbook on a human decision A pack can now say "run these tasks, then wait for someone to approve, then deploy". approve() is a task kind with dependents, a verdict contribution and a name, rather than the scored loop's one global provisioning wait that only an operator at a socket could end. A gate names what may resolve it. An operator on the run always can, over the control bridge or the controller; a gate may additionally name a pull request or a Jira issue, and whichever arrives first wins. Resolution is keyed by a trace id derived from the run and the task and is idempotent under it, so a retrying resolver, a second source, and a resumed process replaying a decision converge on one recorded outcome. Granted settles the gate passing, denied and timeout settle it failing; the lane's existing verdict rule does the rest, so there is no new task status. A run at a gate parks or suspends. Suspending writes the workspace and state dir to the controller's drop-box, reports outcome=suspended and exits zero; a later process restores them through a resume-restore init container and is handed the resolutions the controller settled meanwhile. Parked time does not count against the wall clock. The pod gets a preStop that writes the stop file and a 300s grace period, because ctrlc handles SIGINT only: without it a delete SIGKILLs a parked run mid-gate. ParkPolicy moves to the lib and CRUCIBLE_RESUME_OF to the contract so the renderer can name what the engine parses. The wire stays at version 1: every addition is an optional field or a new token. ADR-0025 records the decision; RFC-0002 gains C-PLAYBOOK-APPROVAL, and C-PLAYBOOK-LANE, C-PLAYBOOK-RESUME, C-PLAYBOOK-SHAPE and RFC-0001 C-WIRE are amended to match. Assisted-by: Claude --- README.md | 2 + crucible-contract/src/gate.rs | 334 +++++++++ crucible-contract/src/lib.rs | 7 +- crucible-contract/src/markers.rs | 4 + crucible/src/admission.rs | 23 + crucible/src/control.rs | 179 ++++- crucible/src/deploy/controller.rs | 3 + crucible/src/deploy/render/kube.rs | 417 +++++++++++- crucible/src/ingest_client.rs | 61 ++ crucible/src/loop_graph.rs | 6 +- crucible/src/main.rs | 59 +- crucible/src/plan/cli.rs | 471 ++++++++++++- crucible/src/plan/exec.rs | 132 +++- crucible/src/plan/gate.rs | 327 +++++++++ crucible/src/plan/gate_host.rs | 320 +++++++++ crucible/src/plan/harness.rs | 29 +- crucible/src/plan/ir.rs | 85 +++ crucible/src/plan/mod.rs | 12 + crucible/src/plan/runner.rs | 13 + crucible/src/plan/starlark.rs | 117 +++- crucible/src/plan/starlark/globals.rs | 70 +- crucible/src/plan/starlark/reference.rs | 118 +++- crucible/src/plan/starlark/values.rs | 40 +- crucible/src/run.rs | 82 ++- crucible/src/suspend.rs | 641 ++++++++++++++++++ crucible/tests/render_parity.rs | 2 + docs/SUMMARY.md | 1 + docs/adr/0029-approval-gates.md | 69 ++ docs/adr/index.md | 1 + docs/dsl-reference.md | 50 ++ docs/rfc/RFC-0001.md | 12 +- docs/rfc/RFC-0002.md | 36 +- ...proval-gates-on-an-event-sourced-loop.toml | 67 ++ gov/rfc/RFC-0001/clauses/C-WIRE.toml | 10 +- .../RFC-0002/clauses/C-PLAYBOOK-APPROVAL.toml | 28 + gov/rfc/RFC-0002/clauses/C-PLAYBOOK-LANE.toml | 4 +- .../RFC-0002/clauses/C-PLAYBOOK-RESUME.toml | 4 +- .../RFC-0002/clauses/C-PLAYBOOK-SHAPE.toml | 4 +- gov/rfc/RFC-0002/rfc.toml | 1 + 39 files changed, 3724 insertions(+), 117 deletions(-) create mode 100644 crucible-contract/src/gate.rs create mode 100644 crucible/src/plan/gate.rs create mode 100644 crucible/src/plan/gate_host.rs create mode 100644 crucible/src/suspend.rs create mode 100644 docs/adr/0029-approval-gates.md create mode 100644 gov/adr/ADR-0025-plan-authored-approval-gates-on-an-event-sourced-loop.toml create mode 100644 gov/rfc/RFC-0002/clauses/C-PLAYBOOK-APPROVAL.toml diff --git a/README.md b/README.md index a97ea987..f98e3fcc 100644 --- a/README.md +++ b/README.md @@ -327,6 +327,8 @@ Running `crucible` without a subcommand starts an optimization loop and requires | `crucible plan show` | Validates and displays a work-graph plan. | | `crucible plan run` | Executes a plan with the shell runner or a manifest-backed agent. | | `crucible watch-pr` | Converts authorized pull-request review comments into live steering or a reseed file. | +| `crucible approve` / `crucible deny` | Resolves the approval gate a live run is parked on, over its control bridge. | +| `crucible fetch-resume` | Restores a suspended run's session log and workspace from the controller before resuming it. | | `crucible fetch` | Downloads one exact S3 object URI to a local file. | | `crucible rank-grounded` | Performs one read-only, code-grounded ranking turn over an existing checkout. | | `crucible build` | Executes a named build configuration and prints the resulting digest-pinned image reference. | diff --git a/crucible-contract/src/gate.rs b/crucible-contract/src/gate.rs new file mode 100644 index 00000000..35e2f96d --- /dev/null +++ b/crucible-contract/src/gate.rs @@ -0,0 +1,334 @@ +//! Approval gates: the wire shapes an `approve(...)` plan task, its resolution, and the +//! artifacts around a parked run share between the engine and the controller. + +use serde::{Deserialize, Serialize}; +use std::fmt; + +/// Where a gate's resolution comes from, as the engine resolved it when the gate was reached. +/// Rides `ApprovalWait.source` on the session log and the `approval-waits` artifact. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum GateSource { + /// A person resolves it through the controller (SPA, MCP) or the control bridge. + Native, + /// A GitHub pull request: an authorized review approval, an `/approve` comment, or a merge. + GithubPr { url: String, until: PrUntil }, + /// A Jira issue reaching a status or carrying a label. + Jira { key: String, until: JiraUntil }, +} + +impl GateSource { + /// The source's wire kind, the token a resolution names in `ApprovalResolved.source`. + pub fn kind(&self) -> &'static str { + match self { + GateSource::Native => "native", + GateSource::GithubPr { .. } => "github_pr", + GateSource::Jira { .. } => "jira", + } + } + + /// The human-facing handle of the thing being waited on. + pub fn handle(&self) -> Option<&str> { + match self { + GateSource::Native => None, + GateSource::GithubPr { url, .. } => Some(url), + GateSource::Jira { key, .. } => Some(key), + } + } +} + +/// What a pull request must reach. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PrUntil { + Approved, + Merged, +} + +impl PrUntil { + pub fn parse(token: &str) -> Option { + match token { + "approved" => Some(PrUntil::Approved), + "merged" => Some(PrUntil::Merged), + _ => None, + } + } + + pub fn as_str(self) -> &'static str { + match self { + PrUntil::Approved => "approved", + PrUntil::Merged => "merged", + } + } +} + +/// What a Jira issue must reach: a status by name, or a label. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum JiraUntil { + Status(String), + Label(String), +} + +/// How a gate was resolved. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum GateDecision { + Granted, + Denied, +} + +impl GateDecision { + pub fn parse(token: &str) -> Option { + match token { + "granted" | "approve" | "approved" => Some(GateDecision::Granted), + "denied" | "deny" => Some(GateDecision::Denied), + _ => None, + } + } + + /// The `ApprovalResolved.outcome` token. + pub fn as_str(self) -> &'static str { + match self { + GateDecision::Granted => "granted", + GateDecision::Denied => "denied", + } + } +} + +/// A gate's resolution, from whichever source produced it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct GateResolution { + pub decision: GateDecision, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub by: Option, + /// The source kind that resolved it (`native`, `github_pr`, `jira`, `timeout`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source: Option, +} + +impl GateResolution { + pub fn granted(by: Option, source: &str) -> Self { + GateResolution { + decision: GateDecision::Granted, + reason: None, + by, + source: Some(source.to_string()), + } + } + + pub fn denied(reason: impl Into, by: Option, source: &str) -> Self { + GateResolution { + decision: GateDecision::Denied, + reason: Some(reason.into()), + by, + source: Some(source.to_string()), + } + } + + /// A park that ran out its ceiling. + pub fn timeout() -> Self { + GateResolution::denied("park timed out waiting for approval", None, "timeout") + } + + /// The `ApprovalResolved.reason` text. + pub fn reason_text(&self) -> String { + self.reason.clone().unwrap_or_default() + } +} + +/// Why an `--approval` argument could not be read. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BadApprovalArg(pub String); + +impl fmt::Display for BadApprovalArg { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "--approval {:?} is not =granted|denied[:reason][@by]", + self.0 + ) + } +} + +impl std::error::Error for BadApprovalArg {} + +/// Parse one `--approval =granted|denied[:reason][@by]` argument. +pub fn parse_approval_arg(raw: &str) -> Result<(String, GateResolution), BadApprovalArg> { + let bad = || BadApprovalArg(raw.to_string()); + let (trace, rest) = raw.split_once('=').ok_or_else(bad)?; + let trace = trace.trim(); + if trace.is_empty() { + return Err(bad()); + } + let (rest, by) = match rest.rsplit_once('@') { + Some((head, by)) if !by.trim().is_empty() => (head, Some(by.trim().to_string())), + _ => (rest, None), + }; + let (decision, reason) = match rest.split_once(':') { + Some((d, reason)) => (d, Some(reason.trim().to_string()).filter(|r| !r.is_empty())), + None => (rest, None), + }; + let decision = GateDecision::parse(decision.trim()).ok_or_else(bad)?; + Ok(( + trace.to_string(), + GateResolution { + decision, + reason, + by, + source: Some("native".to_string()), + }, + )) +} + +/// The trace id an `approve` task waits under: one per task instance per run, and the +/// admission-ledger key a bridge `approve` records against. +pub fn gate_trace_id(run_id: &str, task: &str) -> String { + format!("approve:{run_id}:{task}") +} + +/// Whether a parked run idles in the pod or leaves a snapshot and exits. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ParkMode { + Park, + Suspend, +} + +impl ParkMode { + pub fn as_str(self) -> &'static str { + match self { + ParkMode::Park => "park", + ParkMode::Suspend => "suspend", + } + } +} + +/// One open gate, as the `approval-waits` artifact lists it. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct GateWait { + pub trace_id: String, + pub handle: String, + pub task: String, + pub source: GateSource, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub summary: Option, + pub mode: ParkMode, + /// Unix seconds when the gate was reached. + pub requested_at: f64, +} + +/// The `approval-waits` artifact: the gates a parked run is waiting on, replaced whole on every +/// change so the controller's copy is always the current set. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ApprovalWaits { + pub v: u8, + pub run_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub control_port: Option, + pub waits: Vec, +} + +impl ApprovalWaits { + pub const VERSION: u8 = 1; +} + +/// One resolution the controller holds for a parked pod, as `GET /api/pods/{pod}/approvals` +/// lists them. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PodApproval { + pub trace_id: String, + pub resolution: GateResolution, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sources_round_trip_under_their_kind_tag() { + for (source, kind) in [ + (GateSource::Native, "native"), + ( + GateSource::GithubPr { + url: "https://github.com/o/r/pull/7".into(), + until: PrUntil::Merged, + }, + "github_pr", + ), + ( + GateSource::Jira { + key: "PROJ-1".into(), + until: JiraUntil::Status("Ready".into()), + }, + "jira", + ), + ] { + let json = serde_json::to_value(&source).expect("encode"); + assert_eq!(json["kind"], kind); + assert_eq!(source.kind(), kind); + let back: GateSource = serde_json::from_value(json).expect("decode"); + assert_eq!(back, source); + } + let jira = serde_json::to_value(GateSource::Jira { + key: "PROJ-1".into(), + until: JiraUntil::Label("approved".into()), + }) + .expect("encode"); + assert_eq!(jira["until"]["label"], "approved"); + } + + #[test] + fn approval_args_parse_every_shape_and_refuse_the_rest() { + let (trace, r) = parse_approval_arg("approve:run-1:gate=granted").expect("bare grant"); + assert_eq!(trace, "approve:run-1:gate"); + assert_eq!(r.decision, GateDecision::Granted); + assert_eq!(r.reason, None); + assert_eq!(r.by, None); + let (_, r) = parse_approval_arg("t=denied:changes requested@alice").expect("full deny"); + assert_eq!(r.decision, GateDecision::Denied); + assert_eq!(r.reason.as_deref(), Some("changes requested")); + assert_eq!(r.by.as_deref(), Some("alice")); + let (_, r) = parse_approval_arg("t=granted@bob").expect("grant with by"); + assert_eq!(r.by.as_deref(), Some("bob")); + assert_eq!(r.reason, None); + let (_, r) = parse_approval_arg("t=approve").expect("approve alias"); + assert_eq!(r.decision, GateDecision::Granted); + for bad in ["", "t", "=granted", "t=maybe", "t=", "t=:why"] { + assert!(parse_approval_arg(bad).is_err(), "{bad:?} must not parse"); + } + } + + #[test] + fn trace_ids_are_one_per_task_per_run() { + assert_eq!(gate_trace_id("run-1", "review"), "approve:run-1:review"); + assert_eq!( + gate_trace_id("run-1", "review[item-a]"), + "approve:run-1:review[item-a]" + ); + } + + #[test] + fn the_waits_artifact_round_trips() { + let waits = ApprovalWaits { + v: ApprovalWaits::VERSION, + run_id: "run-1".into(), + control_port: Some(7777), + waits: vec![GateWait { + trace_id: "approve:run-1:gate".into(), + handle: "gate".into(), + task: "gate".into(), + source: GateSource::Native, + summary: Some("ship it?".into()), + mode: ParkMode::Park, + requested_at: 1.5, + }], + }; + let json = serde_json::to_string(&waits).expect("encode"); + let back: ApprovalWaits = serde_json::from_str(&json).expect("decode"); + assert_eq!(back, waits); + assert_eq!(GateResolution::timeout().source.as_deref(), Some("timeout")); + } +} diff --git a/crucible-contract/src/lib.rs b/crucible-contract/src/lib.rs index e8ff2176..b889f649 100644 --- a/crucible-contract/src/lib.rs +++ b/crucible-contract/src/lib.rs @@ -10,6 +10,7 @@ pub mod artifact; pub mod ask; pub mod envelope; pub mod event; +pub mod gate; pub mod identity; pub mod json; pub mod markers; @@ -39,11 +40,15 @@ pub use artifact::{ pub use ask::{Ask, AskKey, AskKeyError}; pub use envelope::{Envelope, EnvelopeKind, SCHEMA_VERSION, TERMINATION_MESSAGE_CAP, Usage}; pub use event::{AgentEvent, ModelUsage, RawStream, Tokens}; +pub use gate::{ + ApprovalWaits, BadApprovalArg, GateDecision, GateResolution, GateSource, GateWait, JiraUntil, + ParkMode, PodApproval, PrUntil, gate_trace_id, parse_approval_arg, +}; pub use identity::{ ComponentIdentity, FORMAT_VERSION as IDENTITY_FORMAT_VERSION, RigIdentity, RunIdentity, }; pub use markers::{ - ENV_INGEST_TOKEN_PATH, ENV_INGEST_URL, ENV_POD_NAME, INGEST_POD_NAME_CLAIM, + ENV_INGEST_TOKEN_PATH, ENV_INGEST_URL, ENV_POD_NAME, ENV_RESUME_OF, INGEST_POD_NAME_CLAIM, INGEST_TOKEN_AUDIENCE, MANAGED_BY_KEY, MANAGED_BY_SELECTOR, MANAGED_BY_VALUE, RANK_ACTIVITY_MARKER, RUN_SESSION_DELIMITER, SCOPE_ACTIVITY_MARKER, SCOPE_PACK_MARKER, SCOPE_PROGRESS_MARKER, SCOPE_REPORT_MARKER, SCOPE_TRANSCRIPT_MARKER, VERDICT_MARKER, diff --git a/crucible-contract/src/markers.rs b/crucible-contract/src/markers.rs index 406318f7..fac63757 100644 --- a/crucible-contract/src/markers.rs +++ b/crucible-contract/src/markers.rs @@ -61,6 +61,10 @@ pub const ENV_INGEST_TOKEN_PATH: &str = "CRUCIBLE_INGEST_TOKEN_PATH"; /// pod name by construction. pub const ENV_POD_NAME: &str = "CRUCIBLE_POD_NAME"; +/// Env var naming the suspended pod whose `run-session` and `run-workspace` artifacts a resumed +/// pod restores before the engine starts. Absent = a fresh run with nothing to restore. +pub const ENV_RESUME_OF: &str = "CRUCIBLE_RESUME_OF"; + /// The audience the ingest projected ServiceAccount token is minted for. The ingest extractor /// requires TokenReview to echo this audience, so the token is useless against the kube API or any /// other service. diff --git a/crucible/src/admission.rs b/crucible/src/admission.rs index b931b994..ed4342be 100644 --- a/crucible/src/admission.rs +++ b/crucible/src/admission.rs @@ -203,6 +203,29 @@ impl AdmissionLedger { replay } + /// The resolution recorded for an approval gate, if an `approve`/`deny` keyed by the gate + /// landed in a previous process. What was written is the fact; whether it was settled is + /// not, since a crash between the admit and the settle still means an operator decided. + pub(crate) fn gate_resolution( + &self, + trace_id: &str, + ) -> Option { + let g = self.lock().ok()?; + let key = AdmissionKey::approve(trace_id); + let at = *g.index.get(&key)?; + match &g.log[at].input { + AdmittedInput::Approve => { + Some(crucible_contract::GateResolution::granted(None, "native")) + } + AdmittedInput::Deny { reason } => Some(crucible_contract::GateResolution::denied( + reason.clone(), + None, + "native", + )), + _ => None, + } + } + fn lock(&self) -> Result> { // A poisoned lock means a prior holder panicked mid-fold; the ledger's in-memory view // can no longer be trusted, so every later call fails the same way. diff --git a/crucible/src/control.rs b/crucible/src/control.rs index 7d3658b7..99b07004 100644 --- a/crucible/src/control.rs +++ b/crucible/src/control.rs @@ -125,6 +125,16 @@ pub(crate) struct ControlState { /// A rejected pending approval, drained by the park as the terminal "not granted" /// outcome. deny: Mutex>, + /// The approval gate a parked playbook is waiting on, and its resolution once an operator + /// `approve`/`deny` lands. Armed by the park, drained by it. + gate: Mutex>, +} + +/// An armed approval gate on the bridge. +#[derive(Debug, Clone)] +pub(crate) struct GateSlot { + pub trace_id: String, + pub resolution: Option, } #[derive(Clone, Debug, Serialize)] @@ -243,6 +253,37 @@ impl ControlState { self.deny.lock().ok()?.take() } + /// Arm the gate a parked run waits on, so an operator `approve`/`deny` resolves it. + pub(crate) fn arm_gate(&self, trace_id: String) { + if let Ok(mut slot) = self.gate.lock() { + *slot = Some(GateSlot { + trace_id, + resolution: None, + }); + } + } + + /// The armed gate's trace id, if one is armed. + pub(crate) fn armed_gate(&self) -> Option { + self.gate.lock().ok()?.as_ref().map(|g| g.trace_id.clone()) + } + + /// Take the gate's resolution once an operator has sent one, disarming the gate. + pub(crate) fn take_gate_resolution(&self) -> Option { + let mut slot = self.gate.lock().ok()?; + if slot.as_ref().is_some_and(|g| g.resolution.is_some()) { + return slot.take().and_then(|g| g.resolution); + } + None + } + + fn resolve_gate(&self, resolution: crucible_contract::GateResolution) -> Option { + let mut slot = self.gate.lock().ok()?; + let gate = slot.as_mut()?; + gate.resolution = Some(resolution); + Some(gate.trace_id.clone()) + } + fn snapshot(&self) -> StatusSnapshot { let mut status = self .status @@ -263,6 +304,34 @@ impl ControlState { } } +/// Send one command line to a live run's control bridge and return its reply. The shape is +/// exactly what [`parse_request`] accepts, so `crucible approve`, `crucible deny`, and the PR +/// watcher all speak to the bridge the same way. +pub(crate) fn send_command(addr: &str, command: &Value) -> Result { + let mut stream = TcpStream::connect(addr) + .with_context(|| format!("connecting to the control bridge at {addr}"))?; + stream + .set_read_timeout(Some(std::time::Duration::from_secs(10))) + .context("setting the reply timeout")?; + writeln!(stream, "{command}").context("sending the command")?; + stream.flush().context("sending the command")?; + let mut reader = BufReader::new(stream); + // The bridge may interleave a status frame; the reply is the first frame carrying `cmd`. + for _ in 0..8 { + let mut line = String::new(); + let n = reader.read_line(&mut line).context("reading the reply")?; + if n == 0 { + break; + } + if let Ok(v) = serde_json::from_str::(line.trim()) + && v.get("cmd").is_some() + { + return Ok(v); + } + } + anyhow::bail!("the control bridge at {addr} sent no reply") +} + /// Start a detached TCP bridge thread. The listener binds to all interfaces so it works /// inside a pod behind `kubectl port-forward`, while still being reachable as localhost /// for local smoke tests. @@ -500,20 +569,64 @@ fn apply_command(req: ControlRequest, state: &ControlState, ledger: &AdmissionLe json!({"ok": true, "cmd": "rescope", "regime": regime}) }, ), - ControlCommand::Approve => admitted(ledger, id, AdmittedInput::Approve, |key| { - grant_approval(state, ledger, key) - }), - ControlCommand::Deny { reason } => admitted( - ledger, - id, - AdmittedInput::Deny { - reason: reason.clone(), - }, - |key| { - supersede(ledger, state.set_deny(key.clone(), reason.clone()), key); - json!({"ok": true, "cmd": "deny", "reason": reason}) - }, - ), + ControlCommand::Approve { by } => { + if let Some(trace) = state.armed_gate() { + // A gate resolution is keyed by the gate itself, so a redelivered approve and a + // resume both converge on the one record. + let key = Some(AdmissionKey::approve(&trace)); + return admitted(ledger, key, AdmittedInput::Approve, |key| { + let resolution = + crucible_contract::GateResolution::granted(by.clone(), "native"); + state.resolve_gate(resolution); + let _ = ledger.settle( + key, + AdmissionOutcome::Applied, + &format!("gate granted{}", by_suffix(by.as_deref())), + ); + json!({"ok": true, "cmd": "approve", "trace_id": trace, "by": by}) + }); + } + admitted(ledger, id, AdmittedInput::Approve, |key| { + grant_approval(state, ledger, key) + }) + } + ControlCommand::Deny { reason, by } => { + if let Some(trace) = state.armed_gate() { + let key = Some(AdmissionKey::approve(&trace)); + return admitted( + ledger, + key, + AdmittedInput::Deny { + reason: reason.clone(), + }, + |key| { + let resolution = crucible_contract::GateResolution::denied( + reason.clone(), + by.clone(), + "native", + ); + state.resolve_gate(resolution); + let _ = ledger.settle( + key, + AdmissionOutcome::Applied, + &format!("gate denied: {reason}{}", by_suffix(by.as_deref())), + ); + json!({"ok": true, "cmd": "deny", "trace_id": trace, "reason": reason, "by": by}) + }, + ); + } + admitted( + ledger, + id, + AdmittedInput::Deny { + reason: reason.clone(), + }, + |key| { + supersede(ledger, state.set_deny(key.clone(), reason.clone()), key); + json!({"ok": true, "cmd": "deny", "reason": reason}) + }, + ) + } ControlCommand::Status => { json!({"ok": true, "cmd": "status", "status": state.snapshot()}) } @@ -639,6 +752,10 @@ fn stop_the_run(id: Option, input: AdmittedInput, ledger: &Admissi } /// Settle the admission a newly-armed one displaced as superseded. +fn by_suffix(by: Option<&str>) -> String { + by.map(|who| format!(" by {who}")).unwrap_or_default() +} + fn supersede(ledger: &AdmissionLedger, displaced: Option, by: &AdmissionKey) { if let Some(old) = displaced { let _ = ledger.settle( @@ -686,9 +803,12 @@ enum ControlCommand { Rescope { regime: String, }, - Approve, + Approve { + by: Option, + }, Deny { reason: String, + by: Option, }, Status, /// Subscribe to the live session broadcast, replaying retained lines with `seq > from_seq` @@ -742,13 +862,25 @@ fn parse_id(raw: Option<&Value>) -> std::result::Result, St Ok(Some(AdmissionKey::new(id))) } +/// Who sent an approve/deny, when the sender says. Free text, bounded like a key. +fn parse_by(value: &Value) -> Option { + value + .get("by") + .and_then(Value::as_str) + .map(str::trim) + .filter(|by| !by.is_empty()) + .map(|by| by.chars().take(MAX_KEY_LEN).collect()) +} + fn command_from_name(cmd: &str, value: &Value) -> std::result::Result { match cmd { "abort" => Ok(ControlCommand::Abort), "stop" => Ok(ControlCommand::Stop), "pause" => Ok(ControlCommand::Pause), "resume" => Ok(ControlCommand::Resume), - "approve" => Ok(ControlCommand::Approve), + "approve" => Ok(ControlCommand::Approve { + by: parse_by(value), + }), "deny" => { // Reason is optional (an operator may just reject); default to a generic note. let reason = value @@ -757,7 +889,10 @@ fn command_from_name(cmd: &str, value: &Value) -> std::result::Result Ok(ControlCommand::Status), "tail" => { @@ -1133,11 +1268,11 @@ mod tests { fn parses_approve_command() { assert_eq!( parse_command(r#"{"cmd":"approve"}"#).unwrap(), - ControlCommand::Approve + ControlCommand::Approve { by: None } ); assert_eq!( parse_command(r#""approve""#).unwrap(), - ControlCommand::Approve + ControlCommand::Approve { by: None } ); } @@ -1146,14 +1281,16 @@ mod tests { assert_eq!( parse_command(r#"{"cmd":"deny","reason":"over budget"}"#).unwrap(), ControlCommand::Deny { - reason: "over budget".into() + reason: "over budget".into(), + by: None, } ); // Bare deny defaults its reason. assert_eq!( parse_command(r#""deny""#).unwrap(), ControlCommand::Deny { - reason: "rejected".into() + reason: "rejected".into(), + by: None, } ); } diff --git a/crucible/src/deploy/controller.rs b/crucible/src/deploy/controller.rs index 9889e279..5548ad65 100644 --- a/crucible/src/deploy/controller.rs +++ b/crucible/src/deploy/controller.rs @@ -352,6 +352,7 @@ mod tests { harness: None, model: None, playbook: None, + ..Default::default() }, ) .expect("render"); @@ -432,6 +433,7 @@ mod tests { harness: None, model: None, playbook: None, + ..Default::default() }, ) .expect("render"); @@ -475,6 +477,7 @@ mod tests { harness: None, model: None, playbook: None, + ..Default::default() }, ) { Ok(_) => panic!("expected an error: no [controller] table"), diff --git a/crucible/src/deploy/render/kube.rs b/crucible/src/deploy/render/kube.rs index b092e60f..bcc9792d 100644 --- a/crucible/src/deploy/render/kube.rs +++ b/crucible/src/deploy/render/kube.rs @@ -132,6 +132,13 @@ pub struct RenderOpts { /// Playbook launch: the wrapper runs `crucible plan run` over the manifest's `[workflow]` rather /// than the agent loop. `None` renders the loop. pub playbook: Option, + /// What a playbook does when an approval gate has parked as long as it may. Rendered as + /// `--park-policy`; the default matches the CLI's. + pub park_policy: crate::plan::ParkPolicy, + /// A resumed run: restore a suspended predecessor's snapshot before the wrapper starts, and + /// hand the engine the gate resolutions the controller already holds. `None` renders a fresh + /// start. + pub resume: Option, } impl Default for RenderOpts { @@ -146,6 +153,8 @@ impl Default for RenderOpts { harness: None, model: None, playbook: None, + park_policy: crate::plan::ParkPolicy::default(), + resume: None, } } } @@ -168,6 +177,30 @@ pub struct PlaybookLaunch { pub params: BTreeMap, } +/// The knobs a resumed run supplies (see [`RenderOpts::resume`]). +#[derive(Debug, Clone)] +pub struct ResumeRender { + /// The suspended pod whose `run-session` and `run-workspace` artifacts this one restores. + pub of_pod: String, + /// Gate resolutions the controller settled while the run was suspended, by trace id. + pub approvals: Vec<(String, crucible_contract::GateResolution)>, +} + +/// One `--approval` argument, in the grammar [`crucible_contract::parse_approval_arg`] reads +/// back. A reason cannot carry `@`, which that grammar reserves for the resolver's name. +fn approval_arg(trace: &str, resolution: &crucible_contract::GateResolution) -> String { + let mut arg = format!("{trace}={}", resolution.decision.as_str()); + if let Some(reason) = resolution.reason.as_deref().filter(|r| !r.is_empty()) { + arg.push(':'); + arg.push_str(&reason.replace('@', " ")); + } + if let Some(by) = resolution.by.as_deref().filter(|b| !b.is_empty()) { + arg.push('@'); + arg.push_str(by); + } + arg +} + /// The in-pod mount dir for the projected Tier 2 ingest ServiceAccount token (Tier 2 ingest). The turn /// reads `/token` and sends it as the bearer to the controller's ingest drop-box. pub(super) const INGEST_TOKEN_DIR: &str = "/var/run/secrets/crucible.io/ingest"; @@ -191,6 +224,10 @@ const PACK_WORKDIR_VOLUME: &str = "pack-workdir"; /// this is a defensive floor so an oversize pack fails the render loudly, not the kubelet silently. const CONFIGMAP_MAX_BYTES: usize = 900 * 1024; +/// How long the kubelet waits between `preStop` and SIGKILL. Long enough for a parked run to +/// notice the stop file, snapshot its workspace, and post it to the drop-box. +const TERMINATION_GRACE_SECS: i64 = 300; + /// POSIX single-quote one argv element for the `/bin/sh -c` wrapper. fn sh_quote(s: &str) -> String { format!("'{}'", s.replace('\'', r"'\''")) @@ -675,6 +712,7 @@ impl Renderer<'_> { env: Some(self.env()?), volume_mounts: Some(self.volume_mounts()), resources: Some(self.resources()), + lifecycle: Some(self.lifecycle()), ..Default::default() }; @@ -718,6 +756,7 @@ impl Renderer<'_> { affinity: node_avoid_affinity(self.profile), host_aliases: self.host_aliases(), init_containers: self.init_containers(), + termination_grace_period_seconds: Some(TERMINATION_GRACE_SECS), containers: vec![container], volumes: Some(self.volumes()), ..Default::default() @@ -741,32 +780,129 @@ impl Renderer<'_> { /// projection into the writable domain-dir emptyDir so the main container reads AND writes there /// (`STEER.md`, `state/`, the workspace clone). `-L` dereferences the ConfigMap's `..data` symlinks /// into real files. Reuses the loop image (it has `/bin/sh` + `cp`), no extra pull. + /// The stop the kubelet delivers before SIGTERM. `ctrlc` handles SIGINT only, so a pod + /// deletion reaches the engine as the cross-process stop file instead: a parked run reads it, + /// settles, and under `park-then-suspend` snapshots itself rather than dying mid-gate. + fn lifecycle(&self) -> core::Lifecycle { + let control = format!("{}/state/control.json", self.domain_dir()); + core::Lifecycle { + pre_stop: Some(core::LifecycleHandler { + exec: Some(core::ExecAction { + command: Some(vec![ + "/bin/sh".to_string(), + "-c".to_string(), + format!( + "mkdir -p {} && printf '{{\"stop\":true}}' > {}", + sh_quote(&format!("{}/state", self.domain_dir())), + sh_quote(&control) + ), + ]), + }), + ..Default::default() + }), + ..Default::default() + } + } + fn init_containers(&self) -> Option> { - let _pack = self.opts.pack.as_ref()?; let domain_dir = self.domain_dir(); - Some(vec![core::Container { - name: "pack-stage".to_string(), + let mut init = Vec::new(); + if self.opts.pack.is_some() { + init.push(core::Container { + name: "pack-stage".to_string(), + image: Some(self.image.clone()), + image_pull_policy: Some("IfNotPresent".to_string()), + command: Some(vec!["/bin/sh".to_string(), "-c".to_string()]), + args: Some(vec![format!( + "set -e\nmkdir -p {domain_dir}\ncp -rL {PACK_SRC_DIR}/. {domain_dir}/\n" + )]), + volume_mounts: Some(vec![ + core::VolumeMount { + name: PACK_CM_VOLUME.to_string(), + mount_path: PACK_SRC_DIR.to_string(), + read_only: Some(true), + ..Default::default() + }, + core::VolumeMount { + name: PACK_WORKDIR_VOLUME.to_string(), + mount_path: domain_dir.clone(), + ..Default::default() + }, + ]), + ..Default::default() + }); + } + // After the pack: the staged domain dir is where the snapshot lands, so restoring first + // would have it copied over. + if let Some(resume) = &self.opts.resume { + init.push(self.resume_restore(resume, &domain_dir)); + } + (!init.is_empty()).then_some(init) + } + + /// Pull the suspended predecessor's `run-session` and `run-workspace` artifacts back into the + /// domain dir, so the wrapper's `--resume` sees only files. The drop-box env is the loop + /// container's own, and the pod token authorizes the read against `resume_of`. + fn resume_restore(&self, resume: &ResumeRender, domain_dir: &str) -> core::Container { + let workspace = Path::new(self.input.workspace_dir) + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_else(|| "workspace".to_string()); + let mut env = vec![ + core::EnvVar { + name: crucible_contract::ENV_RESUME_OF.to_string(), + value: Some(resume.of_pod.clone()), + value_from: None, + }, + core::EnvVar { + name: crucible_contract::ENV_POD_NAME.to_string(), + value: None, + value_from: Some(core::EnvVarSource { + field_ref: Some(core::ObjectFieldSelector { + field_path: "metadata.name".to_string(), + api_version: None, + }), + ..Default::default() + }), + }, + ]; + env.extend(self.ingest_env()); + core::Container { + name: "resume-restore".to_string(), image: Some(self.image.clone()), image_pull_policy: Some("IfNotPresent".to_string()), command: Some(vec!["/bin/sh".to_string(), "-c".to_string()]), args: Some(vec![format!( - "set -e\nmkdir -p {domain_dir}\ncp -rL {PACK_SRC_DIR}/. {domain_dir}/\n" + "set -e\ncrucible fetch-resume --into {} --workspace {}\n", + sh_quote(domain_dir), + sh_quote(&workspace) )]), - volume_mounts: Some(vec![ - core::VolumeMount { - name: PACK_CM_VOLUME.to_string(), - mount_path: PACK_SRC_DIR.to_string(), - read_only: Some(true), - ..Default::default() - }, - core::VolumeMount { - name: PACK_WORKDIR_VOLUME.to_string(), - mount_path: domain_dir, - ..Default::default() - }, - ]), + env: Some(env), + volume_mounts: Some(self.resume_mounts(domain_dir)), ..Default::default() - }]) + } + } + + /// What `resume-restore` writes into: the same domain dir the loop container sees, plus the + /// ingest token it reads the drop-box with. + fn resume_mounts(&self, domain_dir: &str) -> Vec { + let mut mounts = Vec::new(); + if self.opts.pack.is_some() { + mounts.push(core::VolumeMount { + name: PACK_WORKDIR_VOLUME.to_string(), + mount_path: domain_dir.to_string(), + ..Default::default() + }); + } + if self.ingest_url().is_some() { + mounts.push(core::VolumeMount { + name: INGEST_TOKEN_VOLUME.to_string(), + mount_path: INGEST_TOKEN_DIR.to_string(), + read_only: Some(true), + ..Default::default() + }); + } + mounts } /// The broker-child + gate env, projected from the manifest (the source of truth) + the profile. The @@ -967,17 +1103,38 @@ impl Renderer<'_> { // loop where to POST its run-session (and, when the R5 collector produced one, the otel log) // and where its projected `crucible-ingest`-audience token is. Absent = no drop-box; the loop // falls back to the `SESSION` delimiter the wrapper still emits (old controller / local run). - if let Some(ingest_url) = &self.profile.cluster.ingest_url { - env.push(plain(crucible_contract::ENV_INGEST_URL, ingest_url.clone())); - env.push(plain( - crucible_contract::ENV_INGEST_TOKEN_PATH, - format!("{INGEST_TOKEN_DIR}/token"), - )); - } + env.extend(self.ingest_env()); Ok(env) } + /// The controller's drop-box URL, when the profile names one. + fn ingest_url(&self) -> Option<&str> { + self.profile.cluster.ingest_url.as_deref() + } + + /// The drop-box env: where to POST artifacts (and read them back), and the projected + /// `crucible-ingest`-audience token that authorizes both. Empty without a drop-box. + fn ingest_env(&self) -> Vec { + let Some(url) = self.ingest_url() else { + return Vec::new(); + }; + [ + (crucible_contract::ENV_INGEST_URL, url.to_string()), + ( + crucible_contract::ENV_INGEST_TOKEN_PATH, + format!("{INGEST_TOKEN_DIR}/token"), + ), + ] + .into_iter() + .map(|(name, value)| core::EnvVar { + name: name.to_string(), + value: Some(value), + value_from: None, + }) + .collect() + } + /// The container args: the openshell-loop wrapper. Captures the pristine per-component base sha /// (the upstream sha the candidate diff is relative to), then runs crucible. Registry auth is not /// the wrapper's business: `REGISTRY_AUTH_FILE` (see [`Self::env`]) points podman at the mounted @@ -989,6 +1146,17 @@ impl Renderer<'_> { /// When it is absent (an old controller, or a local run) the wrapper falls back to the /// `SESSION (rc=…)` delimiter contract, the controller scrapes everything after the shared /// [`crucible_contract::RUN_SESSION_DELIMITER`] line as the run's session log. + /// The `--park-policy` token, taken from clap's vocabulary so the flag and the enum cannot + /// drift. + fn park_policy_name(&self) -> String { + use clap::ValueEnum as _; + self.opts + .park_policy + .to_possible_value() + .map(|v| v.get_name().to_string()) + .unwrap_or_default() + } + fn wrapper_script(&self) -> Result { let domain_dir = self.domain_dir(); let manifest_file = self.manifest_file; @@ -1007,9 +1175,25 @@ impl Renderer<'_> { }; let harness_flag = crate::deploy::render::turn::harness_flag(self.opts.harness, '='); let model_flag = crate::deploy::render::turn::model_flag(self.opts.model.as_ref(), '='); + let park_policy_flag = format!(" --park-policy={}", self.park_policy_name()); + // A resume is named by the controller, not sniffed from the state dir: the init + // container restores the snapshot, and only the controller knows it did. + let (resume_flag, approval_flags) = match &self.opts.resume { + Some(resume) => ( + " --resume", + resume + .approvals + .iter() + .map(|(trace, r)| { + format!(" --approval {}", sh_quote(&approval_arg(trace, r))) + }) + .collect::(), + ), + None => ("", String::new()), + }; return Ok(format!( r#"D={domain_dir} -crucible plan run --manifest "$D/{manifest_file}" --max-cost {max_cost} --max-time {max_time}{driver_flag}{harness_flag}{model_flag}{param_flags} +crucible plan run --manifest "$D/{manifest_file}" --max-cost {max_cost} --max-time {max_time}{driver_flag}{harness_flag}{model_flag}{park_policy_flag}{resume_flag}{approval_flags}{param_flags} rc=$? if [ -z "${{CRUCIBLE_INGEST_URL:-}}" ]; then echo "=================== {session_delimiter}$rc) ===================" @@ -1818,6 +2002,7 @@ mod tests { harness: None, model: None, playbook: None, + ..Default::default() }, ) .expect("render"); @@ -1977,6 +2162,7 @@ mod tests { harness: None, model: None, playbook: None, + ..Default::default() }, ) .expect("render") @@ -2022,6 +2208,7 @@ mod tests { harness, model: model.map(str::to_string), playbook: None, + ..Default::default() }, ) .expect("render") @@ -2063,6 +2250,7 @@ mod tests { harness: None, model: None, playbook: None, + ..Default::default() }, ) .expect("render"); @@ -2178,6 +2366,7 @@ mod tests { harness: None, model: None, playbook: None, + ..Default::default() }, ) .expect("render"); @@ -2296,6 +2485,7 @@ mod tests { harness: None, model: None, playbook: None, + ..Default::default() }, ) .expect("render"); @@ -2353,6 +2543,7 @@ mod tests { harness: None, model: None, playbook: None, + ..Default::default() }, ) .expect("render") @@ -2545,6 +2736,7 @@ mod tests { harness: None, model: None, playbook: None, + ..Default::default() }, ) .expect("render"); @@ -2622,6 +2814,7 @@ mod tests { harness: None, model: None, playbook: None, + ..Default::default() }, ) .expect("render"); @@ -2708,6 +2901,7 @@ mod tests { harness: None, model: None, playbook: None, + ..Default::default() }, ) .expect("render"); @@ -2803,6 +2997,7 @@ mod tests { harness: None, model: None, playbook: None, + ..Default::default() }, ) .expect("render"); @@ -2913,6 +3108,7 @@ mod tests { harness: None, model: None, playbook: None, + ..Default::default() }, ) .expect("render"); @@ -3028,6 +3224,7 @@ mod tests { harness: None, model: None, playbook: None, + ..Default::default() }, ) .expect("render"); @@ -3079,6 +3276,7 @@ mod tests { harness: None, model: None, playbook: None, + ..Default::default() }, ) { Err(e) => e, @@ -3184,6 +3382,7 @@ mod tests { harness: None, model: None, playbook: None, + ..Default::default() }, ) .expect("render") @@ -3370,6 +3569,7 @@ mod tests { harness: None, model: None, playbook: None, + ..Default::default() }, ) .expect("render"); @@ -4001,6 +4201,7 @@ mod tests { harness: None, model: None, playbook: Some(playbook), + ..Default::default() }, ) .expect("render") @@ -4041,6 +4242,7 @@ mod tests { harness, model: model.map(str::to_string), playbook: Some(playbook_launch()), + ..Default::default() }, ) .expect("render") @@ -4048,7 +4250,7 @@ mod tests { let set = render_with(Some(crate::manifest::Harness::Codex), Some("gpt-5.6-luna")); assert!( set.contains( - "--compute-driver=kubernetes --harness=codex --model=gpt-5.6-luna --param" + "--compute-driver=kubernetes --harness=codex --model=gpt-5.6-luna --park-policy=park-then-deny --param" ), "{set}" ); @@ -4247,6 +4449,7 @@ mod tests { harness: None, model: None, playbook: Some(playbook_launch()), + ..Default::default() }, ) .expect("render"); @@ -4256,4 +4459,164 @@ mod tests { assert!(yaml.contains(PACK_WORKDIR_VOLUME), "{yaml}"); assert!(yaml.contains("crucible plan run --manifest"), "{yaml}"); } + + fn resume_render() -> ResumeRender { + ResumeRender { + of_pod: "crucible-run-42-a".to_string(), + approvals: vec![ + ( + "approve:run-42:ship".to_string(), + crucible_contract::GateResolution { + decision: crucible_contract::GateDecision::Granted, + reason: None, + by: Some("wseaton".to_string()), + source: Some("github_pr".to_string()), + }, + ), + ( + "approve:run-42:deploy".to_string(), + crucible_contract::GateResolution { + decision: crucible_contract::GateDecision::Denied, + reason: Some("not this quarter".to_string()), + by: None, + source: Some("native".to_string()), + }, + ), + ], + } + } + + fn render_playbook_opts(profile: &DeployProfile, opts: RenderOpts, dir: &Path) -> String { + let manifest = playbook_manifest(); + let input = RenderInput::from_playbook_manifest(&manifest, "alpha"); + render(input, dir, "crucible.toml", profile, &opts).expect("render") + } + + /// Every `--approval` the renderer emits must parse back to the resolution it came from, + /// including the `by` the grammar reserves `@` for. + #[test] + fn approval_args_round_trip_through_the_contract_parser() { + for (trace, resolution) in resume_render().approvals { + let arg = approval_arg(&trace, &resolution); + let (back_trace, back) = + crucible_contract::parse_approval_arg(&arg).expect("the render's own arg parses"); + assert_eq!(back_trace, trace, "{arg}"); + assert_eq!(back.decision, resolution.decision, "{arg}"); + assert_eq!(back.reason, resolution.reason, "{arg}"); + assert_eq!(back.by, resolution.by, "{arg}"); + } + } + + /// A reason cannot carry `@`: the grammar reads everything after the last one as the resolver's + /// name, so an unescaped `@` would silently rewrite who denied the gate. + #[test] + fn an_at_sign_in_a_reason_cannot_forge_the_resolver() { + let arg = approval_arg( + "t", + &crucible_contract::GateResolution { + decision: crucible_contract::GateDecision::Denied, + reason: Some("ask ops@example.com".to_string()), + by: Some("real-denier".to_string()), + source: None, + }, + ); + let (_, back) = crucible_contract::parse_approval_arg(&arg).expect("parses"); + assert_eq!(back.by.as_deref(), Some("real-denier"), "{arg}"); + assert_eq!(back.reason.as_deref(), Some("ask ops example.com"), "{arg}"); + } + + /// The park policy is a substrate fact the pod cannot infer: without it every gate takes + /// clap's park-then-deny default, and a run the controller meant to suspend fails instead. + #[test] + fn playbook_wrapper_always_names_the_park_policy() { + let yaml = render_playbook(&k8s_profile(""), playbook_launch()); + assert!( + yaml.contains("--park-policy=park-then-deny"), + "the default policy is still rendered explicitly: {yaml}" + ); + let suspending = render_playbook_opts( + &k8s_profile(""), + RenderOpts { + playbook: Some(playbook_launch()), + park_policy: crate::plan::ParkPolicy::ParkThenSuspend, + ..Default::default() + }, + std::path::Path::new("/opt/crucible/domains/alpha"), + ); + assert!( + suspending.contains("--park-policy=park-then-suspend"), + "{suspending}" + ); + } + + /// A resumed pod restores its predecessor's snapshot in an init container, then runs with + /// `--resume` and the resolutions the controller already settled. + #[test] + fn a_resumed_playbook_restores_the_snapshot_and_replays_its_resolutions() { + let tmp = Scratch::new("resume-pack"); + let dir = tmp.path().join("alpha"); + std::fs::create_dir_all(&dir).expect("mkdir pack"); + write_pack_dir(&dir); + let yaml = render_playbook_opts( + &k8s_profile(r#"ingest_url = "https://controller.example/ingest""#), + RenderOpts { + playbook: Some(playbook_launch()), + pack: Some(PackDelivery { + configmap_name: "crucible-run-42-pack".to_string(), + }), + resume: Some(resume_render()), + ..Default::default() + }, + &dir, + ); + assert!(yaml.contains("name: resume-restore"), "{yaml}"); + assert!(yaml.contains("crucible fetch-resume --into"), "{yaml}"); + assert!( + yaml.contains("value: crucible-run-42-a"), + "the init container is told which pod to restore: {yaml}" + ); + assert!(yaml.contains(crucible_contract::ENV_RESUME_OF), "{yaml}"); + // The snapshot lands in the staged domain dir, so it must be restored after the pack copy + // rather than have the pack overwrite it. + let pack_at = yaml.find("name: pack-stage").expect("pack-stage rendered"); + let resume_at = yaml.find("name: resume-restore").expect("resume-restore"); + assert!(pack_at < resume_at, "pack-stage must run first: {yaml}"); + assert!(yaml.contains(" --resume"), "{yaml}"); + assert!( + yaml.contains("--approval 'approve:run-42:ship=granted@wseaton'"), + "{yaml}" + ); + assert!( + yaml.contains("--approval 'approve:run-42:deploy=denied:not this quarter'"), + "{yaml}" + ); + } + + /// A fresh playbook pod carries neither the init container nor `--resume`: a resume is the + /// controller naming a predecessor, never something the pod discovers for itself. + #[test] + fn a_fresh_playbook_has_no_resume_container_and_no_resume_flag() { + let yaml = render_playbook(&k8s_profile(""), playbook_launch()); + assert!(!yaml.contains("resume-restore"), "{yaml}"); + assert!(!yaml.contains("--resume"), "{yaml}"); + assert!(!yaml.contains("--approval"), "{yaml}"); + } + + /// A pod deletion must reach a parked run as a stop it can act on. `ctrlc` handles SIGINT + /// only, so the preStop hook writes the cross-process stop file and the grace period is long + /// enough for the run to snapshot itself before SIGKILL. + #[test] + fn every_pod_stops_gracefully_through_the_control_file() { + let yaml = render_k8s(&k8s_profile("")); + assert!( + yaml.contains("terminationGracePeriodSeconds: 300"), + "{yaml}" + ); + assert!(yaml.contains("preStop:"), "{yaml}"); + assert!( + yaml.contains("/opt/crucible/domains/alpha/state/control.json"), + "the hook writes the path the engine polls: {yaml}" + ); + assert!(yaml.contains(r#"{"stop":true}"#), "{yaml}"); + } } diff --git a/crucible/src/ingest_client.rs b/crucible/src/ingest_client.rs index e9dc03ef..48035ade 100644 --- a/crucible/src/ingest_client.rs +++ b/crucible/src/ingest_client.rs @@ -162,6 +162,67 @@ fn deliver(cfg: &IngestConfig, kind: ArtifactKind, bytes: &[u8]) -> bool { false } +/// The resolutions the controller holds for this pod's open gates. A controller without the +/// endpoint (404) or an unreachable one yields nothing; the park keeps waiting. +pub fn fetch_pod_approvals(cfg: &IngestConfig) -> Vec { + let url = format!("{}/api/pods/{}/approvals", cfg.base_url, cfg.pod); + match get(cfg, &url) { + Ok(Some(bytes)) => serde_json::from_slice(&bytes).unwrap_or_default(), + Ok(None) => Vec::new(), + Err(e) => { + eprintln!("[crucible] polling {url} for approvals failed: {e}"); + Vec::new() + } + } +} + +/// Pull one artifact a previous pod of this run left in the drop-box. `None` when the +/// controller holds none for it. +pub fn fetch_artifact( + cfg: &IngestConfig, + of_pod: &str, + kind: ArtifactKind, +) -> Result>, String> { + let url = format!( + "{}/api/pods/{}/artifacts/{}?from={}", + cfg.base_url, + cfg.pod, + kind.as_str(), + of_pod + ); + get(cfg, &url) +} + +/// One authenticated GET with the pod's projected token. `Ok(None)` on 404. +fn get(cfg: &IngestConfig, url: &str) -> Result>, String> { + let client = reqwest::blocking::Client::builder() + .timeout(REQUEST_TIMEOUT) + .build() + .map_err(|e| format!("client build failed: {e}"))?; + let token = std::fs::read_to_string(&cfg.token_path) + .map(|t| t.trim().to_string()) + .map_err(|e| format!("reading ingest token {}: {e}", cfg.token_path.display()))?; + let resp = client + .get(url) + .bearer_auth(&token) + .send() + .map_err(|e| format!("GET {url}: {e}"))?; + if resp.status() == reqwest::StatusCode::NOT_FOUND { + return Ok(None); + } + if !resp.status().is_success() { + return Err(format!("GET {url} → HTTP {}", resp.status())); + } + resp.bytes() + .map(|b| Some(b.to_vec())) + .map_err(|e| format!("reading {url}: {e}")) +} + +/// The drop-box config for a run resuming another pod's artifacts, from the pod env. +pub fn resume_of_from_env() -> Option { + non_empty_env(crucible_contract::ENV_RESUME_OF) +} + fn non_empty_env(name: &str) -> Option { std::env::var(name).ok().filter(|s| !s.is_empty()) } diff --git a/crucible/src/loop_graph.rs b/crucible/src/loop_graph.rs index d9a4f0b5..a6520f56 100644 --- a/crucible/src/loop_graph.rs +++ b/crucible/src/loop_graph.rs @@ -83,7 +83,7 @@ impl PriorPlan { /// version, same declared tasks), and only the tasks that passed. Engine operations are /// never seeded: the world a passing `propose` or `apply` produced did not survive the /// process, so they run again. - fn seed(&self, plan: &ValidPlan) -> BTreeMap { + pub(crate) fn seed(&self, plan: &ValidPlan) -> BTreeMap { let declared: Vec = plan.plan().tasks.iter().map(|t| t.name.0.clone()).collect(); if plan.plan().version != self.plan_version || declared != self.declared { return BTreeMap::new(); @@ -175,6 +175,7 @@ pub(crate) fn run_iteration(cx: IterCtx<'_>, r: &mut R) -> Result<( commit_per_task: false, captured_bytes: std::sync::atomic::AtomicU64::new(0), staged: Default::default(), + gate: crate::plan::gate::GateCtx::default(), }, }; // The runner and the on_result hook both need the reporter; collect the wire lines @@ -422,6 +423,7 @@ pub(crate) fn run_epilogue( commit_per_task: false, captured_bytes: std::sync::atomic::AtomicU64::new(0), staged: Default::default(), + gate: crate::plan::gate::GateCtx::default(), }, kept: kept.to_value(), }; @@ -792,7 +794,7 @@ impl TaskRunner for LoopTaskRunner<'_, R> { source, .. } => self.decide(task, source.as_ref()), - TaskKind::TopK { .. } => fail( + TaskKind::TopK { .. } | TaskKind::Approve { .. } => fail( 0.0, format!( "unexpected task kind in the loop template: {}", diff --git a/crucible/src/main.rs b/crucible/src/main.rs index 8fedc5c6..531a17f4 100644 --- a/crucible/src/main.rs +++ b/crucible/src/main.rs @@ -63,6 +63,7 @@ mod scope; mod selftest; mod session; mod stream; +mod suspend; pub(crate) use crucible_harness::stream_json; use crucible::{ @@ -83,9 +84,12 @@ mod openshell { /// The plan runtime: the CLI and the agent-harness task runner over the library's plan IR, /// compiler, and executor. mod plan { - pub use crucible::plan::{TASK_NAME_ENV, exec, ir, runner, starlark, term_img, worktree}; + pub use crucible::plan::{ + ParkPolicy, TASK_NAME_ENV, exec, gate, ir, runner, starlark, term_img, worktree, + }; pub mod cli; + pub mod gate_host; pub mod harness; } @@ -298,6 +302,38 @@ pub(crate) enum Cmd { #[arg(long)] once: bool, }, + /// Grant the approval gate a live run is parked on, over its control bridge. + Approve { + /// The live run's control-bridge address (host:port, from its `--control-port`). + #[arg(long)] + control_addr: String, + /// Who is approving, recorded with the grant. + #[arg(long)] + by: Option, + }, + /// Deny the approval gate a live run is parked on, over its control bridge. + Deny { + /// The live run's control-bridge address (host:port, from its `--control-port`). + #[arg(long)] + control_addr: String, + /// Why, recorded with the denial and shown as the gate's failure. + #[arg(long, default_value = "rejected")] + reason: String, + /// Who is denying, recorded with the denial. + #[arg(long)] + by: Option, + }, + /// Restore a suspended run's snapshot from the controller before resuming it: pulls the + /// `run-session` and `run-workspace` artifacts the pod named by `CRUCIBLE_RESUME_OF` left, + /// into `/state` and `/`. + FetchResume { + /// The manifest directory the run lives in. + #[arg(long)] + into: PathBuf, + /// The workspace directory below `--into` (the manifest's `[workspace].dir`). + #[arg(long, default_value = "workspace")] + workspace: String, + }, /// Download one published object at an exact `s3://bucket/key` URI to a local file, the general /// GetObject the controller's artifact proxy shells so no S3 client leaks into /// `crucible-controller`. Nothing is appended to the URI; the caller passes the exact key. @@ -463,6 +499,27 @@ pub(crate) enum PlanAction { /// that pins its own model keeps it. #[arg(long, requires = "manifest")] model: Option, + /// Continue the session log in the manifest's `state/` instead of starting fresh: tasks + /// the previous process settled are not re-dispatched, and a gate it left open is + /// re-entered. + #[arg(long, requires = "manifest")] + resume: bool, + /// A gate's resolution, `=granted|denied[:reason][@by]` (repeatable). The + /// trace id is on the `approval_wait` line the gate wrote. + #[arg(long = "approval", value_name = "TRACE=DECISION")] + approvals: Vec, + /// What a gate does once it has parked for `--max-park`: fail as timed out, snapshot and + /// exit for a later `--resume`, or (`suspend`) snapshot at once without idling. + #[arg(long, value_enum, default_value_t = plan::ParkPolicy::ParkThenDeny)] + park_policy: plan::ParkPolicy, + /// How long a gate may idle (`30m`, `2h`). Absent: as long as it takes. A gate's own + /// `timeout` clamps it further. + #[arg(long)] + max_park: Option, + /// Serve the control bridge on this port while the run is up, so `crucible approve` + /// can resolve a parked gate. + #[arg(long, requires = "manifest")] + control_port: Option, }, } diff --git a/crucible/src/plan/cli.rs b/crucible/src/plan/cli.rs index f7d65830..2ca4fd28 100644 --- a/crucible/src/plan/cli.rs +++ b/crucible/src/plan/cli.rs @@ -5,6 +5,7 @@ use std::path::Path; use anyhow::{Context, Result}; +use crate::plan::ParkPolicy; use crate::plan::exec::{Substrate, TaskResult, TaskStatus, runnable_set}; use crate::plan::ir::{Direction, Plan, Task, TaskKind, ValidPlan}; use xai_grok_mermaid::{MermaidTheme, RenderLimits, RenderParams, default_engine, render_checked}; @@ -123,6 +124,7 @@ pub fn render(plan: &ValidPlan, caps: &BTreeSet) -> String { .unwrap_or_default() ), TaskKind::TopK { k, .. } => format!("top_k[k={k}]"), + TaskKind::Approve { source, .. } => format!("approve[{}]", source_label(source)), TaskKind::Report { .. } => "report".to_string(), TaskKind::Engine { .. } => t.task.label().to_string(), }; @@ -224,6 +226,7 @@ fn render_mermaid_styled( TaskKind::TopK { .. } => ("{{", "}}", CLASS_STYLES[4]), TaskKind::Report { .. } => ("[[", "]]", CLASS_STYLES[3]), TaskKind::Engine { .. } => ("[[", "]]", CLASS_STYLES[5]), + TaskKind::Approve { .. } => ("{{", "}}", CLASS_STYLES[3]), }; let mut detail = match &t.task { TaskKind::Agent { harness, model, .. } => format!( @@ -254,6 +257,9 @@ fn render_mermaid_styled( | TaskKind::Report { .. } | TaskKind::Engine { .. } => String::new(), TaskKind::TopK { k, .. } => format!("
k={k}"), + TaskKind::Approve { source, .. } => { + format!("
{}", mermaid_label(&source_label(source))) + } }; if let Some(session) = &t.session { detail.push_str(&format!("
session: {}", mermaid_label(session))); @@ -394,6 +400,28 @@ pub(crate) fn task_result_event( } } +/// The gate's source for a label: `native`, or the kind plus what it waits for. +fn source_label(source: &crate::plan::ir::ApprovalSourceSpec) -> String { + use crate::plan::ir::{ApprovalSourceSpec, RefOrLiteral}; + let show = |r: &RefOrLiteral| match r { + RefOrLiteral::Literal(s) => s.clone(), + RefOrLiteral::Output(reference) => reference.to_string(), + }; + match source { + ApprovalSourceSpec::Native => "native".to_string(), + ApprovalSourceSpec::GithubPr { url, until } => { + format!("github_pr {} until {}", show(url), until.as_str()) + } + ApprovalSourceSpec::Jira { key, until } => { + let until = match until { + crucible_contract::JiraUntil::Status(s) => format!("status {s}"), + crucible_contract::JiraUntil::Label(l) => format!("label {l}"), + }; + format!("jira {} until {until}", show(key)) + } + } +} + fn mermaid_label(name: &str) -> String { name.replace('&', "&") .replace('"', """) @@ -541,6 +569,23 @@ pub struct RunOpts { pub ceilings: Ceilings, pub compute_driver: crate::openshell::gateway::ComputeDriver, pub agent: AgentOverride, + pub gates: GateOpts, +} + +/// How a run behaves at an approval gate, and what it already knows about them. +#[derive(Debug, Clone, Default)] +pub struct GateOpts { + /// Continue the session log in `state/` instead of starting a run: settled tasks are not + /// re-dispatched, and an open gate is re-entered. + pub resume: bool, + /// Resolutions handed in on the command line, keyed by trace id. + pub approvals: Vec<(String, crucible_contract::GateResolution)>, + pub policy: ParkPolicy, + /// How long a gate may park before `policy` decides, `None` for as long as it takes. + pub max_park: Option, + /// Serve the control bridge on this port while the run is up, so an operator `approve` + /// resolves a parked gate. + pub control_port: Option, } /// The agent a launch names in place of the manifest's `[agent]` defaults. A task that pins its @@ -598,12 +643,14 @@ pub fn run( manifest: Option<&Path>, opts: RunOpts, ) -> Result<()> { - use crate::plan::exec::{ExecCfg, PlanExit, TaskRunner, execute}; + use crate::plan::exec::{ExecCfg, PlanExit, TaskRunner, execute_from}; + use crate::plan::gate_host as gate; use crate::plan::runner::ShellRunner; let RunOpts { ceilings, compute_driver, agent, + gates, } = opts; if let (None, Some(raw)) = (ceilings.wall_clock, ceilings.wall_clock_raw.as_ref()) { @@ -683,6 +730,7 @@ pub fn run( workdir: std::env::current_dir() .context("resolving the working directory")?, agent_cmd, + gate: crate::plan::gate::GateCtx::new(crate::plan::gate::run_id_from_env()), }), None, ) @@ -743,30 +791,166 @@ pub fn run( } }; write_report(&report); - let out = execute( - &plan, - &substrate, - ExecCfg { - wall_clock: ceilings.wall_clock, - ..ExecCfg::default() - }, - runner.as_mut(), - |task, result| { - report.tasks.push(crucible_contract::TaskReport { + let mut on_result = |task: &Task, result: &TaskResult| { + match report.tasks.iter_mut().find(|t| t.name == task.name.0) { + Some(existing) => { + existing.status = result.status.as_str().to_string(); + existing.cost_usd = result.cost_usd; + } + None => report.tasks.push(crucible_contract::TaskReport { name: task.name.0.clone(), status: result.status.as_str().to_string(), cost_usd: result.cost_usd, - }); - if let Some(selected) = report.results.get_mut(&task.name.0) { - selected.status = result.status.as_str().to_string(); - selected.output = declared_report_output(task, result); + }), + } + if let Some(selected) = report.results.get_mut(&task.name.0) { + selected.status = result.status.as_str().to_string(); + selected.output = declared_report_output(task, result); + } + write_report(&report); + if let Some(f) = &events { + append(f, &task_result_event(plan.plan().version, 0, task, result)); + } + }; + let exec_cfg = ExecCfg { + wall_clock: ceilings.wall_clock, + ..ExecCfg::default() + }; + + // What the run already knows: the results a previous process settled under this same plan, + // the resolutions handed in, and the ones the admission ledger recorded before a death. + let mut host = gate::Host::open(evidence.as_ref(), &plan, &gates)?; + for (trace, resolution) in host.resolutions() { + runner.resolve_gate(&trace, resolution); + } + let mut prior = host.take_prior(); + let mut spent_before = 0.0; + let out = loop { + let mut out = execute_from( + &plan, + &substrate, + exec_cfg.clone(), + runner.as_mut(), + prior, + &mut on_result, + ); + out.spent_usd += spent_before; + let PlanExit::AwaitingApproval(open) = &out.exit else { + break out; + }; + let open = open.clone(); + if let Some(f) = &events { + append(f, &gate::wait_event(&open)); + } + println!( + "gate {} ({}): awaiting approval via {}", + open.task, + open.trace_id, + open.source.kind() + ); + match host.wait(&open, &gates, evidence.as_ref())? { + gate::Waited::Resolved(resolution) => { + if let Some(f) = &events { + append(f, &gate::resolved_event(&open, &resolution)); + } + println!( + "gate {}: {}{}", + open.task, + resolution.decision.as_str(), + resolution + .by + .as_deref() + .map(|by| format!(" by {by}")) + .unwrap_or_default() + ); + runner.resolve_gate(&open.trace_id, resolution); + spent_before = out.spent_usd; + prior = out.results; + } + gate::Waited::Suspend => { + let Some(paths) = &evidence else { + anyhow::bail!("a gate can only suspend a --manifest run"); + }; + match gate::suspend(paths, &plan, &open) { + Ok(()) => { + if let Some(f) = &events { + append( + f, + &crate::session::SessionEvent::Shutdown { + outcome: "suspended".to_string(), + reason: open.trace_id.clone(), + }, + ); + } + println!( + "plan v{}: suspended at gate {} — resume with --resume --approval {}=granted|denied", + plan.plan().version, + open.task, + open.trace_id + ); + return Ok(()); + } + Err(why) => { + // Never lose a run to a failed snapshot: park instead and say why. + eprintln!( + "[crucible] suspend failed ({why:#}); parking on the gate instead" + ); + if let Some(f) = &events { + append( + f, + &crate::session::SessionEvent::Note { + msg: format!("suspend failed, parking instead: {why:#}"), + }, + ); + } + match host.park(&open, None, evidence.as_ref())? { + gate::Parked::Resolved(resolution) => { + if let Some(f) = &events { + append(f, &gate::resolved_event(&open, &resolution)); + } + runner.resolve_gate(&open.trace_id, resolution); + spent_before = out.spent_usd; + prior = out.results; + } + gate::Parked::Stopped => { + if let Some(f) = &events { + append( + f, + &crate::session::SessionEvent::Shutdown { + outcome: "stopped".to_string(), + reason: "stop signal received while parked".into(), + }, + ); + } + anyhow::bail!("stopped while parked on gate {}", open.task); + } + gate::Parked::TimedOut => { + let resolution = crucible_contract::GateResolution::timeout(); + if let Some(f) = &events { + append(f, &gate::resolved_event(&open, &resolution)); + } + runner.resolve_gate(&open.trace_id, resolution); + spent_before = out.spent_usd; + prior = out.results; + } + } + } + } } - write_report(&report); - if let Some(f) = &events { - append(f, &task_result_event(plan.plan().version, 0, task, result)); + gate::Waited::Stopped => { + if let Some(f) = &events { + append( + f, + &crate::session::SessionEvent::Shutdown { + outcome: "stopped".to_string(), + reason: "stop signal received while parked".into(), + }, + ); + } + anyhow::bail!("stopped while parked on gate {}", open.task); } - }, - ); + } + }; for t in plan.tasks_topo() { if let Some(r) = out.results.get(&t.name) { println!( @@ -792,6 +976,7 @@ pub fn run( PlanExit::ShortCircuit { task } => format!("short-circuited at {task}"), PlanExit::BudgetExceeded => "budget exceeded".to_string(), PlanExit::TimeExceeded => "wall-clock ceiling reached".to_string(), + PlanExit::AwaitingApproval(gate) => format!("awaiting approval at {}", gate.task), }; if let Some(f) = &events { append( @@ -825,6 +1010,7 @@ pub fn run( #[cfg(test)] mod tests { use super::*; + use std::time::Duration; const SRC: &str = r#" version = 1 @@ -1417,4 +1603,249 @@ emits = ["verdict", "dirty"] } let _ = std::fs::remove_dir_all(&dir); } + + /// The three-node gate shape, end to end: a deterministic task opens a PR and emits its url, + /// a gate reads that url as what it is waiting on, and a third task runs only if the gate is + /// granted. Every task is a real shell command in a real workspace; the only thing the test + /// supplies is the decision a person would make. + const GATE_WORKFLOW: &str = r#" +open_pr = command( + name = "open-pr", + run = "printf '%s\n' '{\"url\": \"https://github.com/o/r/pull/7\"}'", + emits = ["url"], +) + +ship = approve( + name = "ship", + summary = "Ship the change", + source = github_pr(url = open_pr.url, until = "approved"), + depends_on = [open_pr], +) + +deploy = command( + name = "deploy", + run = "echo shipped > deployed.txt && printf '%s\n' '{}'", + depends_on = [ship], +) + +workflow(type = "playbook", tasks = [open_pr, ship, deploy]) +"#; + + /// A manifest dir holding the three-node playbook. Returns the dir; the caller removes it. + /// Clears the drop-box env: a gate run publishes its session like any other, and inheriting + /// another test's ingest URL would send it to that test's listener. + fn gate_pack(tag: &str) -> std::path::PathBuf { + unsafe { + for key in [ + crucible_contract::ENV_INGEST_URL, + crucible_contract::ENV_INGEST_TOKEN_PATH, + crucible_contract::ENV_POD_NAME, + ] { + std::env::remove_var(key); + } + } + let dir = + std::env::temp_dir().join(format!("crucible-gate-e2e-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("workflow.star"), GATE_WORKFLOW).unwrap(); + std::fs::write( + dir.join("crucible.toml"), + r#" +[repo] +path = "." +[workspace] +dir = "workspace" +setup_cmd = "mkdir -p workspace && git -C workspace init -q && git -C workspace -c user.email=c@l -c user.name=c -c commit.gpgsign=false commit -q --allow-empty -m baseline" +[agent] +backend = "command" +agent_cmd = "true" +goal = "open a PR, wait for approval, deploy" +[workflow] +type = "playbook" +file = "workflow.star" +"#, + ) + .unwrap(); + dir + } + + /// The gate this pack opens. `run_id_from_env` falls back to "local" outside a pod. + fn gate_trace() -> String { + crucible_contract::gate_trace_id("local", "ship") + } + + fn run_gate_pack(dir: &Path, gates: GateOpts) -> Result<()> { + run( + None, + &BTreeMap::new(), + &BTreeSet::new(), + None, + Some(&dir.join("crucible.toml")), + RunOpts { + ceilings: Ceilings { + usd: Some(5.0), + wall_clock: Some(Duration::from_secs(120)), + wall_clock_raw: Some("2m".to_string()), + }, + gates, + ..Default::default() + }, + ) + } + + fn session_log(dir: &Path) -> String { + std::fs::read_to_string(dir.join("state/session.jsonl")).unwrap_or_default() + } + + /// A grant handed to the run before it starts is the resume path: the gate settles from the + /// recorded decision without reopening, and the task after it runs. + #[test] + fn a_granted_gate_lets_the_task_after_it_run() { + let _guard = crate::test_env_lock(); + let dir = gate_pack("granted"); + let out = run_gate_pack( + &dir, + GateOpts { + approvals: vec![( + gate_trace(), + crucible_contract::GateResolution::granted(Some("wseaton".into()), "native"), + )], + ..Default::default() + }, + ); + assert!(out.is_ok(), "a granted run exits clean: {out:?}"); + assert!( + dir.join("workspace/deployed.txt").exists(), + "the task after the gate ran: {}", + session_log(&dir) + ); + let log = session_log(&dir); + assert!( + log.contains("\"url\":\"https://github.com/o/r/pull/7\"") || log.contains("pull/7"), + "the gate names the PR the first task opened: {log}" + ); + let _ = std::fs::remove_dir_all(&dir); + } + + /// A denial blocks the gate's dependents exactly as any other failure does, and the run is + /// invalid: an approval nobody gave must not let the deploy through. + #[test] + fn a_denied_gate_blocks_the_task_after_it() { + let _guard = crate::test_env_lock(); + let dir = gate_pack("denied"); + let out = run_gate_pack( + &dir, + GateOpts { + approvals: vec![( + gate_trace(), + crucible_contract::GateResolution::denied( + "not this quarter", + Some("wseaton".into()), + "native", + ), + )], + ..Default::default() + }, + ); + let err = out.expect_err("a denied required gate invalidates the run"); + assert!( + err.to_string().contains("did not reach a valid verdict"), + "the run fails on its verdict, not on a launch error: {err}" + ); + assert!( + !dir.join("workspace/deployed.txt").exists(), + "nothing after the gate ran: {}", + session_log(&dir) + ); + let _ = std::fs::remove_dir_all(&dir); + } + + /// A gate nobody resolves parks until the run's `max_park`, then settles as a timeout under + /// park-then-deny. The bound is what keeps an unanswered gate from holding a pod forever. + #[test] + fn an_unanswered_gate_times_out_rather_than_waiting_forever() { + let _guard = crate::test_env_lock(); + let dir = gate_pack("timeout"); + let started = std::time::Instant::now(); + let out = run_gate_pack( + &dir, + GateOpts { + max_park: Some(Duration::from_millis(300)), + policy: crate::plan::ParkPolicy::ParkThenDeny, + ..Default::default() + }, + ); + assert!( + out.is_err(), + "a timed-out required gate invalidates the run" + ); + assert!( + started.elapsed() < Duration::from_secs(30), + "the park is bounded by max_park" + ); + assert!(!dir.join("workspace/deployed.txt").exists()); + let log = session_log(&dir); + assert!( + log.contains("timeout"), + "the gate settles as a timeout: {log}" + ); + let _ = std::fs::remove_dir_all(&dir); + } + + /// The operator path, over a real socket: the run parks with its control bridge up, an + /// `approve` arrives the way `crucible approve` sends one, and the deploy runs. Nothing here + /// is stubbed, the bridge is the one the pod serves. + #[test] + fn an_approve_over_the_control_bridge_releases_a_parked_gate() { + let _guard = crate::test_env_lock(); + let dir = gate_pack("bridge"); + let port = { + let probe = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + probe.local_addr().unwrap().port() + }; + let run_dir = dir.clone(); + let runner = std::thread::spawn(move || { + run_gate_pack( + &run_dir, + GateOpts { + control_port: Some(port), + max_park: Some(Duration::from_secs(30)), + ..Default::default() + }, + ) + }); + + // Retry until the bridge is up AND a gate is armed: an approve with no gate armed answers + // without a trace id, which is exactly the "not parked yet" signal to keep waiting on. + let addr = format!("127.0.0.1:{port}"); + let trace = gate_trace(); + let deadline = std::time::Instant::now() + Duration::from_secs(30); + let mut granted = false; + while std::time::Instant::now() < deadline { + if let Ok(reply) = crate::control::send_command( + &addr, + &serde_json::json!({"cmd": "approve", "by": "operator"}), + ) && reply.get("trace_id").and_then(serde_json::Value::as_str) + == Some(trace.as_str()) + { + granted = true; + break; + } + std::thread::sleep(Duration::from_millis(50)); + } + assert!(granted, "the parked run accepted an approve for {trace}"); + + let out = runner.join().expect("the run thread did not panic"); + assert!(out.is_ok(), "the released run finishes clean: {out:?}"); + assert!( + dir.join("workspace/deployed.txt").exists(), + "the task after the gate ran: {}", + session_log(&dir) + ); + let log = session_log(&dir); + assert!(log.contains("approval_wait"), "the gate opened: {log}"); + assert!(log.contains("granted"), "and was granted: {log}"); + let _ = std::fs::remove_dir_all(&dir); + } } diff --git a/crucible/src/plan/exec.rs b/crucible/src/plan/exec.rs index 76a13b99..d6ccb710 100644 --- a/crucible/src/plan/exec.rs +++ b/crucible/src/plan/exec.rs @@ -44,6 +44,21 @@ pub enum AttemptOutcome { Skipped(Value, String), /// Transport failure (infra, not the work). Retried, bounded, every attempt visible. Transport(String), + /// The task is a gate no resolution has reached yet. The walker stops dispatching and + /// returns [`PlanExit::AwaitingApproval`] with everything settled so far; the gate itself + /// settles when the run re-enters with the resolution. + Await(Gate), +} + +/// An open approval gate, as the executor reports it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Gate { + pub task: TaskName, + pub trace_id: String, + pub handle: String, + pub source: crucible_contract::GateSource, + pub summary: Option, + pub timeout_secs: Option, } impl AttemptOutcome { @@ -125,6 +140,10 @@ pub trait TaskRunner { /// producing evidence, so a set from an earlier run cannot outlive its producer's silence. fn drop_captured(&mut self, _task: &Task) {} + /// Hand the runner a gate's resolution, so the next dispatch of that gate settles instead + /// of waiting. A runner with no gates keeps nothing. + fn resolve_gate(&mut self, _trace_id: &str, _resolution: crucible_contract::GateResolution) {} + fn run_many(&mut self, batch: &[BatchItem<'_>]) -> Vec { batch .iter() @@ -290,6 +309,9 @@ pub enum PlanExit { BudgetExceeded, /// The wall-clock ceiling was reached; undispatched tasks were blocked. TimeExceeded, + /// A gate was reached with no resolution. Nothing after it was dispatched or blocked: the + /// results hold exactly the settled tasks, and a re-entry with the resolution continues. + AwaitingApproval(Gate), } impl PlanExit { @@ -301,6 +323,7 @@ impl PlanExit { PlanExit::Completed => "finished", PlanExit::BudgetExceeded | PlanExit::TimeExceeded => "budget", PlanExit::Truncated { .. } | PlanExit::ShortCircuit { .. } => "error", + PlanExit::AwaitingApproval(_) => "suspended", } } } @@ -363,9 +386,9 @@ pub fn execute( } /// [`execute`] with results already settled by an earlier process. A task named in `prior` is -/// never dispatched: its result stands, its output feeds its dependents, and it is reported -/// through `on_result` like any other terminal result so the log accounts for it. This is how a -/// resumed run continues a graph instead of re-running the tasks that already passed. +/// never dispatched: its result stands and its output feeds its dependents. It is not reported +/// again, since the process that settled it already did. This is how a resumed run continues a +/// graph instead of re-running the tasks that already passed. pub fn execute_from( plan: &ValidPlan, substrate: &Substrate, @@ -404,7 +427,6 @@ pub fn execute_from( let mut results: BTreeMap = BTreeMap::new(); for task in plan.tasks_topo() { if let Some(result) = prior.get(&task.name) { - on_result(task, result); results.insert(task.name.clone(), result.clone()); } } @@ -467,6 +489,9 @@ pub fn execute_from( PlanExit::ShortCircuit { task } => format!("required task {task} failed"), PlanExit::BudgetExceeded => "budget ceiling reached".to_string(), PlanExit::TimeExceeded => "wall-clock ceiling reached".to_string(), + PlanExit::AwaitingApproval(gate) => { + format!("gate {} is awaiting approval", gate.task) + } _ => "halted".to_string(), }; let r = TaskResult::undispatched(TaskStatus::Blocked, why); @@ -715,6 +740,20 @@ pub fn execute_from( | TaskKind::Engine { .. } => { run_with_retries(t, &inputs, cfg, runner, &mut spent, budget) } + TaskKind::Approve { .. } => { + let a = runner.run(t, 1, &inputs); + spent += a.cost_usd; + match a.outcome { + // A gate with no resolution stops the walk here. Nothing is recorded + // for it or for anything after it: the re-entry that carries the + // resolution picks up exactly where this left off. + AttemptOutcome::Await(gate) => { + halted = Some(PlanExit::AwaitingApproval(gate)); + break; + } + outcome => (settle_attempt(outcome, 1, a.cost_usd), spent > budget), + } + } }; if budget_exceeded { halted.get_or_insert(PlanExit::BudgetExceeded); @@ -759,6 +798,52 @@ pub fn execute_from( } } +/// One settled attempt as a result, for a task the walker ran exactly once. +fn settle_attempt(outcome: AttemptOutcome, attempts: u32, cost_usd: f64) -> TaskResult { + match outcome { + AttemptOutcome::Pass(output) => TaskResult { + status: TaskStatus::Pass, + attempts, + cost_usd, + output: Some(output), + note: None, + fanout: None, + }, + AttemptOutcome::Skipped(output, note) => TaskResult { + status: TaskStatus::Skipped, + attempts, + cost_usd, + output: Some(output), + note: Some(note), + fanout: None, + }, + AttemptOutcome::Fail { note, output } => TaskResult { + status: TaskStatus::Fail, + attempts, + cost_usd, + output, + note: Some(note), + fanout: None, + }, + AttemptOutcome::Transport(note) => TaskResult { + status: TaskStatus::Transport, + attempts, + cost_usd, + output: None, + note: Some(note), + fanout: None, + }, + AttemptOutcome::Await(gate) => TaskResult { + status: TaskStatus::Fail, + attempts, + cost_usd, + output: None, + note: Some(format!("gate {} left unresolved", gate.trace_id)), + fanout: None, + }, + } +} + /// The reserved input a mapped instance receives its own item under. Reserved like the /// epilogue's kept-candidate input: a task may not declare a dependency by this name. pub const ITEM_INPUT: &str = "item"; @@ -1184,6 +1269,22 @@ fn run_with_retries( *spent > budget, ); } + AttemptOutcome::Await(gate) => { + return ( + TaskResult { + status: TaskStatus::Fail, + attempts, + cost_usd: cost, + output: None, + note: Some(format!( + "task {} reported an approval gate it is not: {}", + t.name, gate.trace_id + )), + fanout: None, + }, + *spent > budget, + ); + } AttemptOutcome::Transport(note) => { if *spent > budget || (*spent >= budget && attempts < max_attempts) { return ( @@ -1299,6 +1400,22 @@ fn run_batch_with_retries<'a>( }, ); } + AttemptOutcome::Await(gate) => { + done.insert( + idx, + TaskResult { + status: TaskStatus::Fail, + attempts: item.attempt, + cost_usd: cost_so_far[idx], + output: None, + note: Some(format!( + "task reported an approval gate it is not: {}", + gate.trace_id + )), + fanout: None, + }, + ); + } AttemptOutcome::Transport(note) => { if item.attempt < max_attempts && !retry_budget_blocked { next.push(( @@ -1555,11 +1672,8 @@ mod tests { assert_eq!(out.exit, PlanExit::Completed); assert_eq!( reported, - vec![ - ("a".to_string(), TaskStatus::Pass), - ("b".to_string(), TaskStatus::Pass) - ], - "the prior result is reported first, then the dispatched one" + vec![("b".to_string(), TaskStatus::Pass)], + "only the dispatched task is reported; the prior one already was" ); assert!( !r.seen_inputs.contains_key("a"), diff --git a/crucible/src/plan/gate.rs b/crucible/src/plan/gate.rs new file mode 100644 index 00000000..ed7e2daf --- /dev/null +++ b/crucible/src/plan/gate.rs @@ -0,0 +1,327 @@ +//! The `approve` task at dispatch time: resolve where its decision comes from, and either +//! settle it from a resolution the run already holds or report it as an open gate. + +use std::collections::BTreeMap; + +use serde_json::Value; + +use crate::plan::exec::{Attempt, AttemptOutcome, Gate}; +use crate::plan::ir::{ApprovalSourceSpec, RefOrLiteral, Task, TaskKind, TaskName}; +use crucible_contract::{GateDecision, GateResolution, GateSource, gate_trace_id}; + +/// What a runner needs to settle gates: the run they belong to, and the resolutions the run +/// holds (from `--approval`, the admission ledger, or a park that just ended). +#[derive(Debug, Clone, Default)] +pub struct GateCtx { + pub run_id: String, + pub resolutions: BTreeMap, +} + +impl GateCtx { + pub fn new(run_id: impl Into) -> Self { + GateCtx { + run_id: run_id.into(), + resolutions: BTreeMap::new(), + } + } + + pub fn resolve(&mut self, trace_id: impl Into, resolution: GateResolution) { + self.resolutions.insert(trace_id.into(), resolution); + } +} + +/// The env var the launcher sets to name the run a gate belongs to. It must be stable across a +/// suspend and its redispatch, or the resolution the controller hands back names a trace the +/// resumed run never opened. +pub const ENV_RUN_ID: &str = "CRUCIBLE_RUN_ID"; + +/// The run id gates are keyed under: `CRUCIBLE_RUN_ID`, else the launcher's `CRUCIBLE_RUN_NAME`, +/// else `local`. +pub fn run_id_from_env() -> String { + for key in [ENV_RUN_ID, "CRUCIBLE_RUN_NAME"] { + if let Ok(v) = std::env::var(key) + && !v.trim().is_empty() + { + return v.trim().to_string(); + } + } + "local".to_string() +} + +/// Why a gate's source could not be resolved from its inputs. +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum SourceError { + #[error("gate reads {field} from {task}, which produced no passing output")] + ProducerMissing { task: String, field: String }, + #[error("gate reads {field} from {task}, whose output has no string {field}")] + FieldMissing { task: String, field: String }, +} + +/// The wire source for a gate, with any upstream reference read from the task's inputs. +pub fn resolve_source( + spec: &ApprovalSourceSpec, + inputs: &BTreeMap, +) -> Result { + let read = |r: &RefOrLiteral| -> Result { + match r { + RefOrLiteral::Literal(s) => Ok(s.clone()), + RefOrLiteral::Output(reference) => { + let producer = + inputs + .get(&reference.task) + .ok_or_else(|| SourceError::ProducerMissing { + task: reference.task.0.clone(), + field: reference.field.0.clone(), + })?; + producer + .get(&reference.field.0) + .and_then(Value::as_str) + .map(str::to_string) + .ok_or_else(|| SourceError::FieldMissing { + task: reference.task.0.clone(), + field: reference.field.0.clone(), + }) + } + } + }; + Ok(match spec { + ApprovalSourceSpec::Native => GateSource::Native, + ApprovalSourceSpec::GithubPr { url, until } => GateSource::GithubPr { + url: read(url)?, + until: *until, + }, + ApprovalSourceSpec::Jira { key, until } => GateSource::Jira { + key: read(key)?, + until: until.clone(), + }, + }) +} + +/// Run one `approve` task: a held resolution settles it, otherwise the gate is reported open. +/// Never spends. +pub fn attempt(ctx: &GateCtx, task: &Task, inputs: &BTreeMap) -> Attempt { + let TaskKind::Approve { + summary, + source, + timeout_secs, + } = &task.task + else { + return Attempt { + outcome: AttemptOutcome::fail(format!("task {} is not an approve task", task.name)), + cost_usd: 0.0, + }; + }; + let source = match resolve_source(source, inputs) { + Ok(source) => source, + Err(why) => { + return Attempt { + outcome: AttemptOutcome::fail(why.to_string()), + cost_usd: 0.0, + }; + } + }; + let trace_id = gate_trace_id(&ctx.run_id, &task.name.0); + let outcome = match ctx.resolutions.get(&trace_id) { + Some(resolution) => settle(resolution, &source), + None => AttemptOutcome::Await(Gate { + task: task.name.clone(), + trace_id, + handle: source + .handle() + .map(str::to_string) + .unwrap_or_else(|| task.name.0.clone()), + source, + summary: summary.clone(), + timeout_secs: *timeout_secs, + }), + }; + Attempt { + outcome, + cost_usd: 0.0, + } +} + +/// A grant passes the gate with the resolution as its output; a denial or timeout fails it +/// with the reason. +fn settle(resolution: &GateResolution, source: &GateSource) -> AttemptOutcome { + match resolution.decision { + GateDecision::Granted => AttemptOutcome::Pass(serde_json::json!({ + "approved_by": resolution.by, + "source": resolution.source.clone().unwrap_or_else(|| source.kind().to_string()), + "reason": resolution.reason, + })), + GateDecision::Denied => AttemptOutcome::Fail { + note: format!( + "denied: {}", + resolution + .reason + .as_deref() + .filter(|r| !r.is_empty()) + .unwrap_or("no reason given") + ), + output: Some(serde_json::json!({ + "denied_by": resolution.by, + "source": resolution.source, + "reason": resolution.reason, + })), + }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::plan::ir::{Join, OutputField, OutputRef, Stage}; + use crucible_contract::{JiraUntil, PrUntil}; + + fn gate(source: ApprovalSourceSpec, deps: &[&str]) -> Task { + Task { + name: TaskName("review".into()), + task: TaskKind::Approve { + summary: Some("ship?".into()), + source, + timeout_secs: None, + }, + depends_on: deps.iter().map(|d| TaskName((*d).into())).collect(), + session: None, + needs: "any".into(), + required: true, + isolation: None, + join: Join::All, + stage: Stage::Iteration, + emits: Vec::new(), + emits_files: Vec::new(), + over: None, + max_fanout: None, + } + } + + fn pr_from(task: &str, field: &str) -> ApprovalSourceSpec { + ApprovalSourceSpec::GithubPr { + url: RefOrLiteral::Output(OutputRef { + task: TaskName(task.into()), + field: OutputField(field.into()), + }), + until: PrUntil::Approved, + } + } + + #[test] + fn an_unresolved_gate_is_reported_open_with_its_resolved_source() { + let ctx = GateCtx::new("run-1"); + let inputs = BTreeMap::from([( + TaskName("open_pr".into()), + serde_json::json!({"pr_url": "https://github.com/o/r/pull/9"}), + )]); + let a = attempt( + &ctx, + &gate(pr_from("open_pr", "pr_url"), &["open_pr"]), + &inputs, + ); + assert_eq!(a.cost_usd, 0.0); + let AttemptOutcome::Await(g) = a.outcome else { + panic!("an unresolved gate awaits"); + }; + assert_eq!(g.trace_id, "approve:run-1:review"); + assert_eq!(g.handle, "https://github.com/o/r/pull/9"); + assert_eq!(g.summary.as_deref(), Some("ship?")); + assert_eq!( + g.source, + GateSource::GithubPr { + url: "https://github.com/o/r/pull/9".into(), + until: PrUntil::Approved + } + ); + } + + #[test] + fn a_native_gate_uses_the_task_name_as_its_handle() { + let ctx = GateCtx::new("run-1"); + let a = attempt( + &ctx, + &gate(ApprovalSourceSpec::Native, &[]), + &BTreeMap::new(), + ); + let AttemptOutcome::Await(g) = a.outcome else { + panic!("awaits"); + }; + assert_eq!(g.handle, "review"); + assert_eq!(g.source, GateSource::Native); + } + + #[test] + fn a_held_grant_passes_and_a_held_denial_fails_with_the_reason() { + let mut ctx = GateCtx::new("run-1"); + ctx.resolve( + "approve:run-1:review", + GateResolution::granted(Some("alice".into()), "native"), + ); + let a = attempt( + &ctx, + &gate(ApprovalSourceSpec::Native, &[]), + &BTreeMap::new(), + ); + let AttemptOutcome::Pass(out) = a.outcome else { + panic!("a grant passes"); + }; + assert_eq!(out["approved_by"], "alice"); + assert_eq!(out["source"], "native"); + + ctx.resolve( + "approve:run-1:review", + GateResolution::denied("changes requested", Some("bob".into()), "github_pr"), + ); + let a = attempt( + &ctx, + &gate(ApprovalSourceSpec::Native, &[]), + &BTreeMap::new(), + ); + let AttemptOutcome::Fail { note, output } = a.outcome else { + panic!("a denial fails"); + }; + assert_eq!(note, "denied: changes requested"); + assert_eq!(output.expect("carries the denial")["denied_by"], "bob"); + + ctx.resolve("approve:run-1:review", GateResolution::timeout()); + let a = attempt( + &ctx, + &gate(ApprovalSourceSpec::Native, &[]), + &BTreeMap::new(), + ); + assert!(matches!(a.outcome, AttemptOutcome::Fail { .. })); + } + + #[test] + fn a_source_read_from_a_missing_producer_or_field_fails_the_gate() { + let ctx = GateCtx::new("run-1"); + let a = attempt( + &ctx, + &gate(pr_from("open_pr", "pr_url"), &["open_pr"]), + &BTreeMap::new(), + ); + let AttemptOutcome::Fail { note, .. } = a.outcome else { + panic!("fails"); + }; + assert!(note.contains("no passing output"), "{note}"); + let inputs = + BTreeMap::from([(TaskName("open_pr".into()), serde_json::json!({"number": 9}))]); + let a = attempt( + &ctx, + &gate(pr_from("open_pr", "pr_url"), &["open_pr"]), + &inputs, + ); + let AttemptOutcome::Fail { note, .. } = a.outcome else { + panic!("fails"); + }; + assert!(note.contains("no string pr_url"), "{note}"); + let jira = ApprovalSourceSpec::Jira { + key: RefOrLiteral::Literal("PROJ-1".into()), + until: JiraUntil::Status("Ready".into()), + }; + let AttemptOutcome::Await(g) = attempt(&ctx, &gate(jira, &[]), &BTreeMap::new()).outcome + else { + panic!("awaits"); + }; + assert_eq!(g.handle, "PROJ-1"); + } +} diff --git a/crucible/src/plan/gate_host.rs b/crucible/src/plan/gate_host.rs new file mode 100644 index 00000000..5d1a0a1e --- /dev/null +++ b/crucible/src/plan/gate_host.rs @@ -0,0 +1,320 @@ +//! What `plan run` does at an approval gate: park on the bridge and the controller, or +//! snapshot and exit; and what it knows on the way in: the results a previous process settled +//! and the resolutions already recorded. + +use std::collections::BTreeMap; +use std::sync::Arc; +use std::sync::atomic::Ordering; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result}; +use crucible_contract::admission::{AdmissionKey, AdmissionOutcome, AdmittedInput}; +use crucible_contract::{ + ApprovalWaits, ArtifactKind, GateDecision, GateResolution, GateWait, LoopState, ParkMode, +}; + +use crate::admission::AdmissionLedger; +use crate::control::ControlState; +use crate::ingest_client::{IngestConfig, fetch_pod_approvals, post_artifact}; +use crate::plan::ParkPolicy; +use crate::plan::cli::GateOpts; +use crate::plan::exec::{Gate, TaskResult}; +use crate::plan::ir::{TaskName, ValidPlan}; +use crate::session::SessionEvent; +use crate::{Paths, STOP}; + +/// How often a parked run asks the controller for a resolution. +const CONTROLLER_POLL: Duration = Duration::from_secs(15); +/// How often a parked run checks the bridge, the stop flag, and the clock. +const PARK_TICK: Duration = Duration::from_millis(250); + +/// How a wait at a gate ended. +pub(crate) enum Waited { + Resolved(GateResolution), + Suspend, + Stopped, +} + +/// How a park ended. +pub(crate) enum Parked { + Resolved(GateResolution), + Stopped, + TimedOut, +} + +pub(crate) struct Host { + run_id: String, + ledger: Option>, + control: Option>, + ingest: Option, + resolutions: BTreeMap, + prior: BTreeMap, +} + +impl Host { + /// Open the run's gate state. With `paths` (a `--manifest` run) the admission ledger is + /// opened, the control bridge started when a port was given, and on `--resume` the session + /// log folded for what the previous process settled and which gate it left open. + pub(crate) fn open(paths: Option<&Paths>, plan: &ValidPlan, gates: &GateOpts) -> Result { + let run_id = crate::plan::gate::run_id_from_env(); + let mut resolutions: BTreeMap = + gates.approvals.iter().cloned().collect(); + let mut prior = BTreeMap::new(); + let mut ledger = None; + let mut control = None; + if let Some(paths) = paths { + let mode = if gates.resume { + forge::ndjson::Open::Fold + } else { + forge::ndjson::Open::Truncate + }; + let opened = Arc::new(AdmissionLedger::open(&paths.admissions, mode)?); + if gates.resume { + let body = std::fs::read_to_string(&paths.session_log).with_context(|| { + format!( + "reading {} to resume (no session log: nothing to resume)", + paths.session_log.display() + ) + })?; + let state = LoopState::from_lines(body.lines()); + prior = prior_from(&state, plan); + if let Some(open) = state.open_approval() + && !resolutions.contains_key(&open.trace_id) + && let Some(recorded) = opened.gate_resolution(&open.trace_id) + { + resolutions.insert(open.trace_id.clone(), recorded); + } + } + if let Some(port) = gates.control_port { + control = Some(crate::control::spawn_bridge( + port, + paths.clone(), + opened.clone(), + )?); + } + ledger = Some(opened); + } + Ok(Host { + run_id, + ledger, + control, + ingest: IngestConfig::from_env(), + resolutions, + prior, + }) + } + + /// Every resolution the run holds on the way in. + pub(crate) fn resolutions(&self) -> Vec<(String, GateResolution)> { + self.resolutions + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect() + } + + /// The results a previous process settled under this plan; empty on a fresh run. + pub(crate) fn take_prior(&mut self) -> BTreeMap { + std::mem::take(&mut self.prior) + } + + /// Wait at `open` under the run's park policy. + pub(crate) fn wait( + &mut self, + open: &Gate, + gates: &GateOpts, + paths: Option<&Paths>, + ) -> Result { + // Every policy idles first; what differs is what the end of the idle means. + self.announce(open, ParkMode::Park); + let timeout = match (gates.max_park, open.timeout_secs) { + (Some(cap), Some(secs)) => Some(cap.min(Duration::from_secs(secs))), + (Some(cap), None) => Some(cap), + (None, Some(secs)) => Some(Duration::from_secs(secs)), + (None, None) => None, + }; + Ok(match self.park(open, timeout, paths)? { + Parked::Resolved(r) => Waited::Resolved(r), + Parked::Stopped if gates.policy == ParkPolicy::ParkThenSuspend => Waited::Suspend, + Parked::Stopped => Waited::Stopped, + Parked::TimedOut if gates.policy == ParkPolicy::ParkThenSuspend => Waited::Suspend, + Parked::TimedOut => Waited::Resolved(GateResolution::timeout()), + }) + } + + /// Idle until the bridge, the controller, a stop, or the clock ends the wait. A resolution + /// from the controller is recorded on the ledger so a later resume finds it; the bridge + /// records its own. + pub(crate) fn park( + &mut self, + open: &Gate, + timeout: Option, + paths: Option<&Paths>, + ) -> Result { + if let Some(control) = &self.control { + control.arm_gate(open.trace_id.clone()); + } + eprintln!( + "[crucible] parked on gate {} ({}): idle, awaiting approval{}", + open.task, + open.trace_id, + timeout + .map(|t| format!(" for up to {}s", t.as_secs())) + .unwrap_or_default() + ); + let start = Instant::now(); + let mut last_poll: Option = None; + loop { + if let Some(resolution) = self.control.as_ref().and_then(|c| c.take_gate_resolution()) { + return Ok(Parked::Resolved(resolution)); + } + // SIGTERM is not an interrupt here (`ctrlc` handles SIGINT only), so a pod deletion + // reaches a parked run as the stop file its preStop hook writes. + if STOP.load(Ordering::SeqCst) + || paths.is_some_and(|p| crate::control::stop_file_says_stop(&p.control)) + { + return Ok(Parked::Stopped); + } + if timeout.is_some_and(|cap| start.elapsed() >= cap) { + return Ok(Parked::TimedOut); + } + let due = last_poll.is_none_or(|t| t.elapsed() >= CONTROLLER_POLL); + if due && let Some(cfg) = &self.ingest { + last_poll = Some(Instant::now()); + if let Some(found) = fetch_pod_approvals(cfg) + .into_iter() + .find(|a| a.trace_id == open.trace_id) + { + self.record(&open.trace_id, &found.resolution); + return Ok(Parked::Resolved(found.resolution)); + } + } + std::thread::sleep(PARK_TICK); + } + } + + /// Tell the controller which gates this run is parked on. Best effort: a run with no + /// drop-box (a laptop) has nobody to tell. + fn announce(&self, open: &Gate, mode: ParkMode) { + let Some(cfg) = &self.ingest else { + return; + }; + let waits = ApprovalWaits { + v: ApprovalWaits::VERSION, + run_id: self.run_id.clone(), + control_port: None, + waits: vec![GateWait { + trace_id: open.trace_id.clone(), + handle: open.handle.clone(), + task: open.task.0.clone(), + source: open.source.clone(), + summary: open.summary.clone(), + mode, + requested_at: crate::suspend::now_secs(), + }], + }; + let Ok(json) = serde_json::to_vec(&waits) else { + return; + }; + let Ok(gz) = gzip(&json) else { + return; + }; + post_artifact(cfg, ArtifactKind::ApprovalWaits, &gz); + } + + /// Record a resolution that arrived from outside the bridge, under the gate's key. + fn record(&self, trace_id: &str, resolution: &GateResolution) { + let Some(ledger) = &self.ledger else { + return; + }; + let input = match resolution.decision { + GateDecision::Granted => AdmittedInput::Approve, + GateDecision::Denied => AdmittedInput::Deny { + reason: resolution.reason_text(), + }, + }; + let key = AdmissionKey::approve(trace_id); + if let Ok(crate::admission::Admitted::Fresh(key)) = ledger.admit(Some(key), input) { + let _ = ledger.settle( + &key, + AdmissionOutcome::Applied, + &format!( + "gate {} via {}", + resolution.decision.as_str(), + resolution.source.as_deref().unwrap_or("controller") + ), + ); + } + } +} + +/// Snapshot the run at `open` and deliver it, so a later `--resume` continues from here. +pub(crate) fn suspend(paths: &Paths, _plan: &ValidPlan, open: &Gate) -> Result<()> { + let record = crate::suspend::ResumeRecord { + v: crate::suspend::ResumeRecord::VERSION, + run_id: crate::plan::gate::run_id_from_env(), + gate: open.trace_id.clone(), + head: crate::suspend::head_of(&paths.workspace), + suspended_at: crate::suspend::now_secs(), + }; + let gz = crate::suspend::snapshot(&paths.state, &paths.workspace, &record)?; + let delivered = crate::suspend::deliver(&paths.session_log, &gz)?; + eprintln!( + "[crucible] suspended at gate {} ({}){}", + open.task, + open.trace_id, + if delivered { + ": snapshot delivered to the drop-box" + } else { + ": snapshot kept in the state dir" + } + ); + Ok(()) +} + +/// The results a dead process settled under `plan`: only when the log admitted this same plan +/// (version and task set), and only the passing pack tasks. +fn prior_from(state: &LoopState, plan: &ValidPlan) -> BTreeMap { + let Some(admitted) = &state.plan else { + return BTreeMap::new(); + }; + crate::loop_graph::PriorPlan { + iter: 0, + plan_version: admitted.plan_version, + declared: admitted.tasks.iter().map(|t| t.name.clone()).collect(), + results: state.plan_results.clone(), + } + .seed(plan) +} + +/// The `approval_wait` line a gate opens with. +pub(crate) fn wait_event(open: &Gate) -> SessionEvent { + SessionEvent::ApprovalWait { + handle: open.handle.clone(), + trace_id: open.trace_id.clone(), + mode: "block".to_string(), + task: Some(open.task.0.clone()), + source: serde_json::to_value(&open.source).ok(), + park: Some(ParkMode::Park.as_str().to_string()), + } +} + +/// The `approval_resolved` line that closes a gate. +pub(crate) fn resolved_event(open: &Gate, resolution: &GateResolution) -> SessionEvent { + SessionEvent::ApprovalResolved { + outcome: match resolution.source.as_deref() { + Some("timeout") => "timeout".to_string(), + _ => resolution.decision.as_str().to_string(), + }, + reason: resolution.reason_text(), + trace_id: open.trace_id.clone(), + by: resolution.by.clone(), + source: resolution.source.clone(), + } +} + +fn gzip(bytes: &[u8]) -> std::io::Result> { + use std::io::Write as _; + let mut enc = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + enc.write_all(bytes)?; + enc.finish() +} diff --git a/crucible/src/plan/harness.rs b/crucible/src/plan/harness.rs index a00c5129..90c7bb88 100644 --- a/crucible/src/plan/harness.rs +++ b/crucible/src/plan/harness.rs @@ -187,9 +187,15 @@ pub struct HarnessRunner { /// and materialized into that task's own root when it runs. A task with no entry is one the /// executor never staged, and its `inputs/` is left alone. pub staged: BTreeMap>, + /// The run's gates: which resolutions it holds, under which run id. + pub gate: crate::plan::gate::GateCtx, } impl TaskRunner for HarnessRunner { + fn resolve_gate(&mut self, trace_id: &str, resolution: crucible_contract::GateResolution) { + self.gate.resolve(trace_id, resolution); + } + fn run(&mut self, task: &Task, attempt: u32, inputs: &BTreeMap) -> Attempt { run_task( &Dispatch { @@ -197,6 +203,7 @@ impl TaskRunner for HarnessRunner { paths: &self.paths, captured_bytes: &self.captured_bytes, staged: self.staged.get(&task.name).map(Vec::as_slice), + gate: &self.gate, }, task, attempt, @@ -293,6 +300,7 @@ impl TaskRunner for HarnessRunner { paths: &self.paths, captured_bytes: &self.captured_bytes, staged: self.staged.get(&b.task.name).map(Vec::as_slice), + gate: &self.gate, }, b.task, b.attempt, @@ -317,6 +325,7 @@ impl TaskRunner for HarnessRunner { .map(|b| { let args = self.args.clone(); let paths = self.paths.clone(); + let gate = self.gate.clone(); let pending = pending.as_str(); scope.spawn(move || { run_task( @@ -325,6 +334,7 @@ impl TaskRunner for HarnessRunner { paths: &paths, captured_bytes: captured, staged: staged.get(&b.task.name).map(Vec::as_slice), + gate: &gate, }, b.task, b.attempt, @@ -352,6 +362,7 @@ struct Dispatch<'a> { paths: &'a Paths, captured_bytes: &'a AtomicU64, staged: Option<&'a [StagedInput]>, + gate: &'a crate::plan::gate::GateCtx, } /// Dispatch one task, in the shared workspace or in a private worktree. `pending` is the @@ -369,13 +380,14 @@ fn run_task( paths, captured_bytes, staged, + gate, } = *cx; let Some(Isolation::Worktree) = task.isolation else { if let Err(e) = materialize_inputs(&paths.state, &paths.workspace, staged) { return transport(e); } let before = PriorContents::of(&paths.workspace, &task.emits_files); - let attempt_out = prepare_and_run(args, paths, task, attempt, inputs); + let attempt_out = prepare_and_run(args, paths, gate, task, attempt, inputs); return capture_declared( paths, &paths.workspace, @@ -421,7 +433,7 @@ fn run_task( let iso = Paths::for_worktree(worktree.clone(), paths.skills.clone()); let _ = std::fs::create_dir_all(&iso.state); let before = PriorContents::of(&iso.workspace, &task.emits_files); - let attempt_out = prepare_and_run(args, &iso, task, attempt, inputs); + let attempt_out = prepare_and_run(args, &iso, gate, task, attempt, inputs); // Before the worktree goes: a declared file is part of the task's output, not part of the // workspace state isolation discards, so it has to be taken while the tree is still there. let attempt_out = capture_declared( @@ -461,7 +473,9 @@ fn capture_declared( let failing = match &attempt.outcome { AttemptOutcome::Pass(_) => false, AttemptOutcome::Fail { .. } => true, - AttemptOutcome::Skipped(..) | AttemptOutcome::Transport(_) => return attempt, + AttemptOutcome::Skipped(..) | AttemptOutcome::Transport(_) | AttemptOutcome::Await(_) => { + return attempt; + } }; if task.emits_files.is_empty() { return attempt; @@ -589,6 +603,7 @@ fn materialize_inputs( fn prepare_and_run( args: &Args, paths: &Paths, + gate: &crate::plan::gate::GateCtx, task: &Task, attempt: u32, inputs: &BTreeMap, @@ -602,7 +617,7 @@ fn prepare_and_run( )); } } - let out = run_in(args, paths, task, attempt, inputs); + let out = run_in(args, paths, gate, task, attempt, inputs); Attempt { outcome: out.outcome.settle_declared(), cost_usd: out.cost_usd, @@ -614,6 +629,7 @@ fn prepare_and_run( fn run_in( args: &Args, paths: &Paths, + gate: &crate::plan::gate::GateCtx, task: &Task, attempt: u32, inputs: &BTreeMap, @@ -629,6 +645,7 @@ fn run_in( let mut shell = ShellRunner { workdir: paths.workspace.clone(), agent_cmd: None, + gate: crate::plan::gate::GateCtx::default(), }; return if task.isolation == Some(Isolation::Worktree) { shell.run_in_prepared_worktree(task, inputs) @@ -642,6 +659,9 @@ fn run_in( TaskKind::Engine { .. } => { return fail(0.0, "engine task reached a non-loop runner".to_string()); } + TaskKind::Approve { .. } => { + return crate::plan::gate::attempt(gate, task, inputs); + } }; // Per-task knob overrides on a cloned Args: the heterogeneity axis. Unknown values @@ -2492,6 +2512,7 @@ workflow(type = "playbook", tasks = [analyze, implement, report]) max_fanout: None, }; let mut runner = HarnessRunner { + gate: crate::plan::gate::GateCtx::default(), args: ::try_parse_from(["crucible"]) .unwrap() .run, diff --git a/crucible/src/plan/ir.rs b/crucible/src/plan/ir.rs index cbf83879..6b0147bc 100644 --- a/crucible/src/plan/ir.rs +++ b/crucible/src/plan/ir.rs @@ -193,6 +193,56 @@ pub enum TaskKind { #[serde(default, skip_serializing_if = "Option::is_none")] tiebreak: Option, }, + /// A human gate. The graph waits here until the named source resolves it; a grant passes + /// the task with the resolution as its output, a denial or a timeout fails it. + Approve { + #[serde(default, skip_serializing_if = "Option::is_none")] + summary: Option, + #[serde(default)] + source: ApprovalSourceSpec, + /// A per-gate park ceiling, clamping the run's `--max-park`. + #[serde(default, skip_serializing_if = "Option::is_none")] + timeout_secs: Option, + }, +} + +/// Where an `approve` task's resolution comes from, as authored. A GitHub PR url or a Jira key +/// may be a literal or an upstream task's emitted field, resolved when the gate is dispatched. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum ApprovalSourceSpec { + #[default] + Native, + GithubPr { + url: RefOrLiteral, + until: crucible_contract::PrUntil, + }, + Jira { + key: RefOrLiteral, + until: crucible_contract::JiraUntil, + }, +} + +impl ApprovalSourceSpec { + /// The upstream field the source reads, when it reads one. + pub fn reference(&self) -> Option<&OutputRef> { + match self { + ApprovalSourceSpec::Native => None, + ApprovalSourceSpec::GithubPr { url: r, .. } + | ApprovalSourceSpec::Jira { key: r, .. } => match r { + RefOrLiteral::Output(reference) => Some(reference), + RefOrLiteral::Literal(_) => None, + }, + } + } +} + +/// A value an author wrote out, or one an upstream task emits. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum RefOrLiteral { + Output(OutputRef), + Literal(String), } impl TaskKind { @@ -204,6 +254,7 @@ impl TaskKind { TaskKind::Evaluate { .. } => "evaluate", TaskKind::Report { .. } => "report", TaskKind::TopK { .. } => "top_k", + TaskKind::Approve { .. } => "approve", TaskKind::Engine { op, .. } => match op { EngineOp::Propose => "engine_propose", EngineOp::Apply => "engine_apply", @@ -377,6 +428,12 @@ pub enum PlanError { "task {task:?}: emits is not accepted on {kind} tasks; their outputs are engine-defined" )] EmitsOnEngineTask { task: String, kind: &'static str }, + #[error("approve task {task:?} cannot take {what}: a gate waits, it does not run work")] + ApproveMisuse { task: String, what: &'static str }, + #[error( + "approve task {task:?} reads its source from {from:?}, which is not among its dependencies" + )] + ApproveSourceNotDependency { task: String, from: String }, #[error( "task {task:?} declares invalid output field {field:?}; use 1-64 ASCII letters, digits, or `_`" )] @@ -672,6 +729,34 @@ impl Plan { { return Err(PlanError::ThresholdedEvaluateOmitsScore { task: task() }); } + if let TaskKind::Approve { source, .. } = &t.task { + if !t.emits.is_empty() || !t.emits_files.is_empty() { + return Err(PlanError::EmitsOnEngineTask { + task: task(), + kind: t.task.label(), + }); + } + if t.isolation.is_some() { + return Err(PlanError::ApproveMisuse { + task: task(), + what: "isolation", + }); + } + if t.over.is_some() { + return Err(PlanError::ApproveMisuse { + task: task(), + what: "over", + }); + } + if let Some(reference) = source.reference() + && !t.depends_on.contains(&reference.task) + { + return Err(PlanError::ApproveSourceNotDependency { + task: task(), + from: reference.task.0.clone(), + }); + } + } if let Some(session) = &t.session { if session.is_empty() || session.len() > 64 diff --git a/crucible/src/plan/mod.rs b/crucible/src/plan/mod.rs index fcc5e1db..1e9fc9d1 100644 --- a/crucible/src/plan/mod.rs +++ b/crucible/src/plan/mod.rs @@ -4,12 +4,24 @@ //! and JSON. `validate` checks the supported version and graph structure before execution. pub(crate) mod diag; pub mod exec; +pub mod gate; pub mod ir; pub mod runner; pub mod starlark; pub mod term_img; pub mod worktree; +/// What happens when a gate has parked as long as it may. Lives here rather than beside the CLI +/// because the pod renderer emits it as `--park-policy` and the engine parses it back. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, clap::ValueEnum)] +pub enum ParkPolicy { + /// Idle in place until `--max-park`, then fail the gate as timed out. + #[default] + ParkThenDeny, + /// Idle in place until `--max-park`, then snapshot and exit so a later run can resume. + ParkThenSuspend, +} + /// The environment variable naming the task a turn runs, set by both the command runner and the /// agent harness. Engine-provisioned, so [`crate::exposure`] carries it as standing disclosed /// reach. diff --git a/crucible/src/plan/runner.rs b/crucible/src/plan/runner.rs index a5324533..aeb25d54 100644 --- a/crucible/src/plan/runner.rs +++ b/crucible/src/plan/runner.rs @@ -20,9 +20,15 @@ pub struct ShellRunner { /// Stand-in command for `Agent` tasks (receives the prompt and knobs via env). `None` /// means agent tasks are refused: `plan run` without `--agent-cmd` is command-only. pub agent_cmd: Option, + /// The run's gates: which resolutions it holds, under which run id. + pub gate: crate::plan::gate::GateCtx, } impl TaskRunner for ShellRunner { + fn resolve_gate(&mut self, trace_id: &str, resolution: crucible_contract::GateResolution) { + self.gate.resolve(trace_id, resolution); + } + fn run(&mut self, task: &Task, _attempt: u32, inputs: &BTreeMap) -> Attempt { if task.isolation.is_some() { // Fail loud: a runner that quietly ran an isolation-marked task in the shared @@ -54,6 +60,9 @@ impl ShellRunner { } fn run_in_workdir(&mut self, task: &Task, inputs: &BTreeMap) -> Attempt { + if matches!(task.task, TaskKind::Approve { .. }) { + return crate::plan::gate::attempt(&self.gate, task, inputs); + } let mut cmd = Command::new("sh"); cmd.arg("-c").current_dir(&self.workdir); cmd.env(crate::plan::TASK_NAME_ENV, &task.name.0); @@ -122,6 +131,9 @@ impl ShellRunner { // handles them, this one can't. return fail("engine task reached a non-loop runner".to_string()); } + TaskKind::Approve { .. } => { + return crate::plan::gate::attempt(&self.gate, task, inputs); + } } let out = match cmd.output() { Ok(out) => out, @@ -312,6 +324,7 @@ mod tests { ShellRunner { workdir: std::env::temp_dir(), agent_cmd: None, + gate: crate::plan::gate::GateCtx::default(), } } diff --git a/crucible/src/plan/starlark.rs b/crucible/src/plan/starlark.rs index 5d92d10c..9f531126 100644 --- a/crucible/src/plan/starlark.rs +++ b/crucible/src/plan/starlark.rs @@ -24,8 +24,8 @@ use crate::errors::FileError; use crate::manifest::{WorkflowCfg, WorkflowError, WorkflowType}; use crate::plan::diag; use crate::plan::ir::{ - Direction, EngineOp, Isolation, Join, MAX_FANOUT_CEILING, OutputField, OutputRef, - ReportDestination, SlackDestination, Stage, Task, TaskKind, TaskName, + ApprovalSourceSpec, Direction, EngineOp, Isolation, Join, MAX_FANOUT_CEILING, OutputField, + OutputRef, RefOrLiteral, ReportDestination, SlackDestination, Stage, Task, TaskKind, TaskName, }; use crate::plan::starlark::values::WorkflowValue; @@ -390,6 +390,10 @@ enum Value { Output(OutputRef), Session(SessionDecl), Workflow(WorkflowCfg), + /// `github_pr(...)` / `jira(...)`: where an approval gate's resolution comes from. + GateSource(ApprovalSourceSpec), + /// `status("...")` / `label("...")`: what a Jira issue must reach. + Until(crucible_contract::JiraUntil), /// A starlark value outside the DSL's own space: a dict, a function, a struct. The `take_*` /// helpers report it with the same wrong-type sentence a wrong scalar gets. Opaque, @@ -685,6 +689,10 @@ pub enum CompileError { TopKWithoutDependencies, #[error("stage must be `iteration` or `epilogue`, got {got:?}")] UnknownStage { got: String }, + #[error("github_pr until must be \"approved\" or \"merged\", got {got:?}")] + UnknownPrUntil { got: String }, + #[error("{argument} {got:?} is not a duration (try `90s`, `30m`, `2h`)")] + BadDuration { argument: String, got: String }, #[error("join must be `all`, `passed`, or `settled`, got {got:?}")] UnknownJoin { got: String }, #[error( @@ -875,6 +883,9 @@ const COMMON_FUNCTIONS: &[&str] = &[ "workflow", ]; +/// The playbook lane's own constructors: the approval gate and its sources. +const PLAYBOOK_FUNCTIONS: &[&str] = &["approve", "github_pr", "jira", "status", "label"]; + /// The scored loop's own constructors, absent from a playbook. const SCORED_FUNCTIONS: &[&str] = &[ "apply", @@ -891,7 +902,9 @@ const SCORED_FUNCTIONS: &[&str] = &[ /// constructor cannot be added to one and forgotten in the other. fn dsl_functions(lane: WorkflowType) -> Vec<&'static str> { let mut names = COMMON_FUNCTIONS.to_vec(); - if lane != WorkflowType::Playbook { + if lane == WorkflowType::Playbook { + names.extend_from_slice(PLAYBOOK_FUNCTIONS); + } else { names.extend_from_slice(SCORED_FUNCTIONS); } names.sort_unstable(); @@ -977,6 +990,19 @@ fn known_kwargs(function: &str) -> &'static [&'static str] { "decide" => &["name", "measurement", "depends_on"], "session" => &["name", "harness", "model", "effort"], "workflow" => &["type", "tasks", "result"], + "approve" => &[ + "name", + "summary", + "source", + "timeout", + "depends_on", + "needs", + "required", + "join", + "stage", + ], + "github_pr" => &["url", "until"], + "jira" => &["key", "until"], _ => &[], } } @@ -1098,6 +1124,46 @@ fn constructor( }; dsl_task(&mut named, name, kind, None)? } + "approve" => { + let name = take_declared_name(&mut named, "name")?; + let kind = TaskKind::Approve { + summary: take_optional_string(&mut named, "summary")?, + source: take_gate_source(&mut named)?, + timeout_secs: take_optional_duration_secs(&mut named, "timeout")?, + }; + dsl_task(&mut named, name, kind, None)? + } + "github_pr" => { + let url = take_ref_or_literal(&mut named, "url")?; + let until = match take_string_default(&mut named, "until", "approved")?.as_str() { + "approved" => crucible_contract::PrUntil::Approved, + "merged" => crucible_contract::PrUntil::Merged, + other => { + return Err(CompileError::UnknownPrUntil { + got: other.to_owned(), + }); + } + }; + no_unknown_kwargs(function, &named)?; + return Ok(Value::GateSource(ApprovalSourceSpec::GithubPr { + url, + until, + })); + } + "jira" => { + let key = take_ref_or_literal(&mut named, "key")?; + let until = match named.remove("until") { + Some(Value::Until(until)) => until, + Some(_) => return Err(wrong_type("until", "status(...) or label(...)")), + None => { + return Err(CompileError::MissingArgument { + argument: "until".to_owned(), + }); + } + }; + no_unknown_kwargs(function, &named)?; + return Ok(Value::GateSource(ApprovalSourceSpec::Jira { key, until })); + } "evaluate" => { let name = take_declared_name(&mut named, "name")?; let kind = TaskKind::Evaluate { @@ -1479,6 +1545,49 @@ fn take_emitted_files(named: &mut BTreeMap) -> Result Ok(paths) } +/// `source =` on an `approve` task: `"native"` (the default), or a `github_pr(...)` / +/// `jira(...)` value. +fn take_gate_source(named: &mut BTreeMap) -> Result { + match named.remove("source") { + None | Some(Value::None) => Ok(ApprovalSourceSpec::Native), + Some(Value::String(s)) if s == "native" => Ok(ApprovalSourceSpec::Native), + Some(Value::GateSource(source)) => Ok(source), + Some(_) => Err(wrong_type( + "source", + "\"native\", github_pr(...), or jira(...)", + )), + } +} + +/// A gate source's target: a string, or an upstream task's emitted field. +fn take_ref_or_literal(named: &mut BTreeMap, name: &str) -> Result { + match take_value(named, name)? { + Value::String(s) => Ok(RefOrLiteral::Literal(s)), + Value::Output(reference) => Ok(RefOrLiteral::Output(reference)), + Value::External(_) => Err(CompileError::ExternalOutsidePrompt { + argument: name.to_owned(), + }), + _ => Err(wrong_type(name, "a string or a task's emitted field")), + } +} + +/// `timeout = "30m"`: a duration, in seconds. +fn take_optional_duration_secs( + named: &mut BTreeMap, + name: &str, +) -> Result> { + match named.remove(name) { + None | Some(Value::None) => Ok(None), + Some(Value::String(raw)) => crate::duration::parse_duration(&raw) + .map(|d| Some(d.as_secs())) + .ok_or_else(|| CompileError::BadDuration { + argument: name.to_owned(), + got: raw, + }), + Some(_) => Err(wrong_type(name, "a duration string such as \"30m\"")), + } +} + fn take_over(named: &mut BTreeMap) -> Result> { match named.remove("over") { None | Some(Value::None) => Ok(None), @@ -2400,7 +2509,7 @@ fn catching_panics(evaluate: impl FnOnce() -> T) -> Result { fn lane_globals(lane: WorkflowType) -> Globals { let builder = GlobalsBuilder::standard().with(globals::common); match lane { - WorkflowType::Playbook => builder.build(), + WorkflowType::Playbook => builder.with(globals::playbook).build(), _ => builder.with(globals::scored).build(), } } diff --git a/crucible/src/plan/starlark/globals.rs b/crucible/src/plan/starlark/globals.rs index ef44f9d8..455cb8e2 100644 --- a/crucible/src/plan/starlark/globals.rs +++ b/crucible/src/plan/starlark/globals.rs @@ -19,11 +19,19 @@ use starlark_syntax::codemap::FileSpan; use crate::manifest::{WorkflowCfg, WorkflowType}; use crate::plan::starlark as dsl; use crate::plan::starlark::values::{ - ExternalText, OutputRefValue, SessionValue, TaskValue, WorkflowValue, + ExternalText, GateSourceValue, OutputRefValue, SessionValue, TaskValue, UntilValue, + WorkflowValue, }; /// Constructors that historically took one positional argument. Everything else is named-only. -const POSITIONAL: &[&str] = &["prompt_file", "param", "workflow", "default_autoresearch"]; +const POSITIONAL: &[&str] = &[ + "prompt_file", + "param", + "workflow", + "default_autoresearch", + "status", + "label", +]; /// The constructors every lane has. #[starlark_module] @@ -101,6 +109,50 @@ pub(crate) fn common(builder: &mut GlobalsBuilder) { } } +/// The playbook lane's own constructors: the approval gate and the sources that resolve it. +#[starlark_module] +pub(crate) fn playbook(builder: &mut GlobalsBuilder) { + fn approve<'v>( + #[starlark(args)] args: UnpackTuple>, + #[starlark(kwargs)] kwargs: SmallMap>, + eval: &mut Evaluator<'v, '_, '_>, + ) -> starlark::Result> { + dispatch("approve", args, kwargs, eval) + } + + fn github_pr<'v>( + #[starlark(args)] args: UnpackTuple>, + #[starlark(kwargs)] kwargs: SmallMap>, + eval: &mut Evaluator<'v, '_, '_>, + ) -> starlark::Result> { + dispatch("github_pr", args, kwargs, eval) + } + + fn jira<'v>( + #[starlark(args)] args: UnpackTuple>, + #[starlark(kwargs)] kwargs: SmallMap>, + eval: &mut Evaluator<'v, '_, '_>, + ) -> starlark::Result> { + dispatch("jira", args, kwargs, eval) + } + + fn status<'v>( + #[starlark(args)] args: UnpackTuple>, + #[starlark(kwargs)] kwargs: SmallMap>, + eval: &mut Evaluator<'v, '_, '_>, + ) -> starlark::Result> { + dispatch("status", args, kwargs, eval) + } + + fn label<'v>( + #[starlark(args)] args: UnpackTuple>, + #[starlark(kwargs)] kwargs: SmallMap>, + eval: &mut Evaluator<'v, '_, '_>, + ) -> starlark::Result> { + dispatch("label", args, kwargs, eval) + } +} + /// The scored loop's own constructors. A playbook never sees these, so a playbook author /// cannot name one and cannot be offered one by a did-you-mean. #[starlark_module] @@ -211,6 +263,12 @@ fn call<'v>( .prompt_file(&path) .map(dsl::Value::String), ("param", dsl::Value::String(name)) => state.context_mut().param(&name), + ("status", dsl::Value::String(name)) => Ok(dsl::Value::Until( + crucible_contract::JiraUntil::Status(name), + )), + ("label", dsl::Value::String(name)) => { + Ok(dsl::Value::Until(crucible_contract::JiraUntil::Label(name))) + } ("workflow", dsl::Value::List(tasks)) => { let tasks = dsl::task_list("workflow", tasks)?; let workflow = WorkflowCfg { @@ -338,6 +396,12 @@ fn convert_at(value: Value<'_>, depth: usize) -> dsl::Result { if let Some(workflow) = WorkflowValue::from_value(value) { return Ok(dsl::Value::Workflow(workflow.0.clone())); } + if let Some(source) = GateSourceValue::from_value(value) { + return Ok(dsl::Value::GateSource(source.0.clone())); + } + if let Some(until) = UntilValue::from_value(value) { + return Ok(dsl::Value::Until(until.0.clone())); + } Ok(dsl::Value::Opaque) } @@ -375,5 +439,7 @@ fn alloc_at<'v>(heap: Heap<'v>, value: dsl::Value, depth: usize) -> Value<'v> { }), dsl::Value::Session(session) => heap.alloc(SessionValue(session)), dsl::Value::Workflow(workflow) => heap.alloc(WorkflowValue(workflow)), + dsl::Value::GateSource(source) => heap.alloc(GateSourceValue(source)), + dsl::Value::Until(until) => heap.alloc(UntilValue(until)), } } diff --git a/crucible/src/plan/starlark/reference.rs b/crucible/src/plan/starlark/reference.rs index eeab5ac5..d2b248c2 100644 --- a/crucible/src/plan/starlark/reference.rs +++ b/crucible/src/plan/starlark/reference.rs @@ -18,12 +18,18 @@ pub enum Lane { Common, /// The scored types (`autoresearch` and `custom`) only. Scored, + /// Playbooks only. + Playbook, } impl Lane { #[cfg(test)] fn includes(self, workflow: WorkflowType) -> bool { - self == Lane::Common || workflow != WorkflowType::Playbook + match self { + Lane::Common => true, + Lane::Scored => workflow != WorkflowType::Playbook, + Lane::Playbook => workflow == WorkflowType::Playbook, + } } } @@ -234,6 +240,109 @@ pub fn functions() -> Vec { Kwarg::new("required", "bool", "False makes the report advisory."), ], }, + Function { + name: "approve", + lane: Lane::Playbook, + purpose: "A human gate. The graph waits here until the source resolves it: a grant \ + passes the task with the resolution as its output, a denial or a timeout \ + fails it.", + positional: None, + kwargs: vec![ + name_kwarg(), + Kwarg::new( + "summary", + "str", + "What the approver is deciding, shown with the gate.", + ), + Kwarg::new( + "source", + "\"native\" | github_pr(...) | jira(...)", + "Where the resolution comes from. `native` (the default) is a person acting \ + through the controller or the control bridge.", + ), + Kwarg::new( + "timeout", + "str", + "How long this gate may park before the run's park policy applies, e.g. `30m`.", + ), + Kwarg::new( + "depends_on", + "list[task]", + "Dependencies. A source read from a task's field must name that task here.", + ), + Kwarg::new( + "needs", + "\"any\" | \"all\"", + "How many dependencies must be admitted before the gate is reached.", + ), + Kwarg::new( + "required", + "bool", + "False makes the gate advisory: a denial fails it without invalidating the run.", + ), + Kwarg::new( + "join", + "\"all\" | \"passed\" | \"settled\"", + "Which dependencies must have passed before the gate is reached.", + ), + Kwarg::new( + "stage", + "\"iteration\" | \"epilogue\"", + "`epilogue` gates the run's conclusion instead of its graph.", + ), + ], + }, + Function { + name: "github_pr", + lane: Lane::Playbook, + purpose: "An approval source: a GitHub pull request reaching a state.", + positional: None, + kwargs: vec![ + Kwarg::new( + "url", + "str | task.field", + "The pull request, written out or read from an upstream task's emitted field.", + ), + Kwarg::new( + "until", + "\"approved\" | \"merged\"", + "What resolves the gate: an authorized review approval or `/approve` comment \ + (the default), or the merge.", + ), + ], + }, + Function { + name: "jira", + lane: Lane::Playbook, + purpose: "An approval source: a Jira issue reaching a status or carrying a label.", + positional: None, + kwargs: vec![ + Kwarg::new( + "key", + "str | task.field", + "The issue key, written out or read from an upstream task's emitted field.", + ), + Kwarg::new( + "until", + "status(...) | label(...)", + "What resolves the gate.", + ), + ], + }, + Function { + name: "status", + lane: Lane::Playbook, + purpose: "A Jira predicate: the issue's status name equals the argument.", + positional: Some("name"), + kwargs: vec![], + }, + Function { + name: "label", + lane: Lane::Playbook, + purpose: "A Jira predicate: the issue carries the label.", + positional: Some("name"), + kwargs: vec![], + }, Function { name: "session", lane: Lane::Common, @@ -513,6 +622,12 @@ pub fn markdown() -> String { have these in scope at all, so naming one is an unknown-name error and a \ did-you-mean never offers one.", ), + ( + Lane::Playbook, + "Playbooks only", + "Available to `type = \"playbook\"`. An approval gate parks the run until a person or \ + an external item resolves it; the scored lanes have no gate in scope.", + ), ] { out.push_str(&format!("## {heading}\n\n{blurb}\n\n")); for function in functions().iter().filter(|f| f.lane == lane) { @@ -583,6 +698,7 @@ pub fn json() -> serde_json::Value { "lane": match function.lane { Lane::Common => "common", Lane::Scored => "scored", + Lane::Playbook => "playbook", }, "purpose": function.purpose, "positional": function.positional, diff --git a/crucible/src/plan/starlark/values.rs b/crucible/src/plan/starlark/values.rs index cb9171d9..b8eea7c3 100644 --- a/crucible/src/plan/starlark/values.rs +++ b/crucible/src/plan/starlark/values.rs @@ -17,8 +17,9 @@ use starlark::values::{ use crate::manifest::WorkflowCfg; use crate::plan::diag; -use crate::plan::ir::{OutputField, OutputRef, Task}; +use crate::plan::ir::{ApprovalSourceSpec, OutputField, OutputRef, Task}; use crate::plan::starlark::{CompileError, SessionDecl}; +use crucible_contract::JiraUntil; #[derive(Debug, ProvidesStaticType, NoSerialize, Allocative)] pub(crate) struct TaskValue(#[allocative(skip)] pub(crate) Task); @@ -192,3 +193,40 @@ impl Display for WorkflowValue { #[starlark_value(type = "workflow")] impl<'v> StarlarkValue<'v> for WorkflowValue {} + +/// Where an `approve` task's resolution comes from: `github_pr(...)` or `jira(...)`. +#[derive(Debug, ProvidesStaticType, NoSerialize, Allocative)] +pub(crate) struct GateSourceValue(#[allocative(skip)] pub(crate) ApprovalSourceSpec); + +starlark_simple_value!(GateSourceValue); + +impl Display for GateSourceValue { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.0 { + ApprovalSourceSpec::Native => write!(f, "native"), + ApprovalSourceSpec::GithubPr { .. } => write!(f, "github_pr(...)"), + ApprovalSourceSpec::Jira { .. } => write!(f, "jira(...)"), + } + } +} + +#[starlark_value(type = "approval_source")] +impl<'v> StarlarkValue<'v> for GateSourceValue {} + +/// What a Jira issue must reach: `status("...")` or `label("...")`. +#[derive(Debug, ProvidesStaticType, NoSerialize, Allocative)] +pub(crate) struct UntilValue(#[allocative(skip)] pub(crate) JiraUntil); + +starlark_simple_value!(UntilValue); + +impl Display for UntilValue { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.0 { + JiraUntil::Status(s) => write!(f, "status({s})"), + JiraUntil::Label(l) => write!(f, "label({l})"), + } + } +} + +#[starlark_value(type = "until")] +impl<'v> StarlarkValue<'v> for UntilValue {} diff --git a/crucible/src/run.rs b/crucible/src/run.rs index 839c312f..3c56bea1 100644 --- a/crucible/src/run.rs +++ b/crucible/src/run.rs @@ -70,6 +70,8 @@ pub(crate) enum RunError { #[source] source: fs_extra::error::Error, }, + #[error("--max-park `{raw}` is not a duration (e.g. 30m, 4h, 7d)")] + BadMaxPark { raw: String }, #[error(transparent)] File(#[from] FileError), } @@ -139,12 +141,6 @@ pub(crate) fn dispatch(cli: Cli) -> Result<()> { once, }) = cli.command { - let sink = match (control_addr, reseed) { - (Some(addr), None) => crate::pr_watch::Sink::Steer(addr), - (None, Some(path)) => crate::pr_watch::Sink::Reseed(path), - (None, None) => return Err(RunError::WatchPrNoSink.into()), - (Some(_), Some(_)) => return Err(RunError::WatchPrTwoSinks.into()), - }; let opts = crate::pr_watch::WatchOpts { poll: std::time::Duration::from_secs(poll_secs), bot_user, @@ -154,9 +150,55 @@ pub(crate) fn dispatch(cli: Cli) -> Result<()> { }, once, }; + let sink = match (control_addr, reseed) { + (Some(addr), None) => crate::pr_watch::Sink::Steer(addr), + (None, Some(path)) => crate::pr_watch::Sink::Reseed(path), + (None, None) => return Err(RunError::WatchPrNoSink.into()), + (Some(_), Some(_)) => return Err(RunError::WatchPrTwoSinks.into()), + }; return crate::pr_watch::watch_and_steer(&pr, &sink, &opts); } + if let Some(Cmd::Approve { control_addr, by }) = &cli.command { + let reply = control::send_command( + control_addr, + &serde_json::json!({"cmd": "approve", "by": by}), + )?; + println!("{reply}"); + return Ok(()); + } + + if let Some(Cmd::Deny { + control_addr, + reason, + by, + }) = &cli.command + { + let reply = control::send_command( + control_addr, + &serde_json::json!({"cmd": "deny", "reason": reason, "by": by}), + )?; + println!("{reply}"); + return Ok(()); + } + + if let Some(Cmd::FetchResume { into, workspace }) = &cli.command { + let record = crate::suspend::fetch_resume(into, workspace)?; + match record { + Some(r) => println!( + "[crucible fetch-resume] restored run {} at gate {} into {}", + r.run_id, + r.gate, + into.display() + ), + None => println!( + "[crucible fetch-resume] restored a snapshot with no resume record into {}", + into.display() + ), + } + return Ok(()); + } + if let Some(Cmd::Ps { namespace, json }) = cli.command { return crate::ps::run(namespace.as_deref(), json); } @@ -238,8 +280,25 @@ pub(crate) fn dispatch(cli: Cli) -> Result<()> { compute_driver, harness, model, + resume, + approvals, + park_policy, + max_park, + control_port, } => { let _engine = crate::engine::EngineCtx::new()?; + install_ctrlc()?; + let approvals = approvals + .iter() + .map(|raw| crucible_contract::parse_approval_arg(raw)) + .collect::, _>>()?; + let max_park = match max_park { + Some(raw) => Some( + crate::duration::parse_duration(raw) + .ok_or_else(|| RunError::BadMaxPark { raw: raw.clone() })?, + ), + None => None, + }; crate::plan::cli::run( file.as_deref(), &crate::plan::cli::parse_params(params)?, @@ -259,6 +318,13 @@ pub(crate) fn dispatch(cli: Cli) -> Result<()> { harness: *harness, model: model.clone(), }, + gates: crate::plan::cli::GateOpts { + resume: *resume, + approvals, + policy: *park_policy, + max_park, + control_port: *control_port, + }, }, ) } @@ -337,6 +403,9 @@ pub(crate) fn dispatch(cli: Cli) -> Result<()> { harness: args.harness, model: args.model.clone(), playbook, + // The controller sets these when it dispatches; a hand render has no gate to resume. + park_policy: crate::plan::ParkPolicy::default(), + resume: None, }; if args.controller { // Deprecated in favor of the `crucible-controller` Helm chart (the one packaging path). @@ -565,6 +634,7 @@ pub(crate) fn prep_plan_runner_with_params( commit_per_task, captured_bytes: std::sync::atomic::AtomicU64::new(0), staged: Default::default(), + gate: crate::plan::gate::GateCtx::new(crate::plan::gate::run_id_from_env()), }, m, )) diff --git a/crucible/src/suspend.rs b/crucible/src/suspend.rs new file mode 100644 index 00000000..0aa2de58 --- /dev/null +++ b/crucible/src/suspend.rs @@ -0,0 +1,641 @@ +//! Leaving and re-entering a run at an approval gate. +//! +//! A suspended run writes everything a later process needs beside the session log: the +//! `run-workspace` artifact (the state dir minus the session log, a bundle of the workspace +//! repo, and `resume.json`), and posts it with the session log to the controller's drop-box +//! when the pod carries the ingest env. A resumed pod restores the two artifacts into place +//! before the engine starts (`crucible fetch-resume`), so `plan run --resume` only ever sees +//! files. + +use std::io::Write as _; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use anyhow::{Context, Result}; +use crucible_contract::ArtifactKind; +use serde::{Deserialize, Serialize}; + +use crate::ingest_client::{IngestConfig, fetch_artifact, post_artifact, resume_of_from_env}; + +/// The bundle of the workspace repository inside the `run-workspace` tar. +pub const WORKSPACE_BUNDLE: &str = "workspace.bundle"; +/// The resume record inside `state/` and the `run-workspace` tar. +pub const RESUME_FILE: &str = "resume.json"; + +/// What a suspended run leaves for its successor. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ResumeRecord { + pub v: u8, + pub run_id: String, + /// The gate the run suspended on. + pub gate: String, + /// `HEAD` of the workspace at suspend time; the bundle carries it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub head: Option, + /// Unix seconds. + pub suspended_at: f64, +} + +impl ResumeRecord { + pub const VERSION: u8 = 1; +} + +#[derive(Debug, thiserror::Error)] +pub enum SuspendError { + #[error("the run-workspace snapshot is {bytes} bytes, over the {cap} byte cap")] + Oversize { bytes: u64, cap: u64 }, + #[error("posting {kind} to the drop-box failed")] + Undelivered { kind: ArtifactKind }, + #[error( + "fetch-resume needs the pod ingest env ({})", + crucible_contract::ENV_INGEST_URL + )] + NoIngestEnv, + #[error( + "fetch-resume needs {} to name the pod to resume", + crucible_contract::ENV_RESUME_OF + )] + NoResumeOf, + #[error("fetching {kind} of pod {of}: {message}")] + Fetch { + kind: ArtifactKind, + of: String, + message: String, + }, + #[error("pod {of} left no {kind} artifact")] + MissingArtifact { kind: ArtifactKind, of: String }, +} + +/// Attempts a drop-box fetch gets before the resume gives up; the controller may still be +/// storing the artifact when a fast scheduler starts the successor pod. +const FETCH_ATTEMPTS: u32 = 3; +const FETCH_BACKOFF: std::time::Duration = std::time::Duration::from_secs(2); + +/// Build the `run-workspace` artifact (gzipped tar) for `state` and `workspace`, and write the +/// resume record into `state/` so a local resume finds it too. +pub fn snapshot(state: &Path, workspace: &Path, record: &ResumeRecord) -> Result> { + let record_json = serde_json::to_vec_pretty(record).context("encoding resume.json")?; + std::fs::write(state.join(RESUME_FILE), &record_json) + .with_context(|| format!("writing {}", state.join(RESUME_FILE).display()))?; + let bundle = bundle_workspace(workspace)?; + + let mut builder = tar::Builder::new(Vec::new()); + append_dir(&mut builder, state, Path::new("state"), &|name| { + name != "session.jsonl" && name != "files.tmp" + })?; + if let Some(bundle) = bundle { + let mut header = tar::Header::new_gnu(); + header.set_size(bundle.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder + .append_data(&mut header, WORKSPACE_BUNDLE, bundle.as_slice()) + .context("adding the workspace bundle")?; + } + let tar = builder.into_inner().context("finishing the snapshot tar")?; + let mut enc = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + enc.write_all(&tar).context("gzipping the snapshot")?; + let gz = enc.finish().context("gzipping the snapshot")?; + let cap = ArtifactKind::RunWorkspace.max_bytes(); + if gz.len() as u64 > cap { + return Err(SuspendError::Oversize { + bytes: gz.len() as u64, + cap, + } + .into()); + } + Ok(gz) +} + +/// Post the session log and the snapshot to the drop-box. `Ok(false)` when the pod carries no +/// ingest env (a local run keeps its state dir), `Err` when a post did not land. +pub fn deliver(session_log: &Path, snapshot: &[u8]) -> Result { + let Some(cfg) = IngestConfig::from_env() else { + return Ok(false); + }; + let session = + std::fs::read(session_log).with_context(|| format!("reading {}", session_log.display()))?; + let session_gz = gzip(&session)?; + for (kind, bytes) in [ + (ArtifactKind::RunSession, session_gz.as_slice()), + (ArtifactKind::RunWorkspace, snapshot), + ] { + if !post_artifact(&cfg, kind, bytes).delivered { + return Err(SuspendError::Undelivered { kind }.into()); + } + } + Ok(true) +} + +/// Restore a `run-workspace` artifact into `state` and `workspace`: the state files go back +/// where they were, the bundle is fetched into the workspace repo, and its `HEAD` checked out. +pub fn restore(snapshot_gz: &[u8], state: &Path, workspace: &Path) -> Result> { + let tar = { + let mut dec = flate2::read::GzDecoder::new(snapshot_gz); + let mut out = Vec::new(); + std::io::Read::read_to_end(&mut dec, &mut out).context("gunzipping the snapshot")?; + out + }; + let mut archive = tar::Archive::new(tar.as_slice()); + let mut bundle: Option> = None; + std::fs::create_dir_all(state).with_context(|| format!("creating {}", state.display()))?; + for entry in archive.entries().context("reading the snapshot tar")? { + let mut entry = entry.context("reading a snapshot entry")?; + let path = entry.path().context("snapshot entry path")?.into_owned(); + if path.components().any(|c| { + matches!( + c, + std::path::Component::ParentDir | std::path::Component::RootDir + ) + }) { + anyhow::bail!("snapshot entry {} escapes the state dir", path.display()); + } + if path == Path::new(WORKSPACE_BUNDLE) { + let mut bytes = Vec::new(); + std::io::Read::read_to_end(&mut entry, &mut bytes).context("reading the bundle")?; + bundle = Some(bytes); + continue; + } + let Ok(rel) = path.strip_prefix("state") else { + continue; + }; + let to = state.join(rel); + if let Some(parent) = to.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("creating {}", parent.display()))?; + } + entry + .unpack(&to) + .with_context(|| format!("restoring {}", to.display()))?; + } + let record = std::fs::read(state.join(RESUME_FILE)) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); + if let Some(bundle) = bundle { + restore_bundle( + workspace, + &bundle, + record.as_ref().and_then(|r| r.head.as_deref()), + )?; + } + Ok(record) +} + +/// Restore a suspended run's artifacts from the drop-box into `into`, the receiving half of +/// [`deliver`]: `run-session` becomes `/state/session.jsonl` and `run-workspace` is +/// unpacked over `/state` and `/`. Reads the pod ingest env and +/// `CRUCIBLE_RESUME_OF`, so it runs as an init container beside the engine it feeds. +pub fn fetch_resume(into: &Path, workspace: &str) -> Result> { + let cfg = IngestConfig::from_env().ok_or(SuspendError::NoIngestEnv)?; + let of = resume_of_from_env().ok_or(SuspendError::NoResumeOf)?; + let state = into.join("state"); + std::fs::create_dir_all(&state).with_context(|| format!("creating {}", state.display()))?; + + let session = fetch_required(&cfg, &of, ArtifactKind::RunSession)?; + let log = state.join("session.jsonl"); + std::fs::write(&log, gunzip(&session)?) + .with_context(|| format!("writing {}", log.display()))?; + + let snapshot = fetch_required(&cfg, &of, ArtifactKind::RunWorkspace)?; + restore(&snapshot, &state, &into.join(workspace)) +} + +/// One artifact, retried: a 404 means the controller has not stored it yet, an error means the +/// call itself failed. Both are worth another attempt before refusing to resume. +fn fetch_required(cfg: &IngestConfig, of: &str, kind: ArtifactKind) -> Result> { + let mut last: Option = None; + for attempt in 1..=FETCH_ATTEMPTS { + match fetch_artifact(cfg, of, kind) { + Ok(Some(bytes)) => return Ok(bytes), + Ok(None) => last = None, + Err(e) => last = Some(e), + } + if attempt < FETCH_ATTEMPTS { + std::thread::sleep(FETCH_BACKOFF); + } + } + match last { + Some(message) => Err(SuspendError::Fetch { + kind, + of: of.to_string(), + message, + } + .into()), + None => Err(SuspendError::MissingArtifact { + kind, + of: of.to_string(), + } + .into()), + } +} + +fn bundle_workspace(workspace: &Path) -> Result>> { + if !workspace.join(".git").exists() { + return Ok(None); + } + let out = Command::new("git") + .arg("-C") + .arg(workspace) + .args(["bundle", "create", "-", "--all"]) + .output() + .context("running git bundle")?; + if !out.status.success() { + let stderr = String::from_utf8_lossy(&out.stderr); + // A repository with no commits cannot be bundled; the resumed run starts from the + // pack's own setup, which is what it had. + if stderr.contains("Refusing to create empty bundle") { + return Ok(None); + } + anyhow::bail!("git bundle failed: {}", stderr.trim()); + } + Ok(Some(out.stdout)) +} + +fn restore_bundle(workspace: &Path, bundle: &[u8], head: Option<&str>) -> Result<()> { + std::fs::create_dir_all(workspace) + .with_context(|| format!("creating {}", workspace.display()))?; + let path = workspace.join(".crucible-resume.bundle"); + std::fs::write(&path, bundle).with_context(|| format!("writing {}", path.display()))?; + if !workspace.join(".git").exists() { + run_git(workspace, &["init", "-q"])?; + } + run_git( + workspace, + &[ + "fetch", + "-q", + path.to_string_lossy().as_ref(), + "+refs/heads/*:refs/resume/*", + "+refs/tags/*:refs/tags/*", + ], + )?; + if let Some(head) = head { + run_git(workspace, &["checkout", "-q", "--force", head])?; + } + let _ = std::fs::remove_file(&path); + Ok(()) +} + +fn run_git(workspace: &Path, args: &[&str]) -> Result<()> { + let out = Command::new("git") + .arg("-C") + .arg(workspace) + .args(args) + .output() + .with_context(|| format!("running git {}", args.join(" ")))?; + if !out.status.success() { + anyhow::bail!( + "git {} failed: {}", + args.join(" "), + String::from_utf8_lossy(&out.stderr).trim() + ); + } + Ok(()) +} + +/// The workspace's `HEAD`, when it is a repository with a commit. +pub fn head_of(workspace: &Path) -> Option { + let out = Command::new("git") + .arg("-C") + .arg(workspace) + .args(["rev-parse", "HEAD"]) + .output() + .ok()?; + out.status + .success() + .then(|| String::from_utf8_lossy(&out.stdout).trim().to_string()) + .filter(|s| !s.is_empty()) +} + +fn append_dir( + builder: &mut tar::Builder>, + dir: &Path, + prefix: &Path, + keep: &dyn Fn(&str) -> bool, +) -> Result<()> { + let mut entries: Vec = std::fs::read_dir(dir) + .with_context(|| format!("reading {}", dir.display()))? + .filter_map(|e| e.ok().map(|e| e.path())) + .collect(); + entries.sort(); + for path in entries { + let Some(name) = path.file_name().and_then(|n| n.to_str()) else { + continue; + }; + if !keep(name) { + continue; + } + let rel = prefix.join(name); + if path.is_dir() { + append_dir(builder, &path, &rel, keep)?; + } else if path.is_file() { + builder + .append_path_with_name(&path, &rel) + .with_context(|| format!("adding {}", path.display()))?; + } + } + Ok(()) +} + +fn gunzip(bytes: &[u8]) -> Result> { + let mut out = Vec::new(); + let mut dec = flate2::read::GzDecoder::new(bytes); + std::io::Read::read_to_end(&mut dec, &mut out).context("gunzipping the session log")?; + Ok(out) +} + +fn gzip(bytes: &[u8]) -> Result> { + let mut enc = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + enc.write_all(bytes).context("gzip")?; + enc.finish().context("gzip") +} + +/// Unix seconds now, for records and events. +pub fn now_secs() -> f64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs_f64()) + .unwrap_or(0.0) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn temp(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "crucible-suspend-{tag}-{}-{}", + std::process::id(), + now_secs() as u64 + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir + } + + fn git(dir: &Path, args: &[&str]) { + let out = Command::new("git") + .arg("-C") + .arg(dir) + .args(args) + .output() + .unwrap(); + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + } + + /// A one-shot HTTP server over a real socket: serves each artifact GET from `bodies` keyed by + /// the artifact kind in the path, closing the connection after every reply. + fn serve_artifacts( + bodies: Vec<(&'static str, Vec)>, + ) -> (String, std::thread::JoinHandle>) { + use std::io::{BufRead, BufReader, Write as _}; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = format!("http://{}", listener.local_addr().unwrap()); + let handle = std::thread::spawn(move || { + let mut paths = Vec::new(); + for _ in 0..bodies.len() { + let (mut stream, _) = listener.accept().unwrap(); + let mut reader = BufReader::new(stream.try_clone().unwrap()); + let mut request = String::new(); + reader.read_line(&mut request).unwrap(); + loop { + let mut header = String::new(); + if reader.read_line(&mut header).unwrap() == 0 || header.trim().is_empty() { + break; + } + } + let path = request + .split_whitespace() + .nth(1) + .unwrap_or_default() + .to_string(); + let body = bodies + .iter() + .find(|(kind, _)| path.contains(kind)) + .map(|(_, b)| b.clone()); + match body { + Some(body) => { + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .unwrap(); + stream.write_all(&body).unwrap(); + } + None => write!( + stream, + "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ) + .unwrap(), + } + stream.flush().unwrap(); + paths.push(path); + } + paths + }); + (addr, handle) + } + + #[test] + fn fetch_resume_restores_the_session_log_and_the_workspace_from_the_drop_box() { + let _guard = crate::test_env_lock(); + let root = temp("fetch-src"); + let state = root.join("state"); + let workspace = root.join("workspace"); + std::fs::create_dir_all(&state).unwrap(); + std::fs::create_dir_all(&workspace).unwrap(); + std::fs::write(state.join("session.jsonl"), "{\"kind\":\"note\"}\n").unwrap(); + std::fs::write(state.join("admissions.jsonl"), "admitted\n").unwrap(); + git(&workspace, &["init", "-q"]); + std::fs::write(workspace.join("a.txt"), "one").unwrap(); + git(&workspace, &["add", "-A"]); + git( + &workspace, + &[ + "-c", + "user.email=c@l", + "-c", + "user.name=c", + "-c", + "commit.gpgsign=false", + "commit", + "-qm", + "one", + ], + ); + let head = head_of(&workspace).expect("head"); + let record = ResumeRecord { + v: ResumeRecord::VERSION, + run_id: "run-7".into(), + gate: "approve:run-7:ship".into(), + head: Some(head.clone()), + suspended_at: 2.0, + }; + let snapshot_gz = snapshot(&state, &workspace, &record).expect("snapshot"); + let session_gz = gzip(b"{\"kind\":\"note\"}\n").expect("gzip"); + + let (base_url, server) = serve_artifacts(vec![ + (ArtifactKind::RunSession.as_str(), session_gz), + (ArtifactKind::RunWorkspace.as_str(), snapshot_gz), + ]); + + let token = root.join("token"); + std::fs::write(&token, "pod-token").unwrap(); + unsafe { + std::env::set_var(crucible_contract::ENV_INGEST_URL, &base_url); + std::env::set_var(crucible_contract::ENV_INGEST_TOKEN_PATH, &token); + std::env::set_var(crucible_contract::ENV_POD_NAME, "crucible-run-7-b"); + std::env::set_var(crucible_contract::ENV_RESUME_OF, "crucible-run-7-a"); + } + let into = temp("fetch-dst"); + let back = fetch_resume(&into, "workspace").expect("fetch-resume"); + unsafe { + for k in [ + crucible_contract::ENV_INGEST_URL, + crucible_contract::ENV_INGEST_TOKEN_PATH, + crucible_contract::ENV_POD_NAME, + crucible_contract::ENV_RESUME_OF, + ] { + std::env::remove_var(k); + } + } + + assert_eq!(back.as_ref(), Some(&record)); + assert_eq!( + std::fs::read_to_string(into.join("state").join("session.jsonl")).unwrap(), + "{\"kind\":\"note\"}\n", + "the session log is restored from its own artifact" + ); + assert_eq!( + std::fs::read_to_string(into.join("state").join("admissions.jsonl")).unwrap(), + "admitted\n" + ); + assert_eq!( + head_of(&into.join("workspace")).as_deref(), + Some(head.as_str()) + ); + + let paths = server.join().expect("server"); + assert!( + paths.iter().all(|p| p.contains("from=crucible-run-7-a")), + "each fetch names the pod being resumed: {paths:?}" + ); + let _ = std::fs::remove_dir_all(&root); + let _ = std::fs::remove_dir_all(&into); + } + + #[test] + fn a_snapshot_round_trips_state_files_and_the_workspace_history() { + let root = temp("roundtrip"); + let state = root.join("state"); + let workspace = root.join("workspace"); + std::fs::create_dir_all(&state).unwrap(); + std::fs::create_dir_all(&workspace).unwrap(); + std::fs::write(state.join("session.jsonl"), "{\"kind\":\"note\"}\n").unwrap(); + std::fs::write(state.join("admissions.jsonl"), "admitted\n").unwrap(); + std::fs::create_dir_all(state.join("files").join("draft")).unwrap(); + std::fs::write(state.join("files").join("draft").join("NOTES.md"), "notes").unwrap(); + git(&workspace, &["init", "-q"]); + std::fs::write(workspace.join("a.txt"), "one").unwrap(); + git(&workspace, &["add", "-A"]); + git( + &workspace, + &[ + "-c", + "user.email=c@l", + "-c", + "user.name=c", + "-c", + "commit.gpgsign=false", + "commit", + "-qm", + "one", + ], + ); + let head = head_of(&workspace).expect("head"); + let record = ResumeRecord { + v: ResumeRecord::VERSION, + run_id: "run-1".into(), + gate: "approve:run-1:gate".into(), + head: Some(head.clone()), + suspended_at: 1.0, + }; + let gz = snapshot(&state, &workspace, &record).expect("snapshot"); + assert!( + state.join(RESUME_FILE).exists(), + "the record is left in state/ for a local resume" + ); + + let other = temp("restored"); + let state2 = other.join("state"); + let workspace2 = other.join("workspace"); + let back = restore(&gz, &state2, &workspace2).expect("restore"); + assert_eq!(back.as_ref(), Some(&record)); + assert_eq!( + std::fs::read_to_string(state2.join("admissions.jsonl")).unwrap(), + "admitted\n" + ); + assert_eq!( + std::fs::read_to_string(state2.join("files").join("draft").join("NOTES.md")).unwrap(), + "notes" + ); + assert!( + !state2.join("session.jsonl").exists(), + "the session log travels as its own artifact" + ); + assert_eq!(head_of(&workspace2).as_deref(), Some(head.as_str())); + assert_eq!( + std::fs::read_to_string(workspace2.join("a.txt")).unwrap(), + "one" + ); + let _ = std::fs::remove_dir_all(&root); + let _ = std::fs::remove_dir_all(&other); + } + + #[test] + fn a_workspace_without_commits_snapshots_without_a_bundle() { + let root = temp("nobundle"); + let state = root.join("state"); + let workspace = root.join("workspace"); + std::fs::create_dir_all(&state).unwrap(); + std::fs::create_dir_all(&workspace).unwrap(); + git(&workspace, &["init", "-q"]); + let record = ResumeRecord { + v: ResumeRecord::VERSION, + run_id: "run-1".into(), + gate: "g".into(), + head: None, + suspended_at: 1.0, + }; + let gz = snapshot(&state, &workspace, &record).expect("snapshot"); + let other = temp("nobundle-restored"); + let back = restore(&gz, &other.join("state"), &other.join("workspace")).expect("restore"); + assert_eq!(back.map(|r| r.gate), Some("g".to_string())); + let _ = std::fs::remove_dir_all(&root); + let _ = std::fs::remove_dir_all(&other); + } + + #[test] + fn a_snapshot_entry_that_escapes_is_refused() { + let mut builder = tar::Builder::new(Vec::new()); + let mut header = tar::Header::new_gnu(); + header.set_size(2); + header.set_mode(0o644); + // `append_data` refuses to write `..`, so the traversal goes straight into the name field: + // the archive under attack is one an attacker wrote, not one this builder produced. + let name = b"state/../../evil"; + header.as_old_mut().name[..name.len()].copy_from_slice(name); + header.set_cksum(); + builder.append(&header, b"hi".as_slice()).unwrap(); + let tar = builder.into_inner().unwrap(); + let gz = gzip(&tar).unwrap(); + let root = temp("escape"); + assert!(restore(&gz, &root.join("state"), &root.join("workspace")).is_err()); + let _ = std::fs::remove_dir_all(&root); + } +} diff --git a/crucible/tests/render_parity.rs b/crucible/tests/render_parity.rs index 9f3abe38..445641ae 100644 --- a/crucible/tests/render_parity.rs +++ b/crucible/tests/render_parity.rs @@ -82,6 +82,7 @@ fn deploy_render_matches_render_yaml() { harness: Some(Harness::Hermes), model: Some("hermes-4-70b".to_string()), playbook: None, + ..Default::default() }, ) .expect("library render"); @@ -173,6 +174,7 @@ fn deploy_render_pack_playbook_matches_render_yaml() { ("depth".to_string(), "--deep".to_string()), ]), }), + ..Default::default() }, ) .expect("library render"); diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index e4697413..a877c2b4 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -54,3 +54,4 @@ - [ADR 0025: Durable tool steps for broker builds and measures](./adr/0025-durable-tool-steps.md) - [ADR 0026: The no-judge task lane](./adr/0026-no-judge-task-lane.md) - [ADR 0028: Retire the wide tournament](./adr/0028-retire-the-wide-tournament.md) + - [ADR 0029: Plan-authored approval gates on an event-sourced loop](./adr/0029-approval-gates.md) diff --git a/docs/adr/0029-approval-gates.md b/docs/adr/0029-approval-gates.md new file mode 100644 index 00000000..3cd51f01 --- /dev/null +++ b/docs/adr/0029-approval-gates.md @@ -0,0 +1,69 @@ +# ADR 0029: Plan-authored approval gates on an event-sourced loop + +**Status:** Accepted; implemented (2026-09-04). Governance source: `gov/adr/ADR-0025`. +**Date:** 2026-09-04 +**Related:** [ADR-0018](./0018-session-log-as-the-record.md) (the log this makes authoritative), +[ADR-0004](./0004-core-loop-state-model.md) (amended: dispatch may outlive its process), +[ADR-0003](./0003-frozen-judge.md), [RFC-0002 C-PLAYBOOK-APPROVAL](../rfc/RFC-0002.md) + +## Context + +A playbook could already ask a human for something. The provisioning ask parks the loop on a +marker and waits for a re-scope over the control bridge. That path belongs to the scored loop, +it is one global wait rather than a step in the graph, and the only thing that can end it is an +operator at a socket. + +The work that needs gating is not shaped like that. A pack wants to say "run these tasks, then +wait for someone to approve the change, then deploy", and the approval usually already exists +somewhere else: a pull request got a review, a tracker issue moved to Ready. It is a node in the +graph, with dependents, a verdict contribution, and a name. + +Waiting is expensive in the shape the controller runs playbooks. A pod idling for a day on a +human holds a scheduled slot and a node. A run that can leave and come back is worth more than +one that can only idle, but leaving requires state that outlives the process. + +By this point three folds of the session log had grown up independently: the resume fold rebuilt +counters, a tail scan classified crashes, and the controller's ingest had its own event enum. A +gate adds events all three would have to learn. ADR-0018 made the session log the run's record; +nothing had yet made it the run's state. + +## Decision + +Add an approval gate as a task kind the author declares, and make the session log the single +fold every consumer reads. + +The fold moves into the contract: one exhaustive `apply` over the event vocabulary, one +classifier, one resume view. The engine and the controller read the same code, so a new event +kind is taught to one place. The scored loop's decisions move out of the driver into a state +machine with no I/O, which the driver hosts by performing effects and handing back results. + +A gate names what may resolve it. An operator acting on the run always can; a gate may +additionally name a pull request or a tracker issue, and whichever arrives first wins. +Resolution is keyed by a trace id derived from the run and the task, and is idempotent under +that key, so a retrying resolver, a second source, and a resumed process replaying a decision +all converge on one recorded outcome. Granted settles the gate passing; denied and timeout +settle it failing, and the ordinary verdict rule does the rest. No fourth outcome, no new task +status. + +A run at a gate either waits in place or suspends. Suspending writes the workspace and the state +dir to the controller's existing artifact drop-box and exits zero under a distinct shutdown +outcome; a later process restores them and is handed the decisions the controller settled +meanwhile. Waiting time does not count against the wall-clock ceiling. + +## Consequences + +A pack can express "a person decides here" as a node rather than as a task that polls something. +The four folds become one, so the controller and the engine cannot disagree about what a log +says. Moving the loop's decisions somewhere testable turned up an attempt bound that had been +counting per process rather than per iteration. + +The feature is half usable until the controller carries it. The engine can suspend, but nothing +resumes it without the controller's approval rows, its suspended status, and its artifact +endpoints, so the core pin bump and that work are one delivery rather than two. + +A gate makes a run's wall clock unbounded in practice: the ceiling stops counting while a gate +is open, and the only backstop is the gate's own timeout. + +The wire stays at one version. Every addition is an optional field or a new token, and readers +must skip event kinds they do not know, which is what makes an older consumer safe against a +newer writer. diff --git a/docs/adr/index.md b/docs/adr/index.md index 45c6e605..1175b1c6 100644 --- a/docs/adr/index.md +++ b/docs/adr/index.md @@ -31,3 +31,4 @@ Expand this section in the sidebar to browse the full list. | [0026](./0026-no-judge-task-lane.md) | The no-judge task lane | Implemented | | [0027](./0027-measurement-sessions.md) | Measurement sessions, one warm engine and many observations | Proposed | | [0028](./0028-retire-the-wide-tournament.md) | Retire the wide tournament | Implemented | +| [0029](./0029-approval-gates.md) | Plan-authored approval gates on an event-sourced loop | Implemented | diff --git a/docs/dsl-reference.md b/docs/dsl-reference.md index ebf34960..4a188c9a 100644 --- a/docs/dsl-reference.md +++ b/docs/dsl-reference.md @@ -218,6 +218,56 @@ Expand the built-in propose/apply/measure/decide loop into visible nodes, plus t Takes one positional argument, `extra_tasks`. +## Playbooks only + +Available to `type = "playbook"`. An approval gate parks the run until a person or an external item resolves it; the scored lanes have no gate in scope. + +### `approve()` + +A human gate. The graph waits here until the source resolves it: a grant passes the task with the resolution as its output, a denial or a timeout fails it. + +| Argument | Type | Purpose | +| --- | --- | --- | +| `name` | `str` | Task identity, unique within the workflow. | +| `summary` | `str` | What the approver is deciding, shown with the gate. | +| `source` | `"native" \| github_pr(...) \| jira(...)` | Where the resolution comes from. `native` (the default) is a person acting through the controller or the control bridge. | +| `timeout` | `str` | How long this gate may park before the run's park policy applies, e.g. `30m`. | +| `depends_on` | `list[task]` | Dependencies. A source read from a task's field must name that task here. | +| `needs` | `"any" \| "all"` | How many dependencies must be admitted before the gate is reached. | +| `required` | `bool` | False makes the gate advisory: a denial fails it without invalidating the run. | +| `join` | `"all" \| "passed" \| "settled"` | Which dependencies must have passed before the gate is reached. | +| `stage` | `"iteration" \| "epilogue"` | `epilogue` gates the run's conclusion instead of its graph. | + +### `github_pr()` + +An approval source: a GitHub pull request reaching a state. + +| Argument | Type | Purpose | +| --- | --- | --- | +| `url` | `str \| task.field` | The pull request, written out or read from an upstream task's emitted field. | +| `until` | `"approved" \| "merged"` | What resolves the gate: an authorized review approval or `/approve` comment (the default), or the merge. | + +### `jira()` + +An approval source: a Jira issue reaching a status or carrying a label. + +| Argument | Type | Purpose | +| --- | --- | --- | +| `key` | `str \| task.field` | The issue key, written out or read from an upstream task's emitted field. | +| `until` | `status(...) \| label(...)` | What resolves the gate. | + +### `status()` + +A Jira predicate: the issue's status name equals the argument. + +Takes one positional argument, `name`. + +### `label()` + +A Jira predicate: the issue carries the label. + +Takes one positional argument, `name`. + ## Reserved fields Names the engine reads and writes for itself. They are not constructor arguments; they appear in a task's own JSON output and in the inputs it receives. diff --git a/docs/rfc/RFC-0001.md b/docs/rfc/RFC-0001.md index f286738d..47d63f04 100644 --- a/docs/rfc/RFC-0001.md +++ b/docs/rfc/RFC-0001.md @@ -1,5 +1,5 @@ - + # RFC-0001: Crucible implementation contract @@ -205,14 +205,18 @@ The NDJSON session log (state/session.jsonl) must keep its existing event kinds Additive event kinds: - identity: the run's RunIdentity, emitted once at setup and again on --resume. -- shutdown: { outcome, reason }, emitted exactly once as the last line of every run (after finished/summary). outcome must be one of finished/solved/budget/complete/stopped/escalated/stalled/error. finished means the graph or the loop ran out of work; complete means a task declared there was no work left to do and dispatch stopped short of that, the unscored counterpart of solved. The outcome says how dispatch stopped, not whether the run succeeded. A dead stream with no shutdown line means the pod died mid-run. +- shutdown: { outcome, reason }, emitted exactly once as the last line of every run (after finished/summary). outcome must be one of finished/solved/budget/complete/stopped/escalated/stalled/suspended/error. suspended means the run stopped at an approval gate intending to be continued by a later process, and is the one outcome a consumer must not treat as terminal. finished means the graph or the loop ran out of work; complete means a task declared there was no work left to do and dispatch stopped short of that, the unscored counterpart of solved. The outcome says how dispatch stopped, not whether the run succeeded. A dead stream with no shutdown line means the pod died mid-run. - agent_session: { session, action, turn }, emitted before a persistent agent turn. Must not contain the provider cursor or native transcript content. -- approval_wait: { handle, trace_id, mode }, emitted when the loop reads a pending-provisioning marker. Every approval_wait must be closed by an approval_resolved except on stop-while-parked and process death. -- approval_resolved: { outcome, reason } with outcome one of granted/denied/timeout. +- approval_wait: { handle, trace_id, mode }, emitted when the loop reads a pending-provisioning marker, and when a playbook opens an approval gate. It may additionally carry task, naming the gate task; source, describing what the gate named as able to resolve it; and park, saying whether the run intends to wait in place or suspend. Every approval_wait must be closed by an approval_resolved except on stop-while-parked, suspension, and process death. +- approval_resolved: { outcome, reason } with outcome one of granted/denied/timeout. It may additionally carry trace_id, naming which open wait it closes; by, the deciding identity where the source supplied one; and source, naming what produced the decision. A reader must match a resolution to its wait by trace_id where one is present, because a run may hold more than one gate open at a time. - plan_admitted: { plan_version, reason, budget_usd, tasks }, emitted once after the graph is admitted and before any dispatch. Each task carries its name, kind, dependencies, session, needs, required flag, join, and stage. This is the consumer's route to the graph, so no gate label has to encode it. A consumer computing a verdict must skip tasks whose stage is epilogue, which is why the field is on the wire rather than inferable. - asks_emitted: { task, asks }, emitted once per task that emitted any, as that task settles. Each ask carries an emitter-supplied key, the workflow it names, and that workflow's parameter values. The key must be rejected on decode, not only on construction, so the wire is where the key rules are enforced rather than a way around them. - recovery: { class, iter, detail }, emitted once per --resume. +Every event may carry ts, the Unix time at which it was written. A reader must not require it: logs written before the field existed do not carry it, and a reader that computes elapsed or waiting time from it must tolerate its absence rather than refuse the log. + +A reader must not refuse a session log because it carries an event kind it does not know. It must skip that event and continue. This is what lets a newer writer's log be read by an older consumer at all: the event kinds above are additive by design, and a reader that failed on the first unfamiliar one would make every addition a breaking change. + RunIdentity is the comparability key: two runs' scores are comparable only if it matches. It must be a hash-of-hashes over: repo URL/path + pristine base commit SHA, frozen manifest text hash, inject content+destination hashes, measure_cmd, and direction. *Since: v0.1.0* diff --git a/docs/rfc/RFC-0002.md b/docs/rfc/RFC-0002.md index 842cdbc3..53b3194f 100644 --- a/docs/rfc/RFC-0002.md +++ b/docs/rfc/RFC-0002.md @@ -1,5 +1,5 @@ - + # RFC-0002: Playbook workflows @@ -47,11 +47,11 @@ A manifest with no [judge] MUST accept a [workflow] whose type is "playbook". A A playbook graph MUST NOT contain a task naming an engine operation: an operation the scored loop's orchestrator owns rather than the plan author, namely producing a candidate, applying one, measuring one, grading evidence, deciding keep or discard, and measuring one differentially against another. A task that runs an author-supplied command and grades its own result is not an engine operation and MUST be permitted, because it asserts something about the work rather than advancing a scored loop. -A playbook MUST execute its graph exactly once per run. The engine MUST reject a requested iteration count greater than one rather than silently ignoring it. +A playbook MUST execute its graph exactly once per run. The engine MUST reject a requested iteration count greater than one rather than silently ignoring it. A run interrupted at an approval gate under [RFC-0002:C-PLAYBOOK-APPROVAL](../rfc/RFC-0002.md#rfc-0002c-playbook-approval) and continued by a later process is one run, not two: the obligation is on the graph, not on the process, and the tasks a prior process settled MUST NOT be dispatched again. A playbook's verdict MUST be invalid when any required task settled failing, was truncated, or was left blocked, and when a cost or wall-clock ceiling was exhausted, whatever else the run reports. Otherwise the verdict MUST be valid when every required task settled passing, and when a task declared early completion. A required task left undispatched by early completion MUST NOT affect the verdict; a required task that had already failed MUST invalidate the run whatever a later early-completion signal says, so that a concurrent task cannot launder a failure into a success. The verdict MUST NOT depend on the outcome of any advisory task or of any epilogue task. -The run MUST exit zero if and only if its verdict is valid, and MUST exit nonzero otherwise. This is a stronger obligation than the task lane's, where exit zero means only that the run completed. The exit code alone does not say why a run ended, so the shutdown outcome MUST distinguish an exhausted graph, an early completion, a failure, an operator stop, and an exhausted ceiling. That outcome MUST be drawn from the vocabulary [RFC-0001:C-WIRE](../rfc/RFC-0001.md#rfc-0001c-wire) defines for the shutdown event; the value that vocabulary carries for early completion is "complete". +The run MUST exit zero if and only if its verdict is valid, and MUST exit nonzero otherwise. A run that suspended at an approval gate is the one exception: it has reached no verdict, MUST exit zero, and MUST be distinguished by its shutdown outcome rather than by its exit code, because the alternative is to report a run that is merely waiting as one that failed. This is a stronger obligation than the task lane's, where exit zero means only that the run completed. The exit code alone does not say why a run ended, so the shutdown outcome MUST distinguish an exhausted graph, an early completion, a failure, an operator stop, an exhausted ceiling, and a suspension. That outcome MUST be drawn from the vocabulary [RFC-0001:C-WIRE](../rfc/RFC-0001.md#rfc-0001c-wire) defines for the shutdown event; the value that vocabulary carries for early completion is "complete". Any task MAY declare early completion by returning the boolean field "complete" as true in its output, optionally alongside a "reason" string. Both names are reserved, and a task MUST NOT name either in its declared outputs: a declared output must be present on every passing attempt, whereas these appear only on the attempt that ends the run, so declaring them would fail every ordinary run. The engine MUST reject a graph that declares either. The engine MUST then stop dispatching further tasks, MUST record the reason where one was given, and MUST NOT treat undispatched tasks as blocked. Determining that there is nothing to do is a successful outcome, and a playbook that cannot say so has no way to end a scheduled run quietly. The shutdown outcome says how dispatch stopped and the verdict says whether the run succeeded; they are independent axes, so a run whose dispatch stopped on early completion after a required task had already failed records the early-completion outcome and an invalid verdict. @@ -73,11 +73,11 @@ A playbook MUST carry the task lane's gate label, written under the key [RFC-000 ### [RFC-0002:C-PLAYBOOK-SHAPE] What the graph expresses (Informative) -A playbook's graph says what runs before what, and what must pass before what runs. It does not say how many times anything runs. The graph is dispatched once, so a task that must retry, poll, or converge does that inside its own execution, where the author's command or the agent's own turn loop owns the repetition. A repair loop is therefore one task, not an edge back into the graph, and the engine never sees the iteration at all. +A playbook's graph says what runs before what, and what must pass before what runs. It does not say how many times anything runs. An approval gate under [RFC-0002:C-PLAYBOOK-APPROVAL](../rfc/RFC-0002.md#rfc-0002c-playbook-approval) is the one node that runs nothing at all: it is an edge condition given a name, so that "wait for a person here" is something the graph can express and a reader can see, rather than something buried inside a task that also does work. The graph is dispatched once, so a task that must retry, poll, or converge does that inside its own execution, where the author's command or the agent's own turn loop owns the repetition. A repair loop is therefore one task, not an edge back into the graph, and the engine never sees the iteration at all. This is also why an item set discovered at run time cannot become new nodes. The only two things a run can do with such a set are handle it inside one task or emit it as asks under [RFC-0002:C-ASKS](../rfc/RFC-0002.md#rfc-0002c-asks), and which one is right turns on whether the items deserve their own budget, isolation, and verdict. -The payoff is that the compiled graph is fully renderable before any spend and identical across runs of the same pack at the same parameters: the node set and the edges between them do not depend on what a task finds. Which of those nodes are dispatched still varies. [RFC-0002:C-PLAYBOOK-LANE](../rfc/RFC-0002.md#rfc-0002c-playbook-lane) leaves tasks undispatched after an early completion, and [RFC-0002:C-PLAYBOOK-CAPS](../rfc/RFC-0002.md#rfc-0002c-playbook-caps) truncates the plan or skips an advisory subtree when a capability is unavailable. Dispatch order among concurrent tasks is not fixed either. What is fixed is the shape. [RFC-0002:C-PLAYBOOK-LANE](../rfc/RFC-0002.md#rfc-0002c-playbook-lane) states the once-per-run obligation; this clause records what follows from it for an author deciding where to put a loop. +The payoff is that the compiled graph is fully renderable before any spend and identical across runs of the same pack at the same parameters: the node set and the edges between them do not depend on what a task finds. Which of those nodes are dispatched still varies. [RFC-0002:C-PLAYBOOK-LANE](../rfc/RFC-0002.md#rfc-0002c-playbook-lane) leaves tasks undispatched after an early completion, and [RFC-0002:C-PLAYBOOK-CAPS](../rfc/RFC-0002.md#rfc-0002c-playbook-caps) truncates the plan or skips an advisory subtree when a capability is unavailable. Dispatch order among concurrent tasks is not fixed either, and a gate may pause dispatch for as long as a person takes, or split it across two processes when the run suspends. What is fixed is the shape. [RFC-0002:C-PLAYBOOK-LANE](../rfc/RFC-0002.md#rfc-0002c-playbook-lane) states the once-per-run obligation; this clause records what follows from it for an author deciding where to put a loop. *Since: v0.1.0* @@ -175,7 +175,9 @@ A resumed playbook MUST NOT re-dispatch a task that already settled. The engine A resumed playbook MUST reconstruct its workspace from the pristine checkout plus the declared file outputs of every folded task that settled passing, staged under the rules in [RFC-0002:C-TASK-FILES](../rfc/RFC-0002.md#rfc-0002c-task-files). Files captured from a task that settled failing MUST NOT be restored into the workspace; they reach a dependent joining "settled" through its staged inputs only. It MUST NOT attempt to reconstruct the workspace as some particular task left it: when concurrent tasks were in flight at the interruption, no such state is well defined, and a task's declared output is the only part of its work the contract ever promised would survive. -Spend and elapsed time recorded before the interruption MUST count against the ceilings on resumption. A crash MUST NOT reset either. +A run continued after a suspension at an approval gate MUST instead restore the workspace as the suspending process left it. The rule above exists because a crash can strike with concurrent tasks in flight, leaving no well-defined state to restore; a gate is reached only when the tasks before it have settled, so there is exactly one such state and rebuilding a weaker approximation of it would discard work the run had already paid for. A resumed run MUST fall back to the reconstruction above when the suspending process left no usable snapshot, rather than refusing to continue. + +Spend and elapsed time recorded before the interruption MUST count against the ceilings on resumption. A crash MUST NOT reset either. Time a prior process spent waiting at a gate MUST NOT count, under [RFC-0002:C-PLAYBOOK-APPROVAL](../rfc/RFC-0002.md#rfc-0002c-playbook-approval). A resumed run MUST NOT re-dispatch the main graph when the folded results cover every main-graph task, and equally when any folded result declared early completion. Early completion leaves tasks undispatched by design, so a resume that only checked for full coverage would dispatch them again on every restart. It MUST still dispatch any epilogue task that has no folded result, under the conditions in [RFC-0002:C-PLAYBOOK-LANE](../rfc/RFC-0002.md#rfc-0002c-playbook-lane). The run's summary, shutdown, and publication MUST each happen exactly once across the original run and all of its resumptions. @@ -223,6 +225,28 @@ The admitted-plan event MUST carry "settled" under the key it already carries a *Since: v0.1.0* +### [RFC-0002:C-PLAYBOOK-APPROVAL] Approval gates (Normative) + +A playbook MAY declare a task that waits for a human decision before its dependents run. Such a gate task MUST NOT run a command, dispatch an agent, or change the workspace: its whole effect is the decision it carries, so a run that reaches one has nothing to undo if the decision never arrives. + +Every gate MUST carry a trace id that is unique within the run and stable across the run's resumptions, so that a decision recorded against it before an interruption is still the same decision afterwards. The trace id MUST be derivable from the run and the task alone, because a resolver that learned it from one process must be able to name it to another. + +A gate MUST settle in exactly one of three ways. Granted settles the gate task passing, and its dependents run. Denied settles it failing with the denier's reason, and its dependents are blocked exactly as any other failure blocks them. Timeout settles it failing when the gate has waited as long as it was permitted. The engine MUST NOT introduce a fourth outcome, and MUST NOT report a gate as settled without one of these three: a run whose dependents ran because nobody said no would make an approval meaningless. + +A denied or timed-out gate MUST invalidate the run's verdict when the gate task is required, under the ordinary rule in [RFC-0002:C-PLAYBOOK-LANE](../rfc/RFC-0002.md#rfc-0002c-playbook-lane). Approval is not a separate axis from success; a required step that was refused is a step that did not pass. + +Resolution MUST be idempotent by trace id. A decision delivered more than once, whether by a retrying resolver, by a second source, or by a resume replaying one already recorded, MUST converge on the first decision recorded rather than settling the gate twice or reversing it. This is what lets a resolver retry freely and a resumed run replay a decision it cannot tell it already applied. + +A gate MUST accept a decision from more than one source: an operator acting directly on the run, and an external system the gate names, such as a tracker issue reaching a state or a pull request being approved. Whichever arrives first resolves the gate, and the recorded resolution MUST name the source and, where the source supplies one, the deciding identity. A run MUST NOT require any particular source to be reachable: a gate whose named source cannot be consulted MUST still be resolvable by an operator, or a run becomes unfinishable whenever an external system is down. + +While a gate is open the run MUST NOT count the waiting against any wall-clock ceiling. A ceiling bounds the work a run may do, and a run waiting on a person is doing none; charging it would make a ceiling a function of how promptly a human answered. + +A run that reaches a gate MUST either wait in place or suspend. A run that waits MUST remain able to accept a decision for as long as it waits. A run that suspends MUST first record enough state for a later process to continue the same run, MUST report the shutdown outcome [RFC-0001:C-WIRE](../rfc/RFC-0001.md#rfc-0001c-wire) reserves for a suspension, and MUST exit zero: a suspension is not a failure, and a consumer that treated it as one would retry a run that is merely waiting. A suspended run MUST NOT be reported as having produced a verdict; its verdict is determined only when a later process finishes the graph. + +The engine MUST accept, at the start of a run, decisions already reached for gates the run has not yet opened, and MUST apply each to its gate rather than reopening it. This is what makes a suspension resumable: the process that resumes learns the decision from its caller, not from the gate it never saw opened. + +*Since: v0.1.0* + --- ## Changelog diff --git a/gov/adr/ADR-0025-plan-authored-approval-gates-on-an-event-sourced-loop.toml b/gov/adr/ADR-0025-plan-authored-approval-gates-on-an-event-sourced-loop.toml new file mode 100644 index 00000000..a18c6f79 --- /dev/null +++ b/gov/adr/ADR-0025-plan-authored-approval-gates-on-an-event-sourced-loop.toml @@ -0,0 +1,67 @@ +#:schema ../schema/adr.schema.json + +[govctl] +id = "ADR-0025" +title = "Plan-authored approval gates on an event-sourced loop" +status = "accepted" +date = "2026-09-04" +refs = [ + "ADR-0003", + "ADR-0004", + "ADR-0018", + "RFC-0001", + "RFC-0002", +] + +[content] +context = """ +A playbook can already ask a human for something: the provisioning ask parks the loop on a marker and waits for a re-scope over the control bridge. That path is the scored loop's, it is one global wait rather than a step in the graph, and the only thing that can end it is an operator at a socket. + +The work that needs gating is not shaped like that. A pack wants to say "run these tasks, then wait for someone to approve the change, then deploy", and the approval usually already exists somewhere else: a pull request got a review, a tracker issue moved to Ready. It is a node in the graph, with dependents, a verdict contribution, and a name. + +Waiting is also expensive in the shape the controller runs playbooks: a pod idling for a day on a human costs a scheduled slot and a node. A run that can leave and come back is worth more than a run that can only idle, but leaving requires state that outlives the process. + +Three folds of the session log had grown up independently by this point: the resume fold rebuilt counters, a tail scan classified crashes, and the controller's ingest had its own enum. A gate adds events all three would have to learn. [[ADR-0018]] made the session log the run's record; nothing had made it the run's state. +""" +decision = """ +Add an approval gate as a task kind a playbook author declares, and make the session log the single fold every consumer reads. + +The fold moves into the contract: one exhaustive apply over the event vocabulary, one classifier, one resume view. The engine and the controller read the same code, so a new event kind is taught to one place. The scored loop's decisions move out of the driver into a state machine with no I/O, which the driver hosts by performing effects and handing back results. + +A gate names what may resolve it. An operator acting on the run always can; a gate may additionally name a pull request or a tracker issue, and whichever arrives first wins. Resolution is keyed by a trace id derived from the run and the task, and is idempotent under that key, so a retrying resolver, a second source, and a resumed process replaying a decision all converge on one recorded outcome. Granted settles the gate passing; denied and timeout settle it failing, and the ordinary verdict rule in [[RFC-0002:C-PLAYBOOK-LANE]] does the rest. There is no fourth outcome and no new task status. + +A run at a gate either waits in place or suspends. Suspending writes the workspace and the state dir to the controller's existing artifact drop-box and exits zero under a distinct shutdown outcome; a later process restores them and is handed the decisions the controller settled meanwhile. Waiting time does not count against the wall-clock ceiling. + +The obligations are in [[RFC-0002:C-PLAYBOOK-APPROVAL]], with the wire additions in [[RFC-0001:C-WIRE]] and the resume rule in [[RFC-0002:C-PLAYBOOK-RESUME]]. This amends [[ADR-0004]], which assumed a run's dispatch ends with the process that started it. +""" +consequences = """ +Positive: a pack can express "a person decides here" as a node rather than as a task that polls something. The four folds become one, so the controller and the engine cannot disagree about what a log says. The scored loop's decisions became testable without a harness, and doing that turned up an attempt bound that had been counting per process rather than per iteration. + +Negative: the feature is only half usable until the controller carries it. The engine can suspend, but nothing resumes it without the controller's approval rows, its suspended status, and its artifact endpoints, so the pin bump and that work are one delivery rather than two. A gate also makes a run's wall-clock unbounded in practice: the ceiling stops counting, and the only backstop is the gate's own timeout. + +Neutral: the wire stays at one version. Every addition is an optional field or a new token, and readers are required to skip event kinds they do not know, which is what makes an older consumer safe against a newer writer. +""" + +[[content.alternatives]] +text = "A gate task in the graph, resolvable by an operator or by a named external source, with park and suspend" +status = "accepted" + +[[content.alternatives]] +text = "Extend the existing provisioning ask to cover approvals" +status = "rejected" +rejection_reason = "It is one global wait on the scored loop, not a node with dependents and a verdict contribution, and only an operator at a socket can end it." + +[[content.alternatives]] +text = "Let a task poll the tracker or the forge itself and pass when it sees approval" +status = "rejected" +rejection_reason = "Puts the forge token and the wait policy in the pack, makes the wait indistinguishable from work in the log, and charges the wall-clock ceiling for time spent waiting on a person." + +[[content.alternatives]] +text = "Host the durable wait on a workflow engine rather than building it" +status = "rejected" +rejection_reason = "The controller is already a durable workflow engine for this purpose, and the loop must keep running from a laptop with no server; revisited once the reducer is the thing a workflow would step." + +[[content.alternatives]] +text = "Park only, never suspend" +status = "rejected" +rejection_reason = "An idle pod holds a scheduled slot and a node for as long as the human takes, which is the common case rather than the exception." diff --git a/gov/rfc/RFC-0001/clauses/C-WIRE.toml b/gov/rfc/RFC-0001/clauses/C-WIRE.toml index 5fda5300..6b2390a4 100644 --- a/gov/rfc/RFC-0001/clauses/C-WIRE.toml +++ b/gov/rfc/RFC-0001/clauses/C-WIRE.toml @@ -13,12 +13,16 @@ The NDJSON session log (state/session.jsonl) must keep its existing event kinds Additive event kinds: - identity: the run's RunIdentity, emitted once at setup and again on --resume. -- shutdown: { outcome, reason }, emitted exactly once as the last line of every run (after finished/summary). outcome must be one of finished/solved/budget/complete/stopped/escalated/stalled/error. finished means the graph or the loop ran out of work; complete means a task declared there was no work left to do and dispatch stopped short of that, the unscored counterpart of solved. The outcome says how dispatch stopped, not whether the run succeeded. A dead stream with no shutdown line means the pod died mid-run. +- shutdown: { outcome, reason }, emitted exactly once as the last line of every run (after finished/summary). outcome must be one of finished/solved/budget/complete/stopped/escalated/stalled/suspended/error. suspended means the run stopped at an approval gate intending to be continued by a later process, and is the one outcome a consumer must not treat as terminal. finished means the graph or the loop ran out of work; complete means a task declared there was no work left to do and dispatch stopped short of that, the unscored counterpart of solved. The outcome says how dispatch stopped, not whether the run succeeded. A dead stream with no shutdown line means the pod died mid-run. - agent_session: { session, action, turn }, emitted before a persistent agent turn. Must not contain the provider cursor or native transcript content. -- approval_wait: { handle, trace_id, mode }, emitted when the loop reads a pending-provisioning marker. Every approval_wait must be closed by an approval_resolved except on stop-while-parked and process death. -- approval_resolved: { outcome, reason } with outcome one of granted/denied/timeout. +- approval_wait: { handle, trace_id, mode }, emitted when the loop reads a pending-provisioning marker, and when a playbook opens an approval gate. It may additionally carry task, naming the gate task; source, describing what the gate named as able to resolve it; and park, saying whether the run intends to wait in place or suspend. Every approval_wait must be closed by an approval_resolved except on stop-while-parked, suspension, and process death. +- approval_resolved: { outcome, reason } with outcome one of granted/denied/timeout. It may additionally carry trace_id, naming which open wait it closes; by, the deciding identity where the source supplied one; and source, naming what produced the decision. A reader must match a resolution to its wait by trace_id where one is present, because a run may hold more than one gate open at a time. - plan_admitted: { plan_version, reason, budget_usd, tasks }, emitted once after the graph is admitted and before any dispatch. Each task carries its name, kind, dependencies, session, needs, required flag, join, and stage. This is the consumer's route to the graph, so no gate label has to encode it. A consumer computing a verdict must skip tasks whose stage is epilogue, which is why the field is on the wire rather than inferable. - asks_emitted: { task, asks }, emitted once per task that emitted any, as that task settles. Each ask carries an emitter-supplied key, the workflow it names, and that workflow's parameter values. The key must be rejected on decode, not only on construction, so the wire is where the key rules are enforced rather than a way around them. - recovery: { class, iter, detail }, emitted once per --resume. +Every event may carry ts, the Unix time at which it was written. A reader must not require it: logs written before the field existed do not carry it, and a reader that computes elapsed or waiting time from it must tolerate its absence rather than refuse the log. + +A reader must not refuse a session log because it carries an event kind it does not know. It must skip that event and continue. This is what lets a newer writer's log be read by an older consumer at all: the event kinds above are additive by design, and a reader that failed on the first unfamiliar one would make every addition a breaking change. + RunIdentity is the comparability key: two runs' scores are comparable only if it matches. It must be a hash-of-hashes over: repo URL/path + pristine base commit SHA, frozen manifest text hash, inject content+destination hashes, measure_cmd, and direction.""" diff --git a/gov/rfc/RFC-0002/clauses/C-PLAYBOOK-APPROVAL.toml b/gov/rfc/RFC-0002/clauses/C-PLAYBOOK-APPROVAL.toml new file mode 100644 index 00000000..ed5a3849 --- /dev/null +++ b/gov/rfc/RFC-0002/clauses/C-PLAYBOOK-APPROVAL.toml @@ -0,0 +1,28 @@ +#:schema ../../../schema/clause.schema.json + +[govctl] +id = "C-PLAYBOOK-APPROVAL" +title = "Approval gates" +kind = "normative" +status = "active" +since = "0.1.0" + +[content] +text = """ +A playbook MAY declare a task that waits for a human decision before its dependents run. Such a gate task MUST NOT run a command, dispatch an agent, or change the workspace: its whole effect is the decision it carries, so a run that reaches one has nothing to undo if the decision never arrives. + +Every gate MUST carry a trace id that is unique within the run and stable across the run's resumptions, so that a decision recorded against it before an interruption is still the same decision afterwards. The trace id MUST be derivable from the run and the task alone, because a resolver that learned it from one process must be able to name it to another. + +A gate MUST settle in exactly one of three ways. Granted settles the gate task passing, and its dependents run. Denied settles it failing with the denier's reason, and its dependents are blocked exactly as any other failure blocks them. Timeout settles it failing when the gate has waited as long as it was permitted. The engine MUST NOT introduce a fourth outcome, and MUST NOT report a gate as settled without one of these three: a run whose dependents ran because nobody said no would make an approval meaningless. + +A denied or timed-out gate MUST invalidate the run's verdict when the gate task is required, under the ordinary rule in [[RFC-0002:C-PLAYBOOK-LANE]]. Approval is not a separate axis from success; a required step that was refused is a step that did not pass. + +Resolution MUST be idempotent by trace id. A decision delivered more than once, whether by a retrying resolver, by a second source, or by a resume replaying one already recorded, MUST converge on the first decision recorded rather than settling the gate twice or reversing it. This is what lets a resolver retry freely and a resumed run replay a decision it cannot tell it already applied. + +A gate MUST accept a decision from more than one source: an operator acting directly on the run, and an external system the gate names, such as a tracker issue reaching a state or a pull request being approved. Whichever arrives first resolves the gate, and the recorded resolution MUST name the source and, where the source supplies one, the deciding identity. A run MUST NOT require any particular source to be reachable: a gate whose named source cannot be consulted MUST still be resolvable by an operator, or a run becomes unfinishable whenever an external system is down. + +While a gate is open the run MUST NOT count the waiting against any wall-clock ceiling. A ceiling bounds the work a run may do, and a run waiting on a person is doing none; charging it would make a ceiling a function of how promptly a human answered. + +A run that reaches a gate MUST either wait in place or suspend. A run that waits MUST remain able to accept a decision for as long as it waits. A run that suspends MUST first record enough state for a later process to continue the same run, MUST report the shutdown outcome [[RFC-0001:C-WIRE]] reserves for a suspension, and MUST exit zero: a suspension is not a failure, and a consumer that treated it as one would retry a run that is merely waiting. A suspended run MUST NOT be reported as having produced a verdict; its verdict is determined only when a later process finishes the graph. + +The engine MUST accept, at the start of a run, decisions already reached for gates the run has not yet opened, and MUST apply each to its gate rather than reopening it. This is what makes a suspension resumable: the process that resumes learns the decision from its caller, not from the gate it never saw opened.""" diff --git a/gov/rfc/RFC-0002/clauses/C-PLAYBOOK-LANE.toml b/gov/rfc/RFC-0002/clauses/C-PLAYBOOK-LANE.toml index fdb25803..aec8a36d 100644 --- a/gov/rfc/RFC-0002/clauses/C-PLAYBOOK-LANE.toml +++ b/gov/rfc/RFC-0002/clauses/C-PLAYBOOK-LANE.toml @@ -13,11 +13,11 @@ A manifest with no [judge] MUST accept a [workflow] whose type is "playbook". A A playbook graph MUST NOT contain a task naming an engine operation: an operation the scored loop's orchestrator owns rather than the plan author, namely producing a candidate, applying one, measuring one, grading evidence, deciding keep or discard, and measuring one differentially against another. A task that runs an author-supplied command and grades its own result is not an engine operation and MUST be permitted, because it asserts something about the work rather than advancing a scored loop. -A playbook MUST execute its graph exactly once per run. The engine MUST reject a requested iteration count greater than one rather than silently ignoring it. +A playbook MUST execute its graph exactly once per run. The engine MUST reject a requested iteration count greater than one rather than silently ignoring it. A run interrupted at an approval gate under [[RFC-0002:C-PLAYBOOK-APPROVAL]] and continued by a later process is one run, not two: the obligation is on the graph, not on the process, and the tasks a prior process settled MUST NOT be dispatched again. A playbook's verdict MUST be invalid when any required task settled failing, was truncated, or was left blocked, and when a cost or wall-clock ceiling was exhausted, whatever else the run reports. Otherwise the verdict MUST be valid when every required task settled passing, and when a task declared early completion. A required task left undispatched by early completion MUST NOT affect the verdict; a required task that had already failed MUST invalidate the run whatever a later early-completion signal says, so that a concurrent task cannot launder a failure into a success. The verdict MUST NOT depend on the outcome of any advisory task or of any epilogue task. -The run MUST exit zero if and only if its verdict is valid, and MUST exit nonzero otherwise. This is a stronger obligation than the task lane's, where exit zero means only that the run completed. The exit code alone does not say why a run ended, so the shutdown outcome MUST distinguish an exhausted graph, an early completion, a failure, an operator stop, and an exhausted ceiling. That outcome MUST be drawn from the vocabulary [[RFC-0001:C-WIRE]] defines for the shutdown event; the value that vocabulary carries for early completion is "complete". +The run MUST exit zero if and only if its verdict is valid, and MUST exit nonzero otherwise. A run that suspended at an approval gate is the one exception: it has reached no verdict, MUST exit zero, and MUST be distinguished by its shutdown outcome rather than by its exit code, because the alternative is to report a run that is merely waiting as one that failed. This is a stronger obligation than the task lane's, where exit zero means only that the run completed. The exit code alone does not say why a run ended, so the shutdown outcome MUST distinguish an exhausted graph, an early completion, a failure, an operator stop, an exhausted ceiling, and a suspension. That outcome MUST be drawn from the vocabulary [[RFC-0001:C-WIRE]] defines for the shutdown event; the value that vocabulary carries for early completion is "complete". Any task MAY declare early completion by returning the boolean field "complete" as true in its output, optionally alongside a "reason" string. Both names are reserved, and a task MUST NOT name either in its declared outputs: a declared output must be present on every passing attempt, whereas these appear only on the attempt that ends the run, so declaring them would fail every ordinary run. The engine MUST reject a graph that declares either. The engine MUST then stop dispatching further tasks, MUST record the reason where one was given, and MUST NOT treat undispatched tasks as blocked. Determining that there is nothing to do is a successful outcome, and a playbook that cannot say so has no way to end a scheduled run quietly. The shutdown outcome says how dispatch stopped and the verdict says whether the run succeeded; they are independent axes, so a run whose dispatch stopped on early completion after a required task had already failed records the early-completion outcome and an invalid verdict. diff --git a/gov/rfc/RFC-0002/clauses/C-PLAYBOOK-RESUME.toml b/gov/rfc/RFC-0002/clauses/C-PLAYBOOK-RESUME.toml index a3d1eff3..18fab27e 100644 --- a/gov/rfc/RFC-0002/clauses/C-PLAYBOOK-RESUME.toml +++ b/gov/rfc/RFC-0002/clauses/C-PLAYBOOK-RESUME.toml @@ -13,6 +13,8 @@ A resumed playbook MUST NOT re-dispatch a task that already settled. The engine A resumed playbook MUST reconstruct its workspace from the pristine checkout plus the declared file outputs of every folded task that settled passing, staged under the rules in [[RFC-0002:C-TASK-FILES]]. Files captured from a task that settled failing MUST NOT be restored into the workspace; they reach a dependent joining "settled" through its staged inputs only. It MUST NOT attempt to reconstruct the workspace as some particular task left it: when concurrent tasks were in flight at the interruption, no such state is well defined, and a task's declared output is the only part of its work the contract ever promised would survive. -Spend and elapsed time recorded before the interruption MUST count against the ceilings on resumption. A crash MUST NOT reset either. +A run continued after a suspension at an approval gate MUST instead restore the workspace as the suspending process left it. The rule above exists because a crash can strike with concurrent tasks in flight, leaving no well-defined state to restore; a gate is reached only when the tasks before it have settled, so there is exactly one such state and rebuilding a weaker approximation of it would discard work the run had already paid for. A resumed run MUST fall back to the reconstruction above when the suspending process left no usable snapshot, rather than refusing to continue. + +Spend and elapsed time recorded before the interruption MUST count against the ceilings on resumption. A crash MUST NOT reset either. Time a prior process spent waiting at a gate MUST NOT count, under [[RFC-0002:C-PLAYBOOK-APPROVAL]]. A resumed run MUST NOT re-dispatch the main graph when the folded results cover every main-graph task, and equally when any folded result declared early completion. Early completion leaves tasks undispatched by design, so a resume that only checked for full coverage would dispatch them again on every restart. It MUST still dispatch any epilogue task that has no folded result, under the conditions in [[RFC-0002:C-PLAYBOOK-LANE]]. The run's summary, shutdown, and publication MUST each happen exactly once across the original run and all of its resumptions.""" diff --git a/gov/rfc/RFC-0002/clauses/C-PLAYBOOK-SHAPE.toml b/gov/rfc/RFC-0002/clauses/C-PLAYBOOK-SHAPE.toml index d779f5fd..c290d678 100644 --- a/gov/rfc/RFC-0002/clauses/C-PLAYBOOK-SHAPE.toml +++ b/gov/rfc/RFC-0002/clauses/C-PLAYBOOK-SHAPE.toml @@ -9,8 +9,8 @@ since = "0.1.0" [content] text = """ -A playbook's graph says what runs before what, and what must pass before what runs. It does not say how many times anything runs. The graph is dispatched once, so a task that must retry, poll, or converge does that inside its own execution, where the author's command or the agent's own turn loop owns the repetition. A repair loop is therefore one task, not an edge back into the graph, and the engine never sees the iteration at all. +A playbook's graph says what runs before what, and what must pass before what runs. It does not say how many times anything runs. An approval gate under [[RFC-0002:C-PLAYBOOK-APPROVAL]] is the one node that runs nothing at all: it is an edge condition given a name, so that "wait for a person here" is something the graph can express and a reader can see, rather than something buried inside a task that also does work. The graph is dispatched once, so a task that must retry, poll, or converge does that inside its own execution, where the author's command or the agent's own turn loop owns the repetition. A repair loop is therefore one task, not an edge back into the graph, and the engine never sees the iteration at all. This is also why an item set discovered at run time cannot become new nodes. The only two things a run can do with such a set are handle it inside one task or emit it as asks under [[RFC-0002:C-ASKS]], and which one is right turns on whether the items deserve their own budget, isolation, and verdict. -The payoff is that the compiled graph is fully renderable before any spend and identical across runs of the same pack at the same parameters: the node set and the edges between them do not depend on what a task finds. Which of those nodes are dispatched still varies. [[RFC-0002:C-PLAYBOOK-LANE]] leaves tasks undispatched after an early completion, and [[RFC-0002:C-PLAYBOOK-CAPS]] truncates the plan or skips an advisory subtree when a capability is unavailable. Dispatch order among concurrent tasks is not fixed either. What is fixed is the shape. [[RFC-0002:C-PLAYBOOK-LANE]] states the once-per-run obligation; this clause records what follows from it for an author deciding where to put a loop.""" +The payoff is that the compiled graph is fully renderable before any spend and identical across runs of the same pack at the same parameters: the node set and the edges between them do not depend on what a task finds. Which of those nodes are dispatched still varies. [[RFC-0002:C-PLAYBOOK-LANE]] leaves tasks undispatched after an early completion, and [[RFC-0002:C-PLAYBOOK-CAPS]] truncates the plan or skips an advisory subtree when a capability is unavailable. Dispatch order among concurrent tasks is not fixed either, and a gate may pause dispatch for as long as a person takes, or split it across two processes when the run suspends. What is fixed is the shape. [[RFC-0002:C-PLAYBOOK-LANE]] states the once-per-run obligation; this clause records what follows from it for an author deciding where to put a loop.""" diff --git a/gov/rfc/RFC-0002/rfc.toml b/gov/rfc/RFC-0002/rfc.toml index c5134a96..bfe773d3 100644 --- a/gov/rfc/RFC-0002/rfc.toml +++ b/gov/rfc/RFC-0002/rfc.toml @@ -29,6 +29,7 @@ clauses = [ "clauses/C-PLAYBOOK-RESUME.toml", "clauses/C-REPORTS.toml", "clauses/C-SETTLED-JOIN.toml", + "clauses/C-PLAYBOOK-APPROVAL.toml", ] [[changelog]] From 7b83eacf7eaf177863d059c198e68f5d4b042234 Mon Sep 17 00:00:00 2001 From: Will Eaton Date: Fri, 4 Sep 2026 15:52:44 -0400 Subject: [PATCH 06/11] Propose container isolation for deterministic tasks A command task runs sh -c in the loop pod with whatever toolchain the loop image baked, so a pack needing another one either grows the shared image or vendors a build. Nothing new is needed to fix that. Isolation is a one-variant enum, the sandbox module already moves a workdir in and named files out for agent turns, ComputeDriver already picks nested podman or a sibling pod, and emits/emits_files already fix the only channel out of a task, which is what makes where it ran a substrate detail. The proposal is an optional image on command and evaluate, dispatched as a second isolation variant. Proposed, not decided, and sequenced after the gate stack: it wants the pod-dispatch path the controller half is about to change. Assisted-by: Claude --- docs/SUMMARY.md | 1 + .../adr/0030-container-isolation-for-tasks.md | 65 +++++++++++++++++++ docs/adr/index.md | 1 + ...ner-isolation-for-deterministic-tasks.toml | 63 ++++++++++++++++++ 4 files changed, 130 insertions(+) create mode 100644 docs/adr/0030-container-isolation-for-tasks.md create mode 100644 gov/adr/ADR-0026-container-isolation-for-deterministic-tasks.toml diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index a877c2b4..8aba9225 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -55,3 +55,4 @@ - [ADR 0026: The no-judge task lane](./adr/0026-no-judge-task-lane.md) - [ADR 0028: Retire the wide tournament](./adr/0028-retire-the-wide-tournament.md) - [ADR 0029: Plan-authored approval gates on an event-sourced loop](./adr/0029-approval-gates.md) + - [ADR 0030: Container isolation for deterministic tasks](./adr/0030-container-isolation-for-tasks.md) diff --git a/docs/adr/0030-container-isolation-for-tasks.md b/docs/adr/0030-container-isolation-for-tasks.md new file mode 100644 index 00000000..9f614afb --- /dev/null +++ b/docs/adr/0030-container-isolation-for-tasks.md @@ -0,0 +1,65 @@ +# ADR 0030: Container isolation for deterministic tasks + +**Status:** Proposed (2026-09-04). Governance source: `gov/adr/ADR-0026`. +**Date:** 2026-09-04 +**Related:** [ADR-0029](./0029-approval-gates.md) (the pod-dispatch path this waits on), +[ADR-0010](./0010-candidate-portfolios-and-search.md) (the second-runner mistake to avoid), +[work graphs](../work-graphs.md) + +## Context + +A `command` task runs `sh -c` in the run's workspace: in the loop pod, as the loop's user, on +the loop's network, with whatever toolchain the loop image baked. A pack needing a different +toolchain can either get it into the shared loop image or vendor it into the pack. The first +makes the loop image the union of every domain's dependencies; the second makes every pack carry +a build. + +The plan contract already fixes what a task hands back: the fields it declares in `emits`, and +the workspace files it declares in `emits_files`. Nothing else crosses. That is exactly what +makes *where* a task ran a substrate detail rather than a contract one. + +The engine already knows how to run work in a container, twice over. `isolated = true` means +"run somewhere disposable, keep only the declared output" — but `Isolation` is a one-variant +enum whose only member is a git worktree on the same filesystem, image, user and network as the +loop. Separately, an agent turn runs in a real container: `ComputeDriver` picks podman nested in +the loop pod or a sibling pod in-cluster, and the sandbox module already moves a workdir in and +named files out. + +The boundary exists, the transfer exists, and the output contract that makes substitution safe +exists. What is missing is a deterministic task's ability to ask for it. + +## Proposal + +Give `command` and `evaluate` an optional `image`. Supplied, the task runs in a container from +that image, receiving the staged workspace and returning its declared JSON and files. Omitted, +nothing changes — the task runs on the loop pod as today, and every existing pack renders +byte-identically. + +Model it as a second variant of the isolation enum, not an independent flag, so "where does this +run" stays one question with one dispatch point. Placement reuses `ComputeDriver`, so a pack +never names a substrate. The image pins to a digest at render time like every other image the +renderer emits. + +## Consequences + +A pack brings its own toolchain, so the loop image stops accreting one. A slow, flaky, or +hostile command cannot corrupt the shared workspace or reach the loop pod's network. The +declared output stops being a convention and becomes the enforced boundary it was always +described as. + +Against that: a container start per task, image pull as a new failure mode, and a workspace +round-trip that makes a file-heavy task slower than a worktree. `emits_files` becomes +load-bearing — a task that forgets to declare a file silently loses it, where today it would +have left it in the shared workspace and a dependent would have found it anyway. + +The risk worth naming is two dispatchers for one task kind. Landing this as a parallel path +beside the shell runner, rather than a second variant behind one dispatch point, repeats what +the wide tournament cost: a second runner over the same type, and a second thing every later +change has to be taught. + +Open: whether an imaged task may share the workspace instead of being isolated; which secrets +and environment reach it; and whether its changes join the run's git memory, which the playbook +lane commits only for a task that settles passing. + +This wants the pod-dispatch path the approval-gate work is about to change in the controller. It +should be specified after that stack merges, not stacked on it. diff --git a/docs/adr/index.md b/docs/adr/index.md index 1175b1c6..e1308128 100644 --- a/docs/adr/index.md +++ b/docs/adr/index.md @@ -32,3 +32,4 @@ Expand this section in the sidebar to browse the full list. | [0027](./0027-measurement-sessions.md) | Measurement sessions, one warm engine and many observations | Proposed | | [0028](./0028-retire-the-wide-tournament.md) | Retire the wide tournament | Implemented | | [0029](./0029-approval-gates.md) | Plan-authored approval gates on an event-sourced loop | Implemented | +| [0030](./0030-container-isolation-for-tasks.md) | Container isolation for deterministic tasks | Proposed | diff --git a/gov/adr/ADR-0026-container-isolation-for-deterministic-tasks.toml b/gov/adr/ADR-0026-container-isolation-for-deterministic-tasks.toml new file mode 100644 index 00000000..cc6d055d --- /dev/null +++ b/gov/adr/ADR-0026-container-isolation-for-deterministic-tasks.toml @@ -0,0 +1,63 @@ +#:schema ../schema/adr.schema.json + +[govctl] +id = "ADR-0026" +title = "Container isolation for deterministic tasks" +status = "proposed" +date = "2026-09-04" +refs = [ + "ADR-0010", + "ADR-0025", + "RFC-0001", + "RFC-0002", +] + +[content] +context = """ +A `command` task runs `sh -c` in the run's workspace, in the loop pod, as the loop's user, on the loop's network, with whatever toolchain the loop image baked. A pack that needs a different toolchain has two options today: get it into the shared loop image, or vendor it into the pack. The first makes the loop image the union of everything every domain has ever needed; the second makes every pack carry a build. + +The plan contract already says what a task is allowed to hand back: the fields it declares in `emits`, and the workspace files it declares in `emits_files`, staged for dependents under [[RFC-0002:C-TASK-FILES]]. Nothing else crosses. That is what makes where a task ran a substrate detail rather than a contract one. + +The engine also already knows how to run work in a container. `isolated = true` means "run somewhere disposable, keep only the declared output", and `Isolation` is a one-variant enum whose only member is a git worktree, on the same filesystem, image, user, and network as the loop. Separately, an agent turn already runs in a real container: `ComputeDriver` chooses podman nested in the loop pod or a sibling pod in-cluster, and the sandbox module already moves a workdir in and named files out. + +So the boundary exists, the transfer exists, and the output contract that makes substitution safe exists. What is missing is the ability for a deterministic task to ask for it. +""" +decision = """ +Proposed, not decided. + +Give `command` and `evaluate` an optional `image`. Supplied, the task runs in a container built from that image, receiving the staged workspace and returning its declared JSON and files. Omitted, nothing changes: the task runs on the loop pod exactly as today, and every existing pack renders byte-identically. + +Model it as a second variant of the existing isolation enum rather than as an independent flag, so "where does this task run" stays one question with one dispatch point. A separate flag alongside `isolated` would admit combinations that have to be defined and then defended. + +Placement reuses `ComputeDriver`: nested under podman when the loop runs on a laptop or a single host, a sibling pod in-cluster. A pack must not have to know which, for the same reason a pack does not name a cluster today. The image is pinned to a digest at render time like every other image the renderer emits. +""" +consequences = """ +Positive: a pack brings its own toolchain, so the loop image stops accreting one. A command that is slow, flaky, or hostile cannot corrupt the shared workspace or reach the loop pod's network. The task's declared output stops being a convention and becomes the enforced boundary it was always described as. + +Negative: a container start per task, an image pull as a new failure mode, and a workspace round-trip that makes a file-heavy task slower than a worktree. `emits_files` becomes load-bearing: a task that forgets to declare a file silently loses it, where today it would have left it in the shared workspace and a dependent would have found it anyway. That is a behavior change for anyone who was relying on the shared filesystem without saying so. + +The risk worth naming: two dispatchers for the same task kind. If this lands as a parallel path beside the shell runner rather than as a second variant behind one dispatch point, it repeats what [[ADR-0010]]'s wide tournament cost: a second runner over the same type, a second thing every later change has to be taught. + +Open questions this ADR does not answer: whether a task with an image may share the workspace rather than be isolated; which secrets and environment reach it, given that the loop pod's are not automatically the task's; and whether its changes join the run's git memory, which [[RFC-0002:C-PLAYBOOK-LANE]] commits only for a task that settles passing. + +Sequencing: this wants the pod-dispatch path that the approval-gate work in [[ADR-0025]] is about to change in the controller. It should be specified after that stack merges, not stacked on top of it. +""" + +[[content.alternatives]] +text = "An optional image on the existing deterministic task kinds, dispatched as a second isolation variant" +status = "considered" + +[[content.alternatives]] +text = "A distinct container task kind alongside command and evaluate" +status = "rejected" +rejection_reason = "Where a task runs is a substrate choice, not a different kind of work; a third kind would duplicate the whole command surface (run, emits, emits_files, over, join) to change one thing about it." + +[[content.alternatives]] +text = "Keep one loop image and add what packs need to it" +status = "rejected" +rejection_reason = "It is the status quo, and it makes the image the union of every domain's dependencies while giving a deterministic task no isolation from the loop it runs beside." + +[[content.alternatives]] +text = "Let a task shell out to podman itself" +status = "rejected" +rejection_reason = "Puts placement, image pinning, and cleanup in the pack, and requires handing every pack a container socket, which is a larger grant than the isolation it buys." From 314e730dbf75131061cd67343aa854a475f9f254 Mon Sep 17 00:00:00 2001 From: Will Eaton Date: Fri, 4 Sep 2026 16:29:14 -0400 Subject: [PATCH 07/11] Type the gate's errors and box the Starlark task value CI runs clippy from a cold cache with -D warnings; a warm local cache had been replaying stale clean results, so four disallowed anyhow::bail sites, a needless clone, and a large_enum_variant went unseen. The gate's failures become thiserror variants like every other module's: NoBridgeReply, SuspendNeedsManifest, StoppedWhileParked, and the escaping snapshot entry, the failed bundle, and the failed git run on SuspendError. The approve task kind grew Task past the point where holding one inline in the Starlark value enum was reasonable, so that variant is boxed, the same way the loop's decided step already carries its payload. Assisted-by: Claude --- crucible/src/control.rs | 11 ++++++++++- crucible/src/plan/cli.rs | 26 ++++++++++++++++++++++---- crucible/src/plan/starlark.rs | 6 +++--- crucible/src/plan/starlark/globals.rs | 4 ++-- crucible/src/suspend.rs | 23 ++++++++++++++++------- 5 files changed, 53 insertions(+), 17 deletions(-) diff --git a/crucible/src/control.rs b/crucible/src/control.rs index 99b07004..a4011436 100644 --- a/crucible/src/control.rs +++ b/crucible/src/control.rs @@ -304,6 +304,12 @@ impl ControlState { } } +#[derive(Debug, thiserror::Error)] +#[error("the control bridge at {addr} sent no reply")] +pub(crate) struct NoBridgeReply { + pub addr: String, +} + /// Send one command line to a live run's control bridge and return its reply. The shape is /// exactly what [`parse_request`] accepts, so `crucible approve`, `crucible deny`, and the PR /// watcher all speak to the bridge the same way. @@ -329,7 +335,10 @@ pub(crate) fn send_command(addr: &str, command: &Value) -> Result { return Ok(v); } } - anyhow::bail!("the control bridge at {addr} sent no reply") + Err(NoBridgeReply { + addr: addr.to_string(), + } + .into()) } /// Start a detached TCP bridge thread. The listener binds to all interfaces so it works diff --git a/crucible/src/plan/cli.rs b/crucible/src/plan/cli.rs index 2ca4fd28..c8dc5571 100644 --- a/crucible/src/plan/cli.rs +++ b/crucible/src/plan/cli.rs @@ -16,6 +16,18 @@ struct MermaidRenderFailed { detail: String, } +/// A gate asked to suspend from a run with no state dir to snapshot into. +#[derive(Debug, thiserror::Error)] +#[error("a gate can only suspend a --manifest run")] +struct SuspendNeedsManifest; + +/// The run was stopped by an operator while parked on a gate; the gate is still open. +#[derive(Debug, thiserror::Error)] +#[error("stopped while parked on gate {task}")] +struct StoppedWhileParked { + task: String, +} + #[derive(Debug, thiserror::Error)] #[error("plan did not reach a valid verdict ({exit})")] struct NoValidVerdict { @@ -829,7 +841,7 @@ pub fn run( let mut out = execute_from( &plan, &substrate, - exec_cfg.clone(), + exec_cfg, runner.as_mut(), prior, &mut on_result, @@ -869,7 +881,7 @@ pub fn run( } gate::Waited::Suspend => { let Some(paths) = &evidence else { - anyhow::bail!("a gate can only suspend a --manifest run"); + return Err(SuspendNeedsManifest.into()); }; match gate::suspend(paths, &plan, &open) { Ok(()) => { @@ -922,7 +934,10 @@ pub fn run( }, ); } - anyhow::bail!("stopped while parked on gate {}", open.task); + return Err(StoppedWhileParked { + task: open.task.0.clone(), + } + .into()); } gate::Parked::TimedOut => { let resolution = crucible_contract::GateResolution::timeout(); @@ -947,7 +962,10 @@ pub fn run( }, ); } - anyhow::bail!("stopped while parked on gate {}", open.task); + return Err(StoppedWhileParked { + task: open.task.0.clone(), + } + .into()); } } }; diff --git a/crucible/src/plan/starlark.rs b/crucible/src/plan/starlark.rs index 9f531126..f741e79a 100644 --- a/crucible/src/plan/starlark.rs +++ b/crucible/src/plan/starlark.rs @@ -385,7 +385,7 @@ enum Value { /// A dictionary, ordered by key so a rendered prompt is the same on every compile. Map(BTreeMap), List(Vec), - Task(Task), + Task(Box), /// `producer.field`, already checked against the producer's declared emits. Output(OutputRef), Session(SessionDecl), @@ -1311,7 +1311,7 @@ fn constructor( count: constructed, }); } - Ok(Value::Task(task)) + Ok(Value::Task(Box::new(task))) } fn no_unknown_kwargs(function: &str, named: &BTreeMap) -> Result<()> { @@ -1377,7 +1377,7 @@ fn task_list(function: &str, tasks: Vec) -> Result> { tasks .into_iter() .map(|task| match task { - Value::Task(task) => Ok(task), + Value::Task(task) => Ok(*task), _ => Err(CompileError::TaskListEntryNotTask { function: function.to_owned(), }), diff --git a/crucible/src/plan/starlark/globals.rs b/crucible/src/plan/starlark/globals.rs index 455cb8e2..aac842c6 100644 --- a/crucible/src/plan/starlark/globals.rs +++ b/crucible/src/plan/starlark/globals.rs @@ -385,7 +385,7 @@ fn convert_at(value: Value<'_>, depth: usize) -> dsl::Result { return Ok(dsl::Value::External(external.0.clone())); } if let Some(task) = TaskValue::from_value(value) { - return Ok(dsl::Value::Task(task.0.clone())); + return Ok(dsl::Value::Task(Box::new(task.0.clone()))); } if let Some(output) = OutputRefValue::from_value(value) { return output.resolve().map(dsl::Value::Output); @@ -432,7 +432,7 @@ fn alloc_at<'v>(heap: Heap<'v>, value: dsl::Value, depth: usize) -> Value<'v> { dsl::Value::External(segments) => heap.alloc(ExternalText(segments)), // A dictionary never travels back out: the constructors consume it. dsl::Value::Map(_) => Value::new_none(), - dsl::Value::Task(task) => heap.alloc(TaskValue(task)), + dsl::Value::Task(task) => heap.alloc(TaskValue(*task)), dsl::Value::Output(reference) => heap.alloc(OutputRefValue { declared: vec![reference.field.0.clone()], reference, diff --git a/crucible/src/suspend.rs b/crucible/src/suspend.rs index 0aa2de58..f2e3fb1a 100644 --- a/crucible/src/suspend.rs +++ b/crucible/src/suspend.rs @@ -64,6 +64,12 @@ pub enum SuspendError { }, #[error("pod {of} left no {kind} artifact")] MissingArtifact { kind: ArtifactKind, of: String }, + #[error("snapshot entry {} escapes the state dir", .path.display())] + EscapingEntry { path: PathBuf }, + #[error("git bundle failed: {stderr}")] + Bundle { stderr: String }, + #[error("git {args} failed: {stderr}")] + Git { args: String, stderr: String }, } /// Attempts a drop-box fetch gets before the resume gives up; the controller may still be @@ -148,7 +154,7 @@ pub fn restore(snapshot_gz: &[u8], state: &Path, workspace: &Path) -> Result Result>> { if stderr.contains("Refusing to create empty bundle") { return Ok(None); } - anyhow::bail!("git bundle failed: {}", stderr.trim()); + return Err(SuspendError::Bundle { + stderr: stderr.trim().to_string(), + } + .into()); } Ok(Some(out.stdout)) } @@ -284,11 +293,11 @@ fn run_git(workspace: &Path, args: &[&str]) -> Result<()> { .output() .with_context(|| format!("running git {}", args.join(" ")))?; if !out.status.success() { - anyhow::bail!( - "git {} failed: {}", - args.join(" "), - String::from_utf8_lossy(&out.stderr).trim() - ); + return Err(SuspendError::Git { + args: args.join(" "), + stderr: String::from_utf8_lossy(&out.stderr).trim().to_string(), + } + .into()); } Ok(()) } From c341800a4e1a295743689018aa09e3a768308fed Mon Sep 17 00:00:00 2001 From: Will Eaton Date: Fri, 4 Sep 2026 16:37:36 -0400 Subject: [PATCH 08/11] Draw the loop's state machine from the machine itself The loop's decisions became named types when they moved out of the driver, so the state chart is now derivable rather than something to keep redrawing by hand. `crucible loop-reference` prints it, scripts/loop-docs.sh writes docs/loop-machine.md, and a pre-commit hook regenerates the page when machine.rs changes, the same shape the DSL reference already uses. Drift is a compile error, not a stale diagram. Adding a LoopExit breaks shutdown_reason, adding an IterStep breaks step_label, and the coverage tests then hold the new variant to reaching the chart. A third test walks the edges from Head so a state cannot be drawn without a way to reach it. The exits on the chart are the shutdown tokens the session log carries, so a reader can match a node to a run's last line. Assisted-by: Claude --- .pre-commit-config.yaml | 7 + Cargo.lock | 1 + crucible-contract/Cargo.toml | 1 + crucible-contract/src/gate.rs | 121 +++++++++++++- crucible-contract/src/lib.rs | 5 +- crucible/src/control.rs | 47 ++++-- crucible/src/deploy/render/kube.rs | 12 +- crucible/src/machine.rs | 253 +++++++++++++++++++++++++++++ crucible/src/main.rs | 19 ++- crucible/src/plan/cli.rs | 11 +- crucible/src/plan/gate.rs | 168 ++++++++++--------- crucible/src/plan/gate_host.rs | 2 +- crucible/src/plan/harness.rs | 2 +- crucible/src/plan/runner.rs | 4 +- crucible/src/run.rs | 8 + docs/SUMMARY.md | 1 + docs/loop-machine.md | 49 ++++++ scripts/loop-docs.sh | 20 +++ 18 files changed, 612 insertions(+), 119 deletions(-) create mode 100644 docs/loop-machine.md create mode 100755 scripts/loop-docs.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b7329c86..70593585 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,6 +15,13 @@ repos: pass_filenames: false files: ^(crucible/src/plan/starlark|docs/dsl-reference\.md) + - id: loop-docs + name: regenerate the loop state machine page + entry: scripts/loop-docs.sh + language: system + pass_filenames: false + files: ^(crucible/src/machine\.rs|docs/loop-machine\.md) + - id: gov-render name: render RFCs from the governance SSOT entry: govctl render rfc diff --git a/Cargo.lock b/Cargo.lock index 8fc380b0..18213aaf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1528,6 +1528,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.9", + "thiserror", ] [[package]] diff --git a/crucible-contract/Cargo.toml b/crucible-contract/Cargo.toml index 768bd02a..a648dbdf 100644 --- a/crucible-contract/Cargo.toml +++ b/crucible-contract/Cargo.toml @@ -8,6 +8,7 @@ edition.workspace = true [dependencies] serde.workspace = true serde_json.workspace = true +thiserror.workspace = true # Tier 2 content-addressing: engine and controller MUST compute the same `sha256:` digest, # so the one spelling lives here where both sides call it. sha2 = "0.10" diff --git a/crucible-contract/src/gate.rs b/crucible-contract/src/gate.rs index 35e2f96d..0fcb49a0 100644 --- a/crucible-contract/src/gate.rs +++ b/crucible-contract/src/gate.rs @@ -1,9 +1,101 @@ //! Approval gates: the wire shapes an `approve(...)` plan task, its resolution, and the //! artifacts around a parked run share between the engine and the controller. -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; use std::fmt; +/// A non-empty approval actor name bounded to the admission-key size. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Approver(String); + +/// Why an approval actor name cannot cross the wire. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum ApproverError { + #[error("approver {value:?} is empty")] + Empty { value: String }, + #[error("approver {value:?} is {bytes} bytes, maximum is {max} bytes")] + TooLong { + value: String, + bytes: usize, + max: usize, + }, +} + +impl Approver { + /// Return the canonical actor name without its validation wrapper. + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for Approver { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl TryFrom for Approver { + type Error = ApproverError; + + fn try_from(value: String) -> Result { + let value = value.trim(); + if value.is_empty() { + return Err(ApproverError::Empty { + value: value.to_string(), + }); + } + if value.len() > crate::admission::MAX_KEY_LEN { + return Err(ApproverError::TooLong { + value: value.to_string(), + bytes: value.len(), + max: crate::admission::MAX_KEY_LEN, + }); + } + Ok(Approver(value.to_string())) + } +} + +impl TryFrom<&str> for Approver { + type Error = ApproverError; + + fn try_from(value: &str) -> Result { + Self::try_from(value.to_string()) + } +} + +impl std::str::FromStr for Approver { + type Err = ApproverError; + + fn from_str(value: &str) -> Result { + Self::try_from(value) + } +} + +impl From for String { + fn from(value: Approver) -> Self { + value.0 + } +} + +impl Serialize for Approver { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> Deserialize<'de> for Approver { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + Self::try_from(value).map_err(de::Error::custom) + } +} + /// Where a gate's resolution comes from, as the engine resolved it when the gate was reached. /// Rides `ApprovalWait.source` on the session log and the `approval-waits` artifact. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -103,14 +195,14 @@ pub struct GateResolution { #[serde(default, skip_serializing_if = "Option::is_none")] pub reason: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub by: Option, + pub by: Option, /// The source kind that resolved it (`native`, `github_pr`, `jira`, `timeout`). #[serde(default, skip_serializing_if = "Option::is_none")] pub source: Option, } impl GateResolution { - pub fn granted(by: Option, source: &str) -> Self { + pub fn granted(by: Option, source: &str) -> Self { GateResolution { decision: GateDecision::Granted, reason: None, @@ -119,7 +211,7 @@ impl GateResolution { } } - pub fn denied(reason: impl Into, by: Option, source: &str) -> Self { + pub fn denied(reason: impl Into, by: Option, source: &str) -> Self { GateResolution { decision: GateDecision::Denied, reason: Some(reason.into()), @@ -164,7 +256,10 @@ pub fn parse_approval_arg(raw: &str) -> Result<(String, GateResolution), BadAppr return Err(bad()); } let (rest, by) = match rest.rsplit_once('@') { - Some((head, by)) if !by.trim().is_empty() => (head, Some(by.trim().to_string())), + Some((head, by)) if !by.trim().is_empty() => { + let by = Approver::try_from(by).map_err(|_| bad())?; + (head, Some(by)) + } _ => (rest, None), }; let (decision, reason) = match rest.split_once(':') { @@ -290,9 +385,9 @@ mod tests { let (_, r) = parse_approval_arg("t=denied:changes requested@alice").expect("full deny"); assert_eq!(r.decision, GateDecision::Denied); assert_eq!(r.reason.as_deref(), Some("changes requested")); - assert_eq!(r.by.as_deref(), Some("alice")); + assert_eq!(r.by.as_ref().map(Approver::as_str), Some("alice")); let (_, r) = parse_approval_arg("t=granted@bob").expect("grant with by"); - assert_eq!(r.by.as_deref(), Some("bob")); + assert_eq!(r.by.as_ref().map(Approver::as_str), Some("bob")); assert_eq!(r.reason, None); let (_, r) = parse_approval_arg("t=approve").expect("approve alias"); assert_eq!(r.decision, GateDecision::Granted); @@ -301,6 +396,18 @@ mod tests { } } + #[test] + fn approvers_reject_empty_and_overlong_values_without_truncating() { + assert!(Approver::try_from(" ").is_err()); + let value = "é".repeat(crate::admission::MAX_KEY_LEN); + assert!(Approver::try_from(value).is_err()); + let value = "a".repeat(crate::admission::MAX_KEY_LEN); + assert!(Approver::try_from(value).is_ok()); + let value = "a".repeat(crate::admission::MAX_KEY_LEN + 1); + assert!(Approver::try_from(value).is_err()); + assert_eq!(Approver::try_from(" alice ").unwrap().as_str(), "alice"); + } + #[test] fn trace_ids_are_one_per_task_per_run() { assert_eq!(gate_trace_id("run-1", "review"), "approve:run-1:review"); diff --git a/crucible-contract/src/lib.rs b/crucible-contract/src/lib.rs index b889f649..a1ff4c13 100644 --- a/crucible-contract/src/lib.rs +++ b/crucible-contract/src/lib.rs @@ -41,8 +41,9 @@ pub use ask::{Ask, AskKey, AskKeyError}; pub use envelope::{Envelope, EnvelopeKind, SCHEMA_VERSION, TERMINATION_MESSAGE_CAP, Usage}; pub use event::{AgentEvent, ModelUsage, RawStream, Tokens}; pub use gate::{ - ApprovalWaits, BadApprovalArg, GateDecision, GateResolution, GateSource, GateWait, JiraUntil, - ParkMode, PodApproval, PrUntil, gate_trace_id, parse_approval_arg, + ApprovalWaits, Approver, ApproverError, BadApprovalArg, GateDecision, GateResolution, + GateSource, GateWait, JiraUntil, ParkMode, PodApproval, PrUntil, gate_trace_id, + parse_approval_arg, }; pub use identity::{ ComponentIdentity, FORMAT_VERSION as IDENTITY_FORMAT_VERSION, RigIdentity, RunIdentity, diff --git a/crucible/src/control.rs b/crucible/src/control.rs index a4011436..d845d57f 100644 --- a/crucible/src/control.rs +++ b/crucible/src/control.rs @@ -7,6 +7,7 @@ use crate::admission::{AdmissionLedger, Admitted}; use crate::{Paths, STOP}; use anyhow::{Context, Result}; +use crucible_contract::Approver; use crucible_contract::admission::{ AdmissionKey, AdmissionOutcome, AdmittedInput, MAX_KEY_LEN, SteerSource, }; @@ -590,7 +591,7 @@ fn apply_command(req: ControlRequest, state: &ControlState, ledger: &AdmissionLe let _ = ledger.settle( key, AdmissionOutcome::Applied, - &format!("gate granted{}", by_suffix(by.as_deref())), + &format!("gate granted{}", by_suffix(by.as_ref())), ); json!({"ok": true, "cmd": "approve", "trace_id": trace, "by": by}) }); @@ -618,7 +619,7 @@ fn apply_command(req: ControlRequest, state: &ControlState, ledger: &AdmissionLe let _ = ledger.settle( key, AdmissionOutcome::Applied, - &format!("gate denied: {reason}{}", by_suffix(by.as_deref())), + &format!("gate denied: {reason}{}", by_suffix(by.as_ref())), ); json!({"ok": true, "cmd": "deny", "trace_id": trace, "reason": reason, "by": by}) }, @@ -761,7 +762,7 @@ fn stop_the_run(id: Option, input: AdmittedInput, ledger: &Admissi } /// Settle the admission a newly-armed one displaced as superseded. -fn by_suffix(by: Option<&str>) -> String { +fn by_suffix(by: Option<&Approver>) -> String { by.map(|who| format!(" by {who}")).unwrap_or_default() } @@ -813,11 +814,11 @@ enum ControlCommand { regime: String, }, Approve { - by: Option, + by: Option, }, Deny { reason: String, - by: Option, + by: Option, }, Status, /// Subscribe to the live session broadcast, replaying retained lines with `seq > from_seq` @@ -871,14 +872,16 @@ fn parse_id(raw: Option<&Value>) -> std::result::Result, St Ok(Some(AdmissionKey::new(id))) } -/// Who sent an approve/deny, when the sender says. Free text, bounded like a key. -fn parse_by(value: &Value) -> Option { - value - .get("by") - .and_then(Value::as_str) - .map(str::trim) - .filter(|by| !by.is_empty()) - .map(|by| by.chars().take(MAX_KEY_LEN).collect()) +fn parse_by(value: &Value) -> std::result::Result, String> { + let Some(raw) = value.get("by").filter(|value| !value.is_null()) else { + return Ok(None); + }; + let raw = raw + .as_str() + .ok_or_else(|| format!("by must be a string, got {raw:?}"))?; + Approver::try_from(raw) + .map(Some) + .map_err(|error| error.to_string()) } fn command_from_name(cmd: &str, value: &Value) -> std::result::Result { @@ -888,7 +891,7 @@ fn command_from_name(cmd: &str, value: &Value) -> std::result::Result Ok(ControlCommand::Pause), "resume" => Ok(ControlCommand::Resume), "approve" => Ok(ControlCommand::Approve { - by: parse_by(value), + by: parse_by(value)?, }), "deny" => { // Reason is optional (an operator may just reject); default to a generic note. @@ -900,7 +903,7 @@ fn command_from_name(cmd: &str, value: &Value) -> std::result::Result Ok(ControlCommand::Status), @@ -1304,6 +1307,20 @@ mod tests { ); } + #[test] + fn rejects_invalid_approver_values_at_the_control_boundary() { + let ControlCommand::Approve { by: Some(by) } = + parse_command(r#"{"cmd":"approve","by":" alice "}"#).unwrap() + else { + panic!("the approver should be present"); + }; + assert_eq!(by.as_str(), "alice"); + assert!(parse_command(r#"{"cmd":"approve","by":42}"#).is_err()); + let long = "a".repeat(crucible_contract::admission::MAX_KEY_LEN + 1); + let command = serde_json::json!({"cmd": "approve", "by": long}); + assert!(parse_command(&command.to_string()).is_err()); + } + #[test] fn deny_drops_a_recorded_pending_regime_so_approve_cant_resurrect_it() { let (l, _path) = ledger("deny"); diff --git a/crucible/src/deploy/render/kube.rs b/crucible/src/deploy/render/kube.rs index bcc9792d..9e89ed65 100644 --- a/crucible/src/deploy/render/kube.rs +++ b/crucible/src/deploy/render/kube.rs @@ -194,7 +194,7 @@ fn approval_arg(trace: &str, resolution: &crucible_contract::GateResolution) -> arg.push(':'); arg.push_str(&reason.replace('@', " ")); } - if let Some(by) = resolution.by.as_deref().filter(|b| !b.is_empty()) { + if let Some(by) = resolution.by.as_ref().map(|by| by.as_str()) { arg.push('@'); arg.push_str(by); } @@ -4469,7 +4469,7 @@ mod tests { crucible_contract::GateResolution { decision: crucible_contract::GateDecision::Granted, reason: None, - by: Some("wseaton".to_string()), + by: Some("wseaton".parse().unwrap()), source: Some("github_pr".to_string()), }, ), @@ -4516,12 +4516,16 @@ mod tests { &crucible_contract::GateResolution { decision: crucible_contract::GateDecision::Denied, reason: Some("ask ops@example.com".to_string()), - by: Some("real-denier".to_string()), + by: Some("real-denier".parse().unwrap()), source: None, }, ); let (_, back) = crucible_contract::parse_approval_arg(&arg).expect("parses"); - assert_eq!(back.by.as_deref(), Some("real-denier"), "{arg}"); + assert_eq!( + back.by.as_ref().map(|by| by.as_str()), + Some("real-denier"), + "{arg}" + ); assert_eq!(back.reason.as_deref(), Some("ask ops example.com"), "{arg}"); } diff --git a/crucible/src/machine.rs b/crucible/src/machine.rs index 34e8614d..fade6df2 100644 --- a/crucible/src/machine.rs +++ b/crucible/src/machine.rs @@ -437,6 +437,165 @@ impl Machine { } } +/// One transition in the rendered state chart. `label` is the condition that takes the loop +/// from `from` to `to`, in the order the host actually evaluates them. +struct Edge { + from: &'static str, + to: &'static str, + label: &'static str, +} + +/// The loop's transitions, in the order [`crate::loop_driver`] evaluates them. Hand-ordered +/// because the order is the host's, not the machine's; the coverage tests below hold every +/// variant of [`IterStep`] and [`LoopExit`] present, so a new one cannot land undrawn. +const EDGES: &[Edge] = &[ + Edge { + from: "Head", + to: "ApprovalPark", + label: "a block approval is pending", + }, + Edge { + from: "ApprovalPark", + to: "Head", + label: "granted: the re-scope drain re-baselines", + }, + Edge { + from: "ApprovalPark", + to: "escalated", + label: "denied with no fallback", + }, + Edge { + from: "ApprovalPark", + to: "stopped", + label: "stop while parked", + }, + Edge { + from: "Head", + to: "DistressPark", + label: "the agent raised distress(error)", + }, + Edge { + from: "DistressPark", + to: "Head", + label: "the operator cleared the marker", + }, + Edge { + from: "DistressPark", + to: "stopped", + label: "stop, or the park timed out", + }, + Edge { + from: "Head", + to: "stopped", + label: "interrupt at the head", + }, + Edge { + from: "Head", + to: "budget", + label: "a cost or time cap was reached", + }, + Edge { + from: "Head", + to: "finished", + label: "no iterations left", + }, + Edge { + from: "Head", + to: "Iteration", + label: "otherwise: run the turn", + }, + Edge { + from: "Iteration", + to: "Decide", + label: "decided: a measured candidate", + }, + Edge { + from: "Decide", + to: "Head", + label: "kept or discarded", + }, + Edge { + from: "Decide", + to: "solved", + label: "kept and solved, early stop on", + }, + Edge { + from: "Decide", + to: "budget", + label: "a cap was reached deciding", + }, + Edge { + from: "Iteration", + to: "Head", + label: "discarded, or parked for the next head", + }, + Edge { + from: "Iteration", + to: "Iteration", + label: "never-started: re-run, bounded", + }, + Edge { + from: "Iteration", + to: "stalled", + label: "consecutive dead turns hit the bound", + }, + Edge { + from: "Iteration", + to: "escalated", + label: "escalated by the agent", + }, + Edge { + from: "Iteration", + to: "stopped", + label: "stop at the post-turn checkpoint", + }, +]; + +/// The loop's state machine as a mermaid `stateDiagram-v2`. Exits are the shutdown tokens the +/// wire carries, so a reader can match a diagram node to a run's shutdown line. +pub(crate) fn mermaid() -> String { + let mut out = String::from("stateDiagram-v2\n [*] --> Head\n"); + for edge in EDGES { + out.push_str(&format!( + " {} --> {}: {}\n", + edge.from, edge.to, edge.label + )); + } + for exit in EXITS { + out.push_str(&format!(" {} --> [*]\n", exit.shutdown_reason().0)); + } + out +} + +/// Every way the loop can end. Held complete by `every_exit_is_drawn`. +const EXITS: &[LoopExit] = &[ + LoopExit::Finished, + LoopExit::Solved, + LoopExit::Budget, + LoopExit::Stopped, + LoopExit::Escalated, + LoopExit::Stalled, +]; + +/// The published page: the chart plus what each terminal state means on the wire. +pub(crate) fn doc_page() -> String { + let mut out = String::from( + "# The loop state machine\n\n\ + \n\n\ + The scored loop's decisions live in one I/O-free machine: the host performs effects and \ + hands the results back, and the machine answers what happens next. Every edge below is a \ + variant the compiler knows about, so this page cannot drift from the binary that drew it.\n\n\ + ```mermaid\n", + ); + out.push_str(&mermaid()); + out.push_str("```\n\n## How a run ends\n\nThe terminal states are the shutdown tokens the session log carries.\n\n| Token | Meaning |\n| --- | --- |\n"); + for exit in EXITS { + let (token, reason) = exit.shutdown_reason(); + out.push_str(&format!("| `{token}` | {reason} |\n")); + } + out +} + #[cfg(test)] mod tests { use super::*; @@ -811,4 +970,98 @@ mod tests { assert!(!LoopExit::Escalated.concluded()); assert!(!LoopExit::Stalled.concluded()); } + + /// Adding a `LoopExit` breaks `shutdown_reason` first; this holds the new one to reaching + /// the published chart rather than existing only in the type. + #[test] + fn every_exit_is_drawn_and_terminal() { + let chart = mermaid(); + for exit in EXITS { + let token = exit.shutdown_reason().0; + assert!( + chart.contains(&format!("{token} --> [*]")), + "{token}: {chart}" + ); + } + assert_eq!( + EXITS.len(), + 6, + "a new exit needs an EXITS entry and an edge" + ); + } + + /// The label an iteration's outcome is drawn under. Exhaustive on purpose: a new + /// `IterStep` has to be named here before this compiles, and the test below then holds it + /// to appearing on the chart. + fn step_label(step: &IterStep) -> &'static str { + match step { + IterStep::Decided(_) => "decided", + IterStep::Discarded { .. } => "discarded", + IterStep::NeverStarted { .. } => "never-started", + IterStep::Escalated => "escalated", + IterStep::Parked(_) => "parked", + IterStep::Stopped => "stop at the post-turn checkpoint", + } + } + + /// The same for an iteration's outcomes: `step_label` is exhaustive, so a new `IterStep` + /// must be named, and naming it without drawing it fails here. + #[test] + fn every_iteration_outcome_is_drawn() { + let chart = mermaid(); + let steps = [ + IterStep::Discarded { reason: "r".into() }, + IterStep::NeverStarted { reason: "r".into() }, + IterStep::Escalated, + IterStep::Stopped, + ]; + for step in &steps { + let label = step_label(step); + assert!( + chart.contains(label), + "{label} is not on the chart: {chart}" + ); + } + // Decided and Parked carry payloads a test cannot cheaply build; their labels are + // asserted directly against the same exhaustive source. + assert!(chart.contains("decided"), "{chart}"); + assert!(chart.contains("parked"), "{chart}"); + } + + /// Every edge must start somewhere the chart can be entered from, so the diagram is one + /// connected machine rather than a pile of arrows. + #[test] + fn the_chart_has_no_unreachable_state() { + let mut reachable = vec!["Head"]; + let mut grew = true; + while grew { + grew = false; + for edge in EDGES { + if reachable.contains(&edge.from) && !reachable.contains(&edge.to) { + reachable.push(edge.to); + grew = true; + } + } + } + for edge in EDGES { + assert!( + reachable.contains(&edge.from), + "{} is drawn but never reached", + edge.from + ); + } + } + + /// The page carries the chart and the wire vocabulary a reader matches a run against. + #[test] + fn the_doc_page_carries_the_chart_and_the_shutdown_table() { + let page = doc_page(); + assert!(page.contains("```mermaid"), "{page}"); + assert!(page.contains("stateDiagram-v2"), "{page}"); + for exit in EXITS { + let (token, reason) = exit.shutdown_reason(); + assert!(page.contains(&format!("| `{token}` |")), "{page}"); + assert!(page.contains(reason), "{page}"); + } + } } diff --git a/crucible/src/main.rs b/crucible/src/main.rs index 531a17f4..4ec4020d 100644 --- a/crucible/src/main.rs +++ b/crucible/src/main.rs @@ -302,6 +302,14 @@ pub(crate) enum Cmd { #[arg(long)] once: bool, }, + /// Print the scored loop's state machine: every way an iteration ends and every way the + /// run does, as a mermaid state chart. Generated from the machine's own vocabulary, so it + /// describes the binary in hand rather than a diagram someone remembered to redraw. + LoopReference { + /// `markdown` (the published page) or `mermaid` (the bare chart). + #[arg(long, default_value = "markdown")] + format: LoopFormat, + }, /// Grant the approval gate a live run is parked on, over its control bridge. Approve { /// The live run's control-bridge address (host:port, from its `--control-port`). @@ -309,7 +317,7 @@ pub(crate) enum Cmd { control_addr: String, /// Who is approving, recorded with the grant. #[arg(long)] - by: Option, + by: Option, }, /// Deny the approval gate a live run is parked on, over its control bridge. Deny { @@ -321,7 +329,7 @@ pub(crate) enum Cmd { reason: String, /// Who is denying, recorded with the denial. #[arg(long)] - by: Option, + by: Option, }, /// Restore a suspended run's snapshot from the controller before resuming it: pulls the /// `run-session` and `run-workspace` artifacts the pod named by `CRUCIBLE_RESUME_OF` left, @@ -523,6 +531,13 @@ pub(crate) enum PlanAction { }, } +/// How `crucible loop-reference` renders the loop's state machine. +#[derive(Clone, Copy, Debug, clap::ValueEnum)] +pub(crate) enum LoopFormat { + Markdown, + Mermaid, +} + /// How `crucible plan dsl-reference` renders the DSL surface. #[derive(Clone, Copy, Debug, clap::ValueEnum)] pub(crate) enum DslFormat { diff --git a/crucible/src/plan/cli.rs b/crucible/src/plan/cli.rs index c8dc5571..5804bd5c 100644 --- a/crucible/src/plan/cli.rs +++ b/crucible/src/plan/cli.rs @@ -871,8 +871,8 @@ pub fn run( resolution.decision.as_str(), resolution .by - .as_deref() - .map(|by| format!(" by {by}")) + .as_ref() + .map(|by| format!(" by {}", by.as_str())) .unwrap_or_default() ); runner.resolve_gate(&open.trace_id, resolution); @@ -1727,7 +1727,10 @@ file = "workflow.star" GateOpts { approvals: vec![( gate_trace(), - crucible_contract::GateResolution::granted(Some("wseaton".into()), "native"), + crucible_contract::GateResolution::granted( + Some("wseaton".parse().unwrap()), + "native", + ), )], ..Default::default() }, @@ -1759,7 +1762,7 @@ file = "workflow.star" gate_trace(), crucible_contract::GateResolution::denied( "not this quarter", - Some("wseaton".into()), + Some("wseaton".parse().unwrap()), "native", ), )], diff --git a/crucible/src/plan/gate.rs b/crucible/src/plan/gate.rs index ed7e2daf..c01c6599 100644 --- a/crucible/src/plan/gate.rs +++ b/crucible/src/plan/gate.rs @@ -28,6 +28,53 @@ impl GateCtx { pub fn resolve(&mut self, trace_id: impl Into, resolution: GateResolution) { self.resolutions.insert(trace_id.into(), resolution); } + + /// Apply the held resolution, or report that the task is waiting for one. + pub fn attempt( + &self, + task: &Task, + inputs: &BTreeMap, + ) -> Attempt { + let TaskKind::Approve { + summary, + source, + timeout_secs, + } = &task.task + else { + return Attempt { + outcome: AttemptOutcome::fail(format!("task {} is not an approve task", task.name)), + cost_usd: 0.0, + }; + }; + let source = match source.resolve(inputs) { + Ok(source) => source, + Err(why) => { + return Attempt { + outcome: AttemptOutcome::fail(why.to_string()), + cost_usd: 0.0, + }; + } + }; + let trace_id = gate_trace_id(&self.run_id, &task.name.0); + let outcome = match self.resolutions.get(&trace_id) { + Some(resolution) => settle(resolution, &source), + None => AttemptOutcome::Await(Gate { + task: task.name.clone(), + trace_id, + handle: source + .handle() + .map(str::to_string) + .unwrap_or_else(|| task.name.0.clone()), + source, + summary: summary.clone(), + timeout_secs: *timeout_secs, + }), + }; + Attempt { + outcome, + cost_usd: 0.0, + } + } } /// The env var the launcher sets to name the run a gate belongs to. It must be stable across a @@ -57,92 +104,47 @@ pub enum SourceError { FieldMissing { task: String, field: String }, } -/// The wire source for a gate, with any upstream reference read from the task's inputs. -pub fn resolve_source( - spec: &ApprovalSourceSpec, - inputs: &BTreeMap, -) -> Result { - let read = |r: &RefOrLiteral| -> Result { - match r { - RefOrLiteral::Literal(s) => Ok(s.clone()), - RefOrLiteral::Output(reference) => { - let producer = - inputs - .get(&reference.task) - .ok_or_else(|| SourceError::ProducerMissing { +impl ApprovalSourceSpec { + /// Resolve upstream references in a gate source against settled task outputs. + pub fn resolve( + &self, + inputs: &BTreeMap, + ) -> Result { + let read = |r: &RefOrLiteral| -> Result { + match r { + RefOrLiteral::Literal(s) => Ok(s.clone()), + RefOrLiteral::Output(reference) => { + let producer = inputs.get(&reference.task).ok_or_else(|| { + SourceError::ProducerMissing { task: reference.task.0.clone(), field: reference.field.0.clone(), - })?; - producer - .get(&reference.field.0) - .and_then(Value::as_str) - .map(str::to_string) - .ok_or_else(|| SourceError::FieldMissing { - task: reference.task.0.clone(), - field: reference.field.0.clone(), - }) + } + })?; + producer + .get(&reference.field.0) + .and_then(Value::as_str) + .map(str::to_string) + .ok_or_else(|| SourceError::FieldMissing { + task: reference.task.0.clone(), + field: reference.field.0.clone(), + }) + } } - } - }; - Ok(match spec { - ApprovalSourceSpec::Native => GateSource::Native, - ApprovalSourceSpec::GithubPr { url, until } => GateSource::GithubPr { - url: read(url)?, - until: *until, - }, - ApprovalSourceSpec::Jira { key, until } => GateSource::Jira { - key: read(key)?, - until: until.clone(), - }, - }) -} - -/// Run one `approve` task: a held resolution settles it, otherwise the gate is reported open. -/// Never spends. -pub fn attempt(ctx: &GateCtx, task: &Task, inputs: &BTreeMap) -> Attempt { - let TaskKind::Approve { - summary, - source, - timeout_secs, - } = &task.task - else { - return Attempt { - outcome: AttemptOutcome::fail(format!("task {} is not an approve task", task.name)), - cost_usd: 0.0, }; - }; - let source = match resolve_source(source, inputs) { - Ok(source) => source, - Err(why) => { - return Attempt { - outcome: AttemptOutcome::fail(why.to_string()), - cost_usd: 0.0, - }; - } - }; - let trace_id = gate_trace_id(&ctx.run_id, &task.name.0); - let outcome = match ctx.resolutions.get(&trace_id) { - Some(resolution) => settle(resolution, &source), - None => AttemptOutcome::Await(Gate { - task: task.name.clone(), - trace_id, - handle: source - .handle() - .map(str::to_string) - .unwrap_or_else(|| task.name.0.clone()), - source, - summary: summary.clone(), - timeout_secs: *timeout_secs, - }), - }; - Attempt { - outcome, - cost_usd: 0.0, + Ok(match self { + ApprovalSourceSpec::Native => GateSource::Native, + ApprovalSourceSpec::GithubPr { url, until } => GateSource::GithubPr { + url: read(url)?, + until: *until, + }, + ApprovalSourceSpec::Jira { key, until } => GateSource::Jira { + key: read(key)?, + until: until.clone(), + }, + }) } } -/// A grant passes the gate with the resolution as its output; a denial or timeout fails it -/// with the reason. fn settle(resolution: &GateResolution, source: &GateSource) -> AttemptOutcome { match resolution.decision { GateDecision::Granted => AttemptOutcome::Pass(serde_json::json!({ @@ -254,7 +256,7 @@ mod tests { let mut ctx = GateCtx::new("run-1"); ctx.resolve( "approve:run-1:review", - GateResolution::granted(Some("alice".into()), "native"), + GateResolution::granted(Some("alice".parse().unwrap()), "native"), ); let a = attempt( &ctx, @@ -269,7 +271,11 @@ mod tests { ctx.resolve( "approve:run-1:review", - GateResolution::denied("changes requested", Some("bob".into()), "github_pr"), + GateResolution::denied( + "changes requested", + Some("bob".parse().unwrap()), + "github_pr", + ), ); let a = attempt( &ctx, diff --git a/crucible/src/plan/gate_host.rs b/crucible/src/plan/gate_host.rs index 5d1a0a1e..dc08fd74 100644 --- a/crucible/src/plan/gate_host.rs +++ b/crucible/src/plan/gate_host.rs @@ -307,7 +307,7 @@ pub(crate) fn resolved_event(open: &Gate, resolution: &GateResolution) -> Sessio }, reason: resolution.reason_text(), trace_id: open.trace_id.clone(), - by: resolution.by.clone(), + by: resolution.by.as_ref().map(|by| by.as_str().to_string()), source: resolution.source.clone(), } } diff --git a/crucible/src/plan/harness.rs b/crucible/src/plan/harness.rs index 90c7bb88..64705708 100644 --- a/crucible/src/plan/harness.rs +++ b/crucible/src/plan/harness.rs @@ -660,7 +660,7 @@ fn run_in( return fail(0.0, "engine task reached a non-loop runner".to_string()); } TaskKind::Approve { .. } => { - return crate::plan::gate::attempt(gate, task, inputs); + return gate.attempt(task, inputs); } }; diff --git a/crucible/src/plan/runner.rs b/crucible/src/plan/runner.rs index aeb25d54..2cac6745 100644 --- a/crucible/src/plan/runner.rs +++ b/crucible/src/plan/runner.rs @@ -61,7 +61,7 @@ impl ShellRunner { fn run_in_workdir(&mut self, task: &Task, inputs: &BTreeMap) -> Attempt { if matches!(task.task, TaskKind::Approve { .. }) { - return crate::plan::gate::attempt(&self.gate, task, inputs); + return self.gate.attempt(task, inputs); } let mut cmd = Command::new("sh"); cmd.arg("-c").current_dir(&self.workdir); @@ -132,7 +132,7 @@ impl ShellRunner { return fail("engine task reached a non-loop runner".to_string()); } TaskKind::Approve { .. } => { - return crate::plan::gate::attempt(&self.gate, task, inputs); + return self.gate.attempt(task, inputs); } } let out = match cmd.output() { diff --git a/crucible/src/run.rs b/crucible/src/run.rs index 3c56bea1..e25771fb 100644 --- a/crucible/src/run.rs +++ b/crucible/src/run.rs @@ -159,6 +159,14 @@ pub(crate) fn dispatch(cli: Cli) -> Result<()> { return crate::pr_watch::watch_and_steer(&pr, &sink, &opts); } + if let Some(Cmd::LoopReference { format }) = &cli.command { + match format { + crate::LoopFormat::Markdown => print!("{}", crate::machine::doc_page()), + crate::LoopFormat::Mermaid => print!("{}", crate::machine::mermaid()), + } + return Ok(()); + } + if let Some(Cmd::Approve { control_addr, by }) = &cli.command { let reply = control::send_command( control_addr, diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 8aba9225..c67a8b64 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -18,6 +18,7 @@ - [Implementation contract](./crucible-contract.md) - [Work graphs](./work-graphs.md) - [Workflow DSL reference](./dsl-reference.md) +- [The loop state machine](./loop-machine.md) - [Hand-rolled codegen pipelines](./hand-rolled-pipelines.md) - [The codex harness](./codex-harness.md) - [The OpenShell fork](./openshell-fork.md) diff --git a/docs/loop-machine.md b/docs/loop-machine.md new file mode 100644 index 00000000..025e8296 --- /dev/null +++ b/docs/loop-machine.md @@ -0,0 +1,49 @@ +# The loop state machine + + + +The scored loop's decisions live in one I/O-free machine: the host performs effects and hands the results back, and the machine answers what happens next. Every edge below is a variant the compiler knows about, so this page cannot drift from the binary that drew it. + +```mermaid +stateDiagram-v2 + [*] --> Head + Head --> ApprovalPark: a block approval is pending + ApprovalPark --> Head: granted: the re-scope drain re-baselines + ApprovalPark --> escalated: denied with no fallback + ApprovalPark --> stopped: stop while parked + Head --> DistressPark: the agent raised distress(error) + DistressPark --> Head: the operator cleared the marker + DistressPark --> stopped: stop, or the park timed out + Head --> stopped: interrupt at the head + Head --> budget: a cost or time cap was reached + Head --> finished: no iterations left + Head --> Iteration: otherwise: run the turn + Iteration --> Decide: decided: a measured candidate + Decide --> Head: kept or discarded + Decide --> solved: kept and solved, early stop on + Decide --> budget: a cap was reached deciding + Iteration --> Head: discarded, or parked for the next head + Iteration --> Iteration: never-started: re-run, bounded + Iteration --> stalled: consecutive dead turns hit the bound + Iteration --> escalated: escalated by the agent + Iteration --> stopped: stop at the post-turn checkpoint + finished --> [*] + solved --> [*] + budget --> [*] + stopped --> [*] + escalated --> [*] + stalled --> [*] +``` + +## How a run ends + +The terminal states are the shutdown tokens the session log carries. + +| Token | Meaning | +| --- | --- | +| `finished` | all iterations completed | +| `solved` | a kept candidate satisfied the win condition | +| `budget` | a cost or time cap was reached | +| `stopped` | stop signal received | +| `escalated` | the agent declared the harness inadequate — halted for human review | +| `stalled` | the run stalled on consecutive transport failures — no turn could start | \ No newline at end of file diff --git a/scripts/loop-docs.sh b/scripts/loop-docs.sh new file mode 100755 index 00000000..1cf8d667 --- /dev/null +++ b/scripts/loop-docs.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Regenerate docs/loop-machine.md from the loop machine's own vocabulary. +# scripts/loop-docs.sh write the page +# scripts/loop-docs.sh --check fail if the page is stale, write nothing +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +page="$root/docs/loop-machine.md" + +generated="$(cargo run --quiet --manifest-path "$root/Cargo.toml" -p crucible -- loop-reference)" + +if [[ "${1:-}" == "--check" ]]; then + if ! diff -u "$page" <(printf '%s' "$generated"); then + echo "docs/loop-machine.md is stale; run scripts/loop-docs.sh" >&2 + exit 1 + fi + exit 0 +fi + +printf '%s' "$generated" > "$page" From a81ccbed730b00989f6845ccb393e02b08454233 Mon Sep 17 00:00:00 2001 From: Will Eaton Date: Fri, 4 Sep 2026 16:38:06 -0400 Subject: [PATCH 09/11] Format gate.rs Assisted-by: Claude --- crucible/src/plan/gate.rs | 52 ++++++++------------------------------- 1 file changed, 10 insertions(+), 42 deletions(-) diff --git a/crucible/src/plan/gate.rs b/crucible/src/plan/gate.rs index c01c6599..5bece3fc 100644 --- a/crucible/src/plan/gate.rs +++ b/crucible/src/plan/gate.rs @@ -30,11 +30,7 @@ impl GateCtx { } /// Apply the held resolution, or report that the task is waiting for one. - pub fn attempt( - &self, - task: &Task, - inputs: &BTreeMap, - ) -> Attempt { + pub fn attempt(&self, task: &Task, inputs: &BTreeMap) -> Attempt { let TaskKind::Approve { summary, source, @@ -106,10 +102,7 @@ pub enum SourceError { impl ApprovalSourceSpec { /// Resolve upstream references in a gate source against settled task outputs. - pub fn resolve( - &self, - inputs: &BTreeMap, - ) -> Result { + pub fn resolve(&self, inputs: &BTreeMap) -> Result { let read = |r: &RefOrLiteral| -> Result { match r { RefOrLiteral::Literal(s) => Ok(s.clone()), @@ -215,11 +208,7 @@ mod tests { TaskName("open_pr".into()), serde_json::json!({"pr_url": "https://github.com/o/r/pull/9"}), )]); - let a = attempt( - &ctx, - &gate(pr_from("open_pr", "pr_url"), &["open_pr"]), - &inputs, - ); + let a = ctx.attempt(&gate(pr_from("open_pr", "pr_url"), &["open_pr"]), &inputs); assert_eq!(a.cost_usd, 0.0); let AttemptOutcome::Await(g) = a.outcome else { panic!("an unresolved gate awaits"); @@ -239,11 +228,7 @@ mod tests { #[test] fn a_native_gate_uses_the_task_name_as_its_handle() { let ctx = GateCtx::new("run-1"); - let a = attempt( - &ctx, - &gate(ApprovalSourceSpec::Native, &[]), - &BTreeMap::new(), - ); + let a = ctx.attempt(&gate(ApprovalSourceSpec::Native, &[]), &BTreeMap::new()); let AttemptOutcome::Await(g) = a.outcome else { panic!("awaits"); }; @@ -258,11 +243,7 @@ mod tests { "approve:run-1:review", GateResolution::granted(Some("alice".parse().unwrap()), "native"), ); - let a = attempt( - &ctx, - &gate(ApprovalSourceSpec::Native, &[]), - &BTreeMap::new(), - ); + let a = ctx.attempt(&gate(ApprovalSourceSpec::Native, &[]), &BTreeMap::new()); let AttemptOutcome::Pass(out) = a.outcome else { panic!("a grant passes"); }; @@ -277,11 +258,7 @@ mod tests { "github_pr", ), ); - let a = attempt( - &ctx, - &gate(ApprovalSourceSpec::Native, &[]), - &BTreeMap::new(), - ); + let a = ctx.attempt(&gate(ApprovalSourceSpec::Native, &[]), &BTreeMap::new()); let AttemptOutcome::Fail { note, output } = a.outcome else { panic!("a denial fails"); }; @@ -289,19 +266,14 @@ mod tests { assert_eq!(output.expect("carries the denial")["denied_by"], "bob"); ctx.resolve("approve:run-1:review", GateResolution::timeout()); - let a = attempt( - &ctx, - &gate(ApprovalSourceSpec::Native, &[]), - &BTreeMap::new(), - ); + let a = ctx.attempt(&gate(ApprovalSourceSpec::Native, &[]), &BTreeMap::new()); assert!(matches!(a.outcome, AttemptOutcome::Fail { .. })); } #[test] fn a_source_read_from_a_missing_producer_or_field_fails_the_gate() { let ctx = GateCtx::new("run-1"); - let a = attempt( - &ctx, + let a = ctx.attempt( &gate(pr_from("open_pr", "pr_url"), &["open_pr"]), &BTreeMap::new(), ); @@ -311,11 +283,7 @@ mod tests { assert!(note.contains("no passing output"), "{note}"); let inputs = BTreeMap::from([(TaskName("open_pr".into()), serde_json::json!({"number": 9}))]); - let a = attempt( - &ctx, - &gate(pr_from("open_pr", "pr_url"), &["open_pr"]), - &inputs, - ); + let a = ctx.attempt(&gate(pr_from("open_pr", "pr_url"), &["open_pr"]), &inputs); let AttemptOutcome::Fail { note, .. } = a.outcome else { panic!("fails"); }; @@ -324,7 +292,7 @@ mod tests { key: RefOrLiteral::Literal("PROJ-1".into()), until: JiraUntil::Status("Ready".into()), }; - let AttemptOutcome::Await(g) = attempt(&ctx, &gate(jira, &[]), &BTreeMap::new()).outcome + let AttemptOutcome::Await(g) = ctx.attempt(&gate(jira, &[]), &BTreeMap::new()).outcome else { panic!("awaits"); }; From b426878197e57766b9693353f89038c7d5d1b158 Mon Sep 17 00:00:00 2001 From: Will Eaton Date: Fri, 4 Sep 2026 16:45:11 -0400 Subject: [PATCH 10/11] refactor(plan): attach gate-host behavior to Host Move gate events, suspension, resume-state recovery, and compression onto Host so callers invoke behavior through the type that owns gate-host state. Document the domain-method convention and remove the attribution requirement. --- AGENTS.md | 6 +- CLAUDE.md | 1 + crucible/src/plan/cli.rs | 10 +-- crucible/src/plan/gate_host.rs | 132 ++++++++++++++++----------------- 4 files changed, 77 insertions(+), 72 deletions(-) create mode 120000 CLAUDE.md diff --git a/AGENTS.md b/AGENTS.md index 4fa1d974..08694d1e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -55,6 +55,10 @@ whether the code works. values become newtypes whose only constructor is the check (`RepoTarget` via `OrgAllowlist::authorize` is the house style: a checked value is a type, so an unchecked one cannot reach the call). +- Put behavior on the domain type that owns the state or specification it acts + on. Prefer `gate.attempt(...)`, `source.resolve(...)`, and `host.suspend(...)` + over free functions that take the owner as their first argument; keep a free + function only when no domain type owns the operation. - Groups of adjacent scalars in a signature become a struct; mode-dependent knobs become an enum keyed by mode. Repeated inline conversions become `From`/`TryFrom` impls next to the types. @@ -80,7 +84,7 @@ whether the code works. ## PRs, commits, comments - Never put an AI session link in a PR body, PR description, or commit - message. Commits carry a plain `Assisted-by: Claude` trailer. + message. - Agents never post PR comments, review replies, or issue comments; those would appear under the operator's account. Report dispositions in the driving session instead. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 00000000..47dc3e3d --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/crucible/src/plan/cli.rs b/crucible/src/plan/cli.rs index 5804bd5c..a6f7a4ff 100644 --- a/crucible/src/plan/cli.rs +++ b/crucible/src/plan/cli.rs @@ -852,7 +852,7 @@ pub fn run( }; let open = open.clone(); if let Some(f) = &events { - append(f, &gate::wait_event(&open)); + append(f, &host.wait_event(&open)); } println!( "gate {} ({}): awaiting approval via {}", @@ -863,7 +863,7 @@ pub fn run( match host.wait(&open, &gates, evidence.as_ref())? { gate::Waited::Resolved(resolution) => { if let Some(f) = &events { - append(f, &gate::resolved_event(&open, &resolution)); + append(f, &host.resolved_event(&open, &resolution)); } println!( "gate {}: {}{}", @@ -883,7 +883,7 @@ pub fn run( let Some(paths) = &evidence else { return Err(SuspendNeedsManifest.into()); }; - match gate::suspend(paths, &plan, &open) { + match host.suspend(paths, &open) { Ok(()) => { if let Some(f) = &events { append( @@ -918,7 +918,7 @@ pub fn run( match host.park(&open, None, evidence.as_ref())? { gate::Parked::Resolved(resolution) => { if let Some(f) = &events { - append(f, &gate::resolved_event(&open, &resolution)); + append(f, &host.resolved_event(&open, &resolution)); } runner.resolve_gate(&open.trace_id, resolution); spent_before = out.spent_usd; @@ -942,7 +942,7 @@ pub fn run( gate::Parked::TimedOut => { let resolution = crucible_contract::GateResolution::timeout(); if let Some(f) = &events { - append(f, &gate::resolved_event(&open, &resolution)); + append(f, &host.resolved_event(&open, &resolution)); } runner.resolve_gate(&open.trace_id, resolution); spent_before = out.spent_usd; diff --git a/crucible/src/plan/gate_host.rs b/crucible/src/plan/gate_host.rs index dc08fd74..213f6eb5 100644 --- a/crucible/src/plan/gate_host.rs +++ b/crucible/src/plan/gate_host.rs @@ -77,7 +77,7 @@ impl Host { ) })?; let state = LoopState::from_lines(body.lines()); - prior = prior_from(&state, plan); + prior = Self::prior_from(&state, plan); if let Some(open) = state.open_approval() && !resolutions.contains_key(&open.trace_id) && let Some(recorded) = opened.gate_resolution(&open.trace_id) @@ -117,6 +117,19 @@ impl Host { std::mem::take(&mut self.prior) } + fn prior_from(state: &LoopState, plan: &ValidPlan) -> BTreeMap { + let Some(admitted) = &state.plan else { + return BTreeMap::new(); + }; + crate::loop_graph::PriorPlan { + iter: 0, + plan_version: admitted.plan_version, + declared: admitted.tasks.iter().map(|t| t.name.clone()).collect(), + results: state.plan_results.clone(), + } + .seed(plan) + } + /// Wait at `open` under the run's park policy. pub(crate) fn wait( &mut self, @@ -215,12 +228,19 @@ impl Host { let Ok(json) = serde_json::to_vec(&waits) else { return; }; - let Ok(gz) = gzip(&json) else { + let Ok(gz) = Self::gzip(&json) else { return; }; post_artifact(cfg, ArtifactKind::ApprovalWaits, &gz); } + fn gzip(bytes: &[u8]) -> std::io::Result> { + use std::io::Write as _; + let mut enc = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + enc.write_all(bytes)?; + enc.finish() + } + /// Record a resolution that arrived from outside the bridge, under the gate's key. fn record(&self, trace_id: &str, resolution: &GateResolution) { let Some(ledger) = &self.ledger else { @@ -247,74 +267,54 @@ impl Host { } } -/// Snapshot the run at `open` and deliver it, so a later `--resume` continues from here. -pub(crate) fn suspend(paths: &Paths, _plan: &ValidPlan, open: &Gate) -> Result<()> { - let record = crate::suspend::ResumeRecord { - v: crate::suspend::ResumeRecord::VERSION, - run_id: crate::plan::gate::run_id_from_env(), - gate: open.trace_id.clone(), - head: crate::suspend::head_of(&paths.workspace), - suspended_at: crate::suspend::now_secs(), - }; - let gz = crate::suspend::snapshot(&paths.state, &paths.workspace, &record)?; - let delivered = crate::suspend::deliver(&paths.session_log, &gz)?; - eprintln!( - "[crucible] suspended at gate {} ({}){}", - open.task, - open.trace_id, - if delivered { - ": snapshot delivered to the drop-box" - } else { - ": snapshot kept in the state dir" - } - ); - Ok(()) -} - -/// The results a dead process settled under `plan`: only when the log admitted this same plan -/// (version and task set), and only the passing pack tasks. -fn prior_from(state: &LoopState, plan: &ValidPlan) -> BTreeMap { - let Some(admitted) = &state.plan else { - return BTreeMap::new(); - }; - crate::loop_graph::PriorPlan { - iter: 0, - plan_version: admitted.plan_version, - declared: admitted.tasks.iter().map(|t| t.name.clone()).collect(), - results: state.plan_results.clone(), +impl Host { + /// Snapshot the run at an open gate and deliver it for resumption. + pub(crate) fn suspend(&self, paths: &Paths, open: &Gate) -> Result<()> { + let record = crate::suspend::ResumeRecord { + v: crate::suspend::ResumeRecord::VERSION, + run_id: crate::plan::gate::run_id_from_env(), + gate: open.trace_id.clone(), + head: crate::suspend::head_of(&paths.workspace), + suspended_at: crate::suspend::now_secs(), + }; + let gz = crate::suspend::snapshot(&paths.state, &paths.workspace, &record)?; + let delivered = crate::suspend::deliver(&paths.session_log, &gz)?; + eprintln!( + "[crucible] suspended at gate {} ({}){}", + open.task, + open.trace_id, + if delivered { + ": snapshot delivered to the drop-box" + } else { + ": snapshot kept in the state dir" + } + ); + Ok(()) } - .seed(plan) } -/// The `approval_wait` line a gate opens with. -pub(crate) fn wait_event(open: &Gate) -> SessionEvent { - SessionEvent::ApprovalWait { - handle: open.handle.clone(), - trace_id: open.trace_id.clone(), - mode: "block".to_string(), - task: Some(open.task.0.clone()), - source: serde_json::to_value(&open.source).ok(), - park: Some(ParkMode::Park.as_str().to_string()), +impl Host { + pub(crate) fn wait_event(&self, open: &Gate) -> SessionEvent { + SessionEvent::ApprovalWait { + handle: open.handle.clone(), + trace_id: open.trace_id.clone(), + mode: "block".to_string(), + task: Some(open.task.0.clone()), + source: serde_json::to_value(&open.source).ok(), + park: Some(ParkMode::Park.as_str().to_string()), + } } -} -/// The `approval_resolved` line that closes a gate. -pub(crate) fn resolved_event(open: &Gate, resolution: &GateResolution) -> SessionEvent { - SessionEvent::ApprovalResolved { - outcome: match resolution.source.as_deref() { - Some("timeout") => "timeout".to_string(), - _ => resolution.decision.as_str().to_string(), - }, - reason: resolution.reason_text(), - trace_id: open.trace_id.clone(), - by: resolution.by.as_ref().map(|by| by.as_str().to_string()), - source: resolution.source.clone(), + pub(crate) fn resolved_event(&self, open: &Gate, resolution: &GateResolution) -> SessionEvent { + SessionEvent::ApprovalResolved { + outcome: match resolution.source.as_deref() { + Some("timeout") => "timeout".to_string(), + _ => resolution.decision.as_str().to_string(), + }, + reason: resolution.reason_text(), + trace_id: open.trace_id.clone(), + by: resolution.by.as_ref().map(|by| by.as_str().to_string()), + source: resolution.source.clone(), + } } } - -fn gzip(bytes: &[u8]) -> std::io::Result> { - use std::io::Write as _; - let mut enc = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); - enc.write_all(bytes)?; - enc.finish() -} From 00d5f9f2b0a6acdf59bc11600c6a6118632a806a Mon Sep 17 00:00:00 2001 From: Will Eaton Date: Fri, 4 Sep 2026 16:47:40 -0400 Subject: [PATCH 11/11] Dispatch the loop's head from a list the machine owns The head's order of operations existed only as the reading order of run_loop_body: park on a pending approval, drain a re-scope, drain a denial, park on distress, check the interrupt, check the caps. Nothing named that sequence, so the state chart had to assert it by hand and a reordering would have falsified the page without failing anything. machine::HEAD is now that sequence, as data. The host walks it and dispatches each HeadCheck through one function that returns HeadFlow rather than breaking a loop, which is what lets the order live outside the code that runs it. Each check names its own chart edges in an exhaustive match, so the head of the diagram is generated from the same list the host walks, and a new check compiles only once it is described and dispatched. run_loop_body goes from 814 lines to 689. The head's thirteen threaded arguments become one HeadCtx. No behavior change: the checks run in the same order, do the same things, and the 616 engine tests are untouched. The iteration half of the chart is still a written table. IterStep and Settle already name those transitions, but the host performs them inline; moving them behind the same dispatch is the remaining half of this. Assisted-by: Claude --- crucible/src/loop_driver.rs | 372 +++++++++++++++++++++--------------- crucible/src/machine.rs | 205 +++++++++++++++----- docs/loop-machine.md | 2 + 3 files changed, 381 insertions(+), 198 deletions(-) diff --git a/crucible/src/loop_driver.rs b/crucible/src/loop_driver.rs index b4dee4e6..2a1140cc 100644 --- a/crucible/src/loop_driver.rs +++ b/crucible/src/loop_driver.rs @@ -6,10 +6,11 @@ //! workspace prep, front-end choice) lives in [`crate::run`]; this module is just the loop and //! its helpers. +use crate::machine; pub(crate) use crate::machine::IterStep; use crate::machine::{ - BudgetHit, DistressOutcome, LoopCfg, LoopExit, MAX_DEAD_TURN_ATTEMPTS, Machine, ParkOutcome, - RunState, Segment, Settle, + BudgetHit, DistressOutcome, HeadCheck, HeadFlow, LoopCfg, LoopExit, MAX_DEAD_TURN_ATTEMPTS, + Machine, ParkOutcome, RunState, Segment, Settle, }; use crate::reporter::{AgentTurn, Outcome, Phase, Reporter, Row, Stop, TurnBudget}; use crate::{Args, Paths, Prepared, STOP}; @@ -426,6 +427,206 @@ pub(crate) fn run_loop( } } +/// Everything a head check reads that is not the machine or the reporter. One struct so a +/// check takes five arguments instead of thirteen. +struct HeadCtx<'a> { + args: &'a Args, + p: &'a Paths, + prep: &'a Prepared, + world: &'a dyn World, + judge: &'a dyn Judge, + control: Option<&'a control::ControlState>, + ledger: Option<&'a crate::admission::AdmissionLedger>, + preflight: &'a Option, + started: Instant, +} + +/// Perform one head check. The machine owns which checks run and in what order +/// ([`machine::HEAD`]); this owns what each one does. Returning [`HeadFlow`] instead of +/// breaking a loop is what lets that order live in data rather than in this function's shape. +fn head_check( + check: HeadCheck, + m: &mut Machine, + r: &mut R, + ctx: &HeadCtx<'_>, + bad_marker_seen: &mut Option>, +) -> Result { + let it = m.it; + match check { + HeadCheck::WaitIfPaused => { + wait_if_paused(ctx.control, r); + } + HeadCheck::ParkOnPendingBlock => { + // The agent blocked on a pending approval last turn (it had no frozen-regime fallback). + // Park here (idle, budget-paused) until the approval lands as a re-scope (the broker + // fires it over the control bridge) or we're told to stop. The drain below then + // re-baselines into the granted regime. + if let Some(pp) = m.take_pending_block() { + let (outcome, parked) = + park_for_approval(ctx.control, ctx.ledger, r, m.cfg.max_park); + match (m.on_park(parked, &outcome), &outcome) { + (None, _) => {} // the re-scope drain below re-baselines + (Some(LoopExit::Escalated), ParkOutcome::Denied(why)) => { + // `block` means the agent had no frozen-regime fallback, a denial leaves + // nothing to do, so escalate-halt for a human. + r.note(&format!( + "approval denied — escalating (no fallback): provisioning for '{}' was not granted: {why}", + pp.trace_id + )); + update_control_status( + ctx.control, + "escalated", + it, + m.run.segment.best_score, + m.run.spent, + ); + ctx.world.restore(&m.run.segment.best_snap)?; + return Ok(HeadFlow::Exit); + } + (Some(_), _) => return Ok(HeadFlow::Exit), + } + } + } + HeadCheck::DrainRescope => { + // An approved judge-changing grant arrived (via the control channel / MCP): re-baseline + // into the new regime and open a fresh segment before this iteration measures. + if let Some((rescope_key, new_regime)) = ctx.control.and_then(|c| c.take_rescope()) { + // Close any open approval bracket: a rescope IS the grant. Harmless when no + // wait was open (the classifier treats an unmatched resolve as a no-op). + r.approval_resolved("granted", &new_regime); + r.note(&format!( + "control: re-scoping to '{new_regime}' — re-baselining a new comparable segment" + )); + // One atomic swap of the goalpost: the new regime, its fingerprint, the re-baselined + // scores, and the fresh rollback snapshot all land together. The admission + // settles only after the swap: a baseline error leaves it for the resume. + let (segment, _row) = baseline_segment( + ctx.world, + ctx.judge, + &ctx.prep.goal, + new_regime.clone(), + baseline_source(ctx.prep.skip_baseline, ctx.preflight.as_ref()), + )?; + m.rescope(segment); + if let Some(ledger) = ctx.ledger { + let _ = ledger.settle( + &rescope_key, + AdmissionOutcome::Applied, + &format!("re-baselined into '{new_regime}' at iter {it}"), + ); + } + r.segment( + &m.run.segment.fingerprint, + m.run.segment.baseline_score, + &m.run.segment.regime, + ); + write_results(ctx.p, &ctx.prep.goal, &ctx.prep.prior, &m.run.rows)?; + } + } + HeadCheck::DrainDeny => { + // A denial that arrived while *continuing* (the agent had a fallback, so the loop never + // parked) just means the regime change won't happen; note it and stay in the frozen regime. + if let Some((deny_key, reason)) = ctx.control.and_then(|c| c.take_deny()) { + r.approval_resolved("denied", &reason); + r.note(&format!( + "approval not granted ({reason}) — staying in the frozen regime" + )); + if let Some(ledger) = ctx.ledger { + let _ = ledger.settle( + &deny_key, + AdmissionOutcome::Applied, + &format!("drained at the head of iter {it}"), + ); + } + } + } + HeadCheck::ParkOnDistress => { + // The agent raised `distress(severity=error)` during the last turn: that turn finished and + // was decided above, so bookkeeping is complete and this is the safe point to suspend. + // Modeled as an approval wait (same bracket, same parked-time accounting); the operator's + // `rm` of the marker is the grant. + match crate::distress::read_marker() { + Some(Ok(marker)) => { + if let Some(row) = m.on_distress(marker.ts_ms, &marker.reason) { + // An in-place restart re-reads a marker the operator never cleared and + // re-parks (correct: no grant was given), but the row for it is already in + // the resumed log. + if let Some(row) = row { + r.row(&row, false); + m.record(row); + write_results(ctx.p, &ctx.prep.goal, &ctx.prep.prior, &m.run.rows)?; + } + r.note(&format!( + "distress: {}, suspended awaiting the operator (clear {})", + marker.reason, + forge::storage_root().join("distress").display() + )); + for item in &marker.evidence { + r.note(&format!("distress evidence: {item}")); + } + update_control_status( + ctx.control, + "distressed", + it, + m.run.segment.best_score, + m.run.spent, + ); + r.approval_wait( + crate::distress::HANDLE, + crate::distress::HANDLE, + provisioning::WaitMode::Block, + ); + let (outcome, parked) = + park_for_distress(ctx.p, &m.run.rows, r, m.cfg.max_park); + match m.on_distress_park(parked, outcome) { + None => { + r.approval_resolved("granted", "distress cleared by operator"); + r.note("distress cleared, resuming"); + // The head re-checks budget/interrupts before the next turn runs. + return Ok(HeadFlow::Restart); + } + Some(_) => { + if outcome == DistressOutcome::TimedOut { + r.note( + "distress park timed out, stopping with state preserved", + ); + } + return Ok(HeadFlow::Exit); + } + } + } + } + // A marker we cannot parse is a broken handoff, not a suspend order: note it once per + // rewrite and keep iterating. Wedging a paid run on a bad byte is the worse failure. + Some(Err(why)) => { + let mtime = crate::distress::marker_mtime(); + if *bad_marker_seen != Some(mtime) { + *bad_marker_seen = Some(mtime); + r.note(&format!("distress marker unreadable ({why}), not parking")); + } + } + None => {} + } + } + HeadCheck::Interrupt => { + if matches!(r.check_interrupt(ctx.p, &m.run.rows), Stop::Quit) { + m.end(LoopExit::Stopped); + return Ok(HeadFlow::Exit); + } + } + HeadCheck::Budget => { + if let Some(hit) = + m.over_budget(live_max_cost(ctx.args, ctx.control), ctx.started.elapsed()) + { + note_budget_hit(r, ctx.args, hit); + m.end(LoopExit::Budget); + return Ok(HeadFlow::Exit); + } + } + } + Ok(HeadFlow::Continue) +} + /// The host: performs every effect the loop needs and hands the results to the /// [`crate::machine::Machine`], which owns the decisions. Effects happen in the order they /// always have, so the session log a run writes is unchanged. @@ -700,155 +901,28 @@ fn run_loop_body( let mut bad_marker_seen: Option> = None; while m.exit().is_none() && m.has_iterations() { let it = m.it; - wait_if_paused(control, r); - // The agent blocked on a pending approval last turn (it had no frozen-regime fallback). - // Park here (idle, budget-paused) until the approval lands as a re-scope (the broker - // fires it over the control bridge) or we're told to stop. The drain below then - // re-baselines into the granted regime. - if let Some(pp) = m.take_pending_block() { - let (outcome, parked) = park_for_approval(control, ledger, r, m.cfg.max_park); - match (m.on_park(parked, &outcome), &outcome) { - (None, _) => {} // the re-scope drain below re-baselines - (Some(LoopExit::Escalated), ParkOutcome::Denied(why)) => { - // `block` means the agent had no frozen-regime fallback, a denial leaves - // nothing to do, so escalate-halt for a human. - r.note(&format!( - "approval denied — escalating (no fallback): provisioning for '{}' was not granted: {why}", - pp.trace_id - )); - update_control_status( - control, - "escalated", - it, - m.run.segment.best_score, - m.run.spent, - ); - world.restore(&m.run.segment.best_snap)?; - break; - } - (Some(_), _) => break, - } - } - // An approved judge-changing grant arrived (via the control channel / MCP): re-baseline - // into the new regime and open a fresh segment before this iteration measures. - if let Some((rescope_key, new_regime)) = control.and_then(|c| c.take_rescope()) { - // Close any open approval bracket: a rescope IS the grant. Harmless when no - // wait was open (the classifier treats an unmatched resolve as a no-op). - r.approval_resolved("granted", &new_regime); - r.note(&format!( - "control: re-scoping to '{new_regime}' — re-baselining a new comparable segment" - )); - // One atomic swap of the goalpost: the new regime, its fingerprint, the re-baselined - // scores, and the fresh rollback snapshot all land together. The admission - // settles only after the swap: a baseline error leaves it for the resume. - let (segment, _row) = baseline_segment( - world, - judge, - &prep.goal, - new_regime.clone(), - baseline_source(prep.skip_baseline, preflight_baseline.as_ref()), - )?; - m.rescope(segment); - if let Some(ledger) = ledger { - let _ = ledger.settle( - &rescope_key, - AdmissionOutcome::Applied, - &format!("re-baselined into '{new_regime}' at iter {it}"), - ); - } - r.segment( - &m.run.segment.fingerprint, - m.run.segment.baseline_score, - &m.run.segment.regime, - ); - write_results(p, &prep.goal, &prep.prior, &m.run.rows)?; - } - // A denial that arrived while *continuing* (the agent had a fallback, so the loop never - // parked) just means the regime change won't happen; note it and stay in the frozen regime. - if let Some((deny_key, reason)) = control.and_then(|c| c.take_deny()) { - r.approval_resolved("denied", &reason); - r.note(&format!( - "approval not granted ({reason}) — staying in the frozen regime" - )); - if let Some(ledger) = ledger { - let _ = ledger.settle( - &deny_key, - AdmissionOutcome::Applied, - &format!("drained at the head of iter {it}"), - ); - } - } - // The agent raised `distress(severity=error)` during the last turn: that turn finished and - // was decided above, so bookkeeping is complete and this is the safe point to suspend. - // Modeled as an approval wait (same bracket, same parked-time accounting); the operator's - // `rm` of the marker is the grant. - match crate::distress::read_marker() { - Some(Ok(marker)) => { - if let Some(row) = m.on_distress(marker.ts_ms, &marker.reason) { - // An in-place restart re-reads a marker the operator never cleared and - // re-parks (correct: no grant was given), but the row for it is already in - // the resumed log. - if let Some(row) = row { - r.row(&row, false); - m.record(row); - write_results(p, &prep.goal, &prep.prior, &m.run.rows)?; - } - r.note(&format!( - "distress: {}, suspended awaiting the operator (clear {})", - marker.reason, - forge::storage_root().join("distress").display() - )); - for item in &marker.evidence { - r.note(&format!("distress evidence: {item}")); - } - update_control_status( - control, - "distressed", - it, - m.run.segment.best_score, - m.run.spent, - ); - r.approval_wait( - crate::distress::HANDLE, - crate::distress::HANDLE, - provisioning::WaitMode::Block, - ); - let (outcome, parked) = park_for_distress(p, &m.run.rows, r, m.cfg.max_park); - match m.on_distress_park(parked, outcome) { - None => { - r.approval_resolved("granted", "distress cleared by operator"); - r.note("distress cleared, resuming"); - // The head re-checks budget/interrupts before the next turn runs. - continue; - } - Some(_) => { - if outcome == DistressOutcome::TimedOut { - r.note("distress park timed out, stopping with state preserved"); - } - break; - } - } - } - } - // A marker we cannot parse is a broken handoff, not a suspend order: note it once per - // rewrite and keep iterating. Wedging a paid run on a bad byte is the worse failure. - Some(Err(why)) => { - let mtime = crate::distress::marker_mtime(); - if bad_marker_seen != Some(mtime) { - bad_marker_seen = Some(mtime); - r.note(&format!("distress marker unreadable ({why}), not parking")); - } + let mut flow = HeadFlow::Continue; + let ctx = HeadCtx { + args, + p, + prep, + world, + judge, + control, + ledger, + preflight: &preflight_baseline, + started, + }; + for check in machine::HEAD { + flow = head_check(*check, &mut m, r, &ctx, &mut bad_marker_seen)?; + if flow != HeadFlow::Continue { + break; } - None => {} } - if matches!(r.check_interrupt(p, &m.run.rows), Stop::Quit) { - m.end(LoopExit::Stopped); - break; - } - if let Some(hit) = m.over_budget(live_max_cost(args, control), started.elapsed()) { - note_budget_hit(r, args, hit); - m.end(LoopExit::Budget); - break; + match flow { + HeadFlow::Exit => break, + HeadFlow::Restart => continue, + HeadFlow::Continue => {} } r.phase(Phase::Iteration(it)); // One span per loop round, entered for the iteration's whole body on this thread: the diff --git a/crucible/src/machine.rs b/crucible/src/machine.rs index fade6df2..b5f37020 100644 --- a/crucible/src/machine.rs +++ b/crucible/src/machine.rs @@ -437,6 +437,119 @@ impl Machine { } } +/// One check the host performs at the head of an iteration, before any turn runs. The order is +/// the machine's, not the host's reading order: [`HEAD`] is the single place it is written down, +/// the host dispatches over it, and the published state chart is drawn from it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum HeadCheck { + /// Block while an operator holds the run paused. + WaitIfPaused, + /// Park on a `block` approval the last turn raised, budget-paused, until it resolves. + ParkOnPendingBlock, + /// Drain an approved re-scope: re-baseline into the granted regime, opening a segment. + DrainRescope, + /// Drain a denial that arrived while the run continued; the frozen regime stands. + DrainDeny, + /// Park on a distress marker the agent wrote, until the operator clears it. + ParkOnDistress, + /// End the run if a stop or interrupt landed. + Interrupt, + /// End the run if a cost or wall-clock cap is reached. + Budget, +} + +/// The head, in order. Every entry is dispatched by [`crate::loop_driver`]; nothing else may +/// happen before a turn. +pub(crate) const HEAD: &[HeadCheck] = &[ + HeadCheck::WaitIfPaused, + HeadCheck::ParkOnPendingBlock, + HeadCheck::DrainRescope, + HeadCheck::DrainDeny, + HeadCheck::ParkOnDistress, + HeadCheck::Interrupt, + HeadCheck::Budget, +]; + +/// What a head check tells the host to do next. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum HeadFlow { + /// Nothing happened; run the next check. + Continue, + /// Start the head over: state changed underneath the remaining checks. + Restart, + /// The run is over; [`Machine::exit`] carries why. + Exit, +} + +impl HeadCheck { + /// The chart edges this check can take. Exhaustive: a new check names its edges before + /// this compiles, so the head of the chart is the head the host walks. + fn edges(self) -> &'static [Edge] { + match self { + HeadCheck::WaitIfPaused => &[], + HeadCheck::ParkOnPendingBlock => &[ + Edge { + from: "Head", + to: "ApprovalPark", + label: "a block approval is pending", + }, + Edge { + from: "ApprovalPark", + to: "Head", + label: "granted: the re-scope drain re-baselines", + }, + Edge { + from: "ApprovalPark", + to: "escalated", + label: "denied with no fallback", + }, + Edge { + from: "ApprovalPark", + to: "stopped", + label: "stop while parked", + }, + ], + HeadCheck::DrainRescope => &[Edge { + from: "Head", + to: "Head", + label: "an approved re-scope re-baselines a new segment", + }], + HeadCheck::DrainDeny => &[Edge { + from: "Head", + to: "Head", + label: "a denial noted; the frozen regime stands", + }], + HeadCheck::ParkOnDistress => &[ + Edge { + from: "Head", + to: "DistressPark", + label: "the agent raised distress(error)", + }, + Edge { + from: "DistressPark", + to: "Head", + label: "the operator cleared the marker", + }, + Edge { + from: "DistressPark", + to: "stopped", + label: "stop, or the park timed out", + }, + ], + HeadCheck::Interrupt => &[Edge { + from: "Head", + to: "stopped", + label: "interrupt at the head", + }], + HeadCheck::Budget => &[Edge { + from: "Head", + to: "budget", + label: "a cost or time cap was reached", + }], + } + } +} + /// One transition in the rendered state chart. `label` is the condition that takes the loop /// from `from` to `to`, in the order the host actually evaluates them. struct Edge { @@ -445,55 +558,10 @@ struct Edge { label: &'static str, } -/// The loop's transitions, in the order [`crate::loop_driver`] evaluates them. Hand-ordered -/// because the order is the host's, not the machine's; the coverage tests below hold every -/// variant of [`IterStep`] and [`LoopExit`] present, so a new one cannot land undrawn. +/// What happens once the head lets an iteration run. The head's own transitions are not here: +/// they come from [`HEAD`], which is the order the host dispatches, so that half of the chart +/// cannot disagree with the code that walks it. const EDGES: &[Edge] = &[ - Edge { - from: "Head", - to: "ApprovalPark", - label: "a block approval is pending", - }, - Edge { - from: "ApprovalPark", - to: "Head", - label: "granted: the re-scope drain re-baselines", - }, - Edge { - from: "ApprovalPark", - to: "escalated", - label: "denied with no fallback", - }, - Edge { - from: "ApprovalPark", - to: "stopped", - label: "stop while parked", - }, - Edge { - from: "Head", - to: "DistressPark", - label: "the agent raised distress(error)", - }, - Edge { - from: "DistressPark", - to: "Head", - label: "the operator cleared the marker", - }, - Edge { - from: "DistressPark", - to: "stopped", - label: "stop, or the park timed out", - }, - Edge { - from: "Head", - to: "stopped", - label: "interrupt at the head", - }, - Edge { - from: "Head", - to: "budget", - label: "a cost or time cap was reached", - }, Edge { from: "Head", to: "finished", @@ -555,7 +623,7 @@ const EDGES: &[Edge] = &[ /// wire carries, so a reader can match a diagram node to a run's shutdown line. pub(crate) fn mermaid() -> String { let mut out = String::from("stateDiagram-v2\n [*] --> Head\n"); - for edge in EDGES { + for edge in HEAD.iter().flat_map(|c| c.edges()).chain(EDGES) { out.push_str(&format!( " {} --> {}: {}\n", edge.from, edge.to, edge.label @@ -1028,6 +1096,45 @@ mod tests { assert!(chart.contains("parked"), "{chart}"); } + /// The head is dispatched from `HEAD`, so a check that is not listed never runs. `edges` + /// is exhaustive, which makes a new variant compile only once it is described; this holds + /// it to also being dispatched, exactly once. + #[test] + fn head_dispatches_every_check_exactly_once() { + let all = [ + HeadCheck::WaitIfPaused, + HeadCheck::ParkOnPendingBlock, + HeadCheck::DrainRescope, + HeadCheck::DrainDeny, + HeadCheck::ParkOnDistress, + HeadCheck::Interrupt, + HeadCheck::Budget, + ]; + for check in all { + assert_eq!( + HEAD.iter().filter(|h| **h == check).count(), + 1, + "{check:?} is not dispatched exactly once" + ); + } + assert_eq!( + HEAD.len(), + all.len(), + "HEAD has a check this test does not know" + ); + } + + /// The order is the contract the chart is drawn from: a pending approval parks before a + /// re-scope drains it, and both settle before the run can be ended by a cap. + #[test] + fn the_head_parks_before_it_drains_and_ends() { + let at = |c: HeadCheck| HEAD.iter().position(|h| *h == c).expect("dispatched"); + assert!(at(HeadCheck::ParkOnPendingBlock) < at(HeadCheck::DrainRescope)); + assert!(at(HeadCheck::DrainRescope) < at(HeadCheck::DrainDeny)); + assert!(at(HeadCheck::ParkOnDistress) < at(HeadCheck::Interrupt)); + assert!(at(HeadCheck::Interrupt) < at(HeadCheck::Budget)); + } + /// Every edge must start somewhere the chart can be entered from, so the diagram is one /// connected machine rather than a pile of arrows. #[test] diff --git a/docs/loop-machine.md b/docs/loop-machine.md index 025e8296..af99026d 100644 --- a/docs/loop-machine.md +++ b/docs/loop-machine.md @@ -11,6 +11,8 @@ stateDiagram-v2 ApprovalPark --> Head: granted: the re-scope drain re-baselines ApprovalPark --> escalated: denied with no fallback ApprovalPark --> stopped: stop while parked + Head --> Head: an approved re-scope re-baselines a new segment + Head --> Head: a denial noted; the frozen regime stands Head --> DistressPark: the agent raised distress(error) DistressPark --> Head: the operator cleared the marker DistressPark --> stopped: stop, or the park timed out