feat(run): ground a run's success in an independent verification command

cmd/flynn/main.go

@@ -53,6 +53,7 @@ func main() {
 		verbose     = flag.Bool("v", false, "verbose: show tool arguments, outputs, and per-turn detail")
 		verboseLong = flag.Bool("verbose", false, "alias for -v")
 		plain       = flag.Bool("plain", false, "interactive session: use the line-based interface, not the full-screen one")
+		verify      = flag.String("verify", "", "a command that independently checks the goal succeeded; run after the agent stops, its result grounds the run's success in the verifiable record")
 		showVersion = flag.Bool("version", false, "print version and exit")
 	)
 	flag.Parse()
@@ -69,7 +70,7 @@ func main() {
 			fmt.Fprintln(os.Stderr, `usage: flynn goal "<objective>"`)
 			os.Exit(2)
 		}
-		if err := runGoal(*model, objective, *dataDir, !*noLearn, vrb); err != nil {
+		if err := runGoal(*model, objective, *verify, *dataDir, !*noLearn, vrb); err != nil {
 			fmt.Fprintln(os.Stderr, "error:", err)
 			os.Exit(1)
 		}
@@ -185,7 +186,7 @@ func printUsage(w io.Writer) {
   flynn regrade              re-grade learned skills against the working directory
   flynn serve [--telegram-token T] [--signal-tcp ADDR] [--api-addr ADDR]  run as a service: answer chat messages (Telegram, Signal) and/or expose the read-only monitor API
   flynn --version            print the version
-Flags: --model, --data-dir, --no-learn, -v/--verbose, --plain (run with --help for details).`)
+Flags: --model, --data-dir, --no-learn, --verify "<cmd>", -v/--verbose, --plain (run with --help for details).`)
 }
 
 // defaultDataDir is where durable state lives unless overridden: a per-user
@@ -202,7 +203,7 @@ func defaultDataDir() string {
 // completion in the current directory, recalling past learning into the prompt and
 // (unless disabled) distilling the result back out. Progress and the final result
 // are printed; Ctrl-C cancels the run.
-func runGoal(modelSpec, objective, dataDir string, learnEnabled, verbose bool) error {
+func runGoal(modelSpec, objective, verify, dataDir string, learnEnabled, verbose bool) error {
 	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
 	defer stop()
 
@@ -237,7 +238,7 @@ func runGoal(modelSpec, objective, dataDir string, learnEnabled, verbose bool) e
 	// The objective and the final answer are rendered from the run's own events
 	// (session.started and session.converged), so the live transcript and a later
 	// `flynn inspect` of the same run read identically.
-	if _, err := runLearningMission(ctx, os.Stdout, model, plan, distiller, cwd, objective, store, signer, verbose); err != nil {
+	if _, err := runLearningMission(ctx, os.Stdout, model, plan, distiller, cwd, objective, verify, store, signer, verbose); err != nil {
 		return err
 	}
 	return nil

cmd/flynn/run.go

@@ -241,7 +241,7 @@ func missionRegistry() (*resource.Registry, error) {
 // sandboxed toolset, and (when a distiller is supplied) distills the converged run
 // back into skills and memory so the next run starts ahead. Progress is written to
 // out; the model's final summary is returned.
-func runLearningMission(ctx context.Context, out io.Writer, model llm.Model, plan harness.Plan, distiller learn.Distiller, workdir, objective string, store *sqlite.Store, signer chain.RootSigner, verbose bool) (string, error) {
+func runLearningMission(ctx context.Context, out io.Writer, model llm.Model, plan harness.Plan, distiller learn.Distiller, workdir, objective, verify string, store *sqlite.Store, signer chain.RootSigner, verbose bool) (string, error) {
 	reg, err := missionRegistry()
 	if err != nil {
 		return "", err
@@ -278,6 +278,15 @@ func runLearningMission(ctx context.Context, out io.Writer, model llm.Model, pla
 		return "", err
 	}
 
+	// Ground the run's success in an independent check before sealing, when one is
+	// given. The check runs after the agent has stopped and is never seen by the
+	// model, so a sealed run that claims success is backed by a verdict the model
+	// could not have produced. This must run before the seal so the check and outcome
+	// events are part of the verifiable record.
+	if verify != "" && rec != nil {
+		recordGroundTruth(ctx, out, rec, source, workdir, verify)
+	}
+
 	// Seal the run into a signed, verifiable record stored on its own stream, so it
 	// can be checked later from the durable store alone. Best effort: a sealing
 	// failure is reported but never fails the run.
@@ -305,6 +314,58 @@ func runLearningMission(ctx context.Context, out io.Writer, model llm.Model, pla
 	return result, nil
 }
 
+// recordGroundTruth runs the run's verification command independently and records the
+// result on the run's stream: a check event carrying the real exit-code verdict, and
+// an outcome event that binds the run's success to it. The verdict is the system's,
+// produced after the agent stopped and never seen by the model, so a sealed run that
+// claims success is grounded in a check the agent could not have graded itself. A
+// failing or unrunnable check is recorded honestly, which makes the run's own record
+// fail the ground-truth check rather than overstate the outcome.
+func recordGroundTruth(ctx context.Context, out io.Writer, log spine.Log, stream, workdir, verify string) {
+	passed := runVerification(ctx, workdir, verify)
+	if err := appendGroundTruth(ctx, log, stream, passed); err != nil {
+		_, _ = fmt.Fprintf(out, "  (ground-truth not recorded: %v)\n", err)
+		return
+	}
+	if passed {
+		_, _ = fmt.Fprintln(out, "  ground-truth check passed; the run's success is independently verifiable")
+	} else {
+		_, _ = fmt.Fprintln(out, "  ground-truth check did not pass; the run's success is not grounded")
+	}
+}
+
+// runVerification runs command in a confined sandbox at workdir and reports whether it
+// succeeded (exit 0). The command is operator-supplied and run after the agent stops,
+// so its verdict is independent of anything the model produced.
+func runVerification(ctx context.Context, workdir, command string) bool {
+	sb, err := sandbox.NewLocal(workdir, sandbox.WithDefaultConfinement())
+	if err != nil {
+		return false
+	}
+	res, err := sb.Exec(ctx, sandbox.Command{Line: command})
+	return err == nil && res.ExitCode == 0
+}
+
+// appendGroundTruth records the independent check's verdict and binds the run's
+// success to it on the run's stream, using the chain's ground-truth vocabulary.
+func appendGroundTruth(ctx context.Context, log spine.Log, stream string, passed bool) error {
+	if _, err := log.Append(ctx, spine.AppendInput{
+		Stream:  stream,
+		Type:    chain.CheckRecorded,
+		Actor:   spine.ActorSystem,
+		Payload: map[string]any{chain.CheckRefKey: int64(1), chain.CheckPassedKey: passed},
+	}); err != nil {
+		return err
+	}
+	_, err := log.Append(ctx, spine.AppendInput{
+		Stream:  stream,
+		Type:    chain.OutcomeRecorded,
+		Actor:   spine.ActorSystem,
+		Payload: map[string]any{chain.OutcomeResultKey: chain.ResultSuccess, chain.CheckRefKey: int64(1)},
+	})
+	return err
+}
+
 // distillOutcome distills a converged run into durable skills and memory and retires
 // skills that enough runs have proven unhelpful, reporting the tally to out. It is
 // best effort: a capture or decay failure never fails the run. A captured skill's

cmd/flynn/run_test.go

@@ -54,7 +54,7 @@ func TestRunWritesFileThroughSandbox(t *testing.T) {
 	defer cancel()
 	var out bytes.Buffer
 
-	result, err := runLearningMission(ctx, &out, model, harness.Plan{}, nil, dir, "create hello.txt with a greeting", memStore(t), nil, false)
+	result, err := runLearningMission(ctx, &out, model, harness.Plan{}, nil, dir, "create hello.txt with a greeting", "", memStore(t), nil, false)
 	if err != nil {
 		t.Fatalf("run: %v", err)
 	}
@@ -82,7 +82,7 @@ func TestRunRejectsSandboxEscape(t *testing.T) {
 	defer cancel()
 	var out bytes.Buffer
 
-	if _, err := runLearningMission(ctx, &out, model, harness.Plan{}, nil, dir, "try to escape", memStore(t), nil, false); err != nil {
+	if _, err := runLearningMission(ctx, &out, model, harness.Plan{}, nil, dir, "try to escape", "", memStore(t), nil, false); err != nil {
 		t.Fatalf("run: %v", err)
 	}
 	if _, err := os.Stat(filepath.Join(filepath.Dir(dir), "escape.txt")); !os.IsNotExist(err) {
@@ -129,13 +129,13 @@ func TestRunRemembersAcrossRuns(t *testing.T) {
 	distiller := &fakeDistiller{lessons: []learn.Lesson{
 		{Kind: learn.LessonMemory, Body: "the project uses pnpm for installs"},
 	}}
-	if _, err := runLearningMission(ctx, &out, run1, harness.Plan{}, distiller, dir, "set up the project", store, nil, false); err != nil {
+	if _, err := runLearningMission(ctx, &out, run1, harness.Plan{}, distiller, dir, "set up the project", "", store, nil, false); err != nil {
 		t.Fatalf("run 1: %v", err)
 	}
 
 	// Run 2: shares a keyword ("pnpm") with the stored memory, so recall injects it.
 	run2 := llmtest.NewScripted(llmtest.SayText("installed deps"))
-	if _, err := runLearningMission(ctx, &out, run2, harness.Plan{}, nil, dir, "install deps with pnpm", store, nil, false); err != nil {
+	if _, err := runLearningMission(ctx, &out, run2, harness.Plan{}, nil, dir, "install deps with pnpm", "", store, nil, false); err != nil {
 		t.Fatalf("run 2: %v", err)
 	}
 
@@ -196,7 +196,7 @@ func TestRunFeedsTranscriptToDistiller(t *testing.T) {
 		llmtest.SayText("wrote x.txt"),
 	)
 	rec := &recordingDistiller{}
-	if _, err := runLearningMission(ctx, &out, model, harness.Plan{}, rec, dir, "write x.txt", memStore(t), nil, false); err != nil {
+	if _, err := runLearningMission(ctx, &out, model, harness.Plan{}, rec, dir, "write x.txt", "", memStore(t), nil, false); err != nil {
 		t.Fatalf("run: %v", err)
 	}
 
@@ -263,7 +263,7 @@ func TestRunReinforcesRecalledSkill(t *testing.T) {
 		t.Fatal(err)
 	}
 	model := llmtest.NewScripted(llmtest.SayText("done"))
-	if _, err := runLearningMission(ctx, &out, model, harness.Plan{}, &fakeDistiller{}, dir, "deploy with docker", store, nil, false); err != nil {
+	if _, err := runLearningMission(ctx, &out, model, harness.Plan{}, &fakeDistiller{}, dir, "deploy with docker", "", store, nil, false); err != nil {
 		t.Fatalf("run: %v", err)
 	}
 
@@ -291,7 +291,7 @@ func TestRunVerifiesCapturedSkill(t *testing.T) {
 		{Kind: learn.LessonSkill, Title: "Broken skill", Body: "does not work", Check: "exit 1"},
 		{Kind: learn.LessonSkill, Title: "Good skill", Body: "works", Check: "exit 0"},
 	}}
-	if _, err := runLearningMission(ctx, &out, model, harness.Plan{}, distiller, dir, "do the work", store, nil, false); err != nil {
+	if _, err := runLearningMission(ctx, &out, model, harness.Plan{}, distiller, dir, "do the work", "", store, nil, false); err != nil {
 		t.Fatalf("run: %v", err)
 	}
 
@@ -344,7 +344,7 @@ func TestRunSpineIsDurableAndAddressable(t *testing.T) {
 	if err != nil {
 		t.Fatalf("open store: %v", err)
 	}
-	if _, err := runLearningMission(ctx, &out, model, harness.Plan{}, nil, workdir, "do the thing", store1, nil, false); err != nil {
+	if _, err := runLearningMission(ctx, &out, model, harness.Plan{}, nil, workdir, "do the thing", "", store1, nil, false); err != nil {
 		t.Fatalf("run: %v", err)
 	}
 	runID := runIDFromOutput(t, out.String())

cmd/flynn/spine_test.go

@@ -155,3 +155,88 @@ func TestSpineVerifyRejectsUnsealed(t *testing.T) {
 		t.Fatal("verified a nonexistent run")
 	}
 }
+
+// TestRunVerificationIndependent confirms the verification command's verdict is the
+// command's real exit code, run in a sandbox and not derived from anything the model
+// said.
+func TestRunVerificationIndependent(t *testing.T) {
+	ctx := context.Background()
+	if !runVerification(ctx, t.TempDir(), "exit 0") {
+		t.Fatal("a passing command was reported as failed")
+	}
+	if runVerification(ctx, t.TempDir(), "exit 1") {
+		t.Fatal("a failing command was reported as passed")
+	}
+}
+
+// TestRunGroundTruthEndToEnd drives a real run, grounds its success in an independent
+// check, seals it, and verifies the sealed record. A run whose check passed is
+// grounded; a run whose check failed claims success the record can show is not backed,
+// so the run's own sealed record fails the ground-truth check rather than overstating
+// the outcome.
+func TestRunGroundTruthEndToEnd(t *testing.T) {
+	groundTruthOf := func(t *testing.T, verify string) error {
+		t.Helper()
+		ctx := context.Background()
+		dataDir := t.TempDir()
+		store, err := openDataStore(ctx, dataDir)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() { _ = store.Close() }()
+		reg, err := missionRegistry()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		seed := make([]byte, ed25519.SeedSize)
+		priv := ed25519.NewKeyFromSeed(seed)
+		pub := priv.Public().(ed25519.PublicKey)
+		keyID := controlplane.PrincipalID(pub)
+		signer, err := chain.NewEd25519RootSigner(keyID, priv)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		rec := chain.NewRecordingLog(store.Log(), nil)
+		model := llmtest.NewScripted(llmtest.SayText("done"))
+		_, runID, _, err := drive(ctx, io.Discard, model, harness.Plan{}, t.TempDir(),
+			"reply done and stop", defaultSystemPrompt, store.Resources(reg), store.Jobs(), rec, false, "")
+		if err != nil {
+			t.Fatalf("drive: %v", err)
+		}
+		recordGroundTruth(ctx, io.Discard, rec, runID, t.TempDir(), verify)
+		if err := sealRun(ctx, store, rec, runID, signer); err != nil {
+			t.Fatalf("seal: %v", err)
+		}
+
+		events, err := store.Log().Read(ctx, spine.Query{Stream: runID})
+		if err != nil {
+			t.Fatal(err)
+		}
+		record, err := recordFromEvents(events)
+		if err != nil {
+			t.Fatalf("record: %v", err)
+		}
+		ring := chain.NewRootKeyring()
+		if err := ring.Add(keyID, pub); err != nil {
+			t.Fatal(err)
+		}
+		runEvents, err := chain.VerifyRun(record, ring)
+		if err != nil {
+			t.Fatalf("verify run: %v", err)
+		}
+		return chain.VerifyGroundTruth(runEvents)
+	}
+
+	t.Run("passing check grounds the run", func(t *testing.T) {
+		if err := groundTruthOf(t, "exit 0"); err != nil {
+			t.Fatalf("a run with a passing independent check was not grounded: %v", err)
+		}
+	})
+	t.Run("failing check is not grounded", func(t *testing.T) {
+		if err := groundTruthOf(t, "exit 1"); err == nil {
+			t.Fatal("a run whose independent check failed was accepted as grounded")
+		}
+	})
+}