feat(spine): report integrity, governance, and ground truth, and verify a record file

cmd/flynn/main.go

@@ -170,7 +170,7 @@ func printUsage(w io.Writer) {
   flynn status [<run>]       show the live overview, or one run's phase and progress
   flynn resume <run-id>      continue a parked or interrupted run by id
   flynn inspect <run-id>     replay a past run's recorded events (alias: replay)
-  flynn spine verify <run>   check a run's signed, tamper-evident record
+  flynn spine verify <run>   report a run's record tier by tier: integrity, governance, ground truth (or --file <path> for an exported record)
   flynn auth set <provider>  store an API key in the encrypted vault
   flynn models               browse the model catalog (filter with --local, --fit, --vram, ...)
   flynn models bless <ref>   resolve a Hugging Face model into a verified catalog entry and print it for review

cmd/flynn/spine.go

@@ -4,8 +4,11 @@ import (
 	"context"
 	"crypto/ed25519"
 	"encoding/base64"
+	"encoding/hex"
 	"errors"
+	"flag"
 	"fmt"
+	"io"
 	"os"
 
 	"github.com/ionalpha/flynn/chain"
@@ -54,21 +57,39 @@ func sealRun(ctx context.Context, store *sqlite.Store, rec *chain.RecordingLog,
 	return err
 }
 
+// errChecksFailed reports that a record did not pass every check. The per-tier detail
+// is written to stdout; this is the concise error that sets a non-zero exit code so
+// the command can gate a script.
+var errChecksFailed = errors.New("record did not pass all checks")
+
 // dispatchSpine handles the spine subcommands.
 func dispatchSpine(args []string, dataDir string) error {
+	const usage = "usage: flynn spine verify [--file <path> [--key <hex>]] <run-id>"
 	if len(args) >= 1 && args[0] == "verify" {
-		if len(args) < 2 {
-			return errors.New("usage: flynn spine verify <run-id>")
+		fs := flag.NewFlagSet("spine verify", flag.ContinueOnError)
+		file := fs.String("file", "", "verify a record read from this file instead of a stored run")
+		keyHex := fs.String("key", "", "hex-encoded Ed25519 public key to verify a record whose signer is not self-certifying (for example a published conformance vector)")
+		if err := fs.Parse(args[1:]); err != nil {
+			return err
+		}
+		if *file != "" {
+			return verifyRecordFile(*file, *keyHex)
+		}
+		if *keyHex != "" {
+			return errors.New("--key applies only with --file; a stored run names a self-certifying signer")
 		}
-		return verifyRun(dataDir, args[1])
+		if fs.NArg() < 1 {
+			return errors.New(usage)
+		}
+		return verifyRun(dataDir, fs.Arg(0))
 	}
-	return errors.New("usage: flynn spine verify <run-id>")
+	return errors.New(usage)
 }
 
-// verifyRun reads a run's stored signed record and checks it: the record's events
-// rebuild the signed Merkle root and the signature is valid under the key the record
-// names. The key id is self-certifying, so verification needs only the durable store
-// and the record itself.
+// verifyRun reads a run's stored signed record and reports every tier it satisfies:
+// integrity (the events rebuild the signed Merkle root), governance (no action ran
+// without admission), and ground truth (a claimed success is backed by a passing
+// check). The signer is self-certifying, so a stored run needs only the durable store.
 func verifyRun(dataDir, runID string) error {
 	ctx := context.Background()
 	store, err := openDataStore(ctx, dataDir)
@@ -84,34 +105,108 @@ func verifyRun(dataDir, runID string) error {
 	if len(events) == 0 {
 		return fmt.Errorf("no run found with id %q under %s", runID, dataDir)
 	}
-
 	record, err := recordFromEvents(events)
 	if err != nil {
 		return fmt.Errorf("run %q: %w", runID, err)
 	}
+	return verifyRecord(os.Stdout, "run "+runID, record, "")
+}
+
+// verifyRecordFile reads a signed record from a file and reports every tier it
+// satisfies. A record whose signer is self-certifying is verified with no further
+// input; one signed by another key (a published conformance vector) needs that key in
+// hex via --key.
+func verifyRecordFile(path, keyHex string) error {
+	record, err := os.ReadFile(path) //nolint:gosec // the path is an operator-supplied record to verify
+	if err != nil {
+		return err
+	}
+	return verifyRecord(os.Stdout, path, record, keyHex)
+}
 
+// verifyRecord resolves the record's signing key and reports, tier by tier, what the
+// record proves. It returns errChecksFailed if any tier is not satisfied, so the
+// command exits non-zero while still printing the full report.
+func verifyRecord(out io.Writer, label string, record []byte, keyHex string) error {
 	keyID, err := chain.RecordKeyID(record)
 	if err != nil {
 		return err
 	}
-	pub, err := controlplane.ParsePrincipalID(keyID)
+	pub, err := resolveKey(keyID, keyHex)
 	if err != nil {
-		return fmt.Errorf("run %q names an unrecognizable signer: %w", runID, err)
+		return err
 	}
 	ring := chain.NewRootKeyring()
 	if err := ring.Add(keyID, pub); err != nil {
 		return err
 	}
 
-	verified, err := chain.VerifyRun(record, ring)
+	_, _ = fmt.Fprintf(out, "%s\n", label)
+	events, err := chain.VerifyRun(record, ring)
 	if err != nil {
-		_, _ = fmt.Fprintf(os.Stdout, "run %s: NOT VERIFIED: %v\n", runID, err)
-		return err
+		_, _ = fmt.Fprintf(out, "  integrity:    NOT VERIFIED: %v\n", err)
+		return errChecksFailed
+	}
+	_, _ = fmt.Fprintf(out, "  integrity:    VERIFIED (%d events, signed by %s)\n", len(events), keyID)
+
+	failed := false
+	if gerr := chain.VerifyGovernance(events); gerr != nil {
+		_, _ = fmt.Fprintf(out, "  governance:   VIOLATION: %v\n", gerr)
+		failed = true
+	} else {
+		_, _ = fmt.Fprintln(out, "  governance:   OK (no action ran without admission)")
+	}
+
+	gt := chain.VerifyGroundTruth(events)
+	switch {
+	case !claimsSuccess(events):
+		_, _ = fmt.Fprintln(out, "  ground-truth: not asserted (no independent check was bound)")
+	case gt == nil:
+		_, _ = fmt.Fprintln(out, "  ground-truth: GROUNDED (success backed by a passing check)")
+	default:
+		_, _ = fmt.Fprintf(out, "  ground-truth: NOT GROUNDED: %v\n", gt)
+		failed = true
+	}
+	if failed {
+		return errChecksFailed
 	}
-	_, _ = fmt.Fprintf(os.Stdout, "run %s: VERIFIED, %d events, signed by %s\n", runID, len(verified), keyID)
 	return nil
 }
 
+// resolveKey recovers the public key a record is verified against: the supplied hex
+// key when given, otherwise the self-certifying key the record names.
+func resolveKey(keyID, keyHex string) (ed25519.PublicKey, error) {
+	if keyHex != "" {
+		raw, err := hex.DecodeString(keyHex)
+		if err != nil {
+			return nil, fmt.Errorf("--key is not valid hex: %w", err)
+		}
+		if len(raw) != ed25519.PublicKeySize {
+			return nil, fmt.Errorf("--key must be a %d-byte Ed25519 public key, got %d bytes", ed25519.PublicKeySize, len(raw))
+		}
+		return ed25519.PublicKey(raw), nil
+	}
+	pub, err := controlplane.ParsePrincipalID(keyID)
+	if err != nil {
+		return nil, fmt.Errorf("the record is signed by %q, which is not a self-certifying key; supply its public key with --key", keyID)
+	}
+	return pub, nil
+}
+
+// claimsSuccess reports whether the run records a success outcome, which is what the
+// ground-truth tier applies to. A run that claims nothing needs no backing check.
+func claimsSuccess(events []spine.Event) bool {
+	for _, e := range events {
+		if e.Type != chain.OutcomeRecorded {
+			continue
+		}
+		if result, _ := e.Payload[chain.OutcomeResultKey].(string); result == chain.ResultSuccess {
+			return true
+		}
+	}
+	return false
+}
+
 // recordFromEvents extracts the signed record stored on a run's stream.
 func recordFromEvents(events []spine.Event) ([]byte, error) {
 	for _, e := range events {

cmd/flynn/verify_test.go

@@ -0,0 +1,133 @@
+package main
+
+import (
+	"bytes"
+	"crypto/ed25519"
+	"encoding/hex"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/ionalpha/flynn/chain"
+	"github.com/ionalpha/flynn/controlplane"
+	"github.com/ionalpha/flynn/spine"
+)
+
+// vEvent builds a canonical-ready event on the test stream.
+func vEvent(seq int64, typ string, payload map[string]any) spine.Event {
+	return spine.Event{
+		Stream:        "run/test",
+		Seq:           seq,
+		Time:          time.Unix(0, 1_700_000_000_000_000_000).UTC(),
+		Type:          typ,
+		Actor:         spine.ActorSystem,
+		SchemaVersion: 1,
+		Payload:       payload,
+	}
+}
+
+// sealedRecord seals the given events under a signer with keyID and returns the
+// portable record bytes (what `--file` reads).
+func sealedRecord(t *testing.T, keyID string, priv ed25519.PrivateKey, events ...spine.Event) []byte {
+	t.Helper()
+	signer, err := chain.NewEd25519RootSigner(keyID, priv)
+	if err != nil {
+		t.Fatal(err)
+	}
+	b := chain.NewBuilder("run/test")
+	for _, e := range events {
+		if err := b.Add(e); err != nil {
+			t.Fatal(err)
+		}
+	}
+	sealed, err := b.Seal(signer)
+	if err != nil {
+		t.Fatal(err)
+	}
+	rec, err := sealed.Marshal()
+	if err != nil {
+		t.Fatal(err)
+	}
+	return rec
+}
+
+func selfCertKey(t *testing.T) (string, ed25519.PrivateKey) {
+	t.Helper()
+	priv := ed25519.NewKeyFromSeed(make([]byte, ed25519.SeedSize))
+	pub := priv.Public().(ed25519.PublicKey)
+	return controlplane.PrincipalID(pub), priv
+}
+
+func TestVerifyRecordReportsTiers(t *testing.T) {
+	keyID, priv := selfCertKey(t)
+	rec := sealedRecord(t, keyID, priv, vEvent(1, "action.dispatched", map[string]any{}))
+
+	var buf bytes.Buffer
+	if err := verifyRecord(&buf, "rec", rec, ""); err != nil {
+		t.Fatalf("a clean record did not pass: %v", err)
+	}
+	out := buf.String()
+	for _, want := range []string{"integrity:    VERIFIED", "governance:   OK", "ground-truth: not asserted"} {
+		if !strings.Contains(out, want) {
+			t.Fatalf("report missing %q\n%s", want, out)
+		}
+	}
+}
+
+func TestVerifyRecordCatchesTamper(t *testing.T) {
+	keyID, priv := selfCertKey(t)
+	rec := sealedRecord(t, keyID, priv, vEvent(1, "action.dispatched", map[string]any{}))
+	rec[len(rec)-1] ^= 0xff // flip a byte
+
+	var buf bytes.Buffer
+	if err := verifyRecord(&buf, "rec", rec, ""); err == nil {
+		t.Fatal("a tampered record was accepted")
+	}
+	if !strings.Contains(buf.String(), "NOT VERIFIED") {
+		t.Fatalf("tamper not reported:\n%s", buf.String())
+	}
+}
+
+func TestVerifyRecordGroundTruth(t *testing.T) {
+	keyID, priv := selfCertKey(t)
+	run := func(passed bool) (string, error) {
+		rec := sealedRecord(
+			t, keyID, priv,
+			vEvent(1, "action.dispatched", map[string]any{}),
+			vEvent(2, chain.CheckRecorded, map[string]any{chain.CheckRefKey: int64(1), chain.CheckPassedKey: passed}),
+			vEvent(3, chain.OutcomeRecorded, map[string]any{chain.OutcomeResultKey: chain.ResultSuccess, chain.CheckRefKey: int64(1)}),
+		)
+		var buf bytes.Buffer
+		err := verifyRecord(&buf, "rec", rec, "")
+		return buf.String(), err
+	}
+
+	if out, err := run(true); err != nil || !strings.Contains(out, "ground-truth: GROUNDED") {
+		t.Fatalf("a grounded run was not reported grounded: err=%v\n%s", err, out)
+	}
+	out, err := run(false)
+	if err == nil || !strings.Contains(out, "ground-truth: NOT GROUNDED") {
+		t.Fatalf("a run whose check failed was reported grounded: err=%v\n%s", err, out)
+	}
+}
+
+// TestVerifyRecordExternalKey covers a record signed by a key that is not a
+// self-certifying principal id (a published conformance vector): it cannot be verified
+// without the key, and verifies when the key is supplied in hex.
+func TestVerifyRecordExternalKey(t *testing.T) {
+	priv := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{7}, ed25519.SeedSize))
+	pub := priv.Public().(ed25519.PublicKey)
+	rec := sealedRecord(t, "conformance-root", priv, vEvent(1, "action.dispatched", map[string]any{}))
+
+	var buf bytes.Buffer
+	if err := verifyRecord(&buf, "rec", rec, ""); err == nil {
+		t.Fatal("a record with a non-self-certifying key verified without --key")
+	}
+	buf.Reset()
+	if err := verifyRecord(&buf, "rec", rec, hex.EncodeToString(pub)); err != nil {
+		t.Fatalf("a record did not verify with its supplied key: %v\n%s", err, buf.String())
+	}
+	if !strings.Contains(buf.String(), "integrity:    VERIFIED") {
+		t.Fatalf("expected verification with --key:\n%s", buf.String())
+	}
+}