diff --git a/cmd/flynn/main.go b/cmd/flynn/main.go index fa5deba..c943cf9 100644 --- a/cmd/flynn/main.go +++ b/cmd/flynn/main.go @@ -170,7 +170,7 @@ func printUsage(w io.Writer) { flynn status [] show the live overview, or one run's phase and progress flynn resume continue a parked or interrupted run by id flynn inspect replay a past run's recorded events (alias: replay) - flynn spine verify check a run's signed, tamper-evident record + flynn spine verify report a run's record tier by tier: integrity, governance, ground truth (or --file for an exported record) flynn auth set store an API key in the encrypted vault flynn models browse the model catalog (filter with --local, --fit, --vram, ...) flynn models bless resolve a Hugging Face model into a verified catalog entry and print it for review diff --git a/cmd/flynn/spine.go b/cmd/flynn/spine.go index 5e1f9ac..ab4546b 100644 --- a/cmd/flynn/spine.go +++ b/cmd/flynn/spine.go @@ -4,8 +4,11 @@ import ( "context" "crypto/ed25519" "encoding/base64" + "encoding/hex" "errors" + "flag" "fmt" + "io" "os" "github.com/ionalpha/flynn/chain" @@ -54,21 +57,39 @@ func sealRun(ctx context.Context, store *sqlite.Store, rec *chain.RecordingLog, return err } +// errChecksFailed reports that a record did not pass every check. The per-tier detail +// is written to stdout; this is the concise error that sets a non-zero exit code so +// the command can gate a script. +var errChecksFailed = errors.New("record did not pass all checks") + // dispatchSpine handles the spine subcommands. func dispatchSpine(args []string, dataDir string) error { + const usage = "usage: flynn spine verify [--file [--key ]] " if len(args) >= 1 && args[0] == "verify" { - if len(args) < 2 { - return errors.New("usage: flynn spine verify ") + fs := flag.NewFlagSet("spine verify", flag.ContinueOnError) + file := fs.String("file", "", "verify a record read from this file instead of a stored run") + keyHex := fs.String("key", "", "hex-encoded Ed25519 public key to verify a record whose signer is not self-certifying (for example a published conformance vector)") + if err := fs.Parse(args[1:]); err != nil { + return err + } + if *file != "" { + return verifyRecordFile(*file, *keyHex) + } + if *keyHex != "" { + return errors.New("--key applies only with --file; a stored run names a self-certifying signer") } - return verifyRun(dataDir, args[1]) + if fs.NArg() < 1 { + return errors.New(usage) + } + return verifyRun(dataDir, fs.Arg(0)) } - return errors.New("usage: flynn spine verify ") + return errors.New(usage) } -// verifyRun reads a run's stored signed record and checks it: the record's events -// rebuild the signed Merkle root and the signature is valid under the key the record -// names. The key id is self-certifying, so verification needs only the durable store -// and the record itself. +// verifyRun reads a run's stored signed record and reports every tier it satisfies: +// integrity (the events rebuild the signed Merkle root), governance (no action ran +// without admission), and ground truth (a claimed success is backed by a passing +// check). The signer is self-certifying, so a stored run needs only the durable store. func verifyRun(dataDir, runID string) error { ctx := context.Background() store, err := openDataStore(ctx, dataDir) @@ -84,34 +105,108 @@ func verifyRun(dataDir, runID string) error { if len(events) == 0 { return fmt.Errorf("no run found with id %q under %s", runID, dataDir) } - record, err := recordFromEvents(events) if err != nil { return fmt.Errorf("run %q: %w", runID, err) } + return verifyRecord(os.Stdout, "run "+runID, record, "") +} + +// verifyRecordFile reads a signed record from a file and reports every tier it +// satisfies. A record whose signer is self-certifying is verified with no further +// input; one signed by another key (a published conformance vector) needs that key in +// hex via --key. +func verifyRecordFile(path, keyHex string) error { + record, err := os.ReadFile(path) //nolint:gosec // the path is an operator-supplied record to verify + if err != nil { + return err + } + return verifyRecord(os.Stdout, path, record, keyHex) +} +// verifyRecord resolves the record's signing key and reports, tier by tier, what the +// record proves. It returns errChecksFailed if any tier is not satisfied, so the +// command exits non-zero while still printing the full report. +func verifyRecord(out io.Writer, label string, record []byte, keyHex string) error { keyID, err := chain.RecordKeyID(record) if err != nil { return err } - pub, err := controlplane.ParsePrincipalID(keyID) + pub, err := resolveKey(keyID, keyHex) if err != nil { - return fmt.Errorf("run %q names an unrecognizable signer: %w", runID, err) + return err } ring := chain.NewRootKeyring() if err := ring.Add(keyID, pub); err != nil { return err } - verified, err := chain.VerifyRun(record, ring) + _, _ = fmt.Fprintf(out, "%s\n", label) + events, err := chain.VerifyRun(record, ring) if err != nil { - _, _ = fmt.Fprintf(os.Stdout, "run %s: NOT VERIFIED: %v\n", runID, err) - return err + _, _ = fmt.Fprintf(out, " integrity: NOT VERIFIED: %v\n", err) + return errChecksFailed + } + _, _ = fmt.Fprintf(out, " integrity: VERIFIED (%d events, signed by %s)\n", len(events), keyID) + + failed := false + if gerr := chain.VerifyGovernance(events); gerr != nil { + _, _ = fmt.Fprintf(out, " governance: VIOLATION: %v\n", gerr) + failed = true + } else { + _, _ = fmt.Fprintln(out, " governance: OK (no action ran without admission)") + } + + gt := chain.VerifyGroundTruth(events) + switch { + case !claimsSuccess(events): + _, _ = fmt.Fprintln(out, " ground-truth: not asserted (no independent check was bound)") + case gt == nil: + _, _ = fmt.Fprintln(out, " ground-truth: GROUNDED (success backed by a passing check)") + default: + _, _ = fmt.Fprintf(out, " ground-truth: NOT GROUNDED: %v\n", gt) + failed = true + } + if failed { + return errChecksFailed } - _, _ = fmt.Fprintf(os.Stdout, "run %s: VERIFIED, %d events, signed by %s\n", runID, len(verified), keyID) return nil } +// resolveKey recovers the public key a record is verified against: the supplied hex +// key when given, otherwise the self-certifying key the record names. +func resolveKey(keyID, keyHex string) (ed25519.PublicKey, error) { + if keyHex != "" { + raw, err := hex.DecodeString(keyHex) + if err != nil { + return nil, fmt.Errorf("--key is not valid hex: %w", err) + } + if len(raw) != ed25519.PublicKeySize { + return nil, fmt.Errorf("--key must be a %d-byte Ed25519 public key, got %d bytes", ed25519.PublicKeySize, len(raw)) + } + return ed25519.PublicKey(raw), nil + } + pub, err := controlplane.ParsePrincipalID(keyID) + if err != nil { + return nil, fmt.Errorf("the record is signed by %q, which is not a self-certifying key; supply its public key with --key", keyID) + } + return pub, nil +} + +// claimsSuccess reports whether the run records a success outcome, which is what the +// ground-truth tier applies to. A run that claims nothing needs no backing check. +func claimsSuccess(events []spine.Event) bool { + for _, e := range events { + if e.Type != chain.OutcomeRecorded { + continue + } + if result, _ := e.Payload[chain.OutcomeResultKey].(string); result == chain.ResultSuccess { + return true + } + } + return false +} + // recordFromEvents extracts the signed record stored on a run's stream. func recordFromEvents(events []spine.Event) ([]byte, error) { for _, e := range events { diff --git a/cmd/flynn/verify_test.go b/cmd/flynn/verify_test.go new file mode 100644 index 0000000..4e20da5 --- /dev/null +++ b/cmd/flynn/verify_test.go @@ -0,0 +1,133 @@ +package main + +import ( + "bytes" + "crypto/ed25519" + "encoding/hex" + "strings" + "testing" + "time" + + "github.com/ionalpha/flynn/chain" + "github.com/ionalpha/flynn/controlplane" + "github.com/ionalpha/flynn/spine" +) + +// vEvent builds a canonical-ready event on the test stream. +func vEvent(seq int64, typ string, payload map[string]any) spine.Event { + return spine.Event{ + Stream: "run/test", + Seq: seq, + Time: time.Unix(0, 1_700_000_000_000_000_000).UTC(), + Type: typ, + Actor: spine.ActorSystem, + SchemaVersion: 1, + Payload: payload, + } +} + +// sealedRecord seals the given events under a signer with keyID and returns the +// portable record bytes (what `--file` reads). +func sealedRecord(t *testing.T, keyID string, priv ed25519.PrivateKey, events ...spine.Event) []byte { + t.Helper() + signer, err := chain.NewEd25519RootSigner(keyID, priv) + if err != nil { + t.Fatal(err) + } + b := chain.NewBuilder("run/test") + for _, e := range events { + if err := b.Add(e); err != nil { + t.Fatal(err) + } + } + sealed, err := b.Seal(signer) + if err != nil { + t.Fatal(err) + } + rec, err := sealed.Marshal() + if err != nil { + t.Fatal(err) + } + return rec +} + +func selfCertKey(t *testing.T) (string, ed25519.PrivateKey) { + t.Helper() + priv := ed25519.NewKeyFromSeed(make([]byte, ed25519.SeedSize)) + pub := priv.Public().(ed25519.PublicKey) + return controlplane.PrincipalID(pub), priv +} + +func TestVerifyRecordReportsTiers(t *testing.T) { + keyID, priv := selfCertKey(t) + rec := sealedRecord(t, keyID, priv, vEvent(1, "action.dispatched", map[string]any{})) + + var buf bytes.Buffer + if err := verifyRecord(&buf, "rec", rec, ""); err != nil { + t.Fatalf("a clean record did not pass: %v", err) + } + out := buf.String() + for _, want := range []string{"integrity: VERIFIED", "governance: OK", "ground-truth: not asserted"} { + if !strings.Contains(out, want) { + t.Fatalf("report missing %q\n%s", want, out) + } + } +} + +func TestVerifyRecordCatchesTamper(t *testing.T) { + keyID, priv := selfCertKey(t) + rec := sealedRecord(t, keyID, priv, vEvent(1, "action.dispatched", map[string]any{})) + rec[len(rec)-1] ^= 0xff // flip a byte + + var buf bytes.Buffer + if err := verifyRecord(&buf, "rec", rec, ""); err == nil { + t.Fatal("a tampered record was accepted") + } + if !strings.Contains(buf.String(), "NOT VERIFIED") { + t.Fatalf("tamper not reported:\n%s", buf.String()) + } +} + +func TestVerifyRecordGroundTruth(t *testing.T) { + keyID, priv := selfCertKey(t) + run := func(passed bool) (string, error) { + rec := sealedRecord( + t, keyID, priv, + vEvent(1, "action.dispatched", map[string]any{}), + vEvent(2, chain.CheckRecorded, map[string]any{chain.CheckRefKey: int64(1), chain.CheckPassedKey: passed}), + vEvent(3, chain.OutcomeRecorded, map[string]any{chain.OutcomeResultKey: chain.ResultSuccess, chain.CheckRefKey: int64(1)}), + ) + var buf bytes.Buffer + err := verifyRecord(&buf, "rec", rec, "") + return buf.String(), err + } + + if out, err := run(true); err != nil || !strings.Contains(out, "ground-truth: GROUNDED") { + t.Fatalf("a grounded run was not reported grounded: err=%v\n%s", err, out) + } + out, err := run(false) + if err == nil || !strings.Contains(out, "ground-truth: NOT GROUNDED") { + t.Fatalf("a run whose check failed was reported grounded: err=%v\n%s", err, out) + } +} + +// TestVerifyRecordExternalKey covers a record signed by a key that is not a +// self-certifying principal id (a published conformance vector): it cannot be verified +// without the key, and verifies when the key is supplied in hex. +func TestVerifyRecordExternalKey(t *testing.T) { + priv := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{7}, ed25519.SeedSize)) + pub := priv.Public().(ed25519.PublicKey) + rec := sealedRecord(t, "conformance-root", priv, vEvent(1, "action.dispatched", map[string]any{})) + + var buf bytes.Buffer + if err := verifyRecord(&buf, "rec", rec, ""); err == nil { + t.Fatal("a record with a non-self-certifying key verified without --key") + } + buf.Reset() + if err := verifyRecord(&buf, "rec", rec, hex.EncodeToString(pub)); err != nil { + t.Fatalf("a record did not verify with its supplied key: %v\n%s", err, buf.String()) + } + if !strings.Contains(buf.String(), "integrity: VERIFIED") { + t.Fatalf("expected verification with --key:\n%s", buf.String()) + } +}