feat(performance): compare baseline reports

README.md

@@ -108,6 +108,10 @@ The unreleased v1.1.0 development checkout also includes a
 comparing canonical transcript artifacts without re-executing an application
 model.
 
+Use the unreleased [performance baseline comparator](docs/performance-baseline.md)
+for same-host report comparisons; it preserves the existing absolute performance
+budgets.
+
 ## Feedback and contributing
 
 Report bugs or propose features through [GitHub issues](https://github.com/ben-ranford/stave/issues).

cmd/stave-performance-compare/main.go

@@ -0,0 +1,79 @@
+// Command stave-performance-compare compares two saved performance reports.
+// It never writes or refreshes a baseline.
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/ben-ranford/stave/performance"
+)
+
+const (
+	exitSuccess    = 0
+	exitRegression = 2
+	exitInvalid    = 3
+	exitUsage      = 64
+)
+
+func main() { os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) }
+
+func run(args []string, stdout, stderr io.Writer) int {
+	flags := flag.NewFlagSet("stave-performance-compare", flag.ContinueOnError)
+	flags.SetOutput(stderr)
+	baselinePath := flags.String("baseline", "", "checked-in baseline performance report")
+	candidatePath := flags.String("candidate", "", "candidate performance report")
+	if err := flags.Parse(args); err != nil || flags.NArg() != 0 || *baselinePath == "" || *candidatePath == "" {
+		fmt.Fprintln(stderr, "usage: stave-performance-compare -baseline baseline.json -candidate candidate.json")
+		return exitUsage
+	}
+	baseline, err := load(*baselinePath)
+	if err != nil {
+		return invalid(stdout)
+	}
+	candidate, err := load(*candidatePath)
+	if err != nil {
+		return invalid(stdout)
+	}
+	comparison, err := performance.Compare(baseline, candidate)
+	if err != nil {
+		return invalid(stdout)
+	}
+	if err := write(stdout, comparison); err != nil {
+		return exitInvalid
+	}
+	if !comparison.Passed {
+		return exitRegression
+	}
+	return exitSuccess
+}
+
+func load(path string) (performance.Report, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return performance.Report{}, err
+	}
+	defer file.Close()
+	data, err := io.ReadAll(io.LimitReader(file, performance.MaxReportBytes+1))
+	if err != nil {
+		return performance.Report{}, err
+	}
+	return performance.DecodeReport(data)
+}
+
+func invalid(stdout io.Writer) int {
+	_ = write(stdout, map[string]string{"status": "invalid", "error": "report failed bounded structural or compatibility validation"})
+	return exitInvalid
+}
+
+func write(stdout io.Writer, value any) error {
+	data, err := json.Marshal(value)
+	if err != nil {
+		return err
+	}
+	_, err = fmt.Fprintln(stdout, string(data))
+	return err
+}

cmd/stave-performance-compare/main_test.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/ben-ranford/stave/capability"
+	"github.com/ben-ranford/stave/layout"
+	"github.com/ben-ranford/stave/performance"
+)
+
+func TestRunDistinguishesMatchRegressionAndInvalid(t *testing.T) {
+	dir := t.TempDir()
+	baseline, candidate := filepath.Join(dir, "baseline.json"), filepath.Join(dir, "candidate.json")
+	report := fixtureReport()
+	writeFixture(t, baseline, report)
+	writeFixture(t, candidate, report)
+	if code := run([]string{"-baseline", baseline, "-candidate", candidate}, &bytes.Buffer{}, &bytes.Buffer{}); code != exitSuccess {
+		t.Fatalf("match code=%d", code)
+	}
+	report.Measurements[0].P95 = 120 * time.Nanosecond
+	report.Measurements[0].P99 = 120 * time.Nanosecond
+	writeFixture(t, candidate, report)
+	if code := run([]string{"-baseline", baseline, "-candidate", candidate}, &bytes.Buffer{}, &bytes.Buffer{}); code != exitRegression {
+		t.Fatalf("regression code=%d", code)
+	}
+	if err := os.WriteFile(candidate, []byte(`{"host":"secret"`), 0600); err != nil {
+		t.Fatal(err)
+	}
+	if code := run([]string{"-baseline", baseline, "-candidate", candidate}, &bytes.Buffer{}, &bytes.Buffer{}); code != exitInvalid {
+		t.Fatalf("invalid code=%d", code)
+	}
+}
+
+func writeFixture(t *testing.T, path string, report performance.Report) {
+	t.Helper()
+	data, err := json.Marshal(report)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(path, data, 0600); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func fixtureReport() performance.Report {
+	return performance.Report{Host: "host", GoVersion: "go1.22.0", GOOS: "linux", GOARCH: "amd64", CPUs: 8, Nodes: 10000, NodeShape: "balanced", Renderer: "stave.render/v1", Viewport: layout.Size{Width: 120, Height: 40}, Capabilities: capability.Manifest{Width: 120, Height: 40}, Reproducibility: performance.Reproducibility{Invocation: []string{"stave-performance", "-strict"}, SampleCount: 101, Strict: true, GOMAXPROCS: 1}, AllocBytes: 100, AllocLimit: 200, AllocWithin: true, IdleCPU: performance.RatioMeasurement{Name: "idle_cpu.percent_one_core", Window: time.Second, Value: .2, Limit: 1, AllWithinBudget: true}, Measurements: []performance.Measurement{{Name: "render.p95", Samples: 101, P95: 100 * time.Nanosecond, P99: 100 * time.Nanosecond, Limit: time.Microsecond, AllWithinBudget: true}}}
+}

docs/performance-baseline.md

@@ -0,0 +1,40 @@
+# Performance baseline comparison
+
+`stave-performance-compare` is an unreleased v1.1.0 development command. It
+compares two saved reports produced by `stave-performance`; it never updates or
+creates a baseline.
+
+```sh
+go run ./cmd/stave-performance -strict -out baseline.json
+go run ./cmd/stave-performance -strict -out candidate.json
+go run ./cmd/stave-performance-compare -baseline baseline.json -candidate candidate.json
+```
+
+Both reports must be from the same host, Go version, OS/architecture, CPU
+count, declared fixture metadata, capabilities, and exact run parameters. The executable path
+(`os.Args[0]`) is excluded because launch and build locations can differ
+between runs or revisions, and the `-out`/`--out` destination is excluded because it
+names the artifact rather than a measurement parameter. All remaining arguments
+and environment metadata are compared. Declared invariants must match in order,
+and retained-attempt dispositions must match because they declare the fixed
+collector policy. Retained attempts, when present, must contain nonnegative,
+ordered percentile tuples. The aggregate must match the existing collector’s
+selection rule, and the idle CPU value must equal its lowest retained attempt.
+The report does not contain a fixture
+content hash, so matching declared fixture metadata does not prove identical
+fixture source content. Source revisions and dirty-worktree flags are recorded provenance and may
+differ, including when writing the first report changes the next build’s dirty
+flag. The idle CPU window records actual elapsed time, so it is validated as
+positive but is not required to match exactly between runs. The comparator rejects reports that fail their existing absolute
+budgets, change their budget schema, or use incompatible environment or
+reproducibility metadata.
+
+It emits the versioned `stave.performance.comparison/v1` JSON envelope. Each
+metric has an explicit threshold: p95 measurements and allocation allow a 10%
+increase to absorb ordinary same-host noise, while idle CPU allows 0.10
+percentage points. Exit status is `0` when all deltas are within tolerance, `2`
+for a valid regression, `3` for invalid or incompatible input, and `64` for
+usage errors.
+
+`percentDelta` is omitted when its baseline is zero because a relative change
+is undefined in that case.