diff --git a/README.md b/README.md index b2f645f..757d41d 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,10 @@ The unreleased v1.1.0 development checkout also includes a comparing canonical transcript artifacts without re-executing an application model. +Use the unreleased [performance baseline comparator](docs/performance-baseline.md) +for same-host report comparisons; it preserves the existing absolute performance +budgets. + ## Feedback and contributing Report bugs or propose features through [GitHub issues](https://github.com/ben-ranford/stave/issues). diff --git a/cmd/stave-performance-compare/main.go b/cmd/stave-performance-compare/main.go new file mode 100644 index 0000000..dc36632 --- /dev/null +++ b/cmd/stave-performance-compare/main.go @@ -0,0 +1,79 @@ +// Command stave-performance-compare compares two saved performance reports. +// It never writes or refreshes a baseline. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "os" + + "github.com/ben-ranford/stave/performance" +) + +const ( + exitSuccess = 0 + exitRegression = 2 + exitInvalid = 3 + exitUsage = 64 +) + +func main() { os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) } + +func run(args []string, stdout, stderr io.Writer) int { + flags := flag.NewFlagSet("stave-performance-compare", flag.ContinueOnError) + flags.SetOutput(stderr) + baselinePath := flags.String("baseline", "", "checked-in baseline performance report") + candidatePath := flags.String("candidate", "", "candidate performance report") + if err := flags.Parse(args); err != nil || flags.NArg() != 0 || *baselinePath == "" || *candidatePath == "" { + fmt.Fprintln(stderr, "usage: stave-performance-compare -baseline baseline.json -candidate candidate.json") + return exitUsage + } + baseline, err := load(*baselinePath) + if err != nil { + return invalid(stdout) + } + candidate, err := load(*candidatePath) + if err != nil { + return invalid(stdout) + } + comparison, err := performance.Compare(baseline, candidate) + if err != nil { + return invalid(stdout) + } + if err := write(stdout, comparison); err != nil { + return exitInvalid + } + if !comparison.Passed { + return exitRegression + } + return exitSuccess +} + +func load(path string) (performance.Report, error) { + file, err := os.Open(path) + if err != nil { + return performance.Report{}, err + } + defer file.Close() + data, err := io.ReadAll(io.LimitReader(file, performance.MaxReportBytes+1)) + if err != nil { + return performance.Report{}, err + } + return performance.DecodeReport(data) +} + +func invalid(stdout io.Writer) int { + _ = write(stdout, map[string]string{"status": "invalid", "error": "report failed bounded structural or compatibility validation"}) + return exitInvalid +} + +func write(stdout io.Writer, value any) error { + data, err := json.Marshal(value) + if err != nil { + return err + } + _, err = fmt.Fprintln(stdout, string(data)) + return err +} diff --git a/cmd/stave-performance-compare/main_test.go b/cmd/stave-performance-compare/main_test.go new file mode 100644 index 0000000..76023d1 --- /dev/null +++ b/cmd/stave-performance-compare/main_test.go @@ -0,0 +1,52 @@ +package main + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/ben-ranford/stave/capability" + "github.com/ben-ranford/stave/layout" + "github.com/ben-ranford/stave/performance" +) + +func TestRunDistinguishesMatchRegressionAndInvalid(t *testing.T) { + dir := t.TempDir() + baseline, candidate := filepath.Join(dir, "baseline.json"), filepath.Join(dir, "candidate.json") + report := fixtureReport() + writeFixture(t, baseline, report) + writeFixture(t, candidate, report) + if code := run([]string{"-baseline", baseline, "-candidate", candidate}, &bytes.Buffer{}, &bytes.Buffer{}); code != exitSuccess { + t.Fatalf("match code=%d", code) + } + report.Measurements[0].P95 = 120 * time.Nanosecond + report.Measurements[0].P99 = 120 * time.Nanosecond + writeFixture(t, candidate, report) + if code := run([]string{"-baseline", baseline, "-candidate", candidate}, &bytes.Buffer{}, &bytes.Buffer{}); code != exitRegression { + t.Fatalf("regression code=%d", code) + } + if err := os.WriteFile(candidate, []byte(`{"host":"secret"`), 0600); err != nil { + t.Fatal(err) + } + if code := run([]string{"-baseline", baseline, "-candidate", candidate}, &bytes.Buffer{}, &bytes.Buffer{}); code != exitInvalid { + t.Fatalf("invalid code=%d", code) + } +} + +func writeFixture(t *testing.T, path string, report performance.Report) { + t.Helper() + data, err := json.Marshal(report) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0600); err != nil { + t.Fatal(err) + } +} + +func fixtureReport() performance.Report { + return performance.Report{Host: "host", GoVersion: "go1.22.0", GOOS: "linux", GOARCH: "amd64", CPUs: 8, Nodes: 10000, NodeShape: "balanced", Renderer: "stave.render/v1", Viewport: layout.Size{Width: 120, Height: 40}, Capabilities: capability.Manifest{Width: 120, Height: 40}, Reproducibility: performance.Reproducibility{Invocation: []string{"stave-performance", "-strict"}, SampleCount: 101, Strict: true, GOMAXPROCS: 1}, AllocBytes: 100, AllocLimit: 200, AllocWithin: true, IdleCPU: performance.RatioMeasurement{Name: "idle_cpu.percent_one_core", Window: time.Second, Value: .2, Limit: 1, AllWithinBudget: true}, Measurements: []performance.Measurement{{Name: "render.p95", Samples: 101, P95: 100 * time.Nanosecond, P99: 100 * time.Nanosecond, Limit: time.Microsecond, AllWithinBudget: true}}} +} diff --git a/docs/performance-baseline.md b/docs/performance-baseline.md new file mode 100644 index 0000000..b37701d --- /dev/null +++ b/docs/performance-baseline.md @@ -0,0 +1,40 @@ +# Performance baseline comparison + +`stave-performance-compare` is an unreleased v1.1.0 development command. It +compares two saved reports produced by `stave-performance`; it never updates or +creates a baseline. + +```sh +go run ./cmd/stave-performance -strict -out baseline.json +go run ./cmd/stave-performance -strict -out candidate.json +go run ./cmd/stave-performance-compare -baseline baseline.json -candidate candidate.json +``` + +Both reports must be from the same host, Go version, OS/architecture, CPU +count, declared fixture metadata, capabilities, and exact run parameters. The executable path +(`os.Args[0]`) is excluded because launch and build locations can differ +between runs or revisions, and the `-out`/`--out` destination is excluded because it +names the artifact rather than a measurement parameter. All remaining arguments +and environment metadata are compared. Declared invariants must match in order, +and retained-attempt dispositions must match because they declare the fixed +collector policy. Retained attempts, when present, must contain nonnegative, +ordered percentile tuples. The aggregate must match the existing collector’s +selection rule, and the idle CPU value must equal its lowest retained attempt. +The report does not contain a fixture +content hash, so matching declared fixture metadata does not prove identical +fixture source content. Source revisions and dirty-worktree flags are recorded provenance and may +differ, including when writing the first report changes the next build’s dirty +flag. The idle CPU window records actual elapsed time, so it is validated as +positive but is not required to match exactly between runs. The comparator rejects reports that fail their existing absolute +budgets, change their budget schema, or use incompatible environment or +reproducibility metadata. + +It emits the versioned `stave.performance.comparison/v1` JSON envelope. Each +metric has an explicit threshold: p95 measurements and allocation allow a 10% +increase to absorb ordinary same-host noise, while idle CPU allows 0.10 +percentage points. Exit status is `0` when all deltas are within tolerance, `2` +for a valid regression, `3` for invalid or incompatible input, and `64` for +usage errors. + +`percentDelta` is omitted when its baseline is zero because a relative change +is undefined in that case. diff --git a/performance/comparison.go b/performance/comparison.go new file mode 100644 index 0000000..b885171 --- /dev/null +++ b/performance/comparison.go @@ -0,0 +1,441 @@ +package performance + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "reflect" + "sort" + "strings" + "sync" + "unicode/utf8" +) + +const ( + ComparisonSchemaVersion = "stave.performance.comparison/v1" + MaxReportBytes = 16 << 20 +) + +// ComparisonPolicy makes the same-host noise allowance explicit. All values +// are increases: relative fractions for p95/allocation and percentage points +// for idle CPU. +type ComparisonPolicy struct { + MeasurementTolerance float64 + AllocationTolerance float64 + IdleCPUTolerance float64 +} + +var DefaultComparisonPolicy = ComparisonPolicy{MeasurementTolerance: 0.10, AllocationTolerance: 0.10, IdleCPUTolerance: 0.10} + +// Comparison is an opt-in envelope for comparing two existing performance +// report artifacts. Report itself intentionally remains unchanged. +type Comparison struct { + SchemaVersion string `json:"schemaVersion"` + Policy string `json:"policy"` + BaselineRevision string `json:"baselineRevision,omitempty"` + CandidateRevision string `json:"candidateRevision,omitempty"` + Deltas []MetricDelta `json:"deltas"` + Passed bool `json:"passed"` +} + +// MetricDelta describes one deterministic comparison result. Values are raw +// report units: nanoseconds for p95, bytes for allocation, and percentage +// points for idle CPU. +type MetricDelta struct { + Metric string `json:"metric"` + Unit string `json:"unit"` + Baseline float64 `json:"baseline"` + Candidate float64 `json:"candidate"` + Delta float64 `json:"delta"` + PercentDelta float64 `json:"percentDelta,omitempty"` + Tolerance float64 `json:"tolerance"` + ToleranceUnit string `json:"toleranceUnit"` + WithinTolerance bool `json:"withinTolerance"` +} + +// MarshalJSON preserves the public float64 API while making percentDelta +// optional only for relative metrics with a zero baseline. A defined 0% delta +// remains visible, while undefined relative and absolute deltas omit the field. +func (delta MetricDelta) MarshalJSON() ([]byte, error) { + type metricDeltaWire MetricDelta + var percent *float64 + if delta.ToleranceUnit == "relative" && delta.Baseline != 0 { + value := delta.PercentDelta + percent = &value + } + return json.Marshal(struct { + metricDeltaWire + PercentDelta *float64 `json:"percentDelta,omitempty"` + }{metricDeltaWire: metricDeltaWire(delta), PercentDelta: percent}) +} + +// DecodeReport strictly decodes an existing performance report without +// changing its wire schema. It rejects duplicate and unknown fields so a +// baseline cannot silently reinterpret saved measurements. +func DecodeReport(data []byte) (Report, error) { + if len(data) > MaxReportBytes { + return Report{}, fmt.Errorf("performance report exceeds %d-byte limit", MaxReportBytes) + } + data = bytes.TrimSpace(data) + if len(data) == 0 || !utf8.Valid(data) || data[0] != '{' { + return Report{}, errors.New("performance report must be a UTF-8 JSON object") + } + if err := validateReportJSONKeys(data); err != nil { + return Report{}, fmt.Errorf("decode performance report: %w", err) + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + var report Report + if err := decoder.Decode(&report); err != nil { + return Report{}, fmt.Errorf("decode performance report: %w", err) + } + if err := ensureReportEOF(decoder); err != nil { + return Report{}, err + } + if err := ValidateReport(report); err != nil { + return Report{}, err + } + return report, nil +} + +// Compare requires reports from the same declared environment and run policy. +// Source revisions may differ because that is the point of a baseline check. +// It never updates either input or creates a baseline. +func Compare(baseline, candidate Report) (Comparison, error) { + return CompareWithPolicy(baseline, candidate, DefaultComparisonPolicy) +} + +// CompareWithPolicy compares compatible reports with an explicit validated +// same-host noise policy. It never updates either input or creates a baseline. +func CompareWithPolicy(baseline, candidate Report, policy ComparisonPolicy) (Comparison, error) { + if err := validatePolicy(policy); err != nil { + return Comparison{}, err + } + if err := ValidateReport(baseline); err != nil { + return Comparison{}, fmt.Errorf("invalid baseline report: %w", err) + } + if err := ValidateReport(candidate); err != nil { + return Comparison{}, fmt.Errorf("invalid candidate report: %w", err) + } + candidateMeasurements := measurementIndex(candidate.Measurements) + if err := compatibleEnvironment(baseline, candidate, candidateMeasurements); err != nil { + return Comparison{}, err + } + deltas := make([]MetricDelta, 0, len(baseline.Measurements)+2) + for _, metric := range baseline.Measurements { + candidateMetric := candidateMeasurements[metric.Name] + deltas = append(deltas, relativeDelta(metric.Name, "nanoseconds", float64(metric.P95), float64(candidateMetric.P95), policy.MeasurementTolerance)) + } + deltas = append(deltas, + relativeDelta("alloc_bytes", "bytes", float64(baseline.AllocBytes), float64(candidate.AllocBytes), policy.AllocationTolerance), + absoluteDelta("idle_cpu.percent_one_core", "percentage_points", baseline.IdleCPU.Value, candidate.IdleCPU.Value, policy.IdleCPUTolerance), + ) + sort.Slice(deltas, func(i, j int) bool { return deltas[i].Metric < deltas[j].Metric }) + passed := true + for _, delta := range deltas { + passed = passed && delta.WithinTolerance + } + return Comparison{SchemaVersion: ComparisonSchemaVersion, Policy: fmt.Sprintf("same host, Go version, OS/architecture, CPU count, fixture, capabilities, and run parameters; source revisions may differ; %.2f%% p95, %.2f%% allocation, and %.2f percentage-point idle-CPU tolerance", policy.MeasurementTolerance*100, policy.AllocationTolerance*100, policy.IdleCPUTolerance), BaselineRevision: baseline.Reproducibility.VCSRevision, CandidateRevision: candidate.Reproducibility.VCSRevision, Deltas: deltas, Passed: passed}, nil +} + +func validatePolicy(policy ComparisonPolicy) error { + for _, value := range []float64{policy.MeasurementTolerance, policy.AllocationTolerance, policy.IdleCPUTolerance} { + if math.IsNaN(value) || math.IsInf(value, 0) || value < 0 { + return errors.New("comparison tolerances must be finite and non-negative") + } + } + return nil +} + +func ValidateReport(report Report) error { + if report.Host == "" || report.Host == "unknown" || report.GoVersion == "" || report.GOOS == "" || report.GOARCH == "" || report.CPUs < 1 || report.Nodes < 1 || report.NodeShape == "" || report.Renderer == "" || report.Viewport.Width < 1 || report.Viewport.Height < 1 { + return errors.New("performance report has incomplete environment metadata") + } + if report.Reproducibility.SampleCount < 1 || report.Reproducibility.GOMAXPROCS < 1 || len(report.Reproducibility.Invocation) == 0 { + return errors.New("performance report has incomplete reproducibility metadata") + } + if report.AllocBytes > report.AllocLimit || !report.AllocWithin { + return errors.New("performance report fails its allocation budget") + } + if err := validateIdleCPUReport(report.IdleCPU); err != nil { + return err + } + if len(report.Measurements) == 0 { + return errors.New("performance report has no measurements") + } + seen := map[string]struct{}{} + for _, measurement := range report.Measurements { + if measurement.Name == "" || measurement.Samples < 1 || measurement.P50 < 0 || measurement.P50 > measurement.P95 || measurement.P95 > measurement.P99 || measurement.P95 > measurement.Limit || measurement.Limit <= 0 || !measurement.AllWithinBudget { + return fmt.Errorf("performance report fails absolute budget for %q", measurement.Name) + } + if err := validateMeasurementAttempts(measurement); err != nil { + return fmt.Errorf("performance report has invalid retained attempts for %q: %w", measurement.Name, err) + } + if _, duplicate := seen[measurement.Name]; duplicate { + return fmt.Errorf("performance report has duplicate metric %q", measurement.Name) + } + seen[measurement.Name] = struct{}{} + } + return nil +} + +func validateMeasurementAttempts(measurement Measurement) error { + if len(measurement.Attempts) == 0 { + return nil + } + var selected Attempt + for _, attempt := range measurement.Attempts { + if attempt.P50 < 0 || attempt.P50 > attempt.P95 || attempt.P95 > attempt.P99 { + return errors.New("attempt percentiles are not ordered non-negative durations") + } + // This deliberately mirrors the existing collector's zero-P95 sentinel: + // a zero selection is replaced by the next retained attempt. + if selected.P95 == 0 || attempt.P95 < selected.P95 { + selected = attempt + } + } + if measurement.P50 != selected.P50 || measurement.P95 != selected.P95 || measurement.P99 != selected.P99 { + return errors.New("aggregate does not match the selected retained attempt") + } + return nil +} + +func validateIdleCPUReport(metric RatioMeasurement) error { + if metric.Name != "idle_cpu.percent_one_core" || metric.Window <= 0 || math.IsNaN(metric.Value) || math.IsInf(metric.Value, 0) || math.IsNaN(metric.Limit) || math.IsInf(metric.Limit, 0) || metric.Value < 0 || metric.Limit <= 0 || metric.Value >= metric.Limit || !metric.AllWithinBudget { + return errors.New("performance report fails its idle CPU budget") + } + if len(metric.Attempts) == 0 { + return nil + } + minimum := metric.Attempts[0] + for _, attempt := range metric.Attempts { + if math.IsNaN(attempt) || math.IsInf(attempt, 0) || attempt < 0 { + return errors.New("performance report has invalid idle CPU attempts") + } + if attempt < minimum { + minimum = attempt + } + } + if metric.Value != minimum { + return errors.New("idle CPU value does not match the lowest retained attempt") + } + return nil +} + +func compatibleEnvironment(baseline, candidate Report, candidateMeasurements map[string]Measurement) error { + if baseline.Host != candidate.Host || baseline.GoVersion != candidate.GoVersion || baseline.GOOS != candidate.GOOS || baseline.GOARCH != candidate.GOARCH || baseline.CPUs != candidate.CPUs || baseline.Nodes != candidate.Nodes || baseline.NodeShape != candidate.NodeShape || baseline.Renderer != candidate.Renderer || baseline.Viewport != candidate.Viewport || !reflect.DeepEqual(baseline.Capabilities, candidate.Capabilities) || !reflect.DeepEqual(baseline.Invariants, candidate.Invariants) { + return errors.New("performance reports use incompatible environment or fixture schema") + } + if baseline.Reproducibility.SampleCount != candidate.Reproducibility.SampleCount || baseline.Reproducibility.Strict != candidate.Reproducibility.Strict || baseline.Reproducibility.GOMAXPROCS != candidate.Reproducibility.GOMAXPROCS || !reflect.DeepEqual(comparableInvocation(baseline.Reproducibility.Invocation), comparableInvocation(candidate.Reproducibility.Invocation)) { + return errors.New("performance reports use incompatible reproducibility parameters") + } + if baseline.AllocLimit != candidate.AllocLimit || baseline.IdleCPU.Limit != candidate.IdleCPU.Limit || baseline.IdleCPU.Name != candidate.IdleCPU.Name || baseline.IdleCPU.Disposition != candidate.IdleCPU.Disposition || len(baseline.IdleCPU.Attempts) != len(candidate.IdleCPU.Attempts) || len(baseline.Measurements) != len(candidate.Measurements) { + return errors.New("performance reports use incompatible absolute budget schema") + } + for _, measurement := range baseline.Measurements { + candidateMetric := candidateMeasurements[measurement.Name] + if candidateMetric.Name == "" || candidateMetric.Limit != measurement.Limit || candidateMetric.Samples != measurement.Samples || candidateMetric.Disposition != measurement.Disposition || len(candidateMetric.Attempts) != len(measurement.Attempts) { + return fmt.Errorf("performance reports use incompatible metric schema for %q", measurement.Name) + } + } + return nil +} + +// comparableInvocation preserves measured run parameters while excluding +// launch and artifact locations. os.Args[0] can be an ephemeral go-build path, +// while -out names an artifact; neither changes a measurement run. +func comparableInvocation(invocation []string) []string { + result := make([]string, 0, len(invocation)) + for i := 1; i < len(invocation); i++ { + if invocation[i] == "-out" || invocation[i] == "--out" { + i++ + continue + } + if strings.HasPrefix(invocation[i], "-out=") || strings.HasPrefix(invocation[i], "--out=") { + continue + } + result = append(result, invocation[i]) + } + return result +} + +func measurementIndex(measurements []Measurement) map[string]Measurement { + indexed := make(map[string]Measurement, len(measurements)) + for _, measurement := range measurements { + indexed[measurement.Name] = measurement + } + return indexed +} + +func relativeDelta(metric, unit string, baseline, candidate, tolerance float64) MetricDelta { + delta := candidate - baseline + percent := 0.0 + withinTolerance := candidate == 0 + if baseline > 0 { + percent = delta / baseline + withinTolerance = percent <= tolerance + } + return MetricDelta{Metric: metric, Unit: unit, Baseline: baseline, Candidate: candidate, Delta: delta, PercentDelta: percent, Tolerance: tolerance, ToleranceUnit: "relative", WithinTolerance: withinTolerance} +} + +func absoluteDelta(metric, unit string, baseline, candidate, tolerance float64) MetricDelta { + delta := candidate - baseline + return MetricDelta{Metric: metric, Unit: unit, Baseline: baseline, Candidate: candidate, Delta: delta, Tolerance: tolerance, ToleranceUnit: "absolute", WithinTolerance: delta <= tolerance && !math.IsNaN(candidate) && !math.IsInf(candidate, 0)} +} + +func validateReportJSONKeys(data []byte) error { + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.UseNumber() + if err := validateTypedJSONValue(decoder, 0, reflect.TypeOf(Report{})); err != nil { + return err + } + return ensureReportEOF(decoder) +} + +func validateJSONKeys(data []byte, depth int) error { + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.UseNumber() + if err := validateTypedJSONValue(decoder, depth, nil); err != nil { + return err + } + return ensureReportEOF(decoder) +} + +func validateTypedJSONValue(decoder *json.Decoder, depth int, typ reflect.Type) error { + token, err := decoder.Token() + if err != nil { + return err + } + delim, container := token.(json.Delim) + if !container { + return nil + } + if depth > 64 { + return errors.New("performance report JSON nesting exceeds limit") + } + typ = indirectJSONType(typ) + switch delim { + case '{': + return validateTypedJSONObject(decoder, depth, typ) + case '[': + return validateTypedJSONArray(decoder, depth, typ) + default: + return errors.New("invalid performance report JSON") + } +} + +func validateTypedJSONObject(decoder *json.Decoder, depth int, typ reflect.Type) error { + fields := jsonStructFields(typ) + seen := map[string]struct{}{} + for decoder.More() { + key, err := validateJSONObjectKey(decoder, seen) + if err != nil { + return err + } + fieldType, err := typedJSONField(fields, key) + if err != nil { + return err + } + if err := validateTypedJSONValue(decoder, depth+1, fieldType); err != nil { + return err + } + } + _, err := decoder.Token() + return err +} + +func validateTypedJSONArray(decoder *json.Decoder, depth int, typ reflect.Type) error { + var element reflect.Type + if typ != nil && (typ.Kind() == reflect.Array || typ.Kind() == reflect.Slice) { + element = typ.Elem() + } + for decoder.More() { + if err := validateTypedJSONValue(decoder, depth+1, element); err != nil { + return err + } + } + _, err := decoder.Token() + return err +} + +func typedJSONField(fields map[string]reflect.Type, key string) (reflect.Type, error) { + if fields == nil { + return nil, nil + } + if fieldType, exists := fields[key]; exists { + return fieldType, nil + } + for name := range fields { + if strings.EqualFold(key, name) { + return nil, fmt.Errorf("noncanonical performance report key %q, want %q", key, name) + } + } + return nil, nil +} + +func indirectJSONType(typ reflect.Type) reflect.Type { + for typ != nil && typ.Kind() == reflect.Pointer { + typ = typ.Elem() + } + return typ +} + +// jsonStructFieldCache stores immutable field maps keyed by the framework type. +// Typed array entries reuse their schema without allocating a map per object. +var jsonStructFieldCache sync.Map // map[reflect.Type]map[string]reflect.Type + +func jsonStructFields(typ reflect.Type) map[string]reflect.Type { + if typ == nil || typ.Kind() != reflect.Struct { + return nil + } + if cached, ok := jsonStructFieldCache.Load(typ); ok { + return cached.(map[string]reflect.Type) + } + fields := make(map[string]reflect.Type) + for i := 0; i < typ.NumField(); i++ { + field := typ.Field(i) + if field.PkgPath != "" { + continue + } + name, _, _ := strings.Cut(field.Tag.Get("json"), ",") + if name == "-" { + continue + } + if name == "" { + name = field.Name + } + fields[name] = field.Type + } + actual, _ := jsonStructFieldCache.LoadOrStore(typ, fields) + return actual.(map[string]reflect.Type) +} + +func validateJSONObjectKey(decoder *json.Decoder, seen map[string]struct{}) (string, error) { + keyToken, err := decoder.Token() + if err != nil { + return "", err + } + key, ok := keyToken.(string) + if !ok { + return "", errors.New("invalid performance report object key") + } + if _, exists := seen[key]; exists { + return "", fmt.Errorf("duplicate performance report key %q", key) + } + seen[key] = struct{}{} + return key, nil +} + +func ensureReportEOF(decoder *json.Decoder) error { + var extra any + if err := decoder.Decode(&extra); err != io.EOF { + if err == nil { + return errors.New("trailing JSON") + } + return err + } + return nil +} diff --git a/performance/comparison_test.go b/performance/comparison_test.go new file mode 100644 index 0000000..33c8264 --- /dev/null +++ b/performance/comparison_test.go @@ -0,0 +1,418 @@ +package performance + +import ( + "bytes" + "encoding/json" + "math" + "runtime" + "strings" + "testing" + "time" + + "github.com/ben-ranford/stave/capability" + "github.com/ben-ranford/stave/layout" +) + +func TestCompareProducesDeterministicDeltasAndAllowsSourceRevisionChange(t *testing.T) { + baseline := comparisonFixture() + candidate := comparisonFixture() + candidate.Reproducibility.VCSRevision = "candidate" + baseline.Reproducibility.Invocation = append(baseline.Reproducibility.Invocation, "-out", "baseline.json") + candidate.Reproducibility.Invocation = append(candidate.Reproducibility.Invocation, "-out", "candidate.json") + candidate.Measurements[0].P95 = 105 * time.Nanosecond + candidate.Measurements[0].P99 = 105 * time.Nanosecond + comparison, err := Compare(baseline, candidate) + if err != nil { + t.Fatal(err) + } + if !comparison.Passed || comparison.SchemaVersion != ComparisonSchemaVersion || len(comparison.Deltas) != 3 { + t.Fatalf("comparison=%+v", comparison) + } + if comparison.BaselineRevision != "baseline" || comparison.CandidateRevision != "candidate" { + t.Fatalf("revisions=%q/%q", comparison.BaselineRevision, comparison.CandidateRevision) + } +} + +func TestCompareIgnoresNonMeasurementInvocationPaths(t *testing.T) { + baseline := comparisonFixture() + candidate := comparisonFixture() + baseline.Reproducibility.Invocation = []string{"/private/var/folders/a/go-build123/b001/exe/stave-performance", "-strict", "--out=baseline.json"} + candidate.Reproducibility.Invocation = []string{"/private/var/folders/b/go-build456/b001/exe/stave-performance", "-strict", "--out", "candidate.json"} + if _, err := Compare(baseline, candidate); err != nil { + t.Fatalf("Compare() rejected equivalent invocation paths: %v", err) + } +} + +func TestCompareRejectsMeasuredInvocationArgumentChange(t *testing.T) { + baseline := comparisonFixture() + candidate := comparisonFixture() + candidate.Reproducibility.Invocation = []string{"/tmp/go-build/exe/stave-performance", "-strict=false"} + if _, err := Compare(baseline, candidate); err == nil { + t.Fatal("Compare() accepted different invocation arguments") + } +} + +func TestCompareWithPolicyRejectsInvalidTolerancesAndReports(t *testing.T) { + baseline := comparisonFixture() + for _, policy := range []ComparisonPolicy{{MeasurementTolerance: -1}, {MeasurementTolerance: math.NaN()}, {MeasurementTolerance: math.Inf(1)}} { + if _, err := CompareWithPolicy(baseline, baseline, policy); err == nil { + t.Fatal("CompareWithPolicy() accepted invalid tolerance") + } + } + for _, mutate := range []func(*Report){ + func(r *Report) { r.IdleCPU.Value = math.NaN() }, + func(r *Report) { r.IdleCPU.Value = -1 }, + func(r *Report) { r.IdleCPU.Window = 0 }, + func(r *Report) { r.Host = "unknown" }, + func(r *Report) { r.IdleCPU.Attempts = []float64{math.NaN()} }, + func(r *Report) { r.IdleCPU.Attempts = []float64{math.Inf(1)} }, + func(r *Report) { r.IdleCPU.Attempts = []float64{-1} }, + func(r *Report) { r.Measurements[0].P50 = r.Measurements[0].P95 + time.Nanosecond }, + func(r *Report) { r.Measurements[0].P95 = -time.Nanosecond }, + func(r *Report) { r.Measurements[0].P99 = r.Measurements[0].P95 - time.Nanosecond }, + } { + invalid := comparisonFixture() + mutate(&invalid) + if _, err := Compare(baseline, invalid); err == nil { + t.Fatal("Compare() accepted invalid direct API report") + } + } +} + +func TestCompareRejectsRegressionEnvironmentAndBudgetChanges(t *testing.T) { + baseline := comparisonFixture() + cases := []struct { + name string + mutate func(*Report) + wantOK bool + }{ + {"regression", func(r *Report) { + r.Measurements[0].P95, r.Measurements[0].P99 = 120*time.Nanosecond, 120*time.Nanosecond + }, false}, + {"host", func(r *Report) { r.Host = "other" }, true}, + {"run parameters", func(r *Report) { r.Reproducibility.SampleCount++ }, true}, + {"absolute budget", func(r *Report) { r.Measurements[0].Limit++ }, true}, + {"failed absolute budget", func(r *Report) { r.Measurements[0].AllWithinBudget = false }, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + candidate := comparisonFixture() + tc.mutate(&candidate) + comparison, err := Compare(baseline, candidate) + if tc.wantOK { + if err == nil { + t.Fatal("Compare() accepted incompatible reports") + } + return + } + if err != nil || comparison.Passed { + t.Fatalf("Compare() comparison=%+v err=%v", comparison, err) + } + }) + } +} + +func TestCompareRequiresEqualAttemptCounts(t *testing.T) { + baseline, candidate := comparisonFixture(), comparisonFixture() + baseline.Measurements[0].Attempts = []Attempt{ + {P50: time.Nanosecond, P95: 2 * time.Nanosecond, P99: 3 * time.Nanosecond}, + {P50: 4 * time.Nanosecond, P95: 5 * time.Nanosecond, P99: 6 * time.Nanosecond}, + } + candidate.Measurements[0].Attempts = []Attempt{ + {P50: 7 * time.Nanosecond, P95: 8 * time.Nanosecond, P99: 9 * time.Nanosecond}, + {P50: 10 * time.Nanosecond, P95: 11 * time.Nanosecond, P99: 12 * time.Nanosecond}, + } + baseline.IdleCPU.Attempts = []float64{0.01, 0.02} + candidate.IdleCPU.Attempts = []float64{0.03, 0.04} + baseline.Measurements[0].P50, baseline.Measurements[0].P95, baseline.Measurements[0].P99 = time.Nanosecond, 2*time.Nanosecond, 3*time.Nanosecond + candidate.Measurements[0].P50, candidate.Measurements[0].P95, candidate.Measurements[0].P99 = 7*time.Nanosecond, 8*time.Nanosecond, 9*time.Nanosecond + baseline.IdleCPU.Value = 0.01 + candidate.IdleCPU.Value = 0.03 + if _, err := Compare(baseline, candidate); err != nil { + t.Fatalf("Compare() rejected equal attempt counts with different timings: %v", err) + } + + for _, mutate := range []struct { + name string + apply func(*Report) + }{ + {"measurement", func(report *Report) { + report.Measurements[0].Attempts = append(report.Measurements[0].Attempts, Attempt{P50: 12 * time.Nanosecond, P95: 13 * time.Nanosecond, P99: 14 * time.Nanosecond}) + }}, + {"idle CPU", func(report *Report) { + report.IdleCPU.Attempts = append(report.IdleCPU.Attempts, 0.05) + }}, + } { + t.Run(mutate.name, func(t *testing.T) { + mismatched := candidate + mismatched.Measurements = append([]Measurement(nil), candidate.Measurements...) + mismatched.Measurements[0].Attempts = append([]Attempt(nil), candidate.Measurements[0].Attempts...) + mismatched.IdleCPU.Attempts = append([]float64(nil), candidate.IdleCPU.Attempts...) + mutate.apply(&mismatched) + if _, err := Compare(baseline, mismatched); err == nil { + t.Fatal("Compare() accepted mismatched attempt counts") + } + }) + } +} + +func TestCompareRejectsDifferentDeclaredInvariantsAndDisposition(t *testing.T) { + baseline := comparisonFixture() + for _, mutate := range []struct { + name string + apply func(*Report) + }{ + {"invariants", func(report *Report) { report.Invariants = []string{"fixture", "changed"} }}, + {"measurement disposition", func(report *Report) { report.Measurements[0].Disposition = "median retained attempt" }}, + {"idle CPU disposition", func(report *Report) { report.IdleCPU.Disposition = "mean retained window" }}, + } { + t.Run(mutate.name, func(t *testing.T) { + candidate := comparisonFixture() + mutate.apply(&candidate) + if _, err := Compare(baseline, candidate); err == nil { + t.Fatal("Compare() accepted incompatible declared collection contract") + } + }) + } +} + +func TestValidateReportRetainedAttemptsMatchCollectorSelection(t *testing.T) { + valid := comparisonFixture() + valid.Measurements[0].Attempts = []Attempt{ + {P50: 0, P95: 0, P99: 0}, + {P50: 4 * time.Nanosecond, P95: 5 * time.Nanosecond, P99: 6 * time.Nanosecond}, + {P50: 3 * time.Nanosecond, P95: 4 * time.Nanosecond, P99: 7 * time.Nanosecond}, + } + // The collector's zero-P95 sentinel replaces the initial zero attempt, then + // picks the lowest following P95. + valid.Measurements[0].P50, valid.Measurements[0].P95, valid.Measurements[0].P99 = 3*time.Nanosecond, 4*time.Nanosecond, 7*time.Nanosecond + valid.IdleCPU.Attempts = []float64{0.4, 0.2, 0.3} + valid.IdleCPU.Value = 0.2 + if err := ValidateReport(valid); err != nil { + t.Fatalf("ValidateReport() rejected collector-selected attempts: %v", err) + } + + tie := comparisonFixture() + tie.Measurements[0].Attempts = []Attempt{ + {P50: time.Nanosecond, P95: 2 * time.Nanosecond, P99: 3 * time.Nanosecond}, + {P50: 0, P95: 2 * time.Nanosecond, P99: 4 * time.Nanosecond}, + } + tie.Measurements[0].P50, tie.Measurements[0].P95, tie.Measurements[0].P99 = time.Nanosecond, 2*time.Nanosecond, 3*time.Nanosecond + if err := ValidateReport(tie); err != nil { + t.Fatalf("ValidateReport() rejected first retained tie: %v", err) + } + + for _, mutate := range []struct { + name string + apply func(*Report) + }{ + {"wrong selected aggregate", func(report *Report) { report.Measurements[0].P95 = 5 * time.Nanosecond }}, + {"unordered attempt", func(report *Report) { + report.Measurements[0].Attempts[0] = Attempt{P50: 3 * time.Nanosecond, P95: 2 * time.Nanosecond, P99: 4 * time.Nanosecond} + }}, + {"idle CPU not minimum", func(report *Report) { report.IdleCPU.Value = 0.3 }}, + } { + t.Run(mutate.name, func(t *testing.T) { + invalid := valid + invalid.Measurements = append([]Measurement(nil), valid.Measurements...) + invalid.Measurements[0].Attempts = append([]Attempt(nil), valid.Measurements[0].Attempts...) + invalid.IdleCPU.Attempts = append([]float64(nil), valid.IdleCPU.Attempts...) + mutate.apply(&invalid) + if err := ValidateReport(invalid); err == nil { + t.Fatal("ValidateReport() accepted inconsistent retained attempts") + } + }) + } +} + +func TestComparisonJSONDistinguishesDefinedAndUndefinedZeroPercentDelta(t *testing.T) { + baseline, candidate := comparisonFixture(), comparisonFixture() + comparison, err := Compare(baseline, candidate) + if err != nil { + t.Fatal(err) + } + if percent := percentDeltaForMetric(t, comparison, "alloc_bytes"); percent == nil || *percent != 0 { + t.Fatalf("defined zero percent delta = %v, want 0", percent) + } + if percentDeltaFieldCount(t, comparison, "alloc_bytes") != 1 { + t.Fatal("defined percent delta was emitted more than once") + } + + baseline.AllocBytes, candidate.AllocBytes = 0, 0 + comparison, err = Compare(baseline, candidate) + if err != nil { + t.Fatal(err) + } + if percentDeltaForMetric(t, comparison, "alloc_bytes") != nil { + t.Fatal("undefined zero-baseline percent delta was emitted") + } +} + +func percentDeltaForMetric(t *testing.T, comparison Comparison, metric string) *float64 { + t.Helper() + data, err := json.Marshal(comparison) + if err != nil { + t.Fatal(err) + } + var wire struct { + Deltas []struct { + Metric string `json:"metric"` + PercentDelta *float64 `json:"percentDelta"` + } `json:"deltas"` + } + if err := json.Unmarshal(data, &wire); err != nil { + t.Fatal(err) + } + for _, delta := range wire.Deltas { + if delta.Metric == metric { + return delta.PercentDelta + } + } + t.Fatalf("comparison omitted metric %q", metric) + return nil +} + +func percentDeltaFieldCount(t *testing.T, comparison Comparison, metric string) int { + t.Helper() + data, err := json.Marshal(comparison) + if err != nil { + t.Fatal(err) + } + var wire struct { + Deltas []json.RawMessage `json:"deltas"` + } + if err := json.Unmarshal(data, &wire); err != nil { + t.Fatal(err) + } + for _, delta := range wire.Deltas { + var header struct { + Metric string `json:"metric"` + } + if err := json.Unmarshal(delta, &header); err != nil { + t.Fatal(err) + } + if header.Metric == metric { + return bytes.Count(delta, []byte(`"percentDelta"`)) + } + } + t.Fatalf("comparison omitted metric %q", metric) + return 0 +} + +func TestDecodeReportRejectsDuplicateAndUnknownFields(t *testing.T) { + report := comparisonFixture() + data, err := json.Marshal(report) + if err != nil { + t.Fatal(err) + } + if _, err := DecodeReport(data); err != nil { + t.Fatalf("DecodeReport() error=%v", err) + } + for _, invalid := range [][]byte{ + append(append([]byte(nil), data[:len(data)-1]...), []byte(`,"host":"other"}`)...), + append(append([]byte(nil), data[:len(data)-1]...), []byte(`,"unexpected":true}`)...), + append(data, []byte(` {}`)...), + } { + if _, err := DecodeReport(invalid); err == nil { + t.Fatal("DecodeReport() accepted hostile artifact") + } + } +} + +func TestDecodeReportRejectsCaseVariantTypedKeys(t *testing.T) { + data, err := json.Marshal(comparisonFixture()) + if err != nil { + t.Fatal(err) + } + for _, replacement := range []struct { + old, new []byte + }{ + {[]byte(`"host":"host"`), []byte(`"host":"host","Host":"other"`)}, + {[]byte(`"sampleCount":101`), []byte(`"sampleCount":101,"SampleCount":101`)}, + } { + mutated := bytes.Replace(data, replacement.old, replacement.new, 1) + if bytes.Equal(mutated, data) { + t.Fatalf("fixture did not add case-variant key %s", replacement.new) + } + if _, err := DecodeReport(mutated); err == nil { + t.Fatalf("DecodeReport() accepted case-variant typed key %s", replacement.new) + } + } +} + +func comparisonFixture() Report { + return Report{ + Host: "host", GoVersion: "go1.22.0", GOOS: "linux", GOARCH: "amd64", CPUs: 8, + Nodes: 10000, NodeShape: "balanced-binary", Renderer: "stave.render/v1", Viewport: layout.Size{Width: 120, Height: 40}, Capabilities: capability.Manifest{Width: 120, Height: 40}, + Reproducibility: Reproducibility{Invocation: []string{"stave-performance", "-strict"}, SampleCount: 101, Strict: true, GOMAXPROCS: 1, VCSRevision: "baseline"}, + AllocBytes: 100, AllocLimit: 200, AllocWithin: true, + IdleCPU: RatioMeasurement{Name: "idle_cpu.percent_one_core", Window: time.Second, Value: 0.20, Limit: 1, AllWithinBudget: true}, + Measurements: []Measurement{{Name: "render.p95", Samples: 101, P95: 100 * time.Nanosecond, P99: 100 * time.Nanosecond, Limit: time.Microsecond, AllWithinBudget: true}}, + } +} + +func TestComparePreservesP95AbsoluteBudget(t *testing.T) { + report := comparisonFixture() + report.Measurements[0].P99 = report.Measurements[0].Limit + time.Nanosecond + report.IdleCPU.Attempts = []float64{report.IdleCPU.Limit + 1, report.IdleCPU.Value} + if _, err := Compare(report, report); err != nil { + t.Fatalf("valid p95/retry report rejected: %v", err) + } +} + +func TestCompareAllowsBuildProvenanceAndObservedIdleWindowChanges(t *testing.T) { + baseline, candidate := comparisonFixture(), comparisonFixture() + candidate.Reproducibility.VCSModified = !baseline.Reproducibility.VCSModified + candidate.IdleCPU.Window += time.Nanosecond + if _, err := Compare(baseline, candidate); err != nil { + t.Fatalf("source provenance or observed elapsed window rejected: %v", err) + } +} + +func TestReportJSONValidationDoesNotCopyEnclosingSubtrees(t *testing.T) { + data := []byte(`{"unknown":` + strings.Repeat("[", 48) + `"` + strings.Repeat("x", 1<<20) + `"` + strings.Repeat("]", 48) + `}`) + runtime.GC() + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + if err := validateJSONKeys(data, 0); err != nil { + t.Fatal(err) + } + runtime.ReadMemStats(&after) + // Allow decoder buffering and token copies, but not one full value copy per ancestor. + if allocated := after.TotalAlloc - before.TotalAlloc; allocated > uint64(12*len(data)) { + t.Fatalf("nested JSON validation allocated %d bytes for %d input bytes", allocated, len(data)) + } +} + +func TestReportJSONValidationRetainsNestedContracts(t *testing.T) { + for _, data := range []string{`{"a":[{"x":1,"x":2}]}`, `{"a":[1,]}`, `{"a":[]} {}`, strings.Repeat("[", 66) + `0` + strings.Repeat("]", 66)} { + if err := validateJSONKeys([]byte(data), 0); err == nil { + t.Fatalf("invalid nested JSON accepted: %.80s", data) + } + } + if err := validateJSONKeys([]byte(strings.Repeat("[", 65)+`0`+strings.Repeat("]", 65)), 0); err != nil { + t.Fatalf("depth boundary rejected: %v", err) + } +} + +func TestReportJSONValidationCachesTypedFieldMaps(t *testing.T) { + data := []byte(`{"measurements":[` + strings.Repeat(`{},`, 9_999) + `{}]}`) + if err := validateReportJSONKeys(data); err != nil { + t.Fatal(err) + } + measure := func(validate func([]byte) error) uint64 { + runtime.GC() + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + if err := validate(data); err != nil { + t.Fatal(err) + } + runtime.ReadMemStats(&after) + return after.TotalAlloc - before.TotalAlloc + } + generic := measure(func(data []byte) error { return validateJSONKeys(data, 0) }) + typed := measure(validateReportJSONKeys) + if typed > generic+uint64(4*len(data)) { + t.Fatalf("typed field schema validation allocated %d bytes versus generic %d for %d-byte input", typed, generic, len(data)) + } +} diff --git a/scripts/rigor/generated/dependency-inventory.json b/scripts/rigor/generated/dependency-inventory.json index f146df5..1ed81f8 100644 --- a/scripts/rigor/generated/dependency-inventory.json +++ b/scripts/rigor/generated/dependency-inventory.json @@ -160,6 +160,18 @@ "time" ] }, + { + "importPath": "github.com/ben-ranford/stave/cmd/stave-performance-compare", + "dir": "cmd/stave-performance-compare", + "imports": [ + "encoding/json", + "flag", + "fmt", + "github.com/ben-ranford/stave/performance", + "io", + "os" + ] + }, { "importPath": "github.com/ben-ranford/stave/cmd/stave-replay", "dir": "cmd/stave-replay", @@ -404,6 +416,7 @@ "importPath": "github.com/ben-ranford/stave/performance", "dir": "performance", "imports": [ + "bytes", "context", "encoding/json", "errors", @@ -418,10 +431,16 @@ "github.com/ben-ranford/stave/secret", "github.com/ben-ranford/stave/semantic", "github.com/ben-ranford/stave/surface", + "io", + "math", + "reflect", "runtime", "runtime/metrics", "sort", - "time" + "strings", + "sync", + "time", + "unicode/utf8" ] }, { diff --git a/scripts/rigor/generated/public-api.txt b/scripts/rigor/generated/public-api.txt index bba8902..562b4d7 100644 --- a/scripts/rigor/generated/public-api.txt +++ b/scripts/rigor/generated/public-api.txt @@ -428,9 +428,14 @@ type Nop struct { } type Observer interface { Observe(context.Context, Event) } [github.com/ben-ranford/stave/performance] +const ComparisonSchemaVersion untyped string = "stave.performance.comparison/v1" +const MaxReportBytes untyped int = 16777216 const Samples untyped int = 101 func ActionAckLine() ([]byte, error) +func Compare(baseline, candidate Report) (Comparison, error) +func CompareWithPolicy(baseline, candidate Report, policy ComparisonPolicy) (Comparison, error) func Context() context.Context +func DecodeReport(data []byte) (Report, error) func Fixture(n int) (semantic.Tree, error) func Measure(fn func()) time.Duration func MeasureIdleCPU(window time.Duration) RatioMeasurement @@ -440,18 +445,24 @@ func ReduceCounter(ctx context.Context, count int, ev event.Event) (int, []effec func ResizeStormRecovery(count int) error func SurfaceFixture(w, h int) (surface.Surface, error) func TerminalRestoreAfterCancellation() error +func ValidateReport(report Report) error method (*measurementDriver) Close() error method (*measurementDriver) Draw(context.Context, surface.Surface, surface.Patch) error method (*measurementDriver) Events() <-chan event.Event method (*measurementDriver) Open(context.Context, capability.Policy) (capability.Manifest, error) method (*measurementDriver) ReadSecret(context.Context, input.SecretPrompt) (secret.Handle, error) method (*measurementDriver) Restore(context.Context) error +method (MetricDelta) MarshalJSON() ([]byte, error) type Attempt struct { P50 time.Duration `json:"p50"`; P95 time.Duration `json:"p95"`; P99 time.Duration `json:"p99"` } type Budget struct { Name string `json:"name"`; Limit time.Duration `json:"limit"` } +type Comparison struct { SchemaVersion string `json:"schemaVersion"`; Policy string `json:"policy"`; BaselineRevision string `json:"baselineRevision,omitempty"`; CandidateRevision string `json:"candidateRevision,omitempty"`; Deltas []MetricDelta `json:"deltas"`; Passed bool `json:"passed"` } +type ComparisonPolicy struct { MeasurementTolerance float64; AllocationTolerance float64; IdleCPUTolerance float64 } type Measurement struct { Name string `json:"name"`; Samples int `json:"samples"`; P50 time.Duration `json:"p50"`; P95 time.Duration `json:"p95"`; P99 time.Duration `json:"p99"`; AllWithinBudget bool `json:"allWithinBudget"`; Limit time.Duration `json:"limit"`; Disposition string `json:"disposition,omitempty"`; Attempts []Attempt `json:"attempts,omitempty"` } +type MetricDelta struct { Metric string `json:"metric"`; Unit string `json:"unit"`; Baseline float64 `json:"baseline"`; Candidate float64 `json:"candidate"`; Delta float64 `json:"delta"`; PercentDelta float64 `json:"percentDelta,omitempty"`; Tolerance float64 `json:"tolerance"`; ToleranceUnit string `json:"toleranceUnit"`; WithinTolerance bool `json:"withinTolerance"` } type RatioMeasurement struct { Name string `json:"name"`; Window time.Duration `json:"window"`; Value float64 `json:"value"`; Limit float64 `json:"limit"`; AllWithinBudget bool `json:"allWithinBudget"`; Disposition string `json:"disposition,omitempty"`; Attempts []float64 `json:"attempts,omitempty"` } type Report struct { GeneratedAt time.Time `json:"generatedAt"`; Host string `json:"host"`; GoVersion string `json:"goVersion"`; GOOS string `json:"goos"`; GOARCH string `json:"goarch"`; CPUs int `json:"cpus"`; Nodes int `json:"nodes"`; NodeShape string `json:"nodeDistribution"`; Renderer string `json:"renderer"`; Viewport layout.Size `json:"viewport"`; Capabilities capability.Manifest `json:"capabilities"`; Reproducibility Reproducibility `json:"reproducibility"`; AllocBytes uint64 `json:"allocBytes"`; AllocLimit uint64 `json:"allocLimitBytes"`; AllocWithin bool `json:"allocWithinBudget"`; IdleCPU RatioMeasurement `json:"idleCpu"`; Measurements []Measurement `json:"measurements"`; Invariants []string `json:"invariants"` } type Reproducibility struct { Invocation []string `json:"invocation"`; SampleCount int `json:"sampleCount"`; Strict bool `json:"strict"`; GOMAXPROCS int `json:"gomaxprocs"`; VCSRevision string `json:"vcsRevision,omitempty"`; VCSModified bool `json:"vcsModified,omitempty"` } +var DefaultComparisonPolicy ComparisonPolicy [github.com/ben-ranford/stave/primitive] const Grid Direction = "grid"