From 6d94bcb74221bf4970e17e4a644ef5486552eecb Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Tue, 31 Mar 2026 21:03:23 +1030 Subject: [PATCH] preflight: show Test Engine failures from the build --- cmd/preflight/preflight.go | 7 +- cmd/preflight/render.go | 45 ++++++++ go.mod | 3 +- go.sum | 6 +- internal/build/watch/test_tracker.go | 35 ++++++ internal/build/watch/test_tracker_test.go | 129 ++++++++++++++++++++++ internal/build/watch/watch.go | 57 ++++++++++ 7 files changed, 275 insertions(+), 7 deletions(-) create mode 100644 internal/build/watch/test_tracker.go create mode 100644 internal/build/watch/test_tracker_test.go diff --git a/cmd/preflight/preflight.go b/cmd/preflight/preflight.go index b29831fb..f427a2e2 100644 --- a/cmd/preflight/preflight.go +++ b/cmd/preflight/preflight.go @@ -147,7 +147,12 @@ func (c *PreflightCmd) Run(kongCtx *kong.Context, globals cli.GlobalFlags) error return renderStatusError{err: err} } return nil - }) + }, watch.WithTestTracking(func(newFailures []buildkite.BuildTest) error { + if err := renderer.renderTestFailures(newFailures); err != nil { + return renderStatusError{err: err} + } + return nil + })) if err != nil { var renderErr renderStatusError if errors.As(err, &renderErr) { diff --git a/cmd/preflight/render.go b/cmd/preflight/render.go index e86e4738..2aac995c 100644 --- a/cmd/preflight/render.go +++ b/cmd/preflight/render.go @@ -9,6 +9,7 @@ import ( "github.com/buildkite/cli/v3/internal/build/watch" internalpreflight "github.com/buildkite/cli/v3/internal/preflight" + buildkite "github.com/buildkite/go-buildkite/v4" ) const maxTTYRunningJobs = 10 @@ -17,6 +18,7 @@ type renderer interface { appendSnapshotLine(string) setSnapshot(*internalpreflight.SnapshotResult) renderStatus(watch.BuildStatus, string) error + renderTestFailures([]buildkite.BuildTest) error flush() renderFinalFailures(Result, watch.FailedJobs) } @@ -92,6 +94,14 @@ func (r *ttyRenderer) renderStatus(status watch.BuildStatus, buildState string) return nil } +func (r *ttyRenderer) renderTestFailures(tests []buildkite.BuildTest) error { + for _, t := range tests { + line := formatTestFailureLine(t) + r.failedRegion.AppendLine(line) + } + return nil +} + func (r *ttyRenderer) flush() { r.screen.Flush() } @@ -122,6 +132,15 @@ func (r *plainRenderer) setSnapshot(result *internalpreflight.SnapshotResult) { } } +func (r *plainRenderer) renderTestFailures(tests []buildkite.BuildTest) error { + for _, t := range tests { + if _, err := fmt.Fprintf(r.stdout, "%s\n", formatTestFailureLine(t)); err != nil { + return err + } + } + return nil +} + func (r *plainRenderer) renderStatus(status watch.BuildStatus, buildState string) error { var presenter jobPresenter = plainJobPresenter{pipeline: r.pipeline, buildNumber: r.buildNumber} for _, failed := range status.NewlyFailed { @@ -206,6 +225,32 @@ func jobLogCommand(pipeline string, buildNumber int, jobID string) string { return fmt.Sprintf("bk job log -b %d -p %s %s", buildNumber, pipeline, jobID) } +func formatTestFailureLine(t buildkite.BuildTest) string { + name := t.Name + if t.Scope != "" { + name = t.Scope + " " + name + } + line := fmt.Sprintf(" \033[31m✗\033[0m \033[33mtest:\033[0m %s", name) + if t.Location != "" { + line += fmt.Sprintf(" \033[2m%s\033[0m", t.Location) + } + if t.LatestFail == nil { + return line + } + if t.LatestFail.FailureReason != "" { + line += fmt.Sprintf("\n \033[2m%s\033[0m", t.LatestFail.FailureReason) + } + for _, fe := range t.LatestFail.FailureExpanded { + for _, exp := range fe.Expanded { + line += fmt.Sprintf("\n \033[2m%s\033[0m", exp) + } + for _, bt := range fe.Backtrace { + line += fmt.Sprintf("\n \033[2m%s\033[0m", bt) + } + } + return line +} + func formatSummaryLine(s watch.JobSummary) string { var parts []string if s.Passed > 0 { diff --git a/go.mod b/go.mod index 1ee31c04..189bd916 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.25.0 require ( github.com/alecthomas/kong v1.14.0 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be - github.com/buildkite/go-buildkite/v4 v4.17.0 + github.com/buildkite/go-buildkite/v4 v4.18.0 github.com/buildkite/roko v1.4.0 github.com/go-git/go-git/v5 v5.17.1 github.com/goccy/go-yaml v1.19.2 @@ -19,7 +19,6 @@ require ( ) require ( - github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/agnivade/levenshtein v1.2.1 // indirect github.com/alexflint/go-arg v1.5.1 // indirect github.com/alexflint/go-scalar v1.2.0 // indirect diff --git a/go.sum b/go.sum index 3c8a0e36..0413d1a8 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,6 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw= github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= -github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= -github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM= github.com/agnivade/levenshtein v1.2.1/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtCyDIjcwga4/DU= github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= @@ -33,8 +31,8 @@ github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwN github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/bradleyjkemp/cupaloy/v2 v2.6.0 h1:knToPYa2xtfg42U3I6punFEjaGFKWQRXJwj0JTv4mTs= github.com/bradleyjkemp/cupaloy/v2 v2.6.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= -github.com/buildkite/go-buildkite/v4 v4.17.0 h1:OYy9PG5A15K4Up2dkZgvXP7esAqzQskA0VGXvciRUNQ= -github.com/buildkite/go-buildkite/v4 v4.17.0/go.mod h1:8+7GiWBKwEPAWoZnRU/kpNCt46j1iVH8kFMMbD4YDfc= +github.com/buildkite/go-buildkite/v4 v4.18.0 h1:L0zypZi+jo9jqqcsrXSk5Jkyn3Hsdi2fwo5g7ltDbtU= +github.com/buildkite/go-buildkite/v4 v4.18.0/go.mod h1:8+7GiWBKwEPAWoZnRU/kpNCt46j1iVH8kFMMbD4YDfc= github.com/buildkite/roko v1.4.0 h1:DxixoCdpNqxu4/1lXrXbfsKbJSd7r1qoxtef/TT2J80= github.com/buildkite/roko v1.4.0/go.mod h1:0vbODqUFEcVf4v2xVXRfZZRsqJVsCCHTG/TBRByGK4E= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= diff --git a/internal/build/watch/test_tracker.go b/internal/build/watch/test_tracker.go new file mode 100644 index 00000000..f7d6971f --- /dev/null +++ b/internal/build/watch/test_tracker.go @@ -0,0 +1,35 @@ +package watch + +import ( + buildkite "github.com/buildkite/go-buildkite/v4" +) + +// TestTracker tracks which test failure executions have already been reported, +// so that each failure is only surfaced once across polling iterations. +type TestTracker struct { + seen map[string]bool // keyed by latest_fail execution ID +} + +// NewTestTracker creates a new TestTracker. +func NewTestTracker() *TestTracker { + return &TestTracker{ + seen: make(map[string]bool), + } +} + +// Update processes a list of build tests and returns only those with +// a LatestFail execution that has not been seen before. +func (t *TestTracker) Update(tests []buildkite.BuildTest) []buildkite.BuildTest { + var newFailures []buildkite.BuildTest + for _, test := range tests { + if test.LatestFail == nil { + continue + } + if t.seen[test.LatestFail.ID] { + continue + } + t.seen[test.LatestFail.ID] = true + newFailures = append(newFailures, test) + } + return newFailures +} diff --git a/internal/build/watch/test_tracker_test.go b/internal/build/watch/test_tracker_test.go new file mode 100644 index 00000000..54c32ce2 --- /dev/null +++ b/internal/build/watch/test_tracker_test.go @@ -0,0 +1,129 @@ +package watch + +import ( + "testing" + + buildkite "github.com/buildkite/go-buildkite/v4" +) + +func TestTestTracker_Update(t *testing.T) { + t.Run("reports new failures", func(t *testing.T) { + tracker := NewTestTracker() + tests := []buildkite.BuildTest{ + { + ID: "test-1", + Name: "flaky test", + LatestFail: &buildkite.BuildTestLatestFail{ + ID: "exec-1", + FailureReason: "expected 3, got 2", + }, + }, + } + + newFailures := tracker.Update(tests) + if len(newFailures) != 1 { + t.Fatalf("expected 1 new failure, got %d", len(newFailures)) + } + if newFailures[0].Name != "flaky test" { + t.Errorf("expected 'flaky test', got %q", newFailures[0].Name) + } + }) + + t.Run("does not re-report same execution", func(t *testing.T) { + tracker := NewTestTracker() + tests := []buildkite.BuildTest{ + { + ID: "test-1", + Name: "flaky test", + LatestFail: &buildkite.BuildTestLatestFail{ + ID: "exec-1", + FailureReason: "expected 3, got 2", + }, + }, + } + + tracker.Update(tests) + newFailures := tracker.Update(tests) + if len(newFailures) != 0 { + t.Errorf("expected 0 new failures on second poll, got %d", len(newFailures)) + } + }) + + t.Run("reports new execution for same test", func(t *testing.T) { + tracker := NewTestTracker() + tracker.Update([]buildkite.BuildTest{ + { + ID: "test-1", + Name: "flaky test", + LatestFail: &buildkite.BuildTestLatestFail{ + ID: "exec-1", + FailureReason: "first failure", + }, + }, + }) + + newFailures := tracker.Update([]buildkite.BuildTest{ + { + ID: "test-1", + Name: "flaky test", + LatestFail: &buildkite.BuildTestLatestFail{ + ID: "exec-2", + FailureReason: "second failure", + }, + }, + }) + + if len(newFailures) != 1 { + t.Fatalf("expected 1 new failure, got %d", len(newFailures)) + } + if newFailures[0].LatestFail.ID != "exec-2" { + t.Errorf("expected exec-2, got %s", newFailures[0].LatestFail.ID) + } + }) + + t.Run("skips tests without latest_fail", func(t *testing.T) { + tracker := NewTestTracker() + tests := []buildkite.BuildTest{ + {ID: "test-1", Name: "passing test"}, + { + ID: "test-2", + Name: "failing test", + LatestFail: &buildkite.BuildTestLatestFail{ + ID: "exec-1", + FailureReason: "boom", + }, + }, + } + + newFailures := tracker.Update(tests) + if len(newFailures) != 1 { + t.Fatalf("expected 1 new failure, got %d", len(newFailures)) + } + if newFailures[0].ID != "test-2" { + t.Errorf("expected test-2, got %s", newFailures[0].ID) + } + }) + + t.Run("handles multiple new failures at once", func(t *testing.T) { + tracker := NewTestTracker() + tests := []buildkite.BuildTest{ + { + ID: "test-1", + LatestFail: &buildkite.BuildTestLatestFail{ID: "exec-1"}, + }, + { + ID: "test-2", + LatestFail: &buildkite.BuildTestLatestFail{ID: "exec-2"}, + }, + { + ID: "test-3", + LatestFail: &buildkite.BuildTestLatestFail{ID: "exec-3"}, + }, + } + + newFailures := tracker.Update(tests) + if len(newFailures) != 3 { + t.Fatalf("expected 3 new failures, got %d", len(newFailures)) + } + }) +} diff --git a/internal/build/watch/watch.go b/internal/build/watch/watch.go index 8c79b66f..b18cbfab 100644 --- a/internal/build/watch/watch.go +++ b/internal/build/watch/watch.go @@ -22,6 +22,25 @@ const ( // Returning an error aborts the watch loop and propagates that error to the caller. type StatusFunc func(b buildkite.Build) error +// TestStatusFunc is called with newly-seen failed test executions on each poll. +// Returning an error aborts the watch loop. +type TestStatusFunc func(newFailures []buildkite.BuildTest) error + +// WatchOpt configures optional WatchBuild behavior. +type WatchOpt func(*watchConfig) + +type watchConfig struct { + onTestStatus TestStatusFunc +} + +// WithTestTracking enables polling BuildTests.List for failed tests on each +// iteration, calling onTestStatus with any newly-seen failures. +func WithTestTracking(fn TestStatusFunc) WatchOpt { + return func(c *watchConfig) { + c.onTestStatus = fn + } +} + // WatchBuild polls a build until it reaches a terminal state (FinishedAt != nil). // It calls onStatus after each successful poll so callers can render progress. func WatchBuild( @@ -31,7 +50,18 @@ func WatchBuild( buildNumber int, interval time.Duration, onStatus StatusFunc, + opts ...WatchOpt, ) (buildkite.Build, error) { + cfg := &watchConfig{} + for _, opt := range opts { + opt(cfg) + } + + var testTracker *TestTracker + if cfg.onTestStatus != nil { + testTracker = NewTestTracker() + } + var ( consecutiveErrors int lastBuild buildkite.Build @@ -60,6 +90,12 @@ func WatchBuild( } } + if testTracker != nil && b.ID != "" { + if err := pollTestFailures(ctx, client, org, b.ID, testTracker, cfg.onTestStatus); err != nil { + return b, err + } + } + if b.FinishedAt != nil || buildstate.IsTerminal(buildstate.State(b.State)) { return b, nil } @@ -72,3 +108,24 @@ func WatchBuild( } } } + +func pollTestFailures(ctx context.Context, client *buildkite.Client, org, buildID string, tracker *TestTracker, onTestStatus TestStatusFunc) error { + reqCtx, cancel := context.WithTimeout(ctx, DefaultRequestTimeout) + defer cancel() + + tests, _, err := client.BuildTests.List(reqCtx, org, buildID, &buildkite.BuildTestsListOptions{ + Result: "^failed", + State: "enabled", + Include: "latest_fail", + }) + if err != nil { + // Test data may not be available yet; don't treat as fatal. + return nil + } + + newFailures := tracker.Update(tests) + if len(newFailures) > 0 { + return onTestStatus(newFailures) + } + return nil +}