preflight: show Test Engine failures from the build

cmd/preflight/preflight.go

@@ -147,7 +147,12 @@ func (c *PreflightCmd) Run(kongCtx *kong.Context, globals cli.GlobalFlags) error
 			return renderStatusError{err: err}
 		}
 		return nil
-	})
+	}, watch.WithTestTracking(func(newFailures []buildkite.BuildTest) error {
+		if err := renderer.renderTestFailures(newFailures); err != nil {
+			return renderStatusError{err: err}
+		}
+		return nil
+	}))
 	if err != nil {
 		var renderErr renderStatusError
 		if errors.As(err, &renderErr) {

cmd/preflight/render.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/buildkite/cli/v3/internal/build/watch"
 	internalpreflight "github.com/buildkite/cli/v3/internal/preflight"
+	buildkite "github.com/buildkite/go-buildkite/v4"
 )
 
 const maxTTYRunningJobs = 10
@@ -17,6 +18,7 @@ type renderer interface {
 	appendSnapshotLine(string)
 	setSnapshot(*internalpreflight.SnapshotResult)
 	renderStatus(watch.BuildStatus, string) error
+	renderTestFailures([]buildkite.BuildTest) error
 	flush()
 	renderFinalFailures(Result, watch.FailedJobs)
 }
@@ -92,6 +94,14 @@ func (r *ttyRenderer) renderStatus(status watch.BuildStatus, buildState string)
 	return nil
 }
 
+func (r *ttyRenderer) renderTestFailures(tests []buildkite.BuildTest) error {
+	for _, t := range tests {
+		line := formatTestFailureLine(t)
+		r.failedRegion.AppendLine(line)
+	}
+	return nil
+}
+
 func (r *ttyRenderer) flush() {
 	r.screen.Flush()
 }
@@ -122,6 +132,15 @@ func (r *plainRenderer) setSnapshot(result *internalpreflight.SnapshotResult) {
 	}
 }
 
+func (r *plainRenderer) renderTestFailures(tests []buildkite.BuildTest) error {
+	for _, t := range tests {
+		if _, err := fmt.Fprintf(r.stdout, "%s\n", formatTestFailureLine(t)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (r *plainRenderer) renderStatus(status watch.BuildStatus, buildState string) error {
 	var presenter jobPresenter = plainJobPresenter{pipeline: r.pipeline, buildNumber: r.buildNumber}
 	for _, failed := range status.NewlyFailed {
@@ -206,6 +225,32 @@ func jobLogCommand(pipeline string, buildNumber int, jobID string) string {
 	return fmt.Sprintf("bk job log -b %d -p %s %s", buildNumber, pipeline, jobID)
 }
 
+func formatTestFailureLine(t buildkite.BuildTest) string {
+	name := t.Name
+	if t.Scope != "" {
+		name = t.Scope + " " + name
+	}
+	line := fmt.Sprintf("  \033[31m✗\033[0m \033[33mtest:\033[0m %s", name)
+	if t.Location != "" {
+		line += fmt.Sprintf(" \033[2m%s\033[0m", t.Location)
+	}
+	if t.LatestFail == nil {
+		return line
+	}
+	if t.LatestFail.FailureReason != "" {
+		line += fmt.Sprintf("\n    \033[2m%s\033[0m", t.LatestFail.FailureReason)
+	}
+	for _, fe := range t.LatestFail.FailureExpanded {
+		for _, exp := range fe.Expanded {
+			line += fmt.Sprintf("\n    \033[2m%s\033[0m", exp)
+		}
+		for _, bt := range fe.Backtrace {
+			line += fmt.Sprintf("\n    \033[2m%s\033[0m", bt)
+		}
+	}
+	return line
+}
+
 func formatSummaryLine(s watch.JobSummary) string {
 	var parts []string
 	if s.Passed > 0 {

go.mod

@@ -5,7 +5,7 @@ go 1.25.0
 require (
 	github.com/alecthomas/kong v1.14.0
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be
-	github.com/buildkite/go-buildkite/v4 v4.17.0
+	github.com/buildkite/go-buildkite/v4 v4.18.0
 	github.com/buildkite/roko v1.4.0
 	github.com/go-git/go-git/v5 v5.17.1
 	github.com/goccy/go-yaml v1.19.2
@@ -19,7 +19,6 @@ require (
 )
 
 require (
-	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
 	github.com/agnivade/levenshtein v1.2.1 // indirect
 	github.com/alexflint/go-arg v1.5.1 // indirect
 	github.com/alexflint/go-scalar v1.2.0 // indirect

go.sum

@@ -7,8 +7,6 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw=
 github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE=
-github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
-github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
 github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM=
 github.com/agnivade/levenshtein v1.2.1/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtCyDIjcwga4/DU=
 github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0=
@@ -33,8 +31,8 @@ github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwN
 github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
 github.com/bradleyjkemp/cupaloy/v2 v2.6.0 h1:knToPYa2xtfg42U3I6punFEjaGFKWQRXJwj0JTv4mTs=
 github.com/bradleyjkemp/cupaloy/v2 v2.6.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0=
-github.com/buildkite/go-buildkite/v4 v4.17.0 h1:OYy9PG5A15K4Up2dkZgvXP7esAqzQskA0VGXvciRUNQ=
-github.com/buildkite/go-buildkite/v4 v4.17.0/go.mod h1:8+7GiWBKwEPAWoZnRU/kpNCt46j1iVH8kFMMbD4YDfc=
+github.com/buildkite/go-buildkite/v4 v4.18.0 h1:L0zypZi+jo9jqqcsrXSk5Jkyn3Hsdi2fwo5g7ltDbtU=
+github.com/buildkite/go-buildkite/v4 v4.18.0/go.mod h1:8+7GiWBKwEPAWoZnRU/kpNCt46j1iVH8kFMMbD4YDfc=
 github.com/buildkite/roko v1.4.0 h1:DxixoCdpNqxu4/1lXrXbfsKbJSd7r1qoxtef/TT2J80=
 github.com/buildkite/roko v1.4.0/go.mod h1:0vbODqUFEcVf4v2xVXRfZZRsqJVsCCHTG/TBRByGK4E=
 github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=

internal/build/watch/test_tracker.go

@@ -0,0 +1,35 @@
+package watch
+
+import (
+	buildkite "github.com/buildkite/go-buildkite/v4"
+)
+
+// TestTracker tracks which test failure executions have already been reported,
+// so that each failure is only surfaced once across polling iterations.
+type TestTracker struct {
+	seen map[string]bool // keyed by latest_fail execution ID
+}
+
+// NewTestTracker creates a new TestTracker.
+func NewTestTracker() *TestTracker {
+	return &TestTracker{
+		seen: make(map[string]bool),
+	}
+}
+
+// Update processes a list of build tests and returns only those with
+// a LatestFail execution that has not been seen before.
+func (t *TestTracker) Update(tests []buildkite.BuildTest) []buildkite.BuildTest {
+	var newFailures []buildkite.BuildTest
+	for _, test := range tests {
+		if test.LatestFail == nil {
+			continue
+		}
+		if t.seen[test.LatestFail.ID] {
+			continue
+		}
+		t.seen[test.LatestFail.ID] = true
+		newFailures = append(newFailures, test)
+	}
+	return newFailures
+}

internal/build/watch/test_tracker_test.go

@@ -0,0 +1,129 @@
+package watch
+
+import (
+	"testing"
+
+	buildkite "github.com/buildkite/go-buildkite/v4"
+)
+
+func TestTestTracker_Update(t *testing.T) {
+	t.Run("reports new failures", func(t *testing.T) {
+		tracker := NewTestTracker()
+		tests := []buildkite.BuildTest{
+			{
+				ID:   "test-1",
+				Name: "flaky test",
+				LatestFail: &buildkite.BuildTestLatestFail{
+					ID:            "exec-1",
+					FailureReason: "expected 3, got 2",
+				},
+			},
+		}
+
+		newFailures := tracker.Update(tests)
+		if len(newFailures) != 1 {
+			t.Fatalf("expected 1 new failure, got %d", len(newFailures))
+		}
+		if newFailures[0].Name != "flaky test" {
+			t.Errorf("expected 'flaky test', got %q", newFailures[0].Name)
+		}
+	})
+
+	t.Run("does not re-report same execution", func(t *testing.T) {
+		tracker := NewTestTracker()
+		tests := []buildkite.BuildTest{
+			{
+				ID:   "test-1",
+				Name: "flaky test",
+				LatestFail: &buildkite.BuildTestLatestFail{
+					ID:            "exec-1",
+					FailureReason: "expected 3, got 2",
+				},
+			},
+		}
+
+		tracker.Update(tests)
+		newFailures := tracker.Update(tests)
+		if len(newFailures) != 0 {
+			t.Errorf("expected 0 new failures on second poll, got %d", len(newFailures))
+		}
+	})
+
+	t.Run("reports new execution for same test", func(t *testing.T) {
+		tracker := NewTestTracker()
+		tracker.Update([]buildkite.BuildTest{
+			{
+				ID:   "test-1",
+				Name: "flaky test",
+				LatestFail: &buildkite.BuildTestLatestFail{
+					ID:            "exec-1",
+					FailureReason: "first failure",
+				},
+			},
+		})
+
+		newFailures := tracker.Update([]buildkite.BuildTest{
+			{
+				ID:   "test-1",
+				Name: "flaky test",
+				LatestFail: &buildkite.BuildTestLatestFail{
+					ID:            "exec-2",
+					FailureReason: "second failure",
+				},
+			},
+		})
+
+		if len(newFailures) != 1 {
+			t.Fatalf("expected 1 new failure, got %d", len(newFailures))
+		}
+		if newFailures[0].LatestFail.ID != "exec-2" {
+			t.Errorf("expected exec-2, got %s", newFailures[0].LatestFail.ID)
+		}
+	})
+
+	t.Run("skips tests without latest_fail", func(t *testing.T) {
+		tracker := NewTestTracker()
+		tests := []buildkite.BuildTest{
+			{ID: "test-1", Name: "passing test"},
+			{
+				ID:   "test-2",
+				Name: "failing test",
+				LatestFail: &buildkite.BuildTestLatestFail{
+					ID:            "exec-1",
+					FailureReason: "boom",
+				},
+			},
+		}
+
+		newFailures := tracker.Update(tests)
+		if len(newFailures) != 1 {
+			t.Fatalf("expected 1 new failure, got %d", len(newFailures))
+		}
+		if newFailures[0].ID != "test-2" {
+			t.Errorf("expected test-2, got %s", newFailures[0].ID)
+		}
+	})
+
+	t.Run("handles multiple new failures at once", func(t *testing.T) {
+		tracker := NewTestTracker()
+		tests := []buildkite.BuildTest{
+			{
+				ID:         "test-1",
+				LatestFail: &buildkite.BuildTestLatestFail{ID: "exec-1"},
+			},
+			{
+				ID:         "test-2",
+				LatestFail: &buildkite.BuildTestLatestFail{ID: "exec-2"},
+			},
+			{
+				ID:         "test-3",
+				LatestFail: &buildkite.BuildTestLatestFail{ID: "exec-3"},
+			},
+		}
+
+		newFailures := tracker.Update(tests)
+		if len(newFailures) != 3 {
+			t.Fatalf("expected 3 new failures, got %d", len(newFailures))
+		}
+	})
+}

internal/build/watch/watch.go

@@ -22,6 +22,25 @@ const (
 // Returning an error aborts the watch loop and propagates that error to the caller.
 type StatusFunc func(b buildkite.Build) error
 
+// TestStatusFunc is called with newly-seen failed test executions on each poll.
+// Returning an error aborts the watch loop.
+type TestStatusFunc func(newFailures []buildkite.BuildTest) error
+
+// WatchOpt configures optional WatchBuild behavior.
+type WatchOpt func(*watchConfig)
+
+type watchConfig struct {
+	onTestStatus TestStatusFunc
+}
+
+// WithTestTracking enables polling BuildTests.List for failed tests on each
+// iteration, calling onTestStatus with any newly-seen failures.
+func WithTestTracking(fn TestStatusFunc) WatchOpt {
+	return func(c *watchConfig) {
+		c.onTestStatus = fn
+	}
+}
+
 // WatchBuild polls a build until it reaches a terminal state (FinishedAt != nil).
 // It calls onStatus after each successful poll so callers can render progress.
 func WatchBuild(
@@ -31,7 +50,18 @@ func WatchBuild(
 	buildNumber int,
 	interval time.Duration,
 	onStatus StatusFunc,
+	opts ...WatchOpt,
 ) (buildkite.Build, error) {
+	cfg := &watchConfig{}
+	for _, opt := range opts {
+		opt(cfg)
+	}
+
+	var testTracker *TestTracker
+	if cfg.onTestStatus != nil {
+		testTracker = NewTestTracker()
+	}
+
 	var (
 		consecutiveErrors int
 		lastBuild         buildkite.Build
@@ -60,6 +90,12 @@ func WatchBuild(
 				}
 			}
 
+			if testTracker != nil && b.ID != "" {
+				if err := pollTestFailures(ctx, client, org, b.ID, testTracker, cfg.onTestStatus); err != nil {
+					return b, err
+				}
+			}
+
 			if b.FinishedAt != nil || buildstate.IsTerminal(buildstate.State(b.State)) {
 				return b, nil
 			}
@@ -72,3 +108,24 @@ func WatchBuild(
 		}
 	}
 }
+
+func pollTestFailures(ctx context.Context, client *buildkite.Client, org, buildID string, tracker *TestTracker, onTestStatus TestStatusFunc) error {
+	reqCtx, cancel := context.WithTimeout(ctx, DefaultRequestTimeout)
+	defer cancel()
+
+	tests, _, err := client.BuildTests.List(reqCtx, org, buildID, &buildkite.BuildTestsListOptions{
+		Result:  "^failed",
+		State:   "enabled",
+		Include: "latest_fail",
+	})
+	if err != nil {
+		// Test data may not be available yet; don't treat as fatal.
+		return nil
+	}
+
+	newFailures := tracker.Update(tests)
+	if len(newFailures) > 0 {
+		return onTestStatus(newFailures)
+	}
+	return nil
+}