feat: surface BuildKit root-cause on cancellation

builder/solver.go

@@ -21,6 +21,7 @@ import (
 	"github.com/moby/buildkit/util/grpcerrors"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
+	"google.golang.org/grpc/codes"
 )
 
 // statusChanSize is used to ensure we consume all BK status messages without
@@ -90,15 +91,65 @@ func (s *solver) buildMainMulti(
 	})
 	err = eg.Wait()
 
+	chosen := chooseSolveError(buildErr, err)
+	if chosen != nil {
+		return s.withBuildkitFailureContext(chosen)
+	}
+
+	return nil
+}
+
+// chooseSolveError picks the more informative of the two errgroup results.
+// buildErr is from bkClient.Build; monitorErr is from MonitorProgress (which
+// also returns errors from earth's own status processing, e.g. NewCommand).
+//
+// When MonitorProgress aborts the build it cancels the shared errgroup
+// context, so bkClient.Build then returns a bare "context canceled" that
+// masks the real cause. Prefer a non-cancellation monitorErr in that case —
+// otherwise an earth-side self-cancellation is misreported as "BuildKit lost
+// the session".
+func chooseSolveError(buildErr, monitorErr error) error {
+	if buildErr != nil && isCanceledErr(buildErr) && monitorErr != nil && !isCanceledErr(monitorErr) {
+		return errors.Wrap(monitorErr, "earth progress monitor aborted the build")
+	}
+
 	if buildErr != nil {
 		return buildErr
 	}
 
-	if err != nil {
-		return err
+	return monitorErr
+}
+
+func (s *solver) withBuildkitFailureContext(buildErr error) error {
+	if !isCanceledErr(buildErr) {
+		return buildErr
 	}
 
-	return nil
+	if failure, ok := s.logbusSM.FirstFailure(); ok {
+		return solvermon.NewFirstFailureError(buildErr, failure)
+	}
+
+	if cancellation, ok := s.logbusSM.FirstCancellation(); ok {
+		return solvermon.NewFirstCancellationError(buildErr, cancellation)
+	}
+
+	if details, ok := s.logbusSM.CancellationDetails(); ok {
+		return solvermon.NewCancellationDetailsError(buildErr, details)
+	}
+
+	return buildErr
+}
+
+func isCanceledErr(err error) bool {
+	if errors.Is(err, context.Canceled) {
+		return true
+	}
+
+	if grpcErr, ok := grpcerrors.AsGRPCStatus(err); ok && grpcErr.Code() == codes.Canceled {
+		return true
+	}
+
+	return false
 }
 
 func (s *solver) newSolveOptMulti(

builder/solver_test.go

@@ -0,0 +1,69 @@
+package builder
+
+import (
+	"context"
+	"testing"
+
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/require"
+)
+
+func TestChooseSolveError(t *testing.T) {
+	t.Parallel()
+
+	const dupMsg = "duplicate command ID"
+
+	realErr := errors.New("failed to create command: " + dupMsg)
+	canceled := context.Canceled
+
+	tests := map[string]struct {
+		buildErr   error
+		monitorErr error
+		wantSubstr string
+		wantNil    bool
+	}{
+		"both nil": {
+			wantNil: true,
+		},
+		"build error only": {
+			buildErr:   realErr,
+			wantSubstr: dupMsg,
+		},
+		"monitor error only": {
+			monitorErr: realErr,
+			wantSubstr: dupMsg,
+		},
+		"build canceled masks real monitor error": {
+			// The bug: a real earth-side monitor failure cancels the build,
+			// and the resulting bare cancellation must not win.
+			buildErr:   canceled,
+			monitorErr: realErr,
+			wantSubstr: "earth progress monitor aborted the build",
+		},
+		"both canceled stays canceled": {
+			buildErr:   canceled,
+			monitorErr: context.Canceled,
+			wantSubstr: context.Canceled.Error(),
+		},
+		"real build error beats canceled monitor": {
+			buildErr:   realErr,
+			monitorErr: canceled,
+			wantSubstr: dupMsg,
+		},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			got := chooseSolveError(tc.buildErr, tc.monitorErr)
+			if tc.wantNil {
+				require.NoError(t, got)
+				return
+			}
+
+			require.Error(t, got)
+			require.Contains(t, got.Error(), tc.wantSubstr)
+		})
+	}
+}

cmd/earthly/app/run.go

@@ -14,6 +14,7 @@ import (
 	"github.com/EarthBuild/earthbuild/cmd/earthly/helper"
 	"github.com/EarthBuild/earthbuild/earthfile2llb"
 	"github.com/EarthBuild/earthbuild/inputgraph"
+	"github.com/EarthBuild/earthbuild/logbus/solvermon"
 	"github.com/EarthBuild/earthbuild/logstream"
 	"github.com/EarthBuild/earthbuild/util/containerutil"
 	"github.com/EarthBuild/earthbuild/util/errutil"
@@ -139,6 +140,34 @@ func (app *EarthApp) run(ctx context.Context, args []string, lastSignal *syncuti
 	return 0
 }
 
+// printCancellationOrigin reports which side of the client/daemon boundary a
+// canceled solve originated from: if earth's own build context is dead, the
+// cancellation began locally and context.Cause names the culprit; if it is
+// still alive, the daemon (or the session between them) canceled first.
+func (app *EarthApp) printCancellationOrigin(ctx context.Context, lastSignal *syncutil.Signal) {
+	console := app.BaseCLI.Console()
+
+	if sig := lastSignal.Get(); sig != nil {
+		console.Warnf("Local cancellation origin: signal %v received by earth\n", sig)
+		return
+	}
+
+	if ctx.Err() == nil {
+		console.Warnf("Local build context is still alive: " +
+			"the cancellation originated in BuildKit or the session layer, not in earth.\n")
+
+		return
+	}
+
+	cause := context.Cause(ctx)
+	if cause != nil && cause.Error() != context.Canceled.Error() {
+		console.Warnf("Local build context was canceled. Cause: %v\n", cause)
+	} else {
+		console.Warnf("Local build context was canceled with no recorded cause: " +
+			"an earth-side subsystem canceled the build without reporting an error.\n")
+	}
+}
+
 // handleError handles run error, logs it and returns appropriate exit code.
 func (app *EarthApp) handleError(ctx context.Context, err error, args []string, lastSignal *syncutil.Signal) int {
 	ie, isInterpreterError := earthfile2llb.GetInterpreterError(err)
@@ -395,15 +424,89 @@ func (app *EarthApp) handleError(ctx context.Context, err error, args []string,
 		}
 
 		return 6
+	case func() bool {
+		failureErr, ok := solvermon.AsFirstFailureError(err)
+		if !ok {
+			return false
+		}
+
+		app.BaseCLI.Logbus().Run().SetFatalError(
+			failureErr.Failure.End,
+			failureErr.Failure.TargetID,
+			failureErr.Failure.CommandID,
+			failureErr.Failure.FailureType,
+			"",
+			failureErr.Failure.Error,
+		)
+		app.BaseCLI.Console().Warnf("%s\n", failureErr.Failure.String())
+
+		return true
+	}():
+		return 1
+	case func() bool {
+		cancelErr, ok := solvermon.AsFirstCancellationError(err)
+		if !ok {
+			return false
+		}
+
+		app.BaseCLI.Logbus().Run().SetEnd(cancelErr.Cancellation.End, logstream.RunStatus_RUN_STATUS_CANCELED)
+		app.BaseCLI.Console().Warnf(
+			"BuildKit canceled or lost the solve session while running:\n%s\n"+
+				"This usually means the command above was interrupted after an earlier failure, resource event, "+
+				"or buildkit/session failure. "+
+				"Earth did not receive a more specific root cause from BuildKit.\n",
+			cancelErr.Cancellation.String(),
+		)
+		app.printCancellationOrigin(ctx, lastSignal)
+
+		return true
+	}():
+		if containerutil.IsLocal(app.BaseCLI.Flags().BuildkitdSettings.BuildkitAddress) && lastSignal.Get() == nil {
+			app.printCrashLogs(ctx)
+		}
+
+		return 2
+	case func() bool {
+		detailsErr, ok := solvermon.AsCancellationDetailsError(err)
+		if !ok {
+			return false
+		}
+
+		app.BaseCLI.Logbus().Run().SetEnd(detailsErr.Details.End, logstream.RunStatus_RUN_STATUS_CANCELED)
+		app.BaseCLI.Console().Warnf(
+			"BuildKit canceled or lost the solve session.\n%s\n"+
+				"Earth did not receive a more specific root cause from BuildKit.\n",
+			detailsErr.Details.String(),
+		)
+		app.printCancellationOrigin(ctx, lastSignal)
+
+		return true
+	}():
+		if containerutil.IsLocal(app.BaseCLI.Flags().BuildkitdSettings.BuildkitAddress) && lastSignal.Get() == nil {
+			app.printCrashLogs(ctx)
+		}
+
+		return 2
 	case errors.Is(err, context.Canceled), grpcErrOK && grpcErr.Code() == codes.Canceled:
 		app.BaseCLI.Logbus().Run().SetEnd(time.Now(), logstream.RunStatus_RUN_STATUS_CANCELED)
 
-		if app.BaseCLI.Flags().Verbose {
+		showCanceledErr := app.BaseCLI.Flags().Verbose
+		if grpcErrOK && grpcErr.Message() != "" && grpcErr.Message() != context.Canceled.Error() {
+			showCanceledErr = true
+		}
+
+		if !grpcErrOK && err.Error() != context.Canceled.Error() {
+			showCanceledErr = true
+		}
+
+		if showCanceledErr {
 			app.BaseCLI.Console().Warnf("Canceled: %v\n", err)
 		} else {
 			app.BaseCLI.Console().Warn("Canceled\n")
 		}
 
+		app.printCancellationOrigin(ctx, lastSignal)
+
 		if containerutil.IsLocal(app.BaseCLI.Flags().BuildkitdSettings.BuildkitAddress) && lastSignal.Get() == nil {
 			app.printCrashLogs(ctx)
 		}