Check installation script exit codes

server/install.go

@@ -177,7 +177,7 @@ func (s *Server) SetRestoring(state bool) {
 }
 
 func (s *Server) IsInProtectedState() bool {
-	return s.IsInstalling() || s.IsTransferring() || s.IsRestoring()	
+	return s.IsInstalling() || s.IsTransferring() || s.IsRestoring()
 }
 
 // RemoveContainer removes the installation container for the server.
@@ -214,10 +214,10 @@ func (ip *InstallationProcess) Run() error {
 		return err
 	}
 
-	cID, err := ip.Execute()
-	if err != nil {
+	cID, executeErr := ip.Execute()
+	if cID == "" {
 		_ = ip.RemoveContainer()
-		return err
+		return executeErr
 	}
 
 	// If this step fails, log a warning but don't exit out of the process. This is completely
@@ -226,7 +226,7 @@ func (ip *InstallationProcess) Run() error {
 		ip.Server.Log().WithField("error", err).Warn("failed to complete after-execute step of installation process")
 	}
 
-	return nil
+	return executeErr
 }
 
 // Returns the location of the temporary data for the installation process.
@@ -529,18 +529,43 @@ func (ip *InstallationProcess) Execute() (string, error) {
 	}(r.ID)
 
 	sChan, eChan := ip.client.ContainerWait(ctx, r.ID, container.WaitConditionNotRunning)
+	if err := ip.waitForInstallationContainer(sChan, eChan); err != nil {
+		return r.ID, err
+	}
+
+	return r.ID, nil
+}
+
+func (ip *InstallationProcess) waitForInstallationContainer(sChan <-chan container.WaitResponse, eChan <-chan error) error {
 	select {
 	case err := <-eChan:
-		// Once the container has stopped running we can mark the install process as being completed.
-		if err == nil {
-			ip.Server.Events().Publish(DaemonMessageEvent, "Installation process completed.")
-		} else {
-			return "", err
+		ip.Server.Events().Publish(DaemonMessageEvent, "Installation process failed: "+err.Error())
+		return err
+	case response := <-sChan:
+		if err := installationWaitError(response); err != nil {
+			ip.Server.Events().Publish(DaemonMessageEvent, "Installation process failed: "+err.Error())
+			return err
 		}
-	case <-sChan:
 	}
 
-	return r.ID, nil
+	ip.Server.Events().Publish(DaemonMessageEvent, "Installation process completed.")
+	return nil
+}
+
+func installationWaitError(response container.WaitResponse) error {
+	if response.Error != nil {
+		message := strings.TrimSpace(response.Error.Message)
+		if message == "" {
+			message = "unknown container wait error"
+		}
+
+		return errors.Errorf("install: installation container wait failed: %s", message)
+	}
+	if response.StatusCode != 0 {
+		return errors.Errorf("install: installation script exited with code %d", response.StatusCode)
+	}
+
+	return nil
 }
 
 // StreamOutput streams the output of the installation process to a log file in

server/install_test.go

@@ -0,0 +1,100 @@
+package server
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/docker/docker/api/types/container"
+
+	"github.com/pelican/wings/events"
+)
+
+func TestInstallationWaitError(t *testing.T) {
+	tests := []struct {
+		name      string
+		response  container.WaitResponse
+		wantError string
+	}{
+		{
+			name:     "successful installation",
+			response: container.WaitResponse{StatusCode: 0},
+		},
+		{
+			name:      "installation script failure",
+			response:  container.WaitResponse{StatusCode: 1},
+			wantError: "install: installation script exited with code 1",
+		},
+		{
+			name:      "command not found",
+			response:  container.WaitResponse{StatusCode: 127},
+			wantError: "install: installation script exited with code 127",
+		},
+		{
+			name: "container wait failure",
+			response: container.WaitResponse{
+				Error: &container.WaitExitError{Message: "daemon disconnected"},
+			},
+			wantError: "install: installation container wait failed: daemon disconnected",
+		},
+		{
+			name: "container wait failure without message",
+			response: container.WaitResponse{
+				Error: &container.WaitExitError{},
+			},
+			wantError: "install: installation container wait failed: unknown container wait error",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := installationWaitError(tt.response)
+			if tt.wantError == "" {
+				if err != nil {
+					t.Fatalf("installationWaitError() returned unexpected error: %v", err)
+				}
+				return
+			}
+
+			if err == nil {
+				t.Fatalf("installationWaitError() returned nil, expected %q", tt.wantError)
+			}
+			if err.Error() != tt.wantError {
+				t.Fatalf("installationWaitError() error = %q, expected %q", err.Error(), tt.wantError)
+			}
+		})
+	}
+}
+
+func TestWaitForInstallationContainerPublishesWaitFailure(t *testing.T) {
+	s, err := New(nil)
+	if err != nil {
+		t.Fatalf("New() returned unexpected error: %v", err)
+	}
+
+	listener := make(chan []byte, 1)
+	s.Events().On(listener)
+	defer s.Events().Off(listener)
+
+	waitErr := errors.New("daemon disconnected")
+	sChan := make(chan container.WaitResponse)
+	eChan := make(chan error, 1)
+	eChan <- waitErr
+
+	ip := &InstallationProcess{Server: s}
+	if err := ip.waitForInstallationContainer(sChan, eChan); !errors.Is(err, waitErr) {
+		t.Fatalf("waitForInstallationContainer() error = %v, expected %v", err, waitErr)
+	}
+
+	select {
+	case raw := <-listener:
+		event := events.MustDecode(raw)
+		if event.Topic != DaemonMessageEvent {
+			t.Fatalf("event topic = %q, expected %q", event.Topic, DaemonMessageEvent)
+		}
+		if event.Data != "Installation process failed: daemon disconnected" {
+			t.Fatalf("event data = %q, expected failure message", event.Data)
+		}
+	default:
+		t.Fatal("waitForInstallationContainer() did not publish a failure event")
+	}
+}