Add netSoftnetExpose to the VM spec and refuse exposed-port conflicts on a worker

api/openapi.yaml

@@ -741,6 +741,23 @@ components:
             - "66.66.0.0/16"
           items:
             type: string
+        netSoftnetExpose:
+          type: array
+          description: |
+            TCP ports to expose when using Softnet isolation, each in the
+            `EXTERNAL:INTERNAL` format (see `tart run`'s `--net-softnet-expose`),
+            with traffic to the external port on the host forwarded to the
+            internal port on the VM. The scheduler does not place a VM on a
+            worker where another VM already holds one of its external ports or
+            draws them from an endpoint's port range, and the ports of a
+            scheduled VM cannot be changed at all, because its worker keeps them
+            bound until the VM stops. Pick external ports outside the host's
+            ephemeral range, which the operating system hands to whatever asks
+            for a port, endpoint listeners included. Enables `netSoftnet`.
+          example:
+            - "2222:22"
+          items:
+            type: string
         suspendable:
           type: boolean
           description: |

internal/command/create/vm.go

@@ -25,6 +25,7 @@ var diskSize uint64
 var netSoftnet bool
 var netSoftnetAllow []string
 var netSoftnetBlock []string
+var netSoftnetExpose []string
 var netBridged string
 var headless bool
 var nested bool
@@ -69,6 +70,9 @@ func newCreateVMCommand() *cobra.Command {
 	command.Flags().StringSliceVar(&netSoftnetBlock, "net-softnet-block", []string{},
 		"comma-separated list of CIDRs to block the traffic to when using Softnet isolation, see "+
 			"\"tart run\"'s help for \"--net-softnet-block\" for more details; automatically enables --net-softnet")
+	command.Flags().StringSliceVar(&netSoftnetExpose, "net-softnet-expose", []string{},
+		"comma-separated list of TCP ports to expose when using Softnet isolation, in the EXTERNAL:INTERNAL "+
+			"format, see \"tart run\"'s help for \"--net-softnet-expose\" for more details; automatically enables --net-softnet")
 	command.Flags().StringVar(&netBridged, "net-bridged", "", "whether to use Bridged network mode")
 	command.Flags().BoolVar(&headless, "headless", true, "whether to run without graphics")
 	command.Flags().BoolVar(&nested, "nested", false, "enable nested virtualization")
@@ -155,6 +159,7 @@ func runCreateVM(cmd *cobra.Command, args []string) error {
 			NetSoftnet:           netSoftnet,
 			NetSoftnetAllow:      netSoftnetAllow,
 			NetSoftnetBlock:      netSoftnetBlock,
+			NetSoftnetExpose:     netSoftnetExpose,
 			Suspendable:          suspendable,
 		},
 		NetBridged:   netBridged,

internal/command/get/vm.go

@@ -95,6 +95,7 @@ func runGetVM(cmd *cobra.Command, args []string) error {
 	table.AddRow("Softnet enabled", vm.NetSoftnetDeprecated || vm.NetSoftnet)
 	table.AddRow("Softnet allowed CIDRs", strings.Join(vm.NetSoftnetAllow, "\n"))
 	table.AddRow("Softnet blocked CIDRs", strings.Join(vm.NetSoftnetBlock, "\n"))
+	table.AddRow("Softnet exposed ports", strings.Join(vm.NetSoftnetExpose, "\n"))
 	table.AddRow("Bridged networking interface", nonEmptyOrNone(vm.NetBridged))
 	table.AddRow("Headless mode", vm.Headless)
 	table.AddRow("Nested virtualization", vm.Nested)

internal/controller/api_vms.go

@@ -6,11 +6,13 @@ import (
 	"encoding/json"
 	"errors"
 	"net/http"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/cirruslabs/orchard/internal/controller/lifecycle"
+	"github.com/cirruslabs/orchard/internal/controller/scheduler"
 	storepkg "github.com/cirruslabs/orchard/internal/controller/store"
 	"github.com/cirruslabs/orchard/internal/responder"
 	"github.com/cirruslabs/orchard/internal/simplename"
@@ -89,9 +91,10 @@ func (controller *Controller) createVM(ctx *gin.Context) responder.Responder {
 		return responder.JSON(http.StatusPreconditionFailed, NewErrorResponse("%v", err))
 	}
 
-	// Softnet-specific logic: automatically enable Softnet when NetSoftnetAllow or NetSoftnetBlock are set
-	// and propagate deprecated and non-deprecated boolean fields into each other
-	if vm.NetSoftnetDeprecated || vm.NetSoftnet || len(vm.NetSoftnetAllow) != 0 || len(vm.NetSoftnetBlock) != 0 {
+	// Softnet-specific logic: automatically enable Softnet when NetSoftnetAllow, NetSoftnetBlock
+	// or NetSoftnetExpose are set and propagate deprecated and non-deprecated boolean fields into each other
+	if vm.NetSoftnetDeprecated || vm.NetSoftnet || len(vm.NetSoftnetAllow) != 0 || len(vm.NetSoftnetBlock) != 0 ||
+		len(vm.NetSoftnetExpose) != 0 {
 		vm.NetSoftnetDeprecated = true
 		vm.NetSoftnet = true
 	}
@@ -222,9 +225,10 @@ func (controller *Controller) updateVMSpec(ctx *gin.Context) responder.Responder
 			return responder.JSON(http.StatusBadRequest, NewErrorResponse("invalid host processes: %v", err))
 		}
 
-		// Softnet-specific logic: automatically enable Softnet when NetSoftnetAllow or NetSoftnetBlock are set
-		// and propagate deprecated and non-deprecated boolean fields into each other
-		if userVM.NetSoftnetDeprecated || userVM.NetSoftnet || len(userVM.NetSoftnetAllow) != 0 || len(userVM.NetSoftnetBlock) != 0 {
+		// Softnet-specific logic: automatically enable Softnet when NetSoftnetAllow, NetSoftnetBlock
+		// or NetSoftnetExpose are set and propagate deprecated and non-deprecated boolean fields into each other
+		if userVM.NetSoftnetDeprecated || userVM.NetSoftnet || len(userVM.NetSoftnetAllow) != 0 ||
+			len(userVM.NetSoftnetBlock) != 0 || len(userVM.NetSoftnetExpose) != 0 {
 			userVM.NetSoftnetDeprecated = true
 			userVM.NetSoftnet = true
 		}
@@ -258,6 +262,36 @@ func (controller *Controller) updateVMSpec(ctx *gin.Context) responder.Responder
 			return responder.JSON(http.StatusOK, dbVM)
 		}
 
+		// A worker keeps the old ports bound until it stops the VM to apply the new
+		// generation, so releasing them in the controller's view straight away would
+		// let another VM be scheduled onto a port that is still in use
+		if dbVM.IsScheduled() && !slices.Equal(dbVM.NetSoftnetExpose, userVM.NetSoftnetExpose) {
+			return responder.JSON(http.StatusPreconditionFailed, NewErrorResponse("\"netSoftnetExpose\" "+
+				"cannot be changed once the VM is scheduled on worker %q", dbVM.Worker))
+		}
+
+		// Endpoints can be changed on a VM that is already scheduled, which never goes
+		// past the scheduler, so the worker it sits on is checked here instead
+		if dbVM.IsScheduled() && !v1.SemanticallyEqual(dbVM.Endpoints, userVM.Endpoints) {
+			vms, err := txn.ListVMs()
+			if err != nil {
+				controller.logger.Errorf("failed to list VMs in the DB: %v", err)
+
+				return responder.Code(http.StatusInternalServerError)
+			}
+
+			updatedVM := *dbVM
+			updatedVM.VMSpec = userVM.VMSpec
+
+			// the whole specification has to come out clean: a VM being placed is not
+			// refused a worker for a clash it did not make, but this one owns its own
+			if scheduler.WorkerPortConflict(vms, dbVM.Worker, dbVM.Name, updatedVM) {
+				return responder.JSON(http.StatusPreconditionFailed, NewErrorResponse(
+					"\"endpoints\" cannot take a worker port that another VM on worker %q holds",
+					dbVM.Worker))
+			}
+		}
+
 		// VM specification was changed
 		dbVM.VMSpec = userVM.VMSpec
 		dbVM.Generation++

internal/controller/api_vms_test.go

@@ -0,0 +1,212 @@
+//nolint:testpackage // The handler under test is unexported.
+package controller
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/cirruslabs/orchard/internal/controller/notifier"
+	storepkg "github.com/cirruslabs/orchard/internal/controller/store"
+	"github.com/cirruslabs/orchard/internal/controller/store/badger"
+	v1 "github.com/cirruslabs/orchard/pkg/resource/v1"
+	"github.com/gin-gonic/gin"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
+)
+
+func newScheduledVM(name string, worker string, expose []string) v1.VM {
+	var vm v1.VM
+	vm.Name = name
+	vm.UID = name + "-uid"
+	vm.Worker = worker
+	vm.Status = v1.VMStatusRunning
+	vm.PowerState = v1.PowerStateRunning
+	vm.OS = v1.OSDarwin
+	vm.Arch = v1.ArchitectureARM64
+	vm.Runtime = v1.RuntimeTart
+	vm.Image = "ghcr.io/cirruslabs/macos-sequoia-base:latest"
+	vm.NetSoftnetExpose = expose
+	vm.Conditions = []v1.Condition{{Type: v1.ConditionTypeScheduled, State: v1.ConditionStateTrue}}
+
+	return vm
+}
+
+func endpointOn(port uint16) []v1.EndpointSpec {
+	return []v1.EndpointSpec{
+		{
+			Name:            "ssh",
+			Target:          v1.ConnectionTarget{VM: &v1.ConnectionTargetVM{Port: 22}},
+			WorkerPortRange: &v1.PortRange{Min: port, Max: port},
+		},
+	}
+}
+
+// Endpoints may be added to a VM that is already scheduled, so the update is the only
+// place where they can be held against the other VMs on its worker.
+func TestUpdateVMSpecEndpointPortConflict(t *testing.T) {
+	for _, test := range []struct {
+		name        string
+		otherExpose []string
+		port        uint16
+		status      int
+	}{
+		{name: "port another VM exposes", port: 2222, status: http.StatusPreconditionFailed},
+		{name: "free port", port: 2224, status: http.StatusOK},
+		// ports of its own that clash already are no licence to add an endpoint that does
+		{name: "clash on top of a clash", otherExpose: []string{"2222:22", "2223:22"},
+			port: 2222, status: http.StatusPreconditionFailed},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			store, err := badger.NewBadgerStore(t.TempDir(), true, zap.NewNop().Sugar())
+			require.NoError(t, err)
+
+			otherExpose := test.otherExpose
+			if otherExpose == nil {
+				otherExpose = []string{"2222:22"}
+			}
+
+			other := newScheduledVM("other-vm", "worker-a", otherExpose)
+			updated := newScheduledVM("updated-vm", "worker-a", []string{"2223:22"})
+
+			require.NoError(t, store.Update(func(txn storepkg.Transaction) error {
+				if err := txn.SetVM(other); err != nil {
+					return err
+				}
+
+				return txn.SetVM(updated)
+			}))
+
+			logger := zap.NewNop().Sugar()
+			workerNotifier := notifier.NewNotifier(logger)
+
+			_, unregister := workerNotifier.Register(t.Context(), updated.Worker)
+			defer unregister()
+
+			controller := &Controller{
+				insecureAuthDisabled: true,
+				store:                store,
+				workerNotifier:       workerNotifier,
+				logger:               logger,
+			}
+
+			body := updated
+			body.Endpoints = endpointOn(test.port)
+
+			payload, err := json.Marshal(body)
+			require.NoError(t, err)
+
+			recorder := httptest.NewRecorder()
+			ctx, _ := gin.CreateTestContext(recorder)
+			ctx.Params = gin.Params{{Key: "name", Value: updated.Name}}
+			ctx.Request = httptest.NewRequest(http.MethodPut, "/v1/vms/"+updated.Name, bytes.NewReader(payload))
+			ctx.Request.Header.Set("Content-Type", "application/json")
+
+			controller.updateVMSpec(ctx).Respond(ctx)
+			require.Equal(t, test.status, recorder.Code, recorder.Body.String())
+
+			var dbVM *v1.VM
+
+			require.NoError(t, store.View(func(txn storepkg.Transaction) (err error) {
+				dbVM, err = txn.GetVM(updated.Name)
+
+				return
+			}))
+
+			if test.status == http.StatusOK {
+				require.Equal(t, body.Endpoints, dbVM.Endpoints)
+			} else {
+				require.Empty(t, dbVM.Endpoints)
+			}
+		})
+	}
+}
+
+func TestUpdateVMSpecExposedPorts(t *testing.T) {
+	for _, test := range []struct {
+		name      string
+		scheduled bool
+		expose    []string
+		status    int
+	}{
+		// A scheduled VM keeps its ports bound on the worker until it is stopped to apply a new
+		// generation, so they cannot be changed, whether or not the new ones are free
+		{name: "conflicting port", scheduled: true, expose: []string{"02222:80"},
+			status: http.StatusPreconditionFailed},
+		{name: "free port", scheduled: true, expose: []string{"2224:22"}, status: http.StatusPreconditionFailed},
+		{name: "unscheduled", scheduled: false, expose: []string{"2224:22"}, status: http.StatusOK},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			store, err := badger.NewBadgerStore(t.TempDir(), true, zap.NewNop().Sugar())
+			require.NoError(t, err)
+
+			other := newScheduledVM("other-vm", "worker-a", []string{"2222:22"})
+			updated := newScheduledVM("updated-vm", "worker-a", []string{"2223:22"})
+
+			if !test.scheduled {
+				updated.Worker = ""
+				updated.Status = v1.VMStatusPending
+				updated.Conditions = []v1.Condition{
+					{Type: v1.ConditionTypeScheduled, State: v1.ConditionStateFalse},
+				}
+			}
+
+			require.NoError(t, store.Update(func(txn storepkg.Transaction) error {
+				if err := txn.SetVM(other); err != nil {
+					return err
+				}
+
+				return txn.SetVM(updated)
+			}))
+
+			logger := zap.NewNop().Sugar()
+			workerNotifier := notifier.NewNotifier(logger)
+
+			// A successful update notifies the VM's worker; register it, otherwise the
+			// notification waits for the worker to connect before giving up
+			_, unregister := workerNotifier.Register(t.Context(), updated.Worker)
+			defer unregister()
+
+			controller := &Controller{
+				insecureAuthDisabled: true,
+				store:                store,
+				workerNotifier:       workerNotifier,
+				logger:               logger,
+			}
+
+			body := updated
+			body.NetSoftnetExpose = test.expose
+
+			payload, err := json.Marshal(body)
+			require.NoError(t, err)
+
+			recorder := httptest.NewRecorder()
+			ctx, _ := gin.CreateTestContext(recorder)
+			ctx.Params = gin.Params{{Key: "name", Value: updated.Name}}
+			ctx.Request = httptest.NewRequest(http.MethodPut, "/v1/vms/"+updated.Name, bytes.NewReader(payload))
+			ctx.Request.Header.Set("Content-Type", "application/json")
+
+			controller.updateVMSpec(ctx).Respond(ctx)
+			require.Equal(t, test.status, recorder.Code, recorder.Body.String())
+
+			var dbVM *v1.VM
+
+			require.NoError(t, store.View(func(txn storepkg.Transaction) (err error) {
+				dbVM, err = txn.GetVM(updated.Name)
+
+				return
+			}))
+
+			if test.status == http.StatusOK {
+				require.Equal(t, test.expose, dbVM.NetSoftnetExpose)
+				require.Equal(t, uint64(1), dbVM.Generation)
+			} else {
+				require.Contains(t, recorder.Body.String(), "cannot be changed once the VM is scheduled")
+				require.Equal(t, []string{"2223:22"}, dbVM.NetSoftnetExpose)
+				require.Equal(t, uint64(0), dbVM.Generation)
+			}
+		})
+	}
+}