From 0a5808716a11a5e75ed06b9766f6b7ee543ae073 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Tue, 15 Sep 2026 14:24:09 -0700 Subject: [PATCH 1/4] ateomnet: isolate sandbox networking by namespace Use fixed sandbox addresses in private namespaces. gVisor uses a veth pair across two namespaces; microVMs use a tap in one. Redirect TCP egress to atunnel and provide namespace-scoped listeners and dialers. The dialer takes TCP and UDP IP literals only, and restores the worker namespace once the socket exists so a pending connect does not pin a native thread. --- internal/ateomnet/net.go | 34 ++ internal/ateomnet/net_linux_test.go | 17 + internal/ateomnet/resolvconf.go | 75 +++ internal/ateomnet/resolvconf_linux_test.go | 112 ++++ internal/ateomnet/sandbox.go | 538 ++++++++++++++++++ .../ateomnet/sandbox_atunnel_linux_test.go | 97 ++++ internal/ateomnet/sandbox_linux_test.go | 485 ++++++++++++++++ internal/ateompath/ateompath.go | 12 + internal/resources/validate.go | 7 +- 9 files changed, 1373 insertions(+), 4 deletions(-) create mode 100644 internal/ateomnet/resolvconf.go create mode 100644 internal/ateomnet/resolvconf_linux_test.go create mode 100644 internal/ateomnet/sandbox.go create mode 100644 internal/ateomnet/sandbox_atunnel_linux_test.go create mode 100644 internal/ateomnet/sandbox_linux_test.go diff --git a/internal/ateomnet/net.go b/internal/ateomnet/net.go index 831d4859e6..8db27a8226 100644 --- a/internal/ateomnet/net.go +++ b/internal/ateomnet/net.go @@ -171,6 +171,40 @@ func CleanupActorNetwork(ctx context.Context, interiorNetNS netns.NsHandle) erro return cleanupErr } +// AllowUnprivilegedPorts lets this namespace bind ports below 1024 without +// CAP_NET_BIND_SERVICE, which is how atunnel answers a sandbox's DNS on 53. +// The sysctl is per-namespace and grants nothing outside it. +func AllowUnprivilegedPorts() error { + return setNetSysctl("net/ipv4/ip_unprivileged_port_start", "0") +} + +// setNetSysctl writes value to the named sysctl in the current network +// namespace, remounting /proc/sys read-write when the runtime bind-mounted it +// read-only. A no-op when it already reads that way. +func setNetSysctl(key, value string) error { + path := "/proc/sys/" + key + if b, err := os.ReadFile(path); err == nil && strings.TrimSpace(string(b)) == value { + return nil + } + // Only EROFS is worth remounting for; any other error is returned as is. + if err := os.WriteFile(path, []byte(value+"\n"), 0o644); !errors.Is(err, unix.EROFS) { + if err != nil { + return fmt.Errorf("while setting %s in worker pod netns: %w", key, err) + } + return nil + } + if err := unix.Mount("none", "/proc/sys", "", unix.MS_BIND|unix.MS_REMOUNT, ""); err != nil { + return fmt.Errorf("while remounting /proc/sys read-write to set %s: %w", key, err) + } + defer func() { + _ = unix.Mount("none", "/proc/sys", "", unix.MS_BIND|unix.MS_REMOUNT|unix.MS_RDONLY, "") + }() + if err := os.WriteFile(path, []byte(value+"\n"), 0o644); err != nil { + return fmt.Errorf("while setting %s in worker pod netns: %w", key, err) + } + return nil +} + // EnableIPv4Forwarding enables IPv4 forwarding in the current network namespace. func EnableIPv4Forwarding() error { // Forwarding is required because actor packets now enter the worker pod via diff --git a/internal/ateomnet/net_linux_test.go b/internal/ateomnet/net_linux_test.go index 3ee8def3b7..3e5c35675b 100644 --- a/internal/ateomnet/net_linux_test.go +++ b/internal/ateomnet/net_linux_test.go @@ -561,3 +561,20 @@ func TestCreateNetNSWithoutSwitchingReplacesALeftover(t *testing.T) { }) } } + +// Only EROFS takes the remount path: any other error is reported as it is, +// and /proc/sys is left as it was found. Remounting it read-only on the way +// out would break every later write. +func TestSetNetSysctlReportsAnUnrelatedError(t *testing.T) { + err := setNetSysctl("net/ipv4/ateomnet_no_such_sysctl", "0") + if !errors.Is(err, unix.ENOENT) { + t.Fatalf("setNetSysctl() on a missing key: got %v, want ENOENT", err) + } + var st unix.Statfs_t + if err := unix.Statfs("/proc/sys", &st); err != nil { + t.Fatalf("statfs /proc/sys: %v", err) + } + if st.Flags&unix.ST_RDONLY != 0 { + t.Error("/proc/sys was left read-only") + } +} diff --git a/internal/ateomnet/resolvconf.go b/internal/ateomnet/resolvconf.go new file mode 100644 index 0000000000..7fd0f08f79 --- /dev/null +++ b/internal/ateomnet/resolvconf.go @@ -0,0 +1,75 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ateomnet + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// SandboxResolvConf replaces nameservers with the sandbox gateway while +// preserving the pod's search domains and options for Kubernetes DNS. +func SandboxResolvConf(podResolvConf []byte) []byte { + var out strings.Builder + out.WriteString("nameserver " + ActorVethGateway + "\n") + for line := range strings.SplitSeq(string(podResolvConf), "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "nameserver") { + continue + } + if strings.TrimSpace(line) == "" { + continue + } + out.WriteString(line + "\n") + } + return []byte(out.String()) +} + +// WriteRootfsResolvConf installs content at /etc/resolv.conf inside rootfs. +// +// os.Root confines path traversal; unlinking prevents writes through existing links. +func WriteRootfsResolvConf(rootfs string, content []byte) error { + if len(content) == 0 { + return fmt.Errorf("actornet: refusing to write an empty resolv.conf") + } + root, err := os.OpenRoot(rootfs) + if err != nil { + return fmt.Errorf("opening rootfs %q: %w", rootfs, err) + } + defer root.Close() + if err := root.Mkdir("etc", 0o755); err != nil && !errors.Is(err, fs.ErrExist) { + return fmt.Errorf("creating %q: %w", filepath.Join(rootfs, "etc"), err) + } + if err := root.Remove("etc/resolv.conf"); err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("removing existing resolv.conf: %w", err) + } + f, err := root.OpenFile("etc/resolv.conf", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + return fmt.Errorf("creating resolv.conf: %w", err) + } + _, err = f.Write(content) + if closeErr := f.Close(); err == nil { + err = closeErr + } + if err != nil { + return fmt.Errorf("writing resolv.conf: %w", err) + } + return nil +} diff --git a/internal/ateomnet/resolvconf_linux_test.go b/internal/ateomnet/resolvconf_linux_test.go new file mode 100644 index 0000000000..18ef1f2ff1 --- /dev/null +++ b/internal/ateomnet/resolvconf_linux_test.go @@ -0,0 +1,112 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ateomnet + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestSandboxResolvConf(t *testing.T) { + // The shape kubelet writes into a pod. + pod := "nameserver 10.96.0.10\n" + + "search ate-demo.svc.cluster.local svc.cluster.local cluster.local\n" + + "options ndots:5\n" + + got := string(SandboxResolvConf([]byte(pod))) + + want := "nameserver 169.254.17.1\n" + + "search ate-demo.svc.cluster.local svc.cluster.local cluster.local\n" + + "options ndots:5\n" + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("actor resolv.conf mismatch (-want +got):\n%s", diff) + } +} + +// Short service names depend on the search list and on ndots, so an actor given +// only a nameserver line resolves public names but not cluster ones. +func TestSandboxResolvConfKeepsSearchAndOptions(t *testing.T) { + pod := "search svc.cluster.local\nnameserver 10.96.0.10\nnameserver 10.96.0.11\noptions ndots:5 timeout:1\n" + got := string(SandboxResolvConf([]byte(pod))) + + if strings.Contains(got, "10.96.0.10") || strings.Contains(got, "10.96.0.11") { + t.Errorf("a pod resolver survived into the actor's file, so its DNS would bypass atunnel:\n%s", got) + } + if !strings.Contains(got, "search svc.cluster.local") { + t.Errorf("search list lost; cluster short names would stop resolving:\n%s", got) + } + if !strings.Contains(got, "options ndots:5 timeout:1") { + t.Errorf("options lost:\n%s", got) + } + if n := strings.Count(got, "nameserver"); n != 1 { + t.Errorf("got %d nameserver lines, want exactly the gateway", n) + } +} + +func TestWriteRootfsResolvConf(t *testing.T) { + rootfs := t.TempDir() + if err := WriteRootfsResolvConf(rootfs, []byte("nameserver 169.254.17.1\n")); err != nil { + t.Fatalf("WriteRootfsResolvConf: %v", err) + } + got, err := os.ReadFile(filepath.Join(rootfs, "etc", "resolv.conf")) + if err != nil { + t.Fatalf("reading what was written: %v", err) + } + if string(got) != "nameserver 169.254.17.1\n" { + t.Errorf("wrote %q", got) + } + + // Replacing an existing file is the ordinary case: the image usually ships one. + if err := WriteRootfsResolvConf(rootfs, []byte("nameserver 169.254.17.1\nsearch x\n")); err != nil { + t.Fatalf("rewriting: %v", err) + } + + if err := WriteRootfsResolvConf(rootfs, nil); err == nil { + t.Error("an empty resolv.conf was accepted; the actor would resolve nothing") + } +} + +// The rootfs comes from an untrusted image, so a planted symlink must not be +// followed out of it and clobber the worker pod's own file. +func TestWriteRootfsResolvConfDoesNotFollowAPlantedSymlink(t *testing.T) { + rootfs := t.TempDir() + outside := filepath.Join(t.TempDir(), "victim") + if err := os.WriteFile(outside, []byte("original"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(rootfs, "etc"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(rootfs, "etc", "resolv.conf")); err != nil { + t.Fatal(err) + } + + if err := WriteRootfsResolvConf(rootfs, []byte("nameserver 169.254.17.1\n")); err != nil { + t.Fatalf("WriteRootfsResolvConf: %v", err) + } + victim, err := os.ReadFile(outside) + if err != nil { + t.Fatal(err) + } + if string(victim) != "original" { + t.Errorf("the symlink was followed and the file outside the rootfs was overwritten with %q", victim) + } +} diff --git a/internal/ateomnet/sandbox.go b/internal/ateomnet/sandbox.go new file mode 100644 index 0000000000..9fb06ebed8 --- /dev/null +++ b/internal/ateomnet/sandbox.go @@ -0,0 +1,538 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ateomnet + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/netip" + "runtime" + "strconv" + "sync" + "syscall" + + "github.com/agent-substrate/substrate/internal/ateompath" + + "github.com/google/nftables" + "github.com/google/nftables/binaryutil" + "github.com/google/nftables/expr" + "github.com/vishvananda/netlink" + "github.com/vishvananda/netns" + "golang.org/x/sys/unix" +) + +// SandboxNetwork holds a sandbox's runtime and gateway namespaces. +type SandboxNetwork struct { + // ActorUID is the Actor resource's UID. It names the namespaces, so a + // teardown can find them again. + ActorUID string + // RuntimeNetNS is what the sandbox runs in, whichever runtime that is: the + // micro-VM's tap lives here, and gVisor claims every interface here and + // moves their addresses into its own stack. + RuntimeNetNS netns.NsHandle + // GatewayNetNS holds the sandbox's default gateway, DNS relay, and atunnel + // sockets. This is local to the sandbox, not the external egress gateway. + // For microVMs it shares RuntimeNetNS. + // + // TODO: we hope gVisor can take that same single-namespace shape soon, + // once runsc can be given one interface rather than claiming every + // interface in the namespace it runs in. + GatewayNetNS netns.NsHandle + // PodSideIP is identical across sandboxes, isolated by namespace. + PodSideIP net.IP +} + +func (n *SandboxNetwork) holdsNetNS() bool { return n.RuntimeNetNS > 0 } + +// SandboxNetworkConfig describes one actor's private networking. +type SandboxNetworkConfig struct { + ActorUID string + + // Veth separates gVisor's interfaces from the kernel-owned gateway. + // MicroVMs use a tap in a single namespace instead. + Veth bool + + // EgressPort is where atunnel serves this actor. Every TCP connection the + // actor makes is redirected to it, whatever port it was aimed at. + EgressPort uint16 + + // GatewayHWAddr fixes the gateway's MAC, which a micro-VM snapshot freezes + // into the guest's ARP cache. gVisor re-ARPs and can leave it unset. + // + // Applies to the veth path only. The tap path sets its own MAC in + // setupActorTap, after LinkAdd, because tuntap creation ignores the + // hardware address in the link attributes. Both belong here once the two + // runtimes share one shape. + GatewayHWAddr net.HardwareAddr +} + +// SetupSandboxNetwork creates isolated networking with fixed sandbox addresses. +// gVisor uses a veth pair across runtime and gateway namespaces because it takes +// over every interface in its namespace. MicroVMs use a tap in one namespace. +// nftables at the sandbox's default gateway redirect outbound TCP to atunnel, preserving +// SO_ORIGINAL_DST. Traffic to the gateway address is not redirected, so DNS +// over UDP and TCP reaches the relay's own sockets. +func SetupSandboxNetwork(ctx context.Context, cfg SandboxNetworkConfig) (_ *SandboxNetwork, retErr error) { + actorUID := cfg.ActorUID + if actorUID == "" { + return nil, fmt.Errorf("actornet: actor UID is required") + } + + actorNSName := ateompath.ActorNetNSName(actorUID) + actorNS, err := CreateNetNSWithoutSwitching(actorNSName) + if err != nil { + return nil, fmt.Errorf("while creating the actor netns %s: %w", actorNSName, err) + } + defer func() { + if retErr != nil { + actorNS.Close() + _ = removeNamedNetNS(actorNSName) + } + }() + + // Without a veth, atunnel shares the namespace with the runtime's tap. + atunnelNS := actorNS + if cfg.Veth { + outer, err := setupVethPair(ctx, cfg, actorNS) + if err != nil { + return nil, err + } + defer func() { + if retErr != nil { + outer.Close() + _ = removeNamedNetNS(SandboxGatewayNetNSName(cfg.ActorUID)) + } + }() + atunnelNS = outer + } + if err := setupGatewaySide(ctx, atunnelNS, cfg.EgressPort); err != nil { + return nil, err + } + + return &SandboxNetwork{ + ActorUID: actorUID, + RuntimeNetNS: actorNS, + GatewayNetNS: atunnelNS, + // Every actor holds the same address; the namespace is the identity. + PodSideIP: net.ParseIP(ActorVethIP), + }, nil +} + +// setupVethPair creates the gateway namespace and the veth pair joining it to +// actorNS. The caller owns the returned handle and its name. +func setupVethPair(ctx context.Context, cfg SandboxNetworkConfig, actorNS netns.NsHandle) (_ netns.NsHandle, retErr error) { + gatewayNSName := SandboxGatewayNetNSName(cfg.ActorUID) + outer, err := CreateNetNSWithoutSwitching(gatewayNSName) + if err != nil { + return 0, fmt.Errorf("while creating the outer netns %s: %w", gatewayNSName, err) + } + defer func() { + if retErr != nil { + outer.Close() + _ = removeNamedNetNS(gatewayNSName) + } + }() + + // Keep the kernel-owned peer outside gVisor's namespace. + if err := NetNSDo(ctx, outer, func(context.Context) error { + veth := &netlink.Veth{ + LinkAttrs: netlink.LinkAttrs{Name: gatewayVethName}, + PeerName: ActorVethName, + // Create the peer directly in the actor's namespace. Moving a + // netdev across namespaces afterwards costs several times the + // whole setup, all of it under the global RTNL lock, and this + // runs on the resume path. + PeerNamespace: netlink.NsFd(int(actorNS)), + } + if cfg.GatewayHWAddr != nil { + veth.LinkAttrs.HardwareAddr = cfg.GatewayHWAddr + } + if err := netlink.LinkAdd(veth); err != nil { + return fmt.Errorf("while creating the veth pair: %w", err) + } + atSide, err := netlink.LinkByName(gatewayVethName) + if err != nil { + return err + } + if err := netlink.AddrReplace(atSide, HostVethAddr); err != nil { + return fmt.Errorf("while assigning the atunnel-side address: %w", err) + } + if err := netlink.LinkSetUp(atSide); err != nil { + return err + } + return nil + }); err != nil { + return 0, err + } + + // gVisor imports these addresses and routes into its network stack. + if err := NetNSDo(ctx, actorNS, func(context.Context) error { + // Loopback lets the actor reach its own address. + if err := linkUp("lo"); err != nil { + return err + } + eth0, err := netlink.LinkByName(ActorVethName) + if err != nil { + return err + } + if err := netlink.AddrReplace(eth0, ActorVethAddr); err != nil { + return fmt.Errorf("while assigning the actor address: %w", err) + } + if err := netlink.LinkSetUp(eth0); err != nil { + return err + } + return netlink.RouteReplace(&netlink.Route{ + LinkIndex: eth0.Attrs().Index, + Gw: ActorVethGwIP, + }) + }); err != nil { + return 0, err + } + return outer, nil +} + +func linkUp(name string) error { + link, err := netlink.LinkByName(name) + if err != nil { + return err + } + return netlink.LinkSetUp(link) +} + +// setupGatewaySide brings up lo and puts atunnel in front of the actor's TCP. +func setupGatewaySide(ctx context.Context, ns netns.NsHandle, egressPort uint16) error { + if err := NetNSDo(ctx, ns, func(context.Context) error { + // atunnel answers the actor's DNS on 53, and the worker holds no + // CAP_NET_BIND_SERVICE. + if err := AllowUnprivilegedPorts(); err != nil { + return err + } + return linkUp("lo") + }); err != nil { + return err + } + return installEgressRedirect(ns, egressPort) +} + +// installEgressRedirect redirects TCP egress to atunnel, excluding the sandbox's +// own /30: that keeps ingress replies and DNS over TCP to the gateway off the +// redirect, so the relay serves them on its own listener. +func installEgressRedirect(ns netns.NsHandle, egressPort uint16) error { + if egressPort == 0 { + return fmt.Errorf("actornet: atunnel egress port is required") + } + c, err := nftables.New(nftables.WithNetNSFd(int(ns))) + if err != nil { + return fmt.Errorf("while opening nftables in the actor namespace: %w", err) + } + defer func() { _ = c.CloseLasting() }() + + table := c.AddTable(&nftables.Table{Family: nftables.TableFamilyIPv4, Name: "ateom-actor"}) + prerouting := c.AddChain(&nftables.Chain{ + Name: "prerouting", Table: table, Type: nftables.ChainTypeNAT, + Hooknum: nftables.ChainHookPrerouting, Priority: nftables.ChainPriorityNATDest, + }) + + // Offset of the destination address in an IPv4 header. + const ipv4HeaderDst = 16 + exprs := []expr.Any{ + &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: ipv4HeaderDst, Len: 4}, + &expr.Bitwise{ + SourceRegister: 1, DestRegister: 1, Len: 4, + Mask: actorSubnetMask, Xor: []byte{0, 0, 0, 0}, + }, + &expr.Cmp{Op: expr.CmpOpNeq, Register: 1, Data: actorSubnetBase}, + } + exprs = append(exprs, l4ProtocolEqual(unix.IPPROTO_TCP)...) + exprs = append(exprs, + &expr.Immediate{Register: 1, Data: binaryutil.BigEndian.PutUint16(egressPort)}, + &expr.Redir{RegisterProtoMin: 1}, + ) + c.AddRule(&nftables.Rule{Table: table, Chain: prerouting, Exprs: exprs}) + + if err := c.Flush(); err != nil { + return fmt.Errorf("while installing the actor egress redirect: %w", err) + } + return nil +} + +// The actor subnet, in the form the nftables comparison takes. +var actorSubnetBase, actorSubnetMask = func() ([]byte, []byte) { + _, subnet, err := net.ParseCIDR(ActorVethSubnet) + if err != nil { + panic(fmt.Sprintf("parsing constant actor subnet %q: %v", ActorVethSubnet, err)) + } + return subnet.IP.To4(), subnet.Mask +}() + +// gatewayVethName is the veth peer in the gateway namespace. +const gatewayVethName = "atside" + +// SandboxGatewayNetNSName names the namespace holding the veth peer and atunnel's +// sockets for one actor. +func SandboxGatewayNetNSName(actorUID string) string { + return ateompath.ActorNetNSName(actorUID) + "-at" +} + +// CleanupSandboxNetwork closes namespace handles and removes their names. +func CleanupSandboxNetwork(network *SandboxNetwork) error { + if network == nil { + return nil + } + var errs error + // Compare before Close sets the handle to -1; microVMs share one descriptor. + separateNS := network.GatewayNetNS != network.RuntimeNetNS + if network.holdsNetNS() { + if err := network.RuntimeNetNS.Close(); err != nil { + errs = errors.Join(errs, fmt.Errorf("while closing the sandbox netns: %w", err)) + } + } + if separateNS && network.GatewayNetNS > 0 { + if err := network.GatewayNetNS.Close(); err != nil { + errs = errors.Join(errs, fmt.Errorf("while closing the gateway netns: %w", err)) + } + } + // Deleting the namespaces takes any veth pair with them. + for _, name := range []string{ateompath.ActorNetNSName(network.ActorUID), SandboxGatewayNetNSName(network.ActorUID)} { + if err := removeNamedNetNS(name); err != nil { + errs = errors.Join(errs, fmt.Errorf("while deleting netns %s: %w", name, err)) + } + } + return errs +} + +// ListenInNetNS opens wildcard TCP listeners inside ns. +// Sockets retain their namespace and can be served from another namespace. +func ListenInNetNS(ctx context.Context, ns netns.NsHandle, ports []uint16) (_ []net.Listener, retErr error) { + var listeners []net.Listener + defer func() { + if retErr != nil { + for _, l := range listeners { + _ = l.Close() + } + } + }() + if err := NetNSDo(ctx, ns, func(context.Context) error { + for _, port := range ports { + l, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", port)) + if err != nil { + return fmt.Errorf("while listening on port %d: %w", port, err) + } + listeners = append(listeners, l) + } + return nil + }); err != nil { + return nil, err + } + return listeners, nil +} + +// EgressServer serves one actor's captured connections. Satisfied by +// atunnel.Egress; an interface so this package does not depend on it. +type EgressServer interface { + ServeFor(ctx context.Context, actorKey string, listener net.Listener) error +} + +// ServeSandboxEgress puts the egress server's sockets inside the actor's own +// namespace, where the local default route delivers everything it sends. The +// listener is the actor's identity: they all hold the same address, so nothing +// about a connection distinguishes them. +// +// Only ports gets captured. A port with no listener is refused rather than +// escaping, which is the fail-closed half of routing everything through the +// tunnel. Closing the returned listeners stops the actor's egress. +func ServeSandboxEgress(ctx context.Context, e EgressServer, actorKey string, ns netns.NsHandle, ports []uint16) ([]net.Listener, error) { + listeners, err := ListenInNetNS(ctx, ns, ports) + if err != nil { + return nil, fmt.Errorf("while opening actor egress listeners: %w", err) + } + for _, l := range listeners { + go func(l net.Listener) { + // Background rather than the caller's context: these outlive the + // activation and are stopped by closing the listener. + if err := e.ServeFor(context.Background(), actorKey, l); err != nil { + slog.WarnContext(ctx, "Actor egress listener stopped", + slog.String("actorUID", actorKey), slog.Any("err", err)) + } + }(l) + } + return listeners, nil +} + +// DNSServer answers an actor's DNS. Satisfied by atunnel.DNSRelay; an interface +// so this package does not depend on it. +type DNSServer interface { + ServePacket(ctx context.Context, pc net.PacketConn) error + Serve(ctx context.Context, listener net.Listener) error +} + +// ServeSandboxDNS serves UDP and TCP DNS in the gateway namespace. +func ServeSandboxDNS(ctx context.Context, relay DNSServer, ns netns.NsHandle, port uint16) (_ []io.Closer, retErr error) { + // Bind the wildcard because the microVM tap's gateway address is added later. + address := net.JoinHostPort("0.0.0.0", strconv.Itoa(int(port))) + + var packet net.PacketConn + var stream net.Listener + if err := NetNSDo(ctx, ns, func(context.Context) error { + pc, err := net.ListenPacket("udp", address) + if err != nil { + return fmt.Errorf("while opening the actor DNS socket: %w", err) + } + packet = pc + l, err := net.Listen("tcp", address) + if err != nil { + _ = pc.Close() + return fmt.Errorf("while opening the actor DNS listener: %w", err) + } + stream = l + return nil + }); err != nil { + return nil, err + } + + // Detached from the activation RPC's context but cancelable: the relay's + // capacity is the worker's, so teardown must drop queries still in flight. + serveCtx, stopServing := context.WithCancel(context.WithoutCancel(ctx)) + go func() { + if err := relay.ServePacket(serveCtx, packet); err != nil { + slog.WarnContext(ctx, "Actor DNS socket stopped", slog.Any("err", err)) + } + }() + go func() { + if err := relay.Serve(serveCtx, stream); err != nil { + slog.WarnContext(ctx, "Actor DNS listener stopped", slog.Any("err", err)) + } + }() + // Cancel first: closing the sockets alone leaves the queries already being + // resolved holding the relay. + return []io.Closer{closerFunc(func() error { stopServing(); return nil }), packet, stream}, nil +} + +// closerFunc adapts a cancel function to io.Closer, so a caller takes a +// sandbox's sockets and the work behind them down as one list. +type closerFunc func() error + +func (f closerFunc) Close() error { return f() } + +// withNetNS switches to targetNS, calls run, then restores the original namespace. +// run can call restore to switch back and unlock the OS thread before returning. +// Calling restore again after it succeeds has no effect. +// +// A separate goroutine lets us leave the thread locked if restoration fails. +// Go then discards that thread when the goroutine exits. +func withNetNS(targetNS netns.NsHandle, run func(restore func() error) error) error { + var resultErr error + var done sync.WaitGroup + done.Add(1) + go func() { + defer done.Done() + runtime.LockOSThread() + originalNS, err := netns.Get() + if err != nil { + runtime.UnlockOSThread() + resultErr = fmt.Errorf("while reading the current netns: %w", err) + return + } + defer originalNS.Close() + if err := netns.Set(targetNS); err != nil { + runtime.UnlockOSThread() + resultErr = fmt.Errorf("while entering the actor netns: %w", err) + return + } + + restored := false + restore := func() error { + if restored { + return nil + } + if err := netns.Set(originalNS); err != nil { + return fmt.Errorf("while restoring the worker netns: %w", err) + } + runtime.UnlockOSThread() + restored = true + return nil + } + + resultErr = run(restore) + if err := restore(); err != nil { + resultErr = err + } + }() + done.Wait() + return resultErr +} + +// NetNSDialer dials TCP or UDP IP literals in ns, pinning a thread only until +// the socket is created. +func NetNSDialer(ns netns.NsHandle) func(context.Context, string, string) (net.Conn, error) { + return func(ctx context.Context, network, addr string) (net.Conn, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if err := validateNetNSDialTarget(network, addr); err != nil { + return nil, err + } + + var conn net.Conn + dialErr := withNetNS(ns, func(restore func() error) error { + // Only creating the socket needs the namespace, and + // ControlContext runs once it exists: restore there rather than + // holding the thread for the whole connect. + socketCreated := false + dialer := net.Dialer{ControlContext: func(context.Context, string, string, syscall.RawConn) error { + if socketCreated { + return errors.New("sandbox dial cannot recreate its socket outside the namespace") + } + socketCreated = true + return restore() + }} + var err error + conn, err = dialer.DialContext(ctx, network, addr) + return err + }) + if dialErr != nil || ctx.Err() != nil { + if conn != nil { + _ = conn.Close() + } + if dialErr != nil { + return nil, dialErr + } + return nil, ctx.Err() + } + return conn, nil + } +} + +func validateNetNSDialTarget(network, addr string) error { + switch network { + case "tcp", "tcp4", "tcp6", "udp", "udp4", "udp6": + default: + return net.UnknownNetworkError(network) + } + hostname, _, err := net.SplitHostPort(addr) + if err != nil { + return err + } + if _, err := netip.ParseAddr(hostname); err != nil { + return fmt.Errorf("NetNSDialer supports only IP literals (got %q): %w", hostname, err) + } + return nil +} diff --git a/internal/ateomnet/sandbox_atunnel_linux_test.go b/internal/ateomnet/sandbox_atunnel_linux_test.go new file mode 100644 index 0000000000..2fc166e6e8 --- /dev/null +++ b/internal/ateomnet/sandbox_atunnel_linux_test.go @@ -0,0 +1,97 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// These integration tests use an external package to avoid an import cycle: +// atunnel imports ateomnet. +package ateomnet_test + +import ( + "context" + "io" + "net" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/ateomnet" + "github.com/agent-substrate/substrate/internal/atunnel" + "github.com/agent-substrate/substrate/internal/roottest" +) + +// Verify redirection preserves the destination, including unconfigured ports. +func TestSandboxEgressReachesAtunnelOnAnyPort(t *testing.T) { + roottest.Require(t, "creates network namespaces") + ctx := context.Background() + const egressPort = 15001 + + n, err := ateomnet.SetupSandboxNetwork(ctx, ateomnet.SandboxNetworkConfig{ + ActorUID: "66666666-6666-6666-6666-666666666666", Veth: true, EgressPort: egressPort, + }) + if err != nil { + t.Fatalf("SetupSandboxNetwork: %v", err) + } + t.Cleanup(func() { ateomnet.CleanupSandboxNetwork(n) }) + + listeners, err := ateomnet.ListenInNetNS(ctx, n.GatewayNetNS, []uint16{egressPort}) + if err != nil { + t.Fatalf("ListenInNetNS: %v", err) + } + defer listeners[0].Close() + + type capture struct{ destination, payload string } + got := make(chan capture, 1) + accept := func() { + c, err := listeners[0].Accept() + if err != nil { + return + } + defer c.Close() + dst, err := atunnel.TCPOriginalDestination(c) + if err != nil { + t.Errorf("TCPOriginalDestination: %v", err) + return + } + buf := make([]byte, len("hello")) + io.ReadFull(c, buf) + got <- capture{destination: dst, payload: string(buf)} + } + go accept() + + for _, want := range []string{"93.184.216.34:443", "93.184.216.34:8080", "93.184.216.34:9999"} { + if err := ateomnet.NetNSDo(ctx, n.RuntimeNetNS, func(context.Context) error { + c, err := net.Dial("tcp", want) + if err != nil { + return err + } + defer c.Close() + _, err = io.WriteString(c, "hello") + return err + }); err != nil { + t.Fatalf("sandbox egress dial to %s: %v", want, err) + } + select { + case c := <-got: + if c.destination != want { + t.Errorf("atunnel saw destination %q, want %q", c.destination, want) + } + if c.payload != "hello" { + t.Errorf("atunnel read %q, want %q", c.payload, "hello") + } + case <-time.After(10 * time.Second): + t.Fatalf("the sandbox's connection to %s never reached atunnel", want) + } + go accept() + } +} diff --git a/internal/ateomnet/sandbox_linux_test.go b/internal/ateomnet/sandbox_linux_test.go new file mode 100644 index 0000000000..1982a13f58 --- /dev/null +++ b/internal/ateomnet/sandbox_linux_test.go @@ -0,0 +1,485 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ateomnet + +import ( + "context" + "errors" + "fmt" + "github.com/agent-substrate/substrate/internal/ateompath" + "github.com/vishvananda/netlink" + "io" + "net" + "net/http" + "net/url" + "os" + "runtime/pprof" + "sync" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/roottest" +) + +const testEgressPort = 15001 + +func TestValidateNetNSDialTarget(t *testing.T) { + for _, target := range []struct { + network, address string + wantErr bool + }{ + {"tcp", "127.0.0.1:80", false}, + {"tcp4", "127.0.0.1:80", false}, + {"tcp6", "[::1]:80", false}, + {"udp", "127.0.0.1:53", false}, + {"udp4", "127.0.0.1:53", false}, + {"udp6", "[fe80::1%eth0]:53", false}, + {"tcp", "localhost:80", true}, + {"tcp", ":80", true}, + {"tcp", "127.0.0.1", true}, + {"unix", "/tmp/socket", true}, + {"ip", "127.0.0.1:80", true}, + } { + t.Run(target.network+"/"+target.address, func(t *testing.T) { + err := validateNetNSDialTarget(target.network, target.address) + if (err != nil) != target.wantErr { + t.Errorf("validateNetNSDialTarget = %v, want error: %t", err, target.wantErr) + } + }) + } + if err := validateNetNSDialTarget("unix", "/tmp/socket"); !errors.Is(err, net.UnknownNetworkError("unix")) { + t.Errorf("unsupported network: got %v, want UnknownNetworkError", err) + } +} + +func TestNetNSDialerRejectsNonIPTargets(t *testing.T) { + for _, target := range []struct{ network, address string }{ + {"tcp", "localhost:80"}, + {"tcp", ":80"}, + {"tcp", "127.0.0.1"}, + {"unix", "/tmp/socket"}, + } { + if conn, err := NetNSDialer(-1)(context.Background(), target.network, target.address); err == nil { + _ = conn.Close() + t.Errorf("accepted %s %s", target.network, target.address) + } + } +} + +func TestNetNSDialerDoesNotPinPendingThreads(t *testing.T) { + roottest.Require(t, "creates network namespaces") + network, err := SetupSandboxNetwork(context.Background(), SandboxNetworkConfig{ + ActorUID: "pending-dial-threads", EgressPort: testEgressPort, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = CleanupSandboxNetwork(network) }) + if err := NetNSDo(context.Background(), network.GatewayNetNS, func(context.Context) error { + link := &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: "silent"}} + if err := netlink.LinkAdd(link); err != nil { + return err + } + if err := netlink.AddrReplace(link, MustParseAddr("192.0.2.1/24")); err != nil { + return err + } + return netlink.LinkSetUp(link) + }); err != nil { + t.Fatal(err) + } + before := pprof.Lookup("threadcreate").Count() + ctx, cancel := context.WithCancel(context.Background()) + var workers sync.WaitGroup + defer func() { cancel(); workers.Wait() }() + finished := make(chan error, 64) + for range 64 { + workers.Add(1) + go func() { + defer workers.Done() + conn, err := NetNSDialer(network.GatewayNetNS)(ctx, "tcp", "192.0.2.2:80") + if conn != nil { + _ = conn.Close() + } + finished <- err + }() + } + select { + case err := <-finished: + t.Fatalf("dial did not remain pending: %v", err) + case <-time.After(250 * time.Millisecond): + } + if growth := pprof.Lookup("threadcreate").Count() - before; growth >= 48 { + t.Errorf("64 pending dials created %d native threads", growth) + } + cancel() + for range 64 { + select { + case err := <-finished: + if !errors.Is(err, context.Canceled) { + t.Errorf("canceled dial: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("dial did not stop after cancellation") + } + } +} + +func TestNetNSDialerCanceledContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := NetNSDialer(-1)(ctx, "tcp", "127.0.0.1:1"); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled dial: got %v, want cancellation", err) + } +} + +func TestSetupSandboxNetwork(t *testing.T) { + roottest.Require(t, "creates network namespaces") + ctx := context.Background() + + type actor struct { + net *SandboxNetwork + body string + } + actors := map[string]*actor{} + for _, uid := range []string{"11111111-1111-1111-1111-111111111111", "22222222-2222-2222-2222-222222222222"} { + n, err := SetupSandboxNetwork(ctx, SandboxNetworkConfig{ActorUID: uid, Veth: true, EgressPort: testEgressPort}) + if err != nil { + t.Fatalf("SetupSandboxNetwork(%s): %v", uid, err) + } + t.Cleanup(func() { + if err := CleanupSandboxNetwork(n); err != nil { + t.Errorf("cleanup %s: %v", uid, err) + } + }) + if got := n.PodSideIP.String(); got != ActorVethIP { + t.Errorf("actor address = %s, want the same %s every actor holds", got, ActorVethIP) + } + + // The actor's app, bound where a real one binds, inside its namespace. + var lis net.Listener + if err := NetNSDo(ctx, n.RuntimeNetNS, func(context.Context) error { + l, err := net.Listen("tcp", net.JoinHostPort(ActorVethIP, "80")) + lis = l + return err + }); err != nil { + t.Fatalf("actor %s listen: %v", uid, err) + } + t.Cleanup(func() { lis.Close() }) + body := "i-am-" + uid[:8] + go http.Serve(lis, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, body) + })) + actors[uid] = &actor{net: n, body: body} + } + + // Reaching each actor is a matter of which namespace the dial is made from. + for uid, a := range actors { + client := &http.Client{Transport: &http.Transport{DialContext: NetNSDialer(a.net.RuntimeNetNS)}, Timeout: 5 * time.Second} + resp, err := client.Get((&url.URL{Scheme: "http", Host: net.JoinHostPort(ActorVethIP, "80")}).String()) + if err != nil { + t.Fatalf("reaching actor %s: %v", uid, err) + } + got, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if string(got) != a.body { + t.Errorf("actor %s answered %q, want %q", uid, got, a.body) + } + } + + // Sandbox addresses must not be reachable from the worker namespace. + direct := &http.Client{Timeout: 2 * time.Second} + if _, err := direct.Get("http://" + net.JoinHostPort(ActorVethIP, "80")); err == nil { + t.Error("the worker namespace reached an actor directly; addresses are not isolated") + } +} + +func TestActorEgressIsFailClosedWithoutAtunnel(t *testing.T) { + roottest.Require(t, "creates network namespaces") + ctx := context.Background() + + n, err := SetupSandboxNetwork(ctx, SandboxNetworkConfig{ + ActorUID: "33333333-3333-3333-3333-333333333333", Veth: true, EgressPort: testEgressPort, + }) + if err != nil { + t.Fatalf("SetupSandboxNetwork: %v", err) + } + t.Cleanup(func() { CleanupSandboxNetwork(n) }) + + for _, destination := range []string{"93.184.216.34:443", "93.184.216.34:8080"} { + if err := NetNSDo(ctx, n.RuntimeNetNS, func(context.Context) error { + c, err := net.DialTimeout("tcp", destination, 3*time.Second) + if err != nil { + return err + } + c.Close() + return nil + }); err == nil { + t.Errorf("the actor reached %s with no atunnel listening; egress is not fail-closed", destination) + } + } +} + +func TestIngressCrossesThePairWhileEgressIsCaptured(t *testing.T) { + roottest.Require(t, "creates network namespaces") + ctx := context.Background() + + n, err := SetupSandboxNetwork(ctx, SandboxNetworkConfig{ActorUID: "44444444-4444-4444-4444-444444444444", Veth: true, EgressPort: testEgressPort}) + if err != nil { + t.Fatalf("SetupSandboxNetwork: %v", err) + } + t.Cleanup(func() { CleanupSandboxNetwork(n) }) + + var app net.Listener + if err := NetNSDo(ctx, n.RuntimeNetNS, func(context.Context) error { + l, e := net.Listen("tcp", net.JoinHostPort(ActorVethIP, "80")) + app = l + return e + }); err != nil { + t.Fatalf("actor listen: %v", err) + } + defer app.Close() + go func() { + for { + c, e := app.Accept() + if e != nil { + return + } + io.WriteString(c, "the-actor") + c.Close() + } + }() + + dial := NetNSDialer(n.GatewayNetNS) + c, err := dial(ctx, "tcp", net.JoinHostPort(ActorVethIP, "80")) + if err != nil { + t.Fatalf("ingress dial: %v", err) + } + got, _ := io.ReadAll(c) + c.Close() + if string(got) != "the-actor" { + t.Errorf("ingress reached %q, want %q", got, "the-actor") + } + + // An unopened port must be refused, not redirected to atunnel. + cctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + if c2, e := dial(cctx, "tcp", net.JoinHostPort(ActorVethIP, "81")); e == nil { + c2.Close() + t.Error("a port with no listener was accepted, so ingress is not reaching the sandbox") + } +} + +// This tests namespace setup only; testing microVM egress requires a tap. +func TestSetupSandboxNetworkWithoutVeth(t *testing.T) { + roottest.Require(t, "creates network namespaces") + ctx := context.Background() + + n, err := SetupSandboxNetwork(ctx, SandboxNetworkConfig{ + ActorUID: "77777777-7777-7777-7777-777777777777", + EgressPort: testEgressPort, + }) + if err != nil { + t.Fatalf("SetupSandboxNetwork: %v", err) + } + t.Cleanup(func() { CleanupSandboxNetwork(n) }) + + if n.GatewayNetNS != n.RuntimeNetNS { + t.Errorf("got a second namespace (%v vs %v); this shape needs one", n.GatewayNetNS, n.RuntimeNetNS) + } + + // No veth was built, so nothing but lo is here until the tap arrives. + if err := NetNSDo(ctx, n.RuntimeNetNS, func(context.Context) error { + links, err := netlink.LinkList() + if err != nil { + return err + } + for _, l := range links { + if l.Attrs().Name != "lo" { + t.Errorf("unexpected interface %q in the actor namespace", l.Attrs().Name) + } + } + return nil + }); err != nil { + t.Fatalf("listing links: %v", err) + } +} + +// A micro-VM snapshot freezes the guest's ARP entry for its gateway, so the +// gateway has to answer with the same MAC on every worker. +func TestGatewayHardwareAddressIsFixedWhenAsked(t *testing.T) { + roottest.Require(t, "creates network namespaces") + ctx := context.Background() + + want, err := net.ParseMAC("02:00:00:00:17:01") + if err != nil { + t.Fatal(err) + } + n, err := SetupSandboxNetwork(ctx, SandboxNetworkConfig{ + ActorUID: "88888888-8888-8888-8888-888888888888", + Veth: true, + EgressPort: testEgressPort, + GatewayHWAddr: want, + }) + if err != nil { + t.Fatalf("SetupSandboxNetwork: %v", err) + } + t.Cleanup(func() { CleanupSandboxNetwork(n) }) + + if err := NetNSDo(ctx, n.GatewayNetNS, func(context.Context) error { + l, err := netlink.LinkByName("atside") + if err != nil { + return err + } + if got := l.Attrs().HardwareAddr.String(); got != want.String() { + t.Errorf("gateway MAC is %s, want %s", got, want) + } + return nil + }); err != nil { + t.Fatalf("reading the gateway link: %v", err) + } +} + +func TestSetupSucceedsOverALeftoverNamespace(t *testing.T) { + roottest.Require(t, "creates network namespaces") + ctx := context.Background() + const uid = "aaaaaaaa-0000-0000-0000-00000000000a" + cfg := SandboxNetworkConfig{ActorUID: uid, Veth: true, EgressPort: testEgressPort} + + first, err := SetupSandboxNetwork(ctx, cfg) + if err != nil { + t.Fatalf("first SetupSandboxNetwork: %v", err) + } + // Simulate interrupted teardown by leaving the namespace names mounted. + first.RuntimeNetNS.Close() + if first.GatewayNetNS != first.RuntimeNetNS { + first.GatewayNetNS.Close() + } + for _, name := range []string{ateompath.ActorNetNSName(uid), SandboxGatewayNetNSName(uid)} { + if _, err := os.Stat("/var/run/netns/" + name); err != nil { + t.Fatalf("expected leftover netns %s: %v", name, err) + } + } + + second, err := SetupSandboxNetwork(ctx, cfg) + if err != nil { + t.Fatalf("the actor is wedged by its own leftover namespace: %v", err) + } + t.Cleanup(func() { CleanupSandboxNetwork(second) }) + + if err := NetNSDo(ctx, second.RuntimeNetNS, func(context.Context) error { + if _, err := netlink.LinkByName(ActorVethName); err != nil { + return fmt.Errorf("actor interface missing after reuse: %w", err) + } + return nil + }); err != nil { + t.Error(err) + } +} + +// Check from the gateway: a successful UDP send from the actor does not prove delivery. +func TestActorUDPHasNowhereToGoBeyondTheNamespacePair(t *testing.T) { + roottest.Require(t, "creates network namespaces") + ctx := context.Background() + + n, err := SetupSandboxNetwork(ctx, SandboxNetworkConfig{ + ActorUID: "bbbbbbbb-0000-0000-0000-00000000000b", Veth: true, EgressPort: testEgressPort, + }) + if err != nil { + t.Fatalf("SetupSandboxNetwork: %v", err) + } + t.Cleanup(func() { CleanupSandboxNetwork(n) }) + + for _, destination := range []string{"93.184.216.34:443", "93.184.216.34:53"} { + if err := NetNSDo(ctx, n.GatewayNetNS, func(context.Context) error { + c, err := net.Dial("udp", destination) + if err != nil { + return err + } + defer c.Close() + _, err = c.Write([]byte("probe")) + return err + }); err == nil { + t.Errorf("the atunnel namespace can reach %s over UDP; actor UDP could follow it out", destination) + } + } +} + +func TestCleanupClosesEachDescriptorOnce(t *testing.T) { + roottest.Require(t, "creates network namespaces") + network, err := SetupSandboxNetwork(context.Background(), SandboxNetworkConfig{ + ActorUID: "close-once", + EgressPort: 15001, + }) + if err != nil { + t.Fatal(err) + } + if network.RuntimeNetNS != network.GatewayNetNS { + t.Fatalf("expected one namespace without a veth, got %d and %d", network.RuntimeNetNS, network.GatewayNetNS) + } + + // A second close of the same descriptor reports EBADF. + if err := CleanupSandboxNetwork(network); err != nil { + t.Fatalf("CleanupSandboxNetwork closed a descriptor twice: %v", err) + } +} + +// stoppableDNS records that its serving contexts were canceled. +type stoppableDNS struct{ packet, stream chan struct{} } + +func (d *stoppableDNS) ServePacket(ctx context.Context, pc net.PacketConn) error { + <-ctx.Done() + close(d.packet) + return pc.Close() +} + +func (d *stoppableDNS) Serve(ctx context.Context, l net.Listener) error { + <-ctx.Done() + close(d.stream) + return l.Close() +} + +func TestClosingSandboxDNSStopsServing(t *testing.T) { + roottest.Require(t, "creates network namespaces") + network, err := SetupSandboxNetwork(context.Background(), SandboxNetworkConfig{ + ActorUID: "dns-teardown", + EgressPort: 15001, + }) + if err != nil { + t.Fatal(err) + } + defer func() { _ = CleanupSandboxNetwork(network) }() + + relay := &stoppableDNS{packet: make(chan struct{}), stream: make(chan struct{})} + closers, err := ServeSandboxDNS(context.Background(), relay, network.GatewayNetNS, 53) + if err != nil { + t.Fatal(err) + } + for _, c := range closers { + _ = c.Close() + } + + for _, tc := range []struct { + name string + stopped chan struct{} + }{{"UDP", relay.packet}, {"TCP", relay.stream}} { + select { + case <-tc.stopped: + case <-time.After(5 * time.Second): + t.Errorf("%s serving outlived the sandbox's sockets", tc.name) + } + } +} diff --git a/internal/ateompath/ateompath.go b/internal/ateompath/ateompath.go index 2a18303c12..277673b276 100644 --- a/internal/ateompath/ateompath.go +++ b/internal/ateompath/ateompath.go @@ -82,6 +82,18 @@ func AteletOTLPSocketPath() string { // AteomsDir is the parent of every per-ateom directory. Each ateom creates // AteomPath(podUID) under it when it boots, so listing this directory is how a +// ActorNetNSName is the named network namespace one actor's sandbox runs in. +// Per actor rather than per pod, so a worker can hold several sandboxes whose +// networks cannot see each other. +func ActorNetNSName(actorUID string) string { + return "ateom-actor:" + actorUID +} + +// ActorNetNSPath is where the kernel exposes that namespace. +func ActorNetNSPath(actorUID string) string { + return filepath.Join("/run/netns", ActorNetNSName(actorUID)) +} + // scraper with no prior knowledge discovers the node's ateoms. func AteomsDir() string { return filepath.Join(BasePath, "ateoms") diff --git a/internal/resources/validate.go b/internal/resources/validate.go index 103c944ed7..85cd37519f 100644 --- a/internal/resources/validate.go +++ b/internal/resources/validate.go @@ -77,10 +77,9 @@ func ValidateGlobalObjectRef(ref *ateapipb.ObjectRef, fldPath *field.Path) field } // ValidateAteomUID rejects a target ateom pod UID that could escape the host -// paths built from it: the netns path (/run/netns/ateom:) and the ateom -// control socket (.../ateoms//ateom.sock). Kubernetes pod UIDs are UUIDs, -// which are valid DNS-1123 labels, so a label check accepts every legitimate -// value while rejecting separators and "..". +// path built from it: the ateom control socket (.../ateoms//ateom.sock). +// Kubernetes pod UIDs are UUIDs, which are valid DNS-1123 labels, so a label +// check accepts every legitimate value while rejecting separators and "..". func ValidateAteomUID(targetAteomUID string) error { if errs := content.IsDNS1123Label(targetAteomUID); len(errs) > 0 { return fmt.Errorf("invalid target ateom UID %q: %s", targetAteomUID, strings.Join(errs, "; ")) From 7e2f7f0225eaa5b7d5baff4229517a0990956e33 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Tue, 15 Sep 2026 14:37:02 -0700 Subject: [PATCH 2/4] atunnel: relay sandbox DNS through pod resolvers Forward UDP and TCP DNS unchanged through the worker pod. Bound concurrency and connection lifetime, and close TCP connections on cancellation. DNS bypasses the actor egress policy. --- docs/egress-trust-bundle.md | 10 + internal/atunnel/dns.go | 348 ++++++++++++++++++++++ internal/atunnel/dns_test.go | 538 +++++++++++++++++++++++++++++++++++ 3 files changed, 896 insertions(+) create mode 100644 internal/atunnel/dns.go create mode 100644 internal/atunnel/dns_test.go diff --git a/docs/egress-trust-bundle.md b/docs/egress-trust-bundle.md index 95e7a3cb57..7249dfe45e 100644 --- a/docs/egress-trust-bundle.md +++ b/docs/egress-trust-bundle.md @@ -10,6 +10,16 @@ certificate error. This guide covers how to project the gateway's CA into an actor's filesystem and how to point the actor's TLS client at it. +## DNS and egress policy + +Actors send DNS queries to a relay at their sandbox's default gateway. The +relay forwards UDP and TCP DNS to the worker pod's configured resolvers, +without passing through the external egress gateway or checking egress policy. +There is currently no per-actor setting to disable this relay or filter queries. + +DNS remains available when no egress gateway is configured. Other outbound TCP +connections are captured by atunnel and refused in that configuration. + ## When you need this You need it when **all** of the following hold: diff --git a/internal/atunnel/dns.go b/internal/atunnel/dns.go new file mode 100644 index 0000000000..24d6541b33 --- /dev/null +++ b/internal/atunnel/dns.go @@ -0,0 +1,348 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package atunnel + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "log/slog" + "net" + "os" + "strings" + "sync" + "time" +) + +const ( + // DNSPort is the relay port on the sandbox's gateway. + DNSPort = 53 + + // Read complete UDP datagrams without truncating EDNS responses. + maxDNSDatagram = 65535 + + // Drop excess UDP queries to bound goroutines and upstream sockets. + maxInFlightDNS = 64 + + // maxDNSConnections bounds open TCP connections. + maxDNSConnections = 16 + + // dnsTCPTimeout limits connection lifetime, including idle clients. + dnsTCPTimeout = 30 * time.Second + + // dnsExchangeTimeout bounds each upstream attempt. + dnsExchangeTimeout = 5 * time.Second +) + +// DNSRelay forwards UDP and TCP DNS unchanged to the worker pod's resolvers. +// It listens in the sandbox's gateway namespace and dials from the worker's. +// DNS bypasses the egress tunnel and is not checked against egress policy. +type DNSRelay struct { + upstreams []string + // dialer reaches upstream resolvers from the worker namespace. + dialer *net.Dialer + + // Limits are shared across all sandboxes using this relay. + inFlight chan struct{} + connections chan struct{} +} + +// NewDNSRelay forwards to upstreams, each "host:port". +func NewDNSRelay(upstreams []string) (*DNSRelay, error) { + if len(upstreams) == 0 { + return nil, fmt.Errorf("atunnel: at least one upstream resolver is required") + } + for _, u := range upstreams { + if _, _, err := net.SplitHostPort(u); err != nil { + return nil, fmt.Errorf("atunnel: invalid upstream resolver %q: %w", u, err) + } + } + return &DNSRelay{ + upstreams: upstreams, + dialer: &net.Dialer{Timeout: dnsExchangeTimeout}, + inFlight: make(chan struct{}, maxInFlightDNS), + connections: make(chan struct{}, maxDNSConnections), + }, nil +} + +// ResolvConfNameservers reads nameservers from resolv.conf as "host:53". +func ResolvConfNameservers(path string) ([]string, error) { + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("atunnel: reading resolv.conf: %w", err) + } + defer f.Close() + + var out []string + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if i := strings.IndexAny(line, "#;"); i >= 0 { + line = strings.TrimSpace(line[:i]) + } + rest, ok := strings.CutPrefix(line, "nameserver") + if !ok { + continue + } + address := strings.TrimSpace(rest) + if address == "" || net.ParseIP(address) == nil { + continue + } + out = append(out, net.JoinHostPort(address, "53")) + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("atunnel: reading resolv.conf: %w", err) + } + if len(out) == 0 { + return nil, fmt.Errorf("atunnel: %s names no usable nameserver", path) + } + return out, nil +} + +// ServePacket answers UDP queries until ctx is canceled or the socket fails. +func (r *DNSRelay) ServePacket(ctx context.Context, pc net.PacketConn) error { + done := make(chan struct{}) + go func() { + select { + case <-ctx.Done(): + _ = pc.Close() + case <-done: + } + }() + defer close(done) + + var wg sync.WaitGroup + defer wg.Wait() + + buf := make([]byte, maxDNSDatagram) + for { + n, from, err := pc.ReadFrom(buf) + if err != nil { + if ctx.Err() != nil || errors.Is(err, net.ErrClosed) { + return nil + } + return fmt.Errorf("atunnel: reading actor DNS query: %w", err) + } + // Copied: the buffer is reused by the next read. + query := make([]byte, n) + copy(query, buf[:n]) + + select { + case r.inFlight <- struct{}{}: + default: + slog.DebugContext(ctx, "atunnel dropped a DNS query; too many in flight") + continue + } + wg.Add(1) + go func() { + defer wg.Done() + defer func() { <-r.inFlight }() + answer, err := r.exchangeUDP(ctx, query) + if err != nil { + slog.WarnContext(ctx, "atunnel could not resolve an actor DNS query", slog.Any("err", err)) + return + } + if _, err := pc.WriteTo(answer, from); err != nil && ctx.Err() == nil { + slog.WarnContext(ctx, "atunnel could not return a DNS answer", slog.Any("err", err)) + } + }() + } +} + +// Serve relays TCP DNS connections until ctx is canceled or the listener closes. +func (r *DNSRelay) Serve(ctx context.Context, listener net.Listener) error { + done := make(chan struct{}) + go func() { + select { + case <-ctx.Done(): + _ = listener.Close() + case <-done: + } + }() + defer close(done) + + // Wait for the relays to drain before returning, so a closed listener + // leaves no goroutine still holding a connection slot. + var wg sync.WaitGroup + defer wg.Wait() + + for { + conn, err := listener.Accept() + if err != nil { + if ctx.Err() != nil || errors.Is(err, net.ErrClosed) { + return nil + } + return fmt.Errorf("atunnel: accepting actor DNS connection: %w", err) + } + select { + case r.connections <- struct{}{}: + default: + slog.DebugContext(ctx, "atunnel refused a DNS connection; too many open") + _ = conn.Close() + continue + } + wg.Add(1) + go func() { + defer wg.Done() + defer func() { <-r.connections }() + r.relayTCP(ctx, conn) + }() + } +} + +func (r *DNSRelay) exchangeUDP(ctx context.Context, query []byte) ([]byte, error) { + var errs error + // deferred holds a server-failure answer to fall back on, see below. + var deferred []byte + for _, upstream := range r.upstreams { + if err := ctx.Err(); err != nil { + return nil, err + } + conn, err := r.dialer.DialContext(ctx, "udp", upstream) + if err != nil { + errs = errors.Join(errs, err) + continue + } + answer, err := func() ([]byte, error) { + defer conn.Close() + stop := context.AfterFunc(ctx, func() { _ = conn.Close() }) + defer stop() + if err := conn.SetDeadline(time.Now().Add(dnsExchangeTimeout)); err != nil { + return nil, err + } + if _, err := conn.Write(query); err != nil { + return nil, err + } + buf := make([]byte, maxDNSDatagram) + n, err := conn.Read(buf) + if err != nil { + return nil, err + } + return buf[:n], nil + }() + if err != nil { + errs = errors.Join(errs, fmt.Errorf("upstream %s: %w", upstream, err)) + continue + } + // SERVFAIL is not an answer, and the sandbox sees only the gateway, so + // it cannot try the pod's other resolvers itself. Keep the last one to + // return if none does better: a real response beats a timeout. + if rcode, ok := failoverRcode(answer); ok { + errs = errors.Join(errs, fmt.Errorf("upstream %s: %w", upstream, rcodeError(rcode))) + deferred = answer + continue + } + return answer, nil + } + if deferred != nil { + return deferred, nil + } + return nil, fmt.Errorf("atunnel: no upstream resolver answered: %w", errs) +} + +// Response codes that say the resolver failed rather than answered. NXDOMAIN +// and NOERROR are answers and are passed back as they are. +const ( + rcodeServFail = 2 + rcodeNotImp = 4 + rcodeRefused = 5 +) + +// failoverRcode reports the response code when the relay should try the next +// upstream. Reads the 12-byte header only; anything shorter is passed through. +func failoverRcode(msg []byte) (byte, bool) { + if len(msg) < 12 { + return 0, false + } + rcode := msg[3] & 0x0f + switch rcode { + case rcodeServFail, rcodeNotImp, rcodeRefused: + return rcode, true + } + return 0, false +} + +func rcodeError(rcode byte) error { + switch rcode { + case rcodeServFail: + return errors.New("answered SERVFAIL") + case rcodeNotImp: + return errors.New("answered NOTIMP") + case rcodeRefused: + return errors.New("answered REFUSED") + } + return fmt.Errorf("answered rcode %d", rcode) +} + +// relayTCP copies a DNS stream without parsing its length-prefixed messages. +func (r *DNSRelay) relayTCP(ctx context.Context, downstream net.Conn) { + defer downstream.Close() + + var upstream net.Conn + var errs error + for _, address := range r.upstreams { + conn, err := r.dialer.DialContext(ctx, "tcp", address) + if err != nil { + errs = errors.Join(errs, err) + continue + } + upstream = conn + break + } + if upstream == nil { + slog.WarnContext(ctx, "atunnel could not reach any resolver for an actor DNS connection", slog.Any("err", errs)) + return + } + defer upstream.Close() + + // Cancel active copies on teardown to release the worker's connection slots. + relayDone := make(chan struct{}) + defer close(relayDone) + go func() { + select { + case <-ctx.Done(): + _ = downstream.Close() + _ = upstream.Close() + case <-relayDone: + } + }() + + deadline := time.Now().Add(dnsTCPTimeout) + if ctxDeadline, ok := ctx.Deadline(); ok && ctxDeadline.Before(deadline) { + deadline = ctxDeadline + } + _ = downstream.SetDeadline(deadline) + _ = upstream.SetDeadline(deadline) + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + _, _ = io.Copy(upstream, downstream) + if c, ok := upstream.(*net.TCPConn); ok { + _ = c.CloseWrite() + } + }() + go func() { + defer wg.Done() + _, _ = io.Copy(downstream, upstream) + if c, ok := downstream.(*net.TCPConn); ok { + _ = c.CloseWrite() + } + }() + wg.Wait() +} diff --git a/internal/atunnel/dns_test.go b/internal/atunnel/dns_test.go new file mode 100644 index 0000000000..67cc3deb9f --- /dev/null +++ b/internal/atunnel/dns_test.go @@ -0,0 +1,538 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package atunnel + +import ( + "bytes" + "context" + "errors" + "io" + "net" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/google/go-cmp/cmp" +) + +func TestResolvConfNameservers(t *testing.T) { + for _, tc := range []struct { + name string + content string + want []string + wantErr bool + }{ + { + name: "cluster resolv.conf", + content: "search ate-system.svc.cluster.local svc.cluster.local\nnameserver 10.96.0.10\noptions ndots:5\n", + want: []string{"10.96.0.10:53"}, + }, + { + name: "several, in order", + content: "nameserver 10.96.0.10\nnameserver 8.8.8.8\n", + want: []string{"10.96.0.10:53", "8.8.8.8:53"}, + }, + { + name: "comments and blanks", + content: "# generated\n\n nameserver 10.96.0.10 # cluster\n;nameserver 1.1.1.1\n", + want: []string{"10.96.0.10:53"}, + }, + {name: "no nameserver", content: "search cluster.local\n", wantErr: true}, + {name: "unparsable address", content: "nameserver not-an-ip\n", wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "resolv.conf") + if err := os.WriteFile(path, []byte(tc.content), 0o644); err != nil { + t.Fatal(err) + } + got, err := ResolvConfNameservers(path) + if tc.wantErr { + if err == nil { + t.Fatalf("ResolvConfNameservers() = %v, want an error", got) + } + return + } + if err != nil { + t.Fatalf("ResolvConfNameservers: %v", err) + } + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("nameservers mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestDNSRelayCancelsUDPExchange(t *testing.T) { + // A resolver that receives the query and never answers, so the exchange is + // blocked on the read when the context is canceled. + silent, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer silent.Close() + asked := make(chan struct{}, 1) + go func() { + buf := make([]byte, maxDNSDatagram) + for { + if _, _, err := silent.ReadFrom(buf); err != nil { + return + } + select { + case asked <- struct{}{}: + default: + } + } + }() + + // Two upstreams: a canceled exchange must not move on to the second. + second := newFakeResolver(t, func(query []byte) []byte { return query }) + relay, err := NewDNSRelay([]string{silent.LocalAddr().String(), second}) + if err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { + _, err := relay.exchangeUDP(ctx, dnsQuery(0x1234)) + done <- err + }() + select { + case <-asked: + case <-time.After(5 * time.Second): + t.Fatal("the resolver never saw the query") + } + cancel() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("exchange returned %v, want cancellation", err) + } + case <-time.After(5 * time.Second): + t.Fatal("canceled exchange remained blocked reading upstream") + } +} + +func TestDNSRelayForwardsUDPVerbatim(t *testing.T) { + upstream := newFakeResolver(t, func(query []byte) []byte { + return append([]byte{0xff}, query...) + }) + + relay, err := NewDNSRelay([]string{upstream}) + if err != nil { + t.Fatal(err) + } + client := serveRelayUDP(t, relay) + + query := []byte{0xab, 0xcd, 0x01, 0x00, 0x00, 0x01} + if _, err := client.Write(query); err != nil { + t.Fatal(err) + } + answer := readWithin(t, client) + if diff := cmp.Diff(append([]byte{0xff}, query...), answer); diff != "" { + t.Errorf("answer mismatch (-want +got):\n%s", diff) + } +} + +func TestDNSRelayFallsBackToTheNextResolver(t *testing.T) { + dead, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + deadAddress := dead.LocalAddr().String() + // Closed, so the exchange fails rather than hanging to its deadline. + dead.Close() + + live := newFakeResolver(t, func(query []byte) []byte { return []byte("answered") }) + relay, err := NewDNSRelay([]string{deadAddress, live}) + if err != nil { + t.Fatal(err) + } + client := serveRelayUDP(t, relay) + + if _, err := client.Write([]byte("query")); err != nil { + t.Fatal(err) + } + if got := string(readWithin(t, client)); got != "answered" { + t.Errorf("answer = %q, want %q", got, "answered") + } +} + +func TestNewDNSRelayRejects(t *testing.T) { + if _, err := NewDNSRelay(nil); err == nil { + t.Error("NewDNSRelay(nil) succeeded; a relay with no upstream can answer nothing") + } + if _, err := NewDNSRelay([]string{"10.96.0.10"}); err == nil { + t.Error("NewDNSRelay accepted an address with no port") + } +} + +// newFakeResolver answers UDP with respond(query), and returns its address. +func newFakeResolver(t *testing.T, respond func([]byte) []byte) string { + t.Helper() + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { pc.Close() }) + go func() { + buf := make([]byte, maxDNSDatagram) + for { + n, from, err := pc.ReadFrom(buf) + if err != nil { + return + } + query := make([]byte, n) + copy(query, buf[:n]) + _, _ = pc.WriteTo(respond(query), from) + } + }() + return pc.LocalAddr().String() +} + +// serveRelayUDP runs the relay on a loopback socket and returns a connection to it. +func serveRelayUDP(t *testing.T, relay *DNSRelay) net.Conn { + t.Helper() + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + go func() { _ = relay.ServePacket(ctx, pc) }() + + client, err := net.Dial("udp", pc.LocalAddr().String()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { client.Close() }) + return client +} + +func readWithin(t *testing.T, conn net.Conn) []byte { + t.Helper() + if err := conn.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil { + t.Fatal(err) + } + buf := make([]byte, maxDNSDatagram) + n, err := conn.Read(buf) + if err != nil { + t.Fatalf("reading the relay's answer: %v", err) + } + return buf[:n] +} + +func TestDNSRelayForwardsAnswersLargerThanTheCommonBuffer(t *testing.T) { + const size = 9000 + answer := make([]byte, size) + for i := range answer { + answer[i] = byte(i) + } + upstream := newFakeResolver(t, func([]byte) []byte { return answer }) + + relay, err := NewDNSRelay([]string{upstream}) + if err != nil { + t.Fatal(err) + } + client := serveRelayUDP(t, relay) + if _, err := client.Write([]byte("query")); err != nil { + t.Fatal(err) + } + + got := readWithin(t, client) + if len(got) != size { + t.Fatalf("answer is %d bytes, want %d: it was cut down in the relay", len(got), size) + } + if !bytes.Equal(got, answer) { + t.Error("answer differs from what the resolver sent") + } +} + +func TestDNSRelayDropsQueriesBeyondItsInFlightLimit(t *testing.T) { + // Hold concurrent queries to exercise the relay's limit. + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { pc.Close() }) + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + var inFlight atomic.Int64 + go func() { + buf := make([]byte, maxDNSDatagram) + for { + n, from, err := pc.ReadFrom(buf) + if err != nil { + return + } + answer := make([]byte, n) + copy(answer, buf[:n]) + go func() { + inFlight.Add(1) + <-release + _, _ = pc.WriteTo(answer, from) + }() + } + }() + + relay, err := NewDNSRelay([]string{pc.LocalAddr().String()}) + if err != nil { + t.Fatal(err) + } + client := serveRelayUDP(t, relay) + + for range maxInFlightDNS * 4 { + if _, err := client.Write([]byte("query")); err != nil { + t.Fatal(err) + } + } + // Wait for the accepted-query count to stabilize. + deadline := time.Now().Add(10 * time.Second) + last := int64(-1) + for time.Now().Before(deadline) { + time.Sleep(100 * time.Millisecond) + if n := inFlight.Load(); n == last { + break + } else { + last = n + } + } + if got := inFlight.Load(); got > maxInFlightDNS { + t.Errorf("the relay had %d queries in flight, want at most %d", got, maxInFlightDNS) + } +} + +// newHeldTCPResolver accepts DNS connections and answers none, holding each +// until the test ends. It reports how many it is holding. +func newHeldTCPResolver(t *testing.T) (address string, accepted *atomic.Int64) { + t.Helper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { lis.Close() }) + + var held atomic.Int64 + var mu sync.Mutex + var conns []net.Conn + t.Cleanup(func() { + mu.Lock() + defer mu.Unlock() + for _, c := range conns { + c.Close() + } + }) + go func() { + for { + conn, err := lis.Accept() + if err != nil { + return + } + mu.Lock() + conns = append(conns, conn) + mu.Unlock() + held.Add(1) + } + }() + return lis.Addr().String(), &held +} + +// serveRelayTCP runs the relay's TCP side on a loopback listener. +func serveRelayTCP(t *testing.T, relay *DNSRelay, ctx context.Context) net.Addr { + t.Helper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { lis.Close() }) + go func() { _ = relay.Serve(ctx, lis) }() + return lis.Addr() +} + +// waitFor polls until cond holds, failing the test if it never does. +func waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatalf("timed out waiting for %s", what) +} + +func TestDNSRelayRefusesTCPConnectionsBeyondItsLimit(t *testing.T) { + upstream, held := newHeldTCPResolver(t) + relay, err := NewDNSRelay([]string{upstream}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + relayAddr := serveRelayTCP(t, relay, ctx) + + for range maxDNSConnections { + conn, err := net.Dial("tcp", relayAddr.String()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close() }) + } + waitFor(t, "the relay to fill up", func() bool { return held.Load() == maxDNSConnections }) + + // The relay should close a connection accepted beyond its limit. + extra, err := net.Dial("tcp", relayAddr.String()) + if err != nil { + t.Fatal(err) + } + defer extra.Close() + if err := extra.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil { + t.Fatal(err) + } + if _, err := extra.Read(make([]byte, 1)); !errors.Is(err, io.EOF) { + t.Errorf("reading the refused connection = %v, want %v", err, io.EOF) + } + if got := held.Load(); got != maxDNSConnections { + t.Errorf("the relay holds %d upstream connections, want %d", got, maxDNSConnections) + } +} + +func TestDNSRelayClosesTCPConnectionsWhenServingEnds(t *testing.T) { + upstream, held := newHeldTCPResolver(t) + relay, err := NewDNSRelay([]string{upstream}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + relayAddr := serveRelayTCP(t, relay, ctx) + + conn, err := net.Dial("tcp", relayAddr.String()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + waitFor(t, "the connection to reach an upstream", func() bool { return held.Load() == 1 }) + + cancel() + + // Cancellation must close the connection before dnsTCPTimeout. + if err := conn.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil { + t.Fatal(err) + } + if _, err := conn.Read(make([]byte, 1)); !errors.Is(err, io.EOF) { + t.Errorf("reading after teardown = %v, want %v: the connection outlived the actor", err, io.EOF) + } +} + +// dnsQuery builds a real query for example.com A, so tests exercise messages a +// resolver would actually send rather than an arbitrary string. +func dnsQuery(id uint16) []byte { + msg := []byte{byte(id >> 8), byte(id), 0x01, 0x00, 0, 1, 0, 0, 0, 0, 0, 0} + for _, label := range []string{"example", "com"} { + msg = append(msg, byte(len(label))) + msg = append(msg, label...) + } + return append(msg, 0, 0, 1, 0, 1) // root label, QTYPE A, QCLASS IN +} + +// dnsAnswer echoes a query back as a response carrying rcode. +func dnsAnswer(query []byte, rcode byte) []byte { + resp := make([]byte, len(query)) + copy(resp, query) + resp[2] |= 0x80 // QR: this is a response + resp[3] = resp[3]&0xf0 | rcode + return resp +} + +// A resolver that answers SERVFAIL has not answered the question, and the +// sandbox can no longer consult the pod's other resolvers itself: it is given +// the gateway as its only nameserver. The relay has to fail over for it. +func TestDNSRelayFailsOverOnServerFailure(t *testing.T) { + var sickCalls, healthyCalls atomic.Int32 + sick := newFakeResolver(t, func(query []byte) []byte { + sickCalls.Add(1) + return dnsAnswer(query, rcodeServFail) + }) + healthy := newFakeResolver(t, func(query []byte) []byte { + healthyCalls.Add(1) + return dnsAnswer(query, 0) + }) + + relay, err := NewDNSRelay([]string{sick, healthy}) + if err != nil { + t.Fatal(err) + } + answer, err := relay.exchangeUDP(context.Background(), dnsQuery(0x1234)) + if err != nil { + t.Fatalf("exchange: %v", err) + } + if got := answer[3] & 0x0f; got != 0 { + t.Errorf("answer rcode = %d, want 0: the relay returned the failing resolver's answer", got) + } + if healthyCalls.Load() != 1 { + t.Errorf("the healthy resolver was asked %d times, want 1", healthyCalls.Load()) + } +} + +// With every resolver failing there is nothing better to return, and a real +// SERVFAIL beats a timeout: the sandbox's resolver can act on it. +func TestDNSRelayReturnsServerFailureWhenAllFail(t *testing.T) { + first := newFakeResolver(t, func(query []byte) []byte { return dnsAnswer(query, rcodeServFail) }) + second := newFakeResolver(t, func(query []byte) []byte { return dnsAnswer(query, rcodeRefused) }) + + relay, err := NewDNSRelay([]string{first, second}) + if err != nil { + t.Fatal(err) + } + answer, err := relay.exchangeUDP(context.Background(), dnsQuery(0x2345)) + if err != nil { + t.Fatalf("exchange: %v", err) + } + if got := answer[3] & 0x0f; got != rcodeRefused { + t.Errorf("answer rcode = %d, want the last resolver's %d", got, rcodeRefused) + } +} + +// NXDOMAIN is an answer, not a failure: failing over would ask every resolver +// about a name that does not exist. +func TestDNSRelayReturnsNXDomainWithoutFailover(t *testing.T) { + const rcodeNXDomain = 3 + var secondCalls atomic.Int32 + first := newFakeResolver(t, func(query []byte) []byte { return dnsAnswer(query, rcodeNXDomain) }) + second := newFakeResolver(t, func(query []byte) []byte { + secondCalls.Add(1) + return dnsAnswer(query, 0) + }) + + relay, err := NewDNSRelay([]string{first, second}) + if err != nil { + t.Fatal(err) + } + answer, err := relay.exchangeUDP(context.Background(), dnsQuery(0x3456)) + if err != nil { + t.Fatalf("exchange: %v", err) + } + if got := answer[3] & 0x0f; got != rcodeNXDomain { + t.Errorf("answer rcode = %d, want NXDOMAIN %d", got, rcodeNXDomain) + } + if secondCalls.Load() != 0 { + t.Error("the relay failed over on NXDOMAIN, which is a valid answer") + } +} From e60919ff26cd8fb809099ce80b35221985cba97c Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Tue, 15 Sep 2026 14:37:50 -0700 Subject: [PATCH 3/4] readyz, ocispec: support sandbox-specific networking A sandbox now lives in its own network namespace, so a readiness probe has to be dialed from there rather than from the worker's. Take the dialer as a parameter, and let a caller name the resolv.conf bound into the sandbox. --- cmd/ateom-gvisor/main.go | 4 ++-- cmd/ateom-microvm/restore.go | 4 +--- cmd/ateom-microvm/run.go | 2 +- internal/atunnel/ingress.go | 4 +--- internal/ocispec/gvisor.go | 10 ++++++++-- internal/ocispec/gvisor_test.go | 31 +++++++++++++++++++++++++++++++ internal/readyz/readyz.go | 30 ++++++++++++++++++++++-------- internal/readyz/readyz_test.go | 12 ++++++------ 8 files changed, 72 insertions(+), 25 deletions(-) diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index 2b965f7217..81400aee61 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -742,7 +742,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload } // Block until every readyz-enabled container reports 200. - if err := readyz.WaitAll(ctx, req.GetSpec().GetContainers(), ateomnet.ActorVethIP); err != nil { + if err := readyz.WaitAll(ctx, req.GetSpec().GetContainers(), ateomnet.ActorVethIP, nil); err != nil { return nil, fmt.Errorf("while waiting for container readyz: %w", err) } if err := s.activateActorNetworking(req.GetAtespace(), req.GetActorName(), egress); err != nil { @@ -1071,7 +1071,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore } // Block until every readyz-enabled container reports 200. - if err := readyz.WaitAll(ctx, req.GetSpec().GetContainers(), ateomnet.ActorVethIP); err != nil { + if err := readyz.WaitAll(ctx, req.GetSpec().GetContainers(), ateomnet.ActorVethIP, nil); err != nil { return nil, fmt.Errorf("while waiting for container readyz: %w", err) } if err := s.activateActorNetworking(req.GetAtespace(), req.GetActorName(), egress); err != nil { diff --git a/cmd/ateom-microvm/restore.go b/cmd/ateom-microvm/restore.go index 186c9ca658..b199537c58 100644 --- a/cmd/ateom-microvm/restore.go +++ b/cmd/ateom-microvm/restore.go @@ -86,8 +86,6 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore return nil, err } - // Same as RunWorkload: a restore is a boot, and graceful shutdown cancels it - // rather than queueing behind it. ctx, cancel := context.WithCancel(ctx) defer cancel() s.setActiveRPC(rpcRestoreWorkload, cancel) @@ -354,7 +352,7 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, tResume := time.Now() // Block until every readyz-enabled container reports 200. - if err := readyz.WaitAll(ctx, containers, ateomnet.ActorVethIP); err != nil { + if err := readyz.WaitAll(ctx, containers, ateomnet.ActorVethIP, nil); err != nil { return fmt.Errorf("while waiting for container readyz: %w", err) } diff --git a/cmd/ateom-microvm/run.go b/cmd/ateom-microvm/run.go index 7422d15f69..dde3f8307b 100644 --- a/cmd/ateom-microvm/run.go +++ b/cmd/ateom-microvm/run.go @@ -594,7 +594,7 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re tContainers := time.Now() // Block until every readyz-enabled container reports 200. - if err := readyz.WaitAll(ctx, containers, ateomnet.ActorVethIP); err != nil { + if err := readyz.WaitAll(ctx, containers, ateomnet.ActorVethIP, nil); err != nil { return fmt.Errorf("while waiting for container readyz: %w", err) } diff --git a/internal/atunnel/ingress.go b/internal/atunnel/ingress.go index a0c167f5e7..3eb5f8d125 100644 --- a/internal/atunnel/ingress.go +++ b/internal/atunnel/ingress.go @@ -71,9 +71,7 @@ type Config struct { Upstream *url.URL } -// Server is an activation-aware HTTPS reverse proxy. It is long-lived across -// actor activations, but only routes requests for the actor currently assigned -// to its worker. +// Server is an HTTPS reverse proxy for the worker's active actors. type Server struct { credentialBundlePath string tlsConfig *tls.Config diff --git a/internal/ocispec/gvisor.go b/internal/ocispec/gvisor.go index 059cfb71b3..ae47f5ad2a 100644 --- a/internal/ocispec/gvisor.go +++ b/internal/ocispec/gvisor.go @@ -26,7 +26,7 @@ import ( // name is drawn from, so no actor container can collide with it. const PauseContainer = "_pause" -// resolvConf is the host resolver config bound into the sandbox. +// resolvConf is the sandbox resolver path and default bind source. const resolvConf = "/etc/resolv.conf" // GVisorOptions describes the gVisor-specific context of one actor container. @@ -35,6 +35,8 @@ type GVisorOptions struct { ContainerName string // DurableVolumes are declared on the sandbox (pause) spec only. DurableVolumes []string + // ResolvConf is the bind source for /etc/resolv.conf; empty uses the pod's file. + ResolvConf string // Size sizes the container's cgroup leaf. Only gVisor applies it; a micro-VM // container's limits come from its own declared resources (see sizing). Size sizing.SandboxSize @@ -65,10 +67,14 @@ func ShapeGVisor(spec *specs.Spec, o GVisorOptions) { if i < 0 { i = len(spec.Mounts) } + source := o.ResolvConf + if source == "" { + source = resolvConf + } spec.Mounts = slices.Insert(spec.Mounts, i, specs.Mount{ Destination: resolvConf, Type: "bind", - Source: resolvConf, + Source: source, Options: []string{"ro"}, }) } diff --git a/internal/ocispec/gvisor_test.go b/internal/ocispec/gvisor_test.go index 6d0ff33566..50f8ca15b9 100644 --- a/internal/ocispec/gvisor_test.go +++ b/internal/ocispec/gvisor_test.go @@ -53,3 +53,34 @@ func TestGVisorCgroupLeafMatchesTheShapedPath(t *testing.T) { t.Errorf("shaped cgroupsPath = %q, want %q", spec.Linux.CgroupsPath, want) } } + +func TestShapeGVisorBindsTheNamedResolvConf(t *testing.T) { + for _, tc := range []struct { + name string + resolvConf string + wantSource string + }{ + {name: "default is the worker pod's", wantSource: "/etc/resolv.conf"}, + { + name: "a sandbox may name its own", + resolvConf: "/var/lib/ateom-gvisor/actors/a/resolv.conf", + wantSource: "/var/lib/ateom-gvisor/actors/a/resolv.conf", + }, + } { + t.Run(tc.name, func(t *testing.T) { + spec := &specs.Spec{} + ShapeGVisor(spec, GVisorOptions{ + ActorUID: "a", ContainerName: "app", ResolvConf: tc.resolvConf, + }) + var got string + for _, m := range spec.Mounts { + if m.Destination == "/etc/resolv.conf" { + got = m.Source + } + } + if got != tc.wantSource { + t.Errorf("resolv.conf bound from %q, want %q", got, tc.wantSource) + } + }) + } +} diff --git a/internal/readyz/readyz.go b/internal/readyz/readyz.go index cbe8c4ea6f..6c84632e9e 100644 --- a/internal/readyz/readyz.go +++ b/internal/readyz/readyz.go @@ -52,20 +52,31 @@ const ( maxIdleConnsHost = 1 ) +// DialFunc reaches the actor, which lives in its own network namespace and is +// not addressable from the caller's. Nil dials from the caller's namespace. +type DialFunc func(ctx context.Context, network, addr string) (net.Conn, error) + // HTTPClient builds a keep-alive HTTP client tuned for fast, repeated // probing of a single endpoint. Exposed as a var so tests can substitute a // transport that targets a test server's loopback address. -var HTTPClient = func() *http.Client { +var HTTPClient = func() *http.Client { return newClient(nil) } + +// newClient probes through dial, or from the caller's namespace when nil. +func newClient(dial DialFunc) *http.Client { + if dial == nil { + dial = (&net.Dialer{Timeout: RequestTimeout}).DialContext + } tr := &http.Transport{ DisableCompression: true, MaxIdleConnsPerHost: maxIdleConnsHost, - DialContext: (&net.Dialer{Timeout: RequestTimeout}).DialContext, + DialContext: dial, ResponseHeaderTimeout: RequestTimeout, } return &http.Client{Transport: tr, Timeout: RequestTimeout} } -// WaitAll blocks until every container with a readyz probe set reports 200, +// WaitAll blocks until every container with a readyz probe set reports 200 +// through dial, // or returns the first error. Containers without a probe are skipped (their // absence means "no readiness gate"). // @@ -73,7 +84,7 @@ var HTTPClient = func() *http.Client { // errors.As cannot cross a process, and the interceptor would flatten it to a // bare codes.Internal, leaving atelet reading UNKNOWN. The ErrorInfo detail is // what carries it. Internal and no crash directive both match today's behavior. -func WaitAll(ctx context.Context, containers []*ateompb.Container, actorIP string) error { +func WaitAll(ctx context.Context, containers []*ateompb.Container, actorIP string, dial DialFunc) error { g, gctx := errgroup.WithContext(ctx) for _, ac := range containers { if ac.GetReadyz() == nil { @@ -81,7 +92,7 @@ func WaitAll(ctx context.Context, containers []*ateompb.Container, actorIP strin } ac := ac g.Go(func() error { - return Wait(gctx, ac.GetName(), ac.GetReadyz(), actorIP) + return Wait(gctx, ac.GetName(), ac.GetReadyz(), actorIP, dial) }) } err := g.Wait() @@ -91,15 +102,18 @@ func WaitAll(ctx context.Context, containers []*ateompb.Container, actorIP strin return err } -// Wait polls the configured HTTP endpoint until it returns 200, the context -// is cancelled, or the overall deadline is exceeded. -func Wait(ctx context.Context, containerName string, probe *ateompb.Readyz, actorIP string) error { +// Wait polls the configured HTTP endpoint through dial until it returns 200, +// the context is cancelled, or the overall deadline is exceeded. +func Wait(ctx context.Context, containerName string, probe *ateompb.Readyz, actorIP string, dial DialFunc) error { url, err := URL(probe, actorIP) if err != nil { return fmt.Errorf("invalid readyz config for %q: %w", containerName, err) } client := HTTPClient() + if dial != nil { + client = newClient(dial) + } defer client.CloseIdleConnections() timeout := overallTimeout(probe) diff --git a/internal/readyz/readyz_test.go b/internal/readyz/readyz_test.go index e1d76655fb..d95106fca2 100644 --- a/internal/readyz/readyz_test.go +++ b/internal/readyz/readyz_test.go @@ -107,7 +107,7 @@ func TestWait_ReturnsOnFirst200(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - if err := Wait(ctx, "main", probe, ip); err != nil { + if err := Wait(ctx, "main", probe, ip, nil); err != nil { t.Fatalf("Wait returned error: %v", err) } } @@ -135,7 +135,7 @@ func TestWait_WaitsForServerToBecomeReady(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() start := time.Now() - if err := Wait(ctx, "main", probe, ip); err != nil { + if err := Wait(ctx, "main", probe, ip, nil); err != nil { t.Fatalf("Wait returned error: %v", err) } elapsed := time.Since(start) @@ -159,7 +159,7 @@ func TestWait_ContextCancellation(t *testing.T) { cancel() }() - err := Wait(ctx, "main", probe, "127.0.0.1") + err := Wait(ctx, "main", probe, "127.0.0.1", nil) if err == nil { t.Fatalf("Wait returned nil, expected cancellation error") } @@ -215,7 +215,7 @@ func TestWait_GivesUpAtProbeTimeout(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() start := time.Now() - err := Wait(ctx, "main", probe, "127.0.0.1") + err := Wait(ctx, "main", probe, "127.0.0.1", nil) if err == nil { t.Fatalf("Wait returned nil, expected a timeout error") } @@ -239,7 +239,7 @@ func TestWaitAll_SkipsContainersWithoutProbe(t *testing.T) { } ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() - if err := WaitAll(ctx, containers, "127.0.0.1"); err != nil { + if err := WaitAll(ctx, containers, "127.0.0.1", nil); err != nil { t.Fatalf("WaitAll with no probes returned error: %v", err) } } @@ -285,7 +285,7 @@ func TestWaitAll_ReasonSurvivesTheRPCBoundary(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - err := WaitAll(ctx, containers, "127.0.0.1") + err := WaitAll(ctx, containers, "127.0.0.1", nil) if err == nil { t.Fatal("WaitAll returned nil, expected a timeout error") } From fc3897ef8bc1de9907b7a166805c414f044894dc Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Tue, 15 Sep 2026 14:57:31 -0700 Subject: [PATCH 4/4] ateom: use per-sandbox network namespaces Wire both runtimes to isolated namespaces, namespace-aware ingress and readiness, and gateway DNS. Preserve fixed addresses across restore. Replace worker-wide routing and microVM TC mirroring with per-sandbox TCP redirects. --- cmd/atelet/main.go | 4 +- cmd/ateom-gvisor/main.go | 99 ++--- cmd/ateom-gvisor/runsc.go | 1 + cmd/ateom-gvisor/sandboxnet.go | 89 +++++ cmd/ateom-microvm/checkpoint.go | 4 +- cmd/ateom-microvm/main.go | 69 ++-- cmd/ateom-microvm/net.go | 106 ++---- cmd/ateom-microvm/net_linux_test.go | 166 +++++++++ cmd/ateom-microvm/restore.go | 19 +- cmd/ateom-microvm/run.go | 75 +--- cmd/ateom-microvm/run_test.go | 65 ---- cmd/ateom-microvm/sandboxnet.go | 76 ++++ internal/ateomnet/dns.go | 80 ++++ internal/ateomnet/net.go | 474 +----------------------- internal/ateomnet/net_linux_test.go | 425 --------------------- internal/ateomnet/sandbox.go | 261 ++++++++----- internal/ateomnet/sandbox_linux_test.go | 168 +++++++-- internal/ateompath/ateompath.go | 30 +- internal/atunnel/egress.go | 15 + internal/atunnel/ingress.go | 84 ++++- internal/atunnel/ingress_test.go | 176 ++++++++- internal/proto/ateletpb/atelet.pb.go | 4 +- internal/proto/ateletpb/atelet.proto | 4 +- internal/proto/ateompb/ateom.pb.go | 4 +- internal/proto/ateompb/ateom.proto | 4 +- 25 files changed, 1116 insertions(+), 1386 deletions(-) create mode 100644 cmd/ateom-gvisor/sandboxnet.go create mode 100644 cmd/ateom-microvm/net_linux_test.go create mode 100644 cmd/ateom-microvm/sandboxnet.go create mode 100644 internal/ateomnet/dns.go diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 6155e49aa0..6ab6138e2a 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -1449,7 +1449,7 @@ func (s *AteomHerder) prepareOCIBundles( []string{"/pause"}, nil, nil, - ateompath.AteomNetNSPath(targetAteomUid), + ateompath.ActorNetNSPath(actorUID), nil, // pause is sandbox infra; it mounts no volumes. nil, nil, // pause only reaps; it needs no capabilities. @@ -1477,7 +1477,7 @@ func (s *AteomHerder) prepareOCIBundles( ctr.GetCommand(), ctr.GetArgs(), envs, - ateompath.AteomNetNSPath(targetAteomUid), + ateompath.ActorNetNSPath(actorUID), spec.GetVolumes(), ctr.GetVolumeMounts(), resolveCapabilities(ctr.GetSecurityContext().GetCapabilities()), diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index 81400aee61..722d11d150 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -54,7 +54,6 @@ import ( "github.com/agent-substrate/substrate/internal/sizing" "github.com/agent-substrate/substrate/internal/version" "github.com/spf13/pflag" - "github.com/vishvananda/netns" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "golang.org/x/sys/unix" "google.golang.org/grpc" @@ -195,26 +194,30 @@ func do(ctx context.Context) error { return fmt.Errorf("while opening unix socket: %w", err) } - // Create a new network namespace that we will pass to gVisor. gVisor will - // read the addresses and routes off of every link in the namespace, then - // remove all the addresses and handle injecting packets into the interfaces - // using AF_PACKET. - interiorNetNS, err := ateomnet.CreateNetNSWithoutSwitching(ateompath.AteomNetNSName(*podUID)) - if err != nil { - return fmt.Errorf("while creating ateom-interior netns: %w", err) - } - actorLogger := actorlog.NewActorLogger(syncedWriter, metadata.OnGCE()) upstream, err := url.Parse(actorHTTPUpstream) if err != nil { return fmt.Errorf("while parsing atunnel upstream: %w", err) } - atunnelIngress, atunnelEgress, atunnelEgressPort, err := runAtunnel(ctx, upstream) + // Use the pod's resolvers for cluster DNS access. + nameservers, err := atunnel.ResolvConfNameservers("/etc/resolv.conf") if err != nil { - return err + return fmt.Errorf("while reading the worker pod resolvers: %w", err) + } + dnsRelay, err := atunnel.NewDNSRelay(nameservers) + if err != nil { + return fmt.Errorf("while building the actor DNS relay: %w", err) } + slog.InfoContext(ctx, "Actor DNS relay ready", slog.Any("upstreams", nameservers)) - ateomService := NewService(interiorNetNS, actorLogger, atunnelIngress, atunnelEgress, atunnelEgressPort, *workerCredentialBundle, *podIdentityTrustBundle, *egressGatewayTrustBundle) + // Construct the service first so atunnel can use its namespace dialer. + ateomService := NewService(dnsRelay, actorLogger, *workerCredentialBundle, *podIdentityTrustBundle, *egressGatewayTrustBundle) + + atunnelIngress, atunnelEgress, atunnelEgressPort, err := runAtunnel(ctx, upstream, ateomService.sandbox.Dialer()) + if err != nil { + return err + } + ateomService.attachAtunnel(atunnelIngress, atunnelEgress, atunnelEgressPort) svr := grpc.NewServer( grpc.StatsHandler(otelgrpc.NewServerHandler()), @@ -266,12 +269,13 @@ func do(ctx context.Context) error { return nil } -func runAtunnel(ctx context.Context, upstream *url.URL) (*atunnel.Server, *atunnel.Egress, uint16, error) { +func runAtunnel(ctx context.Context, upstream *url.URL, dial atunnel.DialFunc) (*atunnel.Server, *atunnel.Egress, uint16, error) { atunnelIngress, err := atunnel.NewServer(atunnel.Config{ CredentialBundlePath: *workerCredentialBundle, TrustBundlePath: *podIdentityTrustBundle, AllowedClientID: *atunnelClientIdentity, Upstream: upstream, + Dial: dial, }) if err != nil { return nil, nil, 0, fmt.Errorf("while configuring atunnel: %w", err) @@ -301,22 +305,11 @@ func runAtunnel(ctx context.Context, upstream *url.URL) (*atunnel.Server, *atunn if err != nil { return nil, nil, 0, fmt.Errorf("while configuring atunnel egress: %w", err) } - egressListener, err := net.Listen("tcp", *atunnelEgressListenAddress) + // Bind egress only in sandbox namespaces. + atunnelEgressPort, err := atunnel.EgressPort(*atunnelEgressListenAddress) if err != nil { - return nil, nil, 0, fmt.Errorf("while opening atunnel egress listener: %w", err) - } - egressTCPAddr, ok := egressListener.Addr().(*net.TCPAddr) - if !ok || egressTCPAddr.Port < 1 || egressTCPAddr.Port > 65535 { - _ = egressListener.Close() - return nil, nil, 0, fmt.Errorf("atunnel egress listener has invalid address %q", egressListener.Addr()) + return nil, nil, 0, err } - atunnelEgressPort := uint16(egressTCPAddr.Port) - go func() { - if err := atunnelEgress.Serve(ctx, egressListener); err != nil { - serverboot.Fatal(ctx, "Failed to serve actor egress", err) - } - }() - slog.InfoContext(ctx, "atunnel egress serving", slog.String("address", *atunnelEgressListenAddress)) return atunnelIngress, atunnelEgress, atunnelEgressPort, nil } @@ -374,11 +367,16 @@ type AteomService struct { // subcommands are probably not safe to call concurrently. lock *cancelableMutex - interiorNetNS netns.NsHandle + // sandbox is the network of the actor this worker is serving. + sandbox ateomnet.SessionHolder + actorLogger *actorlog.ActorLogger atunnelIngress *atunnel.Server atunnelEgress *atunnel.Egress + // dnsRelay answers the sandbox's DNS from inside its own namespace. + dnsRelay *atunnel.DNSRelay + // atunnelEgressPort is the local atunnel listener used as the target of the // actor network's transparent TCP redirect. atunnelEgressPort uint16 @@ -440,14 +438,11 @@ type AteomService struct { var _ ateompb.AteomServer = (*AteomService)(nil) // NewService creates a new AteomService. -func NewService(interiorNetNS netns.NsHandle, actorLogger *actorlog.ActorLogger, atunnelIngress *atunnel.Server, atunnelEgress *atunnel.Egress, atunnelEgressPort uint16, workerCredentialBundlePath, podIdentityTrustBundlePath, egressGatewayTrustBundlePath string) *AteomService { +func NewService(dnsRelay *atunnel.DNSRelay, actorLogger *actorlog.ActorLogger, workerCredentialBundlePath, podIdentityTrustBundlePath, egressGatewayTrustBundlePath string) *AteomService { return &AteomService{ lock: newCancelableMutex(), - interiorNetNS: interiorNetNS, + dnsRelay: dnsRelay, actorLogger: actorLogger, - atunnelIngress: atunnelIngress, - atunnelEgress: atunnelEgress, - atunnelEgressPort: atunnelEgressPort, workerCredentialBundlePath: workerCredentialBundlePath, podIdentityTrustBundlePath: podIdentityTrustBundlePath, egressGatewayTrustBundlePath: egressGatewayTrustBundlePath, @@ -666,15 +661,11 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload if err != nil { return nil, err } - if err := ateomnet.SetupActorNetwork(ctx, ateomnet.NetworkConfig{ - InteriorNetNS: s.interiorNetNS, - DumpNetInfo: true, - EgressRedirectPort: s.egressRedirectPort(req.GetEgressGateway() != nil), - }); err != nil { + if err := s.prepareSandboxNetwork(ctx, req.GetActorUid()); err != nil { // Cleared here as well as in the deferred cleanup below, because that // defer is not registered until after this check. s.activeActor.Store(nil) - return nil, fmt.Errorf("while setting up actor network: %w", err) + return nil, err } rcmd := &runsc{ path: req.GetRunscPath(), @@ -700,7 +691,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload slog.WarnContext(ctx, "Failed to unmount bundle rootfs overlays after Run failure", "actorUID", req.GetActorUid(), "err", err) } - if err := ateomnet.CleanupActorNetwork(cleanupCtx, s.interiorNetNS); err != nil { + if err := s.releaseSandboxNetwork(cleanupCtx); err != nil { slog.WarnContext(cleanupCtx, "Failed to clean up actor network after Run failure", slog.Any("err", err)) } } @@ -742,7 +733,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload } // Block until every readyz-enabled container reports 200. - if err := readyz.WaitAll(ctx, req.GetSpec().GetContainers(), ateomnet.ActorVethIP, nil); err != nil { + if err := readyz.WaitAll(ctx, req.GetSpec().GetContainers(), ateomnet.ActorVethIP, readyz.DialFunc(s.sandbox.Dialer())); err != nil { return nil, fmt.Errorf("while waiting for container readyz: %w", err) } if err := s.activateActorNetworking(req.GetAtespace(), req.GetActorName(), egress); err != nil { @@ -965,14 +956,10 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore if err != nil { return nil, err } - if err := ateomnet.SetupActorNetwork(ctx, ateomnet.NetworkConfig{ - InteriorNetNS: s.interiorNetNS, - DumpNetInfo: true, - EgressRedirectPort: s.egressRedirectPort(req.GetEgressGateway() != nil), - }); err != nil { + if err := s.prepareSandboxNetwork(ctx, req.GetActorUid()); err != nil { // Same as the Run path: the defer below is not registered yet. s.activeActor.Store(nil) - return nil, fmt.Errorf("while setting up actor network: %w", err) + return nil, err } rcmd := &runsc{ path: req.GetRunscPath(), @@ -995,7 +982,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore slog.WarnContext(ctx, "Failed to unmount bundle rootfs overlays after Restore failure", "actorUID", req.GetActorUid(), "err", err) } - if err := ateomnet.CleanupActorNetwork(cleanupCtx, s.interiorNetNS); err != nil { + if err := s.releaseSandboxNetwork(cleanupCtx); err != nil { slog.WarnContext(cleanupCtx, "Failed to clean up actor network after Restore failure", slog.Any("err", err)) } } @@ -1071,7 +1058,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore } // Block until every readyz-enabled container reports 200. - if err := readyz.WaitAll(ctx, req.GetSpec().GetContainers(), ateomnet.ActorVethIP, nil); err != nil { + if err := readyz.WaitAll(ctx, req.GetSpec().GetContainers(), ateomnet.ActorVethIP, readyz.DialFunc(s.sandbox.Dialer())); err != nil { return nil, fmt.Errorf("while waiting for container readyz: %w", err) } if err := s.activateActorNetworking(req.GetAtespace(), req.GetActorName(), egress); err != nil { @@ -1182,7 +1169,7 @@ func (s *AteomService) terminateWorkload(ctx context.Context, actorRef resources errs = append(errs, fmt.Errorf("while unmounting bundle rootfs overlays: %w", err)) } - if err := ateomnet.CleanupActorNetwork(ctx, s.interiorNetNS); err != nil { + if err := s.releaseSandboxNetwork(ctx); err != nil { errs = append(errs, fmt.Errorf("while cleaning up actor network: %w", err)) } @@ -1221,16 +1208,6 @@ func (s *AteomService) deactivateActorNetworking(ctx context.Context) error { return nil } -// egressRedirectPort returns the local atunnel egress listener port when the -// activation arms tunneled egress, and zero otherwise, which leaves the -// prerouting redirect uninstalled and actor egress on the masquerade path. -func (s *AteomService) egressRedirectPort(redirectEgress bool) uint16 { - if !redirectEgress { - return 0 - } - return s.atunnelEgressPort -} - // setupCgroupDelegation prepares the worker pod's cgroup so runsc can create a // per-actor-container leaf under it with real cpu/memory/pids accounting. // diff --git a/cmd/ateom-gvisor/runsc.go b/cmd/ateom-gvisor/runsc.go index 4720b9c4e0..bbf9e13ed5 100644 --- a/cmd/ateom-gvisor/runsc.go +++ b/cmd/ateom-gvisor/runsc.go @@ -69,6 +69,7 @@ func (r *runsc) shapeSpec(containerName string) error { ContainerName: containerName, DurableVolumes: r.durableVolumes, Size: r.size, + ResolvConf: ateompath.ActorResolvConfPath(r.actorUID), }) return ocispec.Save(bundle, spec) } diff --git a/cmd/ateom-gvisor/sandboxnet.go b/cmd/ateom-gvisor/sandboxnet.go new file mode 100644 index 0000000000..00b4a8ee36 --- /dev/null +++ b/cmd/ateom-gvisor/sandboxnet.go @@ -0,0 +1,89 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + "fmt" + "io/fs" + "log/slog" + "os" + "path/filepath" + + "github.com/agent-substrate/substrate/internal/ateomnet" + "github.com/agent-substrate/substrate/internal/ateompath" + "github.com/agent-substrate/substrate/internal/atunnel" +) + +// prepareSandboxNetwork builds the actor's network and starts serving it. +func (s *AteomService) prepareSandboxNetwork(ctx context.Context, actorUID string) error { + if err := s.releaseSandboxNetwork(ctx); err != nil { + return err + } + session, err := ateomnet.ServeSandbox(ctx, ateomnet.SandboxNetworkConfig{ + ActorUID: actorUID, + Veth: true, + EgressPort: s.atunnelEgressPort, + DNSPort: atunnel.DNSPort, + }, s.atunnelEgress, s.dnsRelay) + if err != nil { + return fmt.Errorf("while setting up the sandbox network: %w", err) + } + + // Point the sandbox resolver at its gateway. + if _, err := actorResolvConf(actorUID); err != nil { + _ = session.Close(ctx) + return err + } + + return s.sandbox.Replace(ctx, session) +} + +// releaseSandboxNetwork stops serving the actor and takes its network down, +// along with the resolv.conf atelet's per-activation reset leaves behind. +func (s *AteomService) releaseSandboxNetwork(ctx context.Context) error { + if session := s.sandbox.Session(); session != nil { + if err := os.Remove(ateompath.ActorResolvConfPath(session.Network.ActorUID)); err != nil && !errors.Is(err, fs.ErrNotExist) { + slog.WarnContext(ctx, "Failed to remove the actor resolv.conf", slog.Any("err", err)) + } + } + return s.sandbox.Close(ctx) +} + +// actorResolvConf writes the resolver bind source outside the actor's rootfs. +func actorResolvConf(actorUID string) (string, error) { + pod, err := os.ReadFile("/etc/resolv.conf") + if err != nil { + return "", fmt.Errorf("reading the worker pod resolv.conf: %w", err) + } + path := ateompath.ActorResolvConfPath(actorUID) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return "", fmt.Errorf("creating the actor directory: %w", err) + } + if err := os.WriteFile(path, ateomnet.SandboxResolvConf(pod), 0o644); err != nil { + return "", fmt.Errorf("writing the actor resolv.conf: %w", err) + } + return path, nil +} + +// attachAtunnel completes setup after atunnel receives the service's dialer. +func (s *AteomService) attachAtunnel(ingress *atunnel.Server, egress *atunnel.Egress, egressPort uint16) { + s.atunnelIngress = ingress + s.atunnelEgress = egress + s.atunnelEgressPort = egressPort +} diff --git a/cmd/ateom-microvm/checkpoint.go b/cmd/ateom-microvm/checkpoint.go index 8a4c8c75a7..fa8c4c8e80 100644 --- a/cmd/ateom-microvm/checkpoint.go +++ b/cmd/ateom-microvm/checkpoint.go @@ -25,8 +25,6 @@ import ( "path/filepath" "time" - "github.com/agent-substrate/substrate/internal/ateomnet" - "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/ch" "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata" "github.com/agent-substrate/substrate/internal/ateompath" @@ -398,7 +396,7 @@ func (s *AteomService) terminateWorkload(ctx context.Context, actorUID string) e // the two views of "is an actor here" from disagreeing. s.activeActor.Store(nil) - if err := ateomnet.CleanupActorNetwork(ctx, s.interiorNetNS); err != nil { + if err := s.releaseSandboxNetwork(ctx); err != nil { errs = append(errs, fmt.Errorf("while cleaning up actor network: %w", err)) } return errors.Join(errs...) diff --git a/cmd/ateom-microvm/main.go b/cmd/ateom-microvm/main.go index ba76a18924..dff05e2174 100644 --- a/cmd/ateom-microvm/main.go +++ b/cmd/ateom-microvm/main.go @@ -51,7 +51,6 @@ import ( "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/internal/serverboot" "github.com/agent-substrate/substrate/internal/version" - "github.com/vishvananda/netns" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "golang.org/x/sys/unix" "google.golang.org/grpc" @@ -191,13 +190,6 @@ func do(ctx context.Context) error { return fmt.Errorf("while opening unix socket: %w", err) } - // Networking: create a named interior netns; each activation builds a fresh - // veth pair into it (see net.go) and points kata at it. - interiorNetNS, err := ateomnet.CreateNetNSWithoutSwitching(ateompath.AteomNetNSName(*podUID)) - if err != nil { - return fmt.Errorf("while creating interior netns: %w", err) - } - // Forward the actor container's stdout/stderr to the worker pod's stdout as // JSON with ate.dev/* labels (logging parity with ateom-gvisor). It shares // logWriter with the runtime logger so the two streams to os.Stdout are @@ -207,11 +199,28 @@ func do(ctx context.Context) error { if err != nil { return fmt.Errorf("while parsing atunnel upstream: %w", err) } + // The pod's own resolvers, so an actor resolves exactly what the worker + // resolves -- cluster DNS included. + nameservers, err := atunnel.ResolvConfNameservers("/etc/resolv.conf") + if err != nil { + return fmt.Errorf("while reading the worker pod resolvers: %w", err) + } + dnsRelay, err := atunnel.NewDNSRelay(nameservers) + if err != nil { + return fmt.Errorf("while building the actor DNS relay: %w", err) + } + slog.InfoContext(ctx, "Actor DNS relay ready", slog.Any("upstreams", nameservers)) + + // The service owns the actor's namespace, and atunnel reaches the actor + // through it, so it is built first and handed to atunnel as a dialer. + ateomService := NewService(*podUID, *chBinary, *kataConfig, *kataDebug, *vmmMemReserve, dnsRelay, actorLogger, *workerCredentialBundle, *podIdentityTrustBundle, *egressGatewayTrustBundle) + atunnelIngress, err := atunnel.NewServer(atunnel.Config{ CredentialBundlePath: *workerCredentialBundle, TrustBundlePath: *podIdentityTrustBundle, AllowedClientID: *atunnelClientIdentity, Upstream: upstream, + Dial: ateomService.sandbox.Dialer(), }) if err != nil { return fmt.Errorf("while configuring atunnel: %w", err) @@ -240,24 +249,13 @@ func do(ctx context.Context) error { if err != nil { return fmt.Errorf("while configuring atunnel egress: %w", err) } - egressListener, err := net.Listen("tcp", *atunnelEgressListenAddress) + // Bind egress only in sandbox namespaces. + atunnelEgressPort, err := atunnel.EgressPort(*atunnelEgressListenAddress) if err != nil { - return fmt.Errorf("while opening atunnel egress listener: %w", err) - } - egressTCPAddr, ok := egressListener.Addr().(*net.TCPAddr) - if !ok || egressTCPAddr.Port < 1 || egressTCPAddr.Port > 65535 { - _ = egressListener.Close() - return fmt.Errorf("atunnel egress listener has invalid address %q", egressListener.Addr()) + return err } - atunnelEgressPort := uint16(egressTCPAddr.Port) - go func() { - if err := atunnelEgress.Serve(ctx, egressListener); err != nil { - serverboot.Fatal(ctx, "Failed to serve actor egress", err) - } - }() - slog.InfoContext(ctx, "atunnel egress serving", slog.String("address", *atunnelEgressListenAddress)) - ateomService := NewService(*podUID, *chBinary, *kataConfig, *kataDebug, *vmmMemReserve, interiorNetNS, actorLogger, atunnelIngress, atunnelEgress, atunnelEgressPort, *workerCredentialBundle, *podIdentityTrustBundle, *egressGatewayTrustBundle) + ateomService.attachAtunnel(atunnelIngress, atunnelEgress, atunnelEgressPort) svr := grpc.NewServer( grpc.StatsHandler(otelgrpc.NewServerHandler()), @@ -419,9 +417,11 @@ type AteomService struct { // with the guest RAM). Set from --vmm-mem-reserve-mib. memReserveMiB int - // interiorNetNS hosts the per-activation actor veth peer (see net.go); - // kata is pointed at it. - interiorNetNS netns.NsHandle + // sandbox is the network of the actor this worker is serving. + sandbox ateomnet.SessionHolder + + // dnsRelay answers the actor's DNS from inside its own namespace. + dnsRelay *atunnel.DNSRelay // actorLogger forwards the actor container's stdout/stderr to the worker pod's // stdout as ate.dev/*-labeled JSON and emits actor lifecycle events (parity @@ -493,7 +493,7 @@ type AteomService struct { var _ ateompb.AteomServer = (*AteomService)(nil) // NewService creates a new AteomService. -func NewService(podUID, chBinary, kataConfig string, kataDebug bool, memReserveMiB int, interiorNetNS netns.NsHandle, actorLogger *actorlog.ActorLogger, atunnelIngress *atunnel.Server, atunnelEgress *atunnel.Egress, atunnelEgressPort uint16, workerCredentialBundlePath, podIdentityTrustBundlePath, egressGatewayTrustBundlePath string) *AteomService { +func NewService(podUID, chBinary, kataConfig string, kataDebug bool, memReserveMiB int, dnsRelay *atunnel.DNSRelay, actorLogger *actorlog.ActorLogger, workerCredentialBundlePath, podIdentityTrustBundlePath, egressGatewayTrustBundlePath string) *AteomService { return &AteomService{ lock: newCancelableMutex(), podUID: podUID, @@ -501,11 +501,8 @@ func NewService(podUID, chBinary, kataConfig string, kataDebug bool, memReserveM kataConfig: kataConfig, kataDebug: kataDebug, memReserveMiB: memReserveMiB, - interiorNetNS: interiorNetNS, + dnsRelay: dnsRelay, actorLogger: actorLogger, - atunnelIngress: atunnelIngress, - atunnelEgress: atunnelEgress, - atunnelEgressPort: atunnelEgressPort, workerCredentialBundlePath: workerCredentialBundlePath, podIdentityTrustBundlePath: podIdentityTrustBundlePath, egressGatewayTrustBundlePath: egressGatewayTrustBundlePath, @@ -585,16 +582,6 @@ func (s *AteomService) deactivateActorNetworking(ctx context.Context) error { return nil } -// egressRedirectPort returns the local atunnel egress listener port when the -// activation arms tunneled egress, and zero otherwise, which leaves the -// prerouting redirect uninstalled and actor egress on the masquerade path. -func (s *AteomService) egressRedirectPort(redirectEgress bool) uint16 { - if !redirectEgress { - return 0 - } - return s.atunnelEgressPort -} - // rejectIfDraining returns a codes.Unavailable error if ateom has begun graceful // shutdown, so the control plane reschedules the actor onto a live worker. func (s *AteomService) rejectIfDraining() error { diff --git a/cmd/ateom-microvm/net.go b/cmd/ateom-microvm/net.go index 8d627123c8..3e795c92eb 100644 --- a/cmd/ateom-microvm/net.go +++ b/cmd/ateom-microvm/net.go @@ -23,47 +23,37 @@ import ( "os" "github.com/vishvananda/netlink" - "golang.org/x/sys/unix" + "github.com/vishvananda/netns" "github.com/agent-substrate/substrate/internal/ateomnet" ) const ( - // hostVethMAC is deliberately FIXED (locally administered), unlike - // ateom-gvisor where the kernel's random veth MAC is fine. A CH snapshot - // freezes the guest kernel's ARP cache, including the entry for the - // gateway 169.254.17.1; restoring against a new veth pair with a random - // MAC would blackhole guest egress until that entry expires. A constant - // gateway MAC keeps the frozen entry valid on every pod. - hostVethMAC = "02:a8:1e:00:00:01" + // gatewayMAC is deliberately FIXED (locally administered), unlike + // ateom-gvisor where a random veth MAC is fine. A CH snapshot freezes the + // guest kernel's ARP cache, including the entry for the gateway + // 169.254.17.1; restoring against a tap with a fresh random MAC would + // blackhole guest egress until that entry expired. + gatewayMAC = "02:a8:1e:00:00:01" // actorGuestMAC is the FIXED MAC for the guest's eth0 (the CH virtio-net). - // Fixed for the same reason as hostVethMAC: a cold boot freezes this MAC into - // the guest+snapshot, and restore re-adds the - // virtio-net under the same MAC (SnapshotNetDevices reads it back), so the - // guest's frozen interface config stays valid across pods. Distinct from the - // gateway MAC (…:01). + // Fixed for the same reason as gatewayMAC: a cold boot freezes this MAC into + // the guest+snapshot, and restore re-adds the virtio-net under the same MAC + // (SnapshotNetDevices reads it back), so the guest's frozen interface config + // stays valid across pods. Distinct from the gateway MAC (…:01). actorGuestMAC = "02:a8:1e:00:00:02" -) -var ( - hostVethHWAddr = ateomnet.MustParseMAC(hostVethMAC) + // actorTapMTU is the kernel default the guest was snapshotted against. + actorTapMTU = 1500 ) -// setupRestoreTap recreates, in the interior netns, the tap + TC-mirror wiring -// kata's tcfilter network model builds at boot: a tap device cross-connected to -// eth0 (the actor veth peer) with mirred-redirect ingress filters in both -// directions. Returns the open tap FDs (one per queue pair) for -// cloud-hypervisor to adopt via vm.restore net_fds (the snapshot's virtio-net -// device is fd-backed, so CH requires fresh FDs on restore). Call after -// setupActorNetwork. -func (s *AteomService) setupRestoreTap(ctx context.Context, name string, queuePairs int) ([]*os.File, error) { +var gatewayHWAddr = ateomnet.MustParseMAC(gatewayMAC) + +// setupActorTap creates the guest's tap with a fixed gateway address and MAC. +// Returns the FDs cloud-hypervisor adopts on boot or restore. +func setupActorTap(ctx context.Context, actorNetNS netns.NsHandle, name string, queuePairs int) ([]*os.File, error) { var fds []*os.File - err := ateomnet.NetNSDo(ctx, s.interiorNetNS, func(ctx context.Context) error { - eth0, err := netlink.LinkByName(ateomnet.ActorVethName) - if err != nil { - return fmt.Errorf("acquiring actor veth in interior netns: %w", err) - } + err := ateomnet.NetNSDo(ctx, actorNetNS, func(ctx context.Context) error { if old, lerr := netlink.LinkByName(name); lerr == nil { _ = netlink.LinkDel(old) } @@ -72,7 +62,7 @@ func (s *AteomService) setupRestoreTap(ctx context.Context, name string, queuePa flags |= netlink.TUNTAP_MULTI_QUEUE } tap := &netlink.Tuntap{ - LinkAttrs: netlink.LinkAttrs{Name: name, MTU: eth0.Attrs().MTU}, + LinkAttrs: netlink.LinkAttrs{Name: name, MTU: actorTapMTU}, Mode: netlink.TUNTAP_MODE_TAP, Flags: flags, Queues: queuePairs, @@ -81,34 +71,20 @@ func (s *AteomService) setupRestoreTap(ctx context.Context, name string, queuePa return fmt.Errorf("creating tap %q: %w", name, err) } fds = tap.Fds - if err := netlink.LinkSetUp(tap); err != nil { - return fmt.Errorf("bringing up tap %q: %w", name, err) + // Set the MAC after LinkAdd; tuntap creation ignores LinkAttrs.HardwareAddr. + // It must match the guest's snapshotted ARP entry. + link, err := netlink.LinkByName(name) + if err != nil { + return err + } + if err := netlink.LinkSetHardwareAddr(link, gatewayHWAddr); err != nil { + return fmt.Errorf("setting the gateway MAC on tap %q: %w", name, err) } - // Cross-connect: everything arriving on the veth peer redirects out the - // tap and vice versa (kata's TCFilterModel: ingress qdisc + match-all u32 - // with a mirred egress-redirect action, here via U32.RedirIndex). - for _, pair := range [][2]netlink.Link{{eth0, tap}, {tap, eth0}} { - qdisc := &netlink.Ingress{QdiscAttrs: netlink.QdiscAttrs{ - LinkIndex: pair[0].Attrs().Index, - Parent: netlink.HANDLE_INGRESS, - Handle: netlink.MakeHandle(0xffff, 0), - }} - if err := netlink.QdiscReplace(qdisc); err != nil { - return fmt.Errorf("adding ingress qdisc to %q: %w", pair[0].Attrs().Name, err) - } - filter := &netlink.U32{ - FilterAttrs: netlink.FilterAttrs{ - LinkIndex: pair[0].Attrs().Index, - Parent: netlink.MakeHandle(0xffff, 0), - Priority: 1, - Protocol: unix.ETH_P_ALL, - }, - ClassId: netlink.MakeHandle(1, 1), - RedirIndex: pair[1].Attrs().Index, - } - if err := netlink.FilterAdd(filter); err != nil { - return fmt.Errorf("adding mirred filter %s -> %s: %w", pair[0].Attrs().Name, pair[1].Attrs().Name, err) - } + if err := netlink.AddrReplace(link, ateomnet.HostVethAddr); err != nil { + return fmt.Errorf("assigning the gateway address to tap %q: %w", name, err) + } + if err := netlink.LinkSetUp(link); err != nil { + return fmt.Errorf("bringing up tap %q: %w", name, err) } return nil }) @@ -121,17 +97,15 @@ func (s *AteomService) setupRestoreTap(ctx context.Context, name string, queuePa return fds, nil } -// actorVethMTU reads the MTU of the actor veth (eth0 in the interior netns) so -// ateom can configure the guest eth0 with a matching MTU via the agent -// (UpdateInterface). Defaults to 1500 if the link can't be read. -func (s *AteomService) actorVethMTU(ctx context.Context) int { - mtu := 1500 - _ = ateomnet.NetNSDo(ctx, s.interiorNetNS, func(ctx context.Context) error { - if l, err := netlink.LinkByName(ateomnet.ActorVethName); err == nil { +// actorTapMTUOf reads the tap MTU, falling back to actorTapMTU on error. +func actorTapMTUOf(ctx context.Context, actorNetNS netns.NsHandle, name string) int { + mtu := actorTapMTU + _ = ateomnet.NetNSDo(ctx, actorNetNS, func(ctx context.Context) error { + if l, err := netlink.LinkByName(name); err == nil { mtu = l.Attrs().MTU } else { - slog.WarnContext(ctx, "Failed to read actor veth MTU; using default", - slog.String("link", ateomnet.ActorVethName), slog.Int("default_mtu", mtu), slog.Any("err", err)) + slog.WarnContext(ctx, "Failed to read actor tap MTU; using default", + slog.String("link", name), slog.Int("default_mtu", mtu), slog.Any("err", err)) } return nil }) diff --git a/cmd/ateom-microvm/net_linux_test.go b/cmd/ateom-microvm/net_linux_test.go new file mode 100644 index 0000000000..b54184781f --- /dev/null +++ b/cmd/ateom-microvm/net_linux_test.go @@ -0,0 +1,166 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "testing" + + "github.com/vishvananda/netlink" + "github.com/vishvananda/netns" + + "github.com/agent-substrate/substrate/internal/ateomnet" + "github.com/agent-substrate/substrate/internal/ateompath" + "github.com/agent-substrate/substrate/internal/atunnel" + "github.com/agent-substrate/substrate/internal/roottest" +) + +func TestPrepareSandboxNetworkReplacesSameActor(t *testing.T) { + roottest.Require(t, "creates network namespaces") + ctx := context.Background() + egress, err := atunnel.NewEgress(atunnel.TCPOriginalDestination) + if err != nil { + t.Fatal(err) + } + dns, err := atunnel.NewDNSRelay([]string{"127.0.0.1:53"}) + if err != nil { + t.Fatal(err) + } + service := &AteomService{atunnelEgress: egress, atunnelEgressPort: 15001, dnsRelay: dns} + t.Cleanup(func() { + if err := service.releaseSandboxNetwork(ctx); err != nil { + t.Error(err) + } + }) + const actorUID = "microvm-network-replace" + for range 2 { + if err := service.prepareSandboxNetwork(ctx, actorUID); err != nil { + t.Fatal(err) + } + } + named, err := netns.GetFromName(ateompath.ActorNetNSName(actorUID)) + if err != nil { + t.Fatalf("opening replacement namespace by name: %v", err) + } + defer named.Close() + if !named.Equal(service.sandboxNetNS()) { + t.Fatal("namespace name does not refer to the replacement") + } +} + +// tapNetNS gives a test its own namespace to build a tap in. +func tapNetNS(t *testing.T, name string) netns.NsHandle { + t.Helper() + ns, err := ateomnet.CreateNetNSWithoutSwitching(name) + if err != nil { + t.Fatalf("creating namespace: %v", err) + } + t.Cleanup(func() { + ns.Close() + _ = netns.DeleteNamed(name) + }) + return ns +} + +// The tap is the micro-VM's whole boundary: cloud-hypervisor adopts the +// descriptors, and the kernel keeps the interface side, carrying the gateway +// address and the MAC the guest's snapshot froze into its ARP cache. +func TestSetupActorTap(t *testing.T) { + roottest.Require(t, "creates network namespaces and tap devices") + ctx := context.Background() + ns := tapNetNS(t, "microvm-tap-test") + + fds, err := setupActorTap(ctx, ns, "tap0_kata", 1) + if err != nil { + t.Fatalf("setupActorTap: %v", err) + } + t.Cleanup(func() { + for _, f := range fds { + _ = f.Close() + } + }) + if len(fds) != 1 { + t.Errorf("got %d descriptors, want one per queue pair", len(fds)) + } + + if err := ateomnet.NetNSDo(ctx, ns, func(context.Context) error { + link, err := netlink.LinkByName("tap0_kata") + if err != nil { + return err + } + if got := link.Attrs().HardwareAddr.String(); got != gatewayMAC { + t.Errorf("tap MAC = %s, want the fixed %s the guest's frozen ARP entry names", got, gatewayMAC) + } + if link.Attrs().Flags&1 == 0 { // net.FlagUp + t.Error("tap is down") + } + if got := link.Attrs().MTU; got != actorTapMTU { + t.Errorf("tap MTU = %d, want %d, the value the guest was snapshotted against", got, actorTapMTU) + } + addrs, err := netlink.AddrList(link, netlink.FAMILY_V4) + if err != nil { + return err + } + var found bool + for _, a := range addrs { + if a.IPNet.String() == ateomnet.HostVethAddr.IPNet.String() { + found = true + } + } + if !found { + t.Errorf("tap addresses = %v, want the gateway %s the guest routes to", addrs, ateomnet.HostVethAddr) + } + return nil + }); err != nil { + t.Fatalf("inspecting the tap: %v", err) + } + + if got := actorTapMTUOf(ctx, ns, "tap0_kata"); got != actorTapMTU { + t.Errorf("actorTapMTUOf = %d, want %d", got, actorTapMTU) + } +} + +// A restore rebuilds the tap in a namespace that may still hold the previous +// one, and cloud-hypervisor needs fresh descriptors for its fd-backed +// virtio-net either way. +func TestSetupActorTapReplacesALeftover(t *testing.T) { + roottest.Require(t, "creates network namespaces and tap devices") + ctx := context.Background() + ns := tapNetNS(t, "microvm-tap-replace") + + first, err := setupActorTap(ctx, ns, "tap0_kata", 1) + if err != nil { + t.Fatalf("first setupActorTap: %v", err) + } + for _, f := range first { + defer f.Close() + } + + second, err := setupActorTap(ctx, ns, "tap0_kata", 2) + if err != nil { + t.Fatalf("second setupActorTap over a leftover: %v", err) + } + for _, f := range second { + defer f.Close() + } + if len(second) != 2 { + t.Errorf("got %d descriptors, want one per queue pair", len(second)) + } + if first[0].Fd() == second[0].Fd() { + t.Error("the replacement reused the first tap's descriptor") + } +} diff --git a/cmd/ateom-microvm/restore.go b/cmd/ateom-microvm/restore.go index b199537c58..bbe36899ae 100644 --- a/cmd/ateom-microvm/restore.go +++ b/cmd/ateom-microvm/restore.go @@ -262,15 +262,10 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, tLowers := time.Now() tDurable := tLowers - // Networking: rebuild the per-activation veth + tap; the snapshot's virtio-net - // is fd-backed, so CH needs fresh tap FDs (net_fds) on restore. - if err := ateomnet.SetupActorNetwork(ctx, ateomnet.NetworkConfig{ - InteriorNetNS: s.interiorNetNS, - HostVethHWAddr: hostVethHWAddr, - SweepInteriorLinks: true, - EgressRedirectPort: s.egressRedirectPort(p.egressGateway != nil), - }); err != nil { - return fmt.Errorf("while setting up actor network: %w", err) + // Networking: rebuild the actor's namespace; the snapshot's virtio-net is + // fd-backed, so CH needs fresh tap FDs (net_fds) on restore. + if err := s.prepareSandboxNetwork(ctx, actorUID); err != nil { + return err } defer func() { if retErr != nil { @@ -279,7 +274,7 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, if cleanupErr := s.deactivateActorNetworking(cleanupCtx); cleanupErr != nil { slog.WarnContext(cleanupCtx, "Failed to deactivate actor networking after Restore failure", slog.Any("err", cleanupErr)) } - if cleanupErr := ateomnet.CleanupActorNetwork(cleanupCtx, s.interiorNetNS); cleanupErr != nil { + if cleanupErr := s.releaseSandboxNetwork(cleanupCtx); cleanupErr != nil { slog.WarnContext(cleanupCtx, "Failed to clean up actor network after Restore failure", slog.Any("err", cleanupErr)) } // Detach any bundle rootfs overlays mounted by buildActorContainers @@ -301,7 +296,7 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, } }() for i, nd := range netDevs { - files, terr := s.setupRestoreTap(ctx, fmt.Sprintf("tap%d_kata", i), nd.QueuePairs) + files, terr := setupActorTap(ctx, s.sandboxNetNS(), fmt.Sprintf("tap%d_kata", i), nd.QueuePairs) if terr != nil { return fmt.Errorf("while building restore tap for %s: %w", nd.ID, terr) } @@ -352,7 +347,7 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, tResume := time.Now() // Block until every readyz-enabled container reports 200. - if err := readyz.WaitAll(ctx, containers, ateomnet.ActorVethIP, nil); err != nil { + if err := readyz.WaitAll(ctx, containers, ateomnet.ActorVethIP, readyz.DialFunc(s.sandbox.Dialer())); err != nil { return fmt.Errorf("while waiting for container readyz: %w", err) } diff --git a/cmd/ateom-microvm/run.go b/cmd/ateom-microvm/run.go index dde3f8307b..004c65d5bb 100644 --- a/cmd/ateom-microvm/run.go +++ b/cmd/ateom-microvm/run.go @@ -213,47 +213,6 @@ func (s *AteomService) resolveRuntime(paths map[string]string) resolvedRuntime { } } -// writeGuestResolvConf copies the worker pod's /etc/resolv.conf into a container's -// bundle rootfs (the overlay RO lower) so the guest gets cluster DNS: ateom drops -// atelet's resolv.conf bind and sends no CreateSandbox.Dns, so the guest can -// otherwise reach IPs but not resolve names. -// -// The rootfs is untrusted, so the write goes through os.Root and unlinks rather -// than truncates: an image-planted /etc or /etc/resolv.conf symlink would -// otherwise be followed and clobber that path on the worker pod as root. -func writeGuestResolvConf(rootfs string) error { - content, err := os.ReadFile("/etc/resolv.conf") - if err != nil { - return fmt.Errorf("reading host resolv.conf: %w", err) - } - if len(content) == 0 { - return fmt.Errorf("host /etc/resolv.conf is empty") - } - root, err := os.OpenRoot(rootfs) - if err != nil { - return fmt.Errorf("opening rootfs %q: %w", rootfs, err) - } - defer root.Close() - if err := root.Mkdir("etc", 0o755); err != nil && !errors.Is(err, fs.ErrExist) { - return fmt.Errorf("creating %q: %w", filepath.Join(rootfs, "etc"), err) - } - if err := root.Remove("etc/resolv.conf"); err != nil && !errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("removing existing guest resolv.conf: %w", err) - } - f, err := root.OpenFile("etc/resolv.conf", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) - if err != nil { - return fmt.Errorf("creating guest resolv.conf: %w", err) - } - _, err = f.Write(content) - if closeErr := f.Close(); err == nil { - err = closeErr - } - if err != nil { - return fmt.Errorf("writing guest resolv.conf: %w", err) - } - return nil -} - // RunWorkload boots the actor as a cloud-hypervisor micro-VM and starts its containers. // // ateom boots cloud-hypervisor directly (no kata shim) and gives each container a @@ -409,15 +368,10 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re return err } - // Networking (host side): per-activation veth into the interior netns. The - // tap + TC mirror is built below (after the VM exists) so its FDs are fresh. - if err := ateomnet.SetupActorNetwork(ctx, ateomnet.NetworkConfig{ - InteriorNetNS: s.interiorNetNS, - HostVethHWAddr: hostVethHWAddr, - SweepInteriorLinks: true, - EgressRedirectPort: s.egressRedirectPort(p.egressGateway != nil), - }); err != nil { - return fmt.Errorf("while setting up actor network: %w", err) + // Networking (host side): the actor's own namespace. The tap is built below + // (after the VM exists) so its FDs are fresh. + if err := s.prepareSandboxNetwork(ctx, actorUID); err != nil { + return err } defer func() { if retErr != nil { @@ -426,7 +380,7 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re if cleanupErr := s.deactivateActorNetworking(cleanupCtx); cleanupErr != nil { slog.WarnContext(cleanupCtx, "Failed to deactivate actor networking after Run failure", slog.Any("err", cleanupErr)) } - if cleanupErr := ateomnet.CleanupActorNetwork(cleanupCtx, s.interiorNetNS); cleanupErr != nil { + if cleanupErr := s.releaseSandboxNetwork(cleanupCtx); cleanupErr != nil { slog.WarnContext(cleanupCtx, "Failed to clean up actor network after Run failure", slog.Any("err", cleanupErr)) } // Detach any bundle rootfs overlays mounted by buildActorContainers @@ -536,9 +490,9 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re return fmt.Errorf("while creating VM: %w", err) } - // Network device: build the tap + TC mirror against the actor veth and add a - // virtio-net to the created (pre-boot) VM with the tap FDs (SCM_RIGHTS). - tapFiles, err := s.setupRestoreTap(ctx, "tap0_kata", 1) + // Network device: build the actor's tap and add a virtio-net to the created + // (pre-boot) VM with its FDs (SCM_RIGHTS). + tapFiles, err := setupActorTap(ctx, s.sandboxNetNS(), "tap0_kata", 1) if err != nil { return fmt.Errorf("while building tap: %w", err) } @@ -594,7 +548,7 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re tContainers := time.Now() // Block until every readyz-enabled container reports 200. - if err := readyz.WaitAll(ctx, containers, ateomnet.ActorVethIP, nil); err != nil { + if err := readyz.WaitAll(ctx, containers, ateomnet.ActorVethIP, readyz.DialFunc(s.sandbox.Dialer())); err != nil { return fmt.Errorf("while waiting for container readyz: %w", err) } @@ -661,11 +615,8 @@ func (s *AteomService) buildActorContainers(actorUID string, containers []*ateom return nil, fmt.Errorf("while composing rootfs for %q: %w", cn, err) } bundleRootfs := filepath.Join(bundle, "rootfs") - // Write cluster DNS into the lower before it's served over virtio-fs: ateom - // drops atelet's resolv.conf bind and sends no CreateSandbox.Dns, so without - // this the guest can reach IPs but not resolve names. Doing it here covers both - // run and restore (both reconstruct the lower from the bundle). - if err := writeGuestResolvConf(bundleRootfs); err != nil { + // Set guest DNS before serving the rootfs over virtio-fs, on boot and restore. + if err := writeActorResolvConf(bundleRootfs); err != nil { return nil, fmt.Errorf("while writing guest resolv.conf for %q: %w", cn, err) } ctrs[i] = actorContainer{ @@ -897,7 +848,7 @@ func (s *AteomService) startActorContainers(ctx context.Context, ac *kata.AgentC tSandbox := time.Now() // Configure guest networking (the shim's job): eth0 IP/MAC/MTU, routes, ARP. - mtu := uint64(s.actorVethMTU(ctx)) + mtu := uint64(actorTapMTUOf(ctx, s.sandboxNetNS(), "tap0_kata")) netCtx, netCancel := context.WithTimeout(ctx, 20*time.Second) err = s.configureGuestNetwork(netCtx, ac, mtu) netCancel() @@ -1082,7 +1033,7 @@ func (s *AteomService) configureGuestNetwork(ctx context.Context, ac *kata.Agent return ac.AddARPNeighbors(ctx, []*agentpb.ARPNeighbor{{ ToIPAddress: &agentpb.IPAddress{Family: agentpb.IPFamily_v4, Address: ateomnet.ActorVethGateway}, Device: ateomnet.ActorVethName, - Lladdr: hostVethMAC, + Lladdr: gatewayMAC, State: 0x80, // NUD_PERMANENT }}) } diff --git a/cmd/ateom-microvm/run_test.go b/cmd/ateom-microvm/run_test.go index c511d078a1..6639e38dc2 100644 --- a/cmd/ateom-microvm/run_test.go +++ b/cmd/ateom-microvm/run_test.go @@ -20,7 +20,6 @@ import ( "context" "errors" "net" - "os" "path/filepath" "slices" "strings" @@ -30,70 +29,6 @@ import ( "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata" ) -// A symlink planted in the image at /etc or /etc/resolv.conf must not be followed -// out of the rootfs, or the image picks what ateom overwrites as root. -func TestWriteGuestResolvConfSymlinkEscape(t *testing.T) { - for _, tc := range []struct { - name string - link string // rootfs-relative path planted as a symlink to the canary - // A planted resolv.conf is just unlinked; only an escaping directory fails. - wantErr bool - }{ - {name: "etc dir", link: "etc", wantErr: true}, - {name: "resolv.conf", link: "etc/resolv.conf"}, - } { - t.Run(tc.name, func(t *testing.T) { - dir := t.TempDir() - canary := filepath.Join(dir, "canary") - if err := os.WriteFile(canary, []byte("INITIAL_STATE"), 0o644); err != nil { - t.Fatal(err) - } - rootfs := filepath.Join(dir, "rootfs") - if err := os.MkdirAll(filepath.Join(rootfs, filepath.Dir(tc.link)), 0o755); err != nil { - t.Fatal(err) - } - if err := os.Symlink(canary, filepath.Join(rootfs, tc.link)); err != nil { - t.Fatal(err) - } - - if err := writeGuestResolvConf(rootfs); (err != nil) != tc.wantErr { - t.Errorf("writeGuestResolvConf(%q) error = %v, wantErr %v", rootfs, err, tc.wantErr) - } - got, err := os.ReadFile(canary) - if err != nil { - t.Fatal(err) - } - if string(got) != "INITIAL_STATE" { - t.Errorf("canary = %q, want it untouched: the symlink was followed out of the rootfs", got) - } - }) - } -} - -func TestWriteGuestResolvConf(t *testing.T) { - want, err := os.ReadFile("/etc/resolv.conf") - if err != nil || len(want) == 0 { - t.Skipf("no host /etc/resolv.conf to copy: %v", err) - } - rootfs := t.TempDir() - - if err := writeGuestResolvConf(rootfs); err != nil { - t.Fatalf("writeGuestResolvConf(%q) = %v", rootfs, err) - } - - got, err := os.ReadFile(filepath.Join(rootfs, "etc", "resolv.conf")) - if err != nil { - t.Fatal(err) - } - if string(got) != string(want) { - t.Errorf("guest resolv.conf = %q, want %q", got, want) - } - // A second boot of the same bundle must not fail on the file it just wrote. - if err := writeGuestResolvConf(rootfs); err != nil { - t.Errorf("writeGuestResolvConf(%q) second call = %v", rootfs, err) - } -} - // A vsock socket that has gone missing means cloud-hypervisor stopped the VM // (it unlinks the socket in the vsock device's shutdown), so the poll must give // up at once instead of spending the whole timeout on a guest that is gone. diff --git a/cmd/ateom-microvm/sandboxnet.go b/cmd/ateom-microvm/sandboxnet.go new file mode 100644 index 0000000000..75e342fce7 --- /dev/null +++ b/cmd/ateom-microvm/sandboxnet.go @@ -0,0 +1,76 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "os" + + "github.com/vishvananda/netns" + + "github.com/agent-substrate/substrate/internal/ateomnet" + "github.com/agent-substrate/substrate/internal/atunnel" +) + +// prepareSandboxNetwork builds the actor's network and starts serving it. +func (s *AteomService) prepareSandboxNetwork(ctx context.Context, actorUID string) error { + if err := s.releaseSandboxNetwork(ctx); err != nil { + return err + } + session, err := ateomnet.ServeSandbox(ctx, ateomnet.SandboxNetworkConfig{ + ActorUID: actorUID, + EgressPort: s.atunnelEgressPort, + DNSPort: atunnel.DNSPort, + }, s.atunnelEgress, s.dnsRelay) + if err != nil { + return fmt.Errorf("while setting up the sandbox network: %w", err) + } + + return s.sandbox.Replace(ctx, session) +} + +// releaseSandboxNetwork stops serving the actor and takes its network down. +func (s *AteomService) releaseSandboxNetwork(ctx context.Context) error { + return s.sandbox.Close(ctx) +} + +// sandboxNetNS is where the actor's tap and atunnel's sockets live, or -1 +// between activations. +func (s *AteomService) sandboxNetNS() netns.NsHandle { + session := s.sandbox.Session() + if session == nil { + return -1 + } + return session.Network.GatewayNetNS +} + +// writeActorResolvConf points the guest resolver at its fixed gateway address. +func writeActorResolvConf(rootfs string) error { + pod, err := os.ReadFile("/etc/resolv.conf") + if err != nil { + return fmt.Errorf("reading the worker pod resolv.conf: %w", err) + } + return ateomnet.WriteRootfsResolvConf(rootfs, ateomnet.SandboxResolvConf(pod)) +} + +// attachAtunnel completes setup after atunnel receives the service's dialer. +func (s *AteomService) attachAtunnel(ingress *atunnel.Server, egress *atunnel.Egress, egressPort uint16) { + s.atunnelIngress = ingress + s.atunnelEgress = egress + s.atunnelEgressPort = egressPort +} diff --git a/internal/ateomnet/dns.go b/internal/ateomnet/dns.go new file mode 100644 index 0000000000..b8be5210fa --- /dev/null +++ b/internal/ateomnet/dns.go @@ -0,0 +1,80 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ateomnet + +import ( + "context" + "fmt" + "io" + "log/slog" + "net" + "strconv" + + "github.com/vishvananda/netns" +) + +// dnsServer answers an actor's DNS. Satisfied by atunnel.DNSRelay; an interface +// so this package does not depend on it. +type dnsServer interface { + ServePacket(ctx context.Context, pc net.PacketConn) error + Serve(ctx context.Context, listener net.Listener) error +} + +// serveSandboxDNS serves UDP and TCP DNS in the sandbox's local gateway namespace. +func serveSandboxDNS(ctx context.Context, relay dnsServer, ns netns.NsHandle, port uint16) (_ []io.Closer, _ []func(), retErr error) { + // Bind the wildcard because the microVM tap's gateway address is added later. + address := net.JoinHostPort("0.0.0.0", strconv.Itoa(int(port))) + + var packet net.PacketConn + var stream net.Listener + if err := NetNSDo(ctx, ns, func(context.Context) error { + pc, err := net.ListenPacket("udp", address) + if err != nil { + return fmt.Errorf("while opening the actor DNS socket: %w", err) + } + packet = pc + l, err := net.Listen("tcp", address) + if err != nil { + _ = pc.Close() + return fmt.Errorf("while opening the actor DNS listener: %w", err) + } + stream = l + return nil + }); err != nil { + return nil, nil, err + } + + // Detached from the activation RPC's context but cancelable: the relay's + // capacity is the worker's, so teardown must drop queries still in flight. + serveCtx, stopServing := context.WithCancel(context.WithoutCancel(ctx)) + serve := []func(){ + func() { + if err := relay.ServePacket(serveCtx, packet); err != nil { + slog.WarnContext(ctx, "Actor DNS socket stopped", slog.Any("err", err)) + } + }, + func() { + if err := relay.Serve(serveCtx, stream); err != nil { + slog.WarnContext(ctx, "Actor DNS listener stopped", slog.Any("err", err)) + } + }, + } + // Cancel first: closing the sockets alone leaves the queries already being + // resolved holding the relay. + closers := []io.Closer{closerFunc(func() error { stopServing(); return nil }), packet, stream} + return closers, serve, nil +} diff --git a/internal/ateomnet/net.go b/internal/ateomnet/net.go index 8db27a8226..6800690db8 100644 --- a/internal/ateomnet/net.go +++ b/internal/ateomnet/net.go @@ -21,15 +21,12 @@ import ( "context" "errors" "fmt" - "log/slog" "net" "os" "path/filepath" "runtime" "strings" - "github.com/google/nftables" - "github.com/google/nftables/binaryutil" "github.com/google/nftables/expr" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" @@ -37,14 +34,11 @@ import ( ) const ( - HostVethName = "ateom0" - ActorVethName = "eth0" - HostVethCIDR = "169.254.17.1/30" - ActorVethCIDR = "169.254.17.2/30" - ActorVethGateway = "169.254.17.1" - ActorVethIP = "169.254.17.2" - ActorNftTableName = "ateom_actor" - dnsPort = 53 + ActorVethName = "eth0" + HostVethCIDR = "169.254.17.1/30" + ActorVethCIDR = "169.254.17.2/30" + ActorVethGateway = "169.254.17.1" + ActorVethIP = "169.254.17.2" // ActorVethSubnet is the point-to-point /30 the actor veth lives on. ActorVethSubnet = "169.254.17.0/30" @@ -83,94 +77,6 @@ func MustParseMAC(s string) net.HardwareAddr { return m } -// ConfigureActorVeth configures the actor veth inside the interior netns. -// It assumes it is already running inside the target network namespace. -func ConfigureActorVeth(ctx context.Context) error { - // Run inside the gVisor interior netns. SetupActorNetwork has already created - // the veth peer here, under its final name, so this only has to address it. - // gVisor reads link names, addresses, and routes from this namespace when the - // workload starts, so eth0 is configured like a normal container interface: - // - // * lo is brought up for localhost behavior. - // * eth0 receives the actor-side /30 address. - // * the default route points to the worker-side veth gateway. - loLink, err := netlink.LinkByName("lo") - if err != nil { - return fmt.Errorf("while acquiring lo in interior netns: %w", err) - } - if err := netlink.LinkSetUp(loLink); err != nil { - return fmt.Errorf("while bringing up lo in interior netns: %w", err) - } - - actorLink, err := netlink.LinkByName(ActorVethName) - if err != nil { - return fmt.Errorf("while acquiring actor veth in interior netns: %w", err) - } - - if err := netlink.AddrReplace(actorLink, ActorVethAddr); err != nil { - return fmt.Errorf("while assigning actor veth address: %w", err) - } - if err := netlink.LinkSetUp(actorLink); err != nil { - return fmt.Errorf("while bringing up actor veth: %w", err) - } - - if err := netlink.RouteReplace(&netlink.Route{ - LinkIndex: actorLink.Attrs().Index, - Gw: ActorVethGwIP, - }); err != nil { - return fmt.Errorf("while installing actor default route: %w", err) - } - - return nil -} - -// CleanupActorNetwork removes all per-activation network state owned by ateom. -// Intentionally idempotent. -func CleanupActorNetwork(ctx context.Context, interiorNetNS netns.NsHandle) error { - // Remove all per-activation network state owned by ateom. Deleting the - // worker-side veth also deletes its peer, but the pair is born with its peer - // already in the actor netns, so a setup that failed before the worker side - // was named can leave that peer behind on its own. For that reason cleanup - // also enters the interior netns and deletes the actor interface if present. - // - // This function is intentionally idempotent so it can run before setup, after - // checkpoint, and from setup failure cleanup without requiring the caller to - // know how far network initialization progressed. - var cleanupErr error - if err := RemoveActorNftablesRules(); err != nil { - cleanupErr = errors.Join(cleanupErr, fmt.Errorf("while removing actor nftables rules: %w", err)) - slog.WarnContext(ctx, "Failed to remove actor nftables rules; continuing actor netns cleanup", "err", err) - } - - if link, err := netlink.LinkByName(HostVethName); err == nil { - if err := netlink.LinkDel(link); err != nil { - cleanupErr = errors.Join(cleanupErr, fmt.Errorf("while deleting host veth: %w", err)) - slog.WarnContext(ctx, "Failed to delete host veth; continuing actor netns cleanup", "err", err) - } - } else if _, notFound := errors.AsType[netlink.LinkNotFoundError](err); !notFound { - cleanupErr = errors.Join(cleanupErr, fmt.Errorf("while looking up host veth: %w", err)) - slog.WarnContext(ctx, "Failed to look up host veth; continuing actor netns cleanup", "err", err) - } - - if err := NetNSDo(ctx, interiorNetNS, func(_ context.Context) error { - link, err := netlink.LinkByName(ActorVethName) - if err == nil { - if err := netlink.LinkDel(link); err != nil { - return fmt.Errorf("while deleting interior veth %q: %w", ActorVethName, err) - } - return nil - } - if _, notFound := errors.AsType[netlink.LinkNotFoundError](err); !notFound { - return fmt.Errorf("while looking up interior veth %q: %w", ActorVethName, err) - } - return nil - }); err != nil { - cleanupErr = errors.Join(cleanupErr, fmt.Errorf("while cleaning interior netns links: %w", err)) - } - - return cleanupErr -} - // AllowUnprivilegedPorts lets this namespace bind ports below 1024 without // CAP_NET_BIND_SERVICE, which is how atunnel answers a sandbox's DNS on 53. // The sysctl is per-namespace and grants nothing outside it. @@ -205,160 +111,6 @@ func setNetSysctl(key, value string) error { return nil } -// EnableIPv4Forwarding enables IPv4 forwarding in the current network namespace. -func EnableIPv4Forwarding() error { - // Forwarding is required because actor packets now enter the worker pod via - // the host-side veth and then leave through the pod's eth0. Without this, the - // kernel would not route traffic between those interfaces even though both - // live in the worker pod network namespace. - // - // Without privileged, the container runtime bind-mounts /proc/sys read-only. - // The worker holds CAP_SYS_ADMIN and uses no user namespace, so the ro flag - // is not locked: clear it, write the sysctl, restore ro. - const path = "/proc/sys/net/ipv4/ip_forward" - if b, err := os.ReadFile(path); err == nil && len(b) > 0 && b[0] == '1' { - return nil - } - if err := os.WriteFile(path, []byte("1\n"), 0o644); err == nil { - return nil - } - if err := unix.Mount("none", "/proc/sys", "", unix.MS_BIND|unix.MS_REMOUNT, ""); err != nil { - return fmt.Errorf("while remounting /proc/sys read-write to enable IPv4 forwarding: %w", err) - } - defer func() { - _ = unix.Mount("none", "/proc/sys", "", unix.MS_BIND|unix.MS_REMOUNT|unix.MS_RDONLY, "") - }() - if err := os.WriteFile(path, []byte("1\n"), 0o644); err != nil { - return fmt.Errorf("while enabling IPv4 forwarding in worker pod netns: %w", err) - } - return nil -} - -// InstallActorNftablesRules configures the NAT and filtering rules for the -// actor. egressPort, when non-zero, is the local atunnel egress listener actor -// TCP egress is redirected to; zero leaves the redirect uninstalled. -func InstallActorNftablesRules(egressPort uint16) error { - // Install a dedicated nftables table for the active actor. Keeping all - // rules in an ateom-owned table makes cleanup simple and avoids mutating - // Kubernetes or CNI-managed chains directly. - // - // TODO: Add IPv6 veth addressing, forwarding, and nftables rules once actor - // networking supports dual-stack pods. The current actor network is IPv4-only. - // - // The rules do three things: - // - // * prerouting: redirect new actor TCP connections, other than traffic to - // destination port 53, to atunnel's local listener. REDIRECT preserves - // SO_ORIGINAL_DST for the CONNECT authority. - // * postrouting: masquerade traffic not handled by the TCP tunnel, notably - // traffic to TCP or UDP destination port 53, so hostname resolution - // continues to work. - // * forward: drop actor UDP egress to any port but DNS, and accept the rest - // of the packets forwarded between the actor veth and pod eth0. - if err := RemoveActorNftablesRules(); err != nil { - return err - } - - c := &nftables.Conn{} - table := &nftables.Table{ - Family: nftables.TableFamilyIPv4, - Name: ActorNftTableName, - } - c.AddTable(table) - - prerouting := c.AddChain(&nftables.Chain{ - Name: "prerouting", - Table: table, - Type: nftables.ChainTypeNAT, - Hooknum: nftables.ChainHookPrerouting, - Priority: nftables.ChainPriorityNATDest, - }) - if redirectRule := ActorEgressRedirectRule(table, prerouting, egressPort); redirectRule != nil { - c.AddRule(redirectRule) - } - - postrouting := c.AddChain(&nftables.Chain{ - Name: "postrouting", - Table: table, - Type: nftables.ChainTypeNAT, - Hooknum: nftables.ChainHookPostrouting, - Priority: nftables.ChainPriorityNATSource, - }) - c.AddRule(&nftables.Rule{ - Table: table, - Chain: postrouting, - Exprs: append(IPSourceEqual(ActorVethIP), &expr.Masq{}), - }) - - acceptPolicy := nftables.ChainPolicyAccept - forward := c.AddChain(&nftables.Chain{ - Name: "forward", - Table: table, - Type: nftables.ChainTypeFilter, - Hooknum: nftables.ChainHookForward, - Priority: nftables.ChainPriorityFilter, - Policy: &acceptPolicy, - }) - // Order matters: the accept below is a catch-all, so the drop has to precede - // it. - c.AddRule(actorNonDNSUDPDropRule(table, forward)) - c.AddRule(&nftables.Rule{ - Table: table, - Chain: forward, - Exprs: []expr.Any{ - &expr.Verdict{Kind: expr.VerdictAccept}, - }, - }) - - if err := c.Flush(); err != nil { - return fmt.Errorf("while installing actor nftables rules: %w", err) - } - return nil -} - -// RemoveActorNftablesRules removes the ateom nftables table. -func RemoveActorNftablesRules() error { - // Delete the whole ateom nftables table if it exists. The table is - // per-worker and currently per-active-actor because this worker path runs at - // most one actor at a time. Missing tables are treated as already clean. - c := &nftables.Conn{} - tables, err := c.ListTablesOfFamily(nftables.TableFamilyIPv4) - if err != nil { - return fmt.Errorf("while listing nftables tables: %w", err) - } - for _, table := range tables { - if table.Name != ActorNftTableName { - continue - } - c.DelTable(table) - if err := c.Flush(); err != nil { - return fmt.Errorf("while deleting actor nftables table: %w", err) - } - return nil - } - return nil -} - -func IPSourceEqual(ip string) []expr.Any { - return IPPayloadEqual(12, ip) -} - -func IPPayloadEqual(offset uint32, ip string) []expr.Any { - return []expr.Any{ - &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseNetworkHeader, - Offset: offset, - Len: 4, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: net.ParseIP(ip).To4(), - }, - } -} - func l4ProtocolEqual(proto byte) []expr.Any { return []expr.Any{ &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, @@ -370,65 +122,6 @@ func l4ProtocolEqual(proto byte) []expr.Any { } } -// ActorEgressRedirectRule returns the prerouting rule that redirects actor TCP -// egress, except traffic to [dnsPort], to the local atunnel egress listener on -// port, or nil when port is zero (tunneled egress disabled, so actor egress -// stays on the masquerade path). -func ActorEgressRedirectRule(table *nftables.Table, chain *nftables.Chain, port uint16) *nftables.Rule { - if port == 0 { - return nil - } - exprs := append(IPSourceEqual(ActorVethIP), l4ProtocolEqual(unix.IPPROTO_TCP)...) - exprs = append(exprs, - // Traffic to destination port 53 bypasses atunnel and follows the direct - // masquerade path. - &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseTransportHeader, - Offset: 2, - Len: 2, - }, - &expr.Cmp{ - Op: expr.CmpOpNeq, - Register: 1, - Data: binaryutil.BigEndian.PutUint16(dnsPort), - }, - &expr.Immediate{ - Register: 1, - Data: binaryutil.BigEndian.PutUint16(port), - }, - &expr.Redir{RegisterProtoMin: 1}, - ) - return &nftables.Rule{Table: table, Chain: chain, Exprs: exprs} -} - -// actorNonDNSUDPDropRule returns the forward-chain rule that drops actor UDP -// egress to every destination port but [dnsPort]. -// -// The rule counts what it drops: a workload that legitimately needs UDP shows -// up as a rising counter in `nft list table ip ateom_actor` rather than as an -// unexplained timeout. -func actorNonDNSUDPDropRule(table *nftables.Table, chain *nftables.Chain) *nftables.Rule { - exprs := append(IPSourceEqual(ActorVethIP), l4ProtocolEqual(unix.IPPROTO_UDP)...) - exprs = append(exprs, - // Destination port, at offset 2 of the UDP header. - &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseTransportHeader, - Offset: 2, - Len: 2, - }, - &expr.Cmp{ - Op: expr.CmpOpNeq, - Register: 1, - Data: binaryutil.BigEndian.PutUint16(dnsPort), - }, - &expr.Counter{}, - &expr.Verdict{Kind: expr.VerdictDrop}, - ) - return &nftables.Rule{Table: table, Chain: chain, Exprs: exprs} -} - // CreateNetNSWithoutSwitching creates a named netns and returns its handle, // restoring the caller's current netns before returning. // @@ -510,160 +203,3 @@ func NetNSDo(ctx context.Context, targetNS netns.NsHandle, do func(context.Conte } return nil } - -// DumpNetInfo dumps link and route information for debugging. -func DumpNetInfo(ctx context.Context, prefix string) error { - links, err := netlink.LinkList() - if err != nil { - return fmt.Errorf("in netlink.LinkList(): %w", err) - } - - for _, link := range links { - slog.InfoContext(ctx, prefix+"Link", slog.String("name", link.Attrs().Name), slog.String("type", link.Type()), slog.Any("attrs", link.Attrs())) - - addrs, err := netlink.AddrList(link, netlink.FAMILY_V4) - if err != nil { - return fmt.Errorf("while getting pod eth0 addresses: %w", err) - } - slog.InfoContext(ctx, prefix+"Link Addresses", slog.String("link", link.Attrs().Name), slog.Any("addrs", addrs)) - - rts, err := netlink.RouteList(link, netlink.FAMILY_V4) - if err != nil { - return fmt.Errorf("while getting routes off eth0: %w", err) - } - for _, rt := range rts { - slog.InfoContext(ctx, prefix+"Link Routes", slog.Any("link", link.Attrs().Name), slog.Any("route", rt), slog.Any("route-string", rt.String())) - } - } - - return nil -} - -type NetworkConfig struct { - // InteriorNetNS is the target network namespace for the actor's veth pair peer. - // Used by: Both gVisor and MicroVM. - InteriorNetNS netns.NsHandle - - // HostVethHWAddr is the hardware address to assign to the host veth interface. - // Used by: MicroVM (to ensure consistent MAC addresses across snapshot/restore). - HostVethHWAddr net.HardwareAddr - // SweepInteriorLinks indicates whether to delete existing links in the interior netns (excluding loopback). - // Used by: MicroVM (to clean up stale tap devices). - SweepInteriorLinks bool - - // DumpNetInfo indicates whether to dump network information to the logs for debugging purposes. - // Used by: gVisor. - DumpNetInfo bool - - // EgressRedirectPort is the local atunnel egress listener port actor TCP - // egress is redirected to. Zero installs no redirect, leaving actor egress - // on the masquerade path. - // Used by: Both gVisor and MicroVM. - EgressRedirectPort uint16 -} - -// SetupActorNetwork builds a fresh point-to-point network between the worker -// pod netns and the interior netns. -func SetupActorNetwork(ctx context.Context, cfg NetworkConfig) (retErr error) { - // Build a fresh point-to-point network between the worker pod netns and the - // gVisor interior netns. The worker side keeps the pod's real eth0 and creates - // ateom0 as the gateway; the pair's peer is born inside the actor netns as - // eth0, where it gets the actor-side address and a default route via the - // worker-side veth address. This replaces the old behavior of moving the - // Kubernetes-provided eth0 out of the worker pod. - // - // The nftables rules installed here redirect actor TCP egress to atunnel - // when configured, masquerade traffic the TCP tunnel does not handle - // (notably DNS over UDP), and drop actor UDP egress to any other port. - // - // Clean up stale state from a failed prior activation before creating the - // next actor-side network. The worker currently runs one actor at a time. - if err := CleanupActorNetwork(ctx, cfg.InteriorNetNS); err != nil { - return fmt.Errorf("failed to clean up stale actor network before setup: %w", err) - } - defer func() { - if retErr != nil { - if err := CleanupActorNetwork(ctx, cfg.InteriorNetNS); err != nil { - slog.WarnContext(ctx, "Failed to clean up partially configured actor network", slog.Any("err", err)) - } - } - }() - - if cfg.SweepInteriorLinks { - if err := NetNSDo(ctx, cfg.InteriorNetNS, func(ctx context.Context) error { - links, err := netlink.LinkList() - if err != nil { - return fmt.Errorf("while listing interior netns links: %w", err) - } - for _, l := range links { - if l.Attrs().Name == "lo" { - continue - } - if err := netlink.LinkDel(l); err != nil { - slog.WarnContext(ctx, "Failed to delete leftover interior link", slog.String("link", l.Attrs().Name), slog.Any("err", err)) - } - } - return nil - }); err != nil { - return err - } - } - - // The peer is born in the interior netns under its final name. Do not replace - // this with the obvious "create locally, LinkSetNsFd across, rename to eth0": - // moving and renaming a netdev each cost an RCU grace period under the global - // RTNL lock, which is ~18ms vs ~3ms for all of SetupActorNetwork, on the - // resume path. Naming the peer here is safe because its name is resolved in - // its own netns, so it never collides with the pod's eth0. - veth := &netlink.Veth{ - LinkAttrs: netlink.LinkAttrs{ - Name: HostVethName, - }, - PeerName: ActorVethName, - // netlink.NsFd, not netns.NsHandle: only the netlink type is recognized - // as IFLA_NET_NS_FD on the peer, though both are file descriptors. - PeerNamespace: netlink.NsFd(int(cfg.InteriorNetNS)), - } - if len(cfg.HostVethHWAddr) > 0 { - veth.LinkAttrs.HardwareAddr = cfg.HostVethHWAddr - } - - if err := netlink.LinkAdd(veth); err != nil { - return fmt.Errorf("while creating actor veth pair: %w", err) - } - - hostLink, err := netlink.LinkByName(HostVethName) - if err != nil { - return fmt.Errorf("while getting host veth: %w", err) - } - if err := netlink.AddrReplace(hostLink, HostVethAddr); err != nil { - return fmt.Errorf("while assigning host veth address: %w", err) - } - if err := netlink.LinkSetUp(hostLink); err != nil { - return fmt.Errorf("while bringing up host veth: %w", err) - } - - if err := NetNSDo(ctx, cfg.InteriorNetNS, ConfigureActorVeth); err != nil { - return fmt.Errorf("while configuring actor veth in interior netns: %w", err) - } - - if err := EnableIPv4Forwarding(); err != nil { - return err - } - if err := InstallActorNftablesRules(cfg.EgressRedirectPort); err != nil { - return err - } - - if cfg.DumpNetInfo { - if err := DumpNetInfo(ctx, "Pod NetNS "); err != nil { - return fmt.Errorf("while dumping pod netns links: %w", err) - } - if err := NetNSDo(ctx, cfg.InteriorNetNS, func(ctx context.Context) error { - return DumpNetInfo(ctx, "Interior NetNS ") - }); err != nil { - return fmt.Errorf("while dumping interior netns links: %w", err) - } - } - - return nil -} diff --git a/internal/ateomnet/net_linux_test.go b/internal/ateomnet/net_linux_test.go index 3e5c35675b..2438b54edb 100644 --- a/internal/ateomnet/net_linux_test.go +++ b/internal/ateomnet/net_linux_test.go @@ -17,441 +17,16 @@ package ateomnet import ( - "bytes" "context" "errors" - "net" "os" - "runtime" "testing" "github.com/agent-substrate/substrate/internal/roottest" - "github.com/google/nftables" - "github.com/google/nftables/binaryutil" - "github.com/google/nftables/expr" "github.com/vishvananda/netlink" - "github.com/vishvananda/netns" "golang.org/x/sys/unix" ) -// withTestNetNS runs fn with the calling thread inside a throwaway netns -// standing in for the worker pod's, and hands it a second throwaway netns -// standing in for an actor's interior one. -// -// Both are anonymous (netns.New, not NewNamed) so the test leaves nothing behind -// in /run/netns, and every link, sysctl, and nftables table SetupActorNetwork -// touches is scoped to a namespace that disappears with the test rather than to -// the machine running it. -func withTestNetNS(t *testing.T, fn func(interior netns.NsHandle)) { - t.Helper() - - // Locked for the whole body: netns is a per-thread property, so an unlocked - // goroutine could be rescheduled onto a thread still in the original - // namespace midway through. It also means a t.Fatal inside fn tears the - // thread down instead of returning it to the pool mis-configured. - runtime.LockOSThread() - defer runtime.UnlockOSThread() - - orig, err := netns.Get() - if err != nil { - t.Fatalf("getting current netns: %v", err) - } - defer orig.Close() - // Registered before the namespaces below so it runs after they are closed. - defer func() { - if err := netns.Set(orig); err != nil { - t.Errorf("restoring original netns: %v", err) - } - }() - - pod, err := netns.New() // netns.New switches the thread into the new namespace - if err != nil { - t.Fatalf("creating pod netns: %v", err) - } - defer pod.Close() - interior, err := netns.New() - if err != nil { - t.Fatalf("creating interior netns: %v", err) - } - defer interior.Close() - if err := netns.Set(pod); err != nil { - t.Fatalf("entering pod netns: %v", err) - } - - fn(interior) -} - -// requireNftables skips when the kernel in this environment cannot serve the -// nftables netlink API at all, which SetupActorNetwork needs and which is a -// property of the machine rather than of the code under test. -func requireNftables(t *testing.T) { - t.Helper() - c := &nftables.Conn{} - if _, err := c.ListTablesOfFamily(nftables.TableFamilyIPv4); err != nil { - t.Skipf("nftables unavailable in this environment: %v", err) - } -} - -// linkByName returns the link, or nil when it does not exist. -func linkByName(t *testing.T, name string) netlink.Link { - t.Helper() - link, err := netlink.LinkByName(name) - if err == nil { - return link - } - if _, ok := errors.AsType[netlink.LinkNotFoundError](err); ok { - return nil - } - t.Fatalf("looking up link %q: %v", name, err) - return nil -} - -func hasAddr(t *testing.T, link netlink.Link, cidr string) bool { - t.Helper() - addrs, err := netlink.AddrList(link, netlink.FAMILY_V4) - if err != nil { - t.Fatalf("listing addresses of %q: %v", link.Attrs().Name, err) - } - want := MustParseAddr(cidr) - for _, addr := range addrs { - if addr.IPNet != nil && addr.IPNet.String() == want.IPNet.String() { - return true - } - } - return false -} - -// TestSetupActorNetworkFinalState pins the namespace state gVisor and the -// micro-VM guest read after an activation: what links exist, where, with which -// addresses and routes. It deliberately asserts the end state rather than the -// sequence of netlink calls that produced it, so the setup path stays free to -// get there differently (as it did when the veth peer stopped being created in -// the pod netns and moved across). -func TestSetupActorNetworkFinalState(t *testing.T) { - roottest.Require(t, "creating network namespaces, veth pairs, and nftables rules") - ctx := context.Background() - - withTestNetNS(t, func(interior netns.NsHandle) { - requireNftables(t) - - if err := SetupActorNetwork(ctx, NetworkConfig{InteriorNetNS: interior}); err != nil { - t.Fatalf("SetupActorNetwork: %v", err) - } - - // Worker pod side: the gateway end of the point-to-point link. - host := linkByName(t, HostVethName) - if host == nil { - t.Fatalf("host veth %q missing from the pod netns", HostVethName) - } - if !hasAddr(t, host, HostVethCIDR) { - t.Errorf("host veth %q does not carry %s", HostVethName, HostVethCIDR) - } - if host.Attrs().Flags&1 == 0 { // net.FlagUp - t.Errorf("host veth %q is not up", HostVethName) - } - - // The actor interface must exist ONLY in the interior netns. A peer left - // in the pod netns would mean the pair was built the old way, and worse, - // would collide with the pod's own eth0 on a real worker. - if stray := linkByName(t, ActorVethName); stray != nil { - t.Errorf("actor interface %q must not exist in the pod netns", ActorVethName) - } - - if err := NetNSDo(ctx, interior, func(context.Context) error { - actor := linkByName(t, ActorVethName) - if actor == nil { - t.Fatalf("actor veth %q missing from the interior netns", ActorVethName) - } - if !hasAddr(t, actor, ActorVethCIDR) { - t.Errorf("actor veth %q does not carry %s", ActorVethName, ActorVethCIDR) - } - if actor.Attrs().Flags&1 == 0 { - t.Errorf("actor veth %q is not up", ActorVethName) - } - - if lo := linkByName(t, "lo"); lo == nil { - t.Error("interior netns has no loopback") - } else if lo.Attrs().Flags&1 == 0 { - t.Error("interior loopback is not up") - } - - routes, err := netlink.RouteList(actor, netlink.FAMILY_V4) - if err != nil { - t.Fatalf("listing interior routes: %v", err) - } - // A default route reports its destination either as nil or as an - // explicit 0.0.0.0/0, depending on how the kernel rendered it. - isDefault := func(route netlink.Route) bool { - if route.Dst == nil { - return true - } - ones, _ := route.Dst.Mask.Size() - return ones == 0 - } - var haveDefault bool - for _, route := range routes { - if isDefault(route) && route.Gw.Equal(ActorVethGwIP) { - haveDefault = true - } - } - if !haveDefault { - t.Errorf("interior netns has no default route via %s, got %v", ActorVethGateway, routes) - } - return nil - }); err != nil { - t.Fatalf("inspecting interior netns: %v", err) - } - }) -} - -// TestSetupActorNetworkIsRepeatable covers the activation cycle a reused worker -// runs: set up, tear down, set up again. The second setup has to succeed against -// whatever the first one left behind. -func TestSetupActorNetworkIsRepeatable(t *testing.T) { - roottest.Require(t, "creating network namespaces, veth pairs, and nftables rules") - ctx := context.Background() - - withTestNetNS(t, func(interior netns.NsHandle) { - requireNftables(t) - - for i := range 3 { - if err := SetupActorNetwork(ctx, NetworkConfig{InteriorNetNS: interior}); err != nil { - t.Fatalf("SetupActorNetwork (activation %d): %v", i, err) - } - if linkByName(t, HostVethName) == nil { - t.Fatalf("host veth %q missing after activation %d", HostVethName, i) - } - if err := CleanupActorNetwork(ctx, interior); err != nil { - t.Fatalf("CleanupActorNetwork (activation %d): %v", i, err) - } - } - - // Cleanup is idempotent: the extra call after the loop's last one must - // still succeed, and both ends must be gone. - if err := CleanupActorNetwork(ctx, interior); err != nil { - t.Fatalf("CleanupActorNetwork on an already-clean network: %v", err) - } - if stray := linkByName(t, HostVethName); stray != nil { - t.Errorf("host veth %q survived cleanup", HostVethName) - } - if err := NetNSDo(ctx, interior, func(context.Context) error { - if stray := linkByName(t, ActorVethName); stray != nil { - t.Errorf("actor veth %q survived cleanup", ActorVethName) - } - return nil - }); err != nil { - t.Fatalf("inspecting interior netns: %v", err) - } - }) -} - -// addForwardingTarget gives the pod netns somewhere to forward actor packets -// to, standing in for the real pod's eth0 and default route. Without it a -// forwarded packet is dropped for want of a route before it ever reaches the -// forward hook the rules under test live on. The device is a dummy, so the -// packets go nowhere after that, which is all the assertions need. -func addForwardingTarget(t *testing.T, cidr string) { - t.Helper() - link := &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: "target0"}} - if err := netlink.LinkAdd(link); err != nil { - t.Fatalf("creating the forwarding target link: %v", err) - } - if err := netlink.AddrAdd(link, MustParseAddr(cidr)); err != nil { - t.Fatalf("addressing the forwarding target link: %v", err) - } - if err := netlink.LinkSetUp(link); err != nil { - t.Fatalf("bringing up the forwarding target link: %v", err) - } -} - -// droppedUDPPackets reads the packet count off the forward chain's counted -// rule, which [actorNonDNSUDPDropRule] is. -func droppedUDPPackets(t *testing.T) uint64 { - t.Helper() - c := &nftables.Conn{} - tables, err := c.ListTablesOfFamily(nftables.TableFamilyIPv4) - if err != nil { - t.Fatalf("listing nftables tables: %v", err) - } - for _, table := range tables { - if table.Name != ActorNftTableName { - continue - } - rules, err := c.GetRules(table, &nftables.Chain{Name: "forward", Table: table}) - if err != nil { - t.Fatalf("listing forward chain rules: %v", err) - } - for _, rule := range rules { - for _, e := range rule.Exprs { - if counter, ok := e.(*expr.Counter); ok { - return counter.Packets - } - } - } - t.Fatalf("forward chain has no counted rule, got %d rules", len(rules)) - } - t.Fatalf("nftables table %q is missing", ActorNftTableName) - return 0 -} - -// sendUDP sends one datagram to addr and reports whether the local send -// succeeded. UDP has no acknowledgement, so a successful send says nothing -// about delivery -- the drop is observed through the nftables counter instead. -func sendUDP(t *testing.T, addr string) { - t.Helper() - conn, err := net.Dial("udp4", addr) - if err != nil { - t.Fatalf("dialing %s: %v", addr, err) - } - defer conn.Close() - if _, err := conn.Write([]byte("probe")); err != nil { - t.Fatalf("sending a datagram to %s: %v", addr, err) - } -} - -func TestActorEgressRedirectRuleExcludesDNS(t *testing.T) { - table := &nftables.Table{} - chain := &nftables.Chain{Table: table} - - if rule := ActorEgressRedirectRule(table, chain, 0); rule != nil { - t.Fatal("ActorEgressRedirectRule returned a rule when tunneled egress is disabled") - } - - rule := ActorEgressRedirectRule(table, chain, 15001) - if rule == nil { - t.Fatal("ActorEgressRedirectRule returned nil when tunneled egress is enabled") - } - if len(rule.Exprs) != 8 { - t.Fatalf("redirect rule has %d expressions, want 8", len(rule.Exprs)) - } - payload, ok := rule.Exprs[4].(*expr.Payload) - if !ok { - t.Fatalf("redirect expression 4 is %T, want *expr.Payload", rule.Exprs[4]) - } - if payload.Base != expr.PayloadBaseTransportHeader || payload.Offset != 2 || payload.Len != 2 { - t.Errorf("redirect destination-port payload = %+v, want transport-header offset 2 length 2", payload) - } - cmp, ok := rule.Exprs[5].(*expr.Cmp) - if !ok { - t.Fatalf("redirect expression 5 is %T, want *expr.Cmp", rule.Exprs[5]) - } - if cmp.Op != expr.CmpOpNeq || !bytes.Equal(cmp.Data, binaryutil.BigEndian.PutUint16(dnsPort)) { - t.Errorf("redirect destination-port comparison = %+v, want destination port != %d", cmp, dnsPort) - } -} - -// TestActorNonDNSUDPIsDropped covers the forward-chain rule behaviorally: only -// TCP not destined for port 53 is redirected into atunnel, so UDP on any port -// but 53 must not reach the masquerade, and DNS must still get through or the -// sandbox cannot resolve anything. -func TestActorNonDNSUDPIsDropped(t *testing.T) { - roottest.Require(t, "creating network namespaces, veth pairs, and nftables rules") - ctx := context.Background() - - withTestNetNS(t, func(interior netns.NsHandle) { - requireNftables(t) - - const target = "192.0.2.1" - addForwardingTarget(t, "192.0.2.254/24") - if err := SetupActorNetwork(ctx, NetworkConfig{InteriorNetNS: interior}); err != nil { - t.Fatalf("SetupActorNetwork: %v", err) - } - - before := droppedUDPPackets(t) - if err := NetNSDo(ctx, interior, func(context.Context) error { - sendUDP(t, net.JoinHostPort(target, "53")) - return nil - }); err != nil { - t.Fatalf("sending DNS from the interior netns: %v", err) - } - if got := droppedUDPPackets(t); got != before { - t.Errorf("DNS datagram was dropped: counter went from %d to %d", before, got) - } - - if err := NetNSDo(ctx, interior, func(context.Context) error { - sendUDP(t, net.JoinHostPort(target, "443")) - sendUDP(t, net.JoinHostPort(target, "9999")) - return nil - }); err != nil { - t.Fatalf("sending non-DNS UDP from the interior netns: %v", err) - } - if got := droppedUDPPackets(t); got != before+2 { - t.Errorf("dropped packets = %d, want %d: non-DNS UDP reached the masquerade", got, before+2) - } - }) -} - -// TestSetupActorNetworkHostVethHWAddr covers the micro-VM requirement: a CH -// snapshot freezes the guest's ARP entry for the gateway, so the worker-side -// veth MAC has to be exactly the one the caller asked for, on every pod. -func TestSetupActorNetworkHostVethHWAddr(t *testing.T) { - roottest.Require(t, "creating network namespaces, veth pairs, and nftables rules") - ctx := context.Background() - - withTestNetNS(t, func(interior netns.NsHandle) { - requireNftables(t) - - want := MustParseMAC("02:a8:1e:00:00:01") - if err := SetupActorNetwork(ctx, NetworkConfig{ - InteriorNetNS: interior, - HostVethHWAddr: want, - SweepInteriorLinks: true, - }); err != nil { - t.Fatalf("SetupActorNetwork: %v", err) - } - - host := linkByName(t, HostVethName) - if host == nil { - t.Fatalf("host veth %q missing from the pod netns", HostVethName) - } - if got := host.Attrs().HardwareAddr.String(); got != want.String() { - t.Errorf("host veth MAC = %s, want %s", got, want) - } - }) -} - -// TestSetupActorNetworkSweepsInteriorLinks covers the other half of the micro-VM -// path: SweepInteriorLinks clears a previous activation's leftovers (kata's tap -// device) before the new pair is created, and must not take the loopback or the -// freshly created actor veth with it. -func TestSetupActorNetworkSweepsInteriorLinks(t *testing.T) { - roottest.Require(t, "creating network namespaces, veth pairs, and nftables rules") - ctx := context.Background() - - withTestNetNS(t, func(interior netns.NsHandle) { - requireNftables(t) - - const leftover = "stale-tap0" - if err := NetNSDo(ctx, interior, func(context.Context) error { - return netlink.LinkAdd(&netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: leftover}}) - }); err != nil { - t.Fatalf("planting a leftover interior link: %v", err) - } - - if err := SetupActorNetwork(ctx, NetworkConfig{ - InteriorNetNS: interior, - SweepInteriorLinks: true, - }); err != nil { - t.Fatalf("SetupActorNetwork: %v", err) - } - - if err := NetNSDo(ctx, interior, func(context.Context) error { - if stray := linkByName(t, leftover); stray != nil { - t.Errorf("leftover interior link %q was not swept", leftover) - } - if linkByName(t, ActorVethName) == nil { - t.Errorf("actor veth %q missing after a sweeping setup", ActorVethName) - } - if linkByName(t, "lo") == nil { - t.Error("sweep removed the interior loopback") - } - return nil - }); err != nil { - t.Fatalf("inspecting interior netns: %v", err) - } - }) -} - func TestNamedNetNSRejectsInvalidNames(t *testing.T) { for _, name := range []string{"", ".", "..", "/absolute", "../outside", "nested/name", "ateom-actor:uid/../../outside", "nul\x00name"} { t.Run(name, func(t *testing.T) { diff --git a/internal/ateomnet/sandbox.go b/internal/ateomnet/sandbox.go index 9fb06ebed8..d235c5c2ac 100644 --- a/internal/ateomnet/sandbox.go +++ b/internal/ateomnet/sandbox.go @@ -25,7 +25,6 @@ import ( "net" "net/netip" "runtime" - "strconv" "sync" "syscall" @@ -56,8 +55,6 @@ type SandboxNetwork struct { // once runsc can be given one interface rather than claiming every // interface in the namespace it runs in. GatewayNetNS netns.NsHandle - // PodSideIP is identical across sandboxes, isolated by namespace. - PodSideIP net.IP } func (n *SandboxNetwork) holdsNetNS() bool { return n.RuntimeNetNS > 0 } @@ -74,14 +71,8 @@ type SandboxNetworkConfig struct { // actor makes is redirected to it, whatever port it was aimed at. EgressPort uint16 - // GatewayHWAddr fixes the gateway's MAC, which a micro-VM snapshot freezes - // into the guest's ARP cache. gVisor re-ARPs and can leave it unset. - // - // Applies to the veth path only. The tap path sets its own MAC in - // setupActorTap, after LinkAdd, because tuntap creation ignores the - // hardware address in the link attributes. Both belong here once the two - // runtimes share one shape. - GatewayHWAddr net.HardwareAddr + // DNSPort is where the DNS relay answers, on the sandbox's default gateway. + DNSPort uint16 } // SetupSandboxNetwork creates isolated networking with fixed sandbox addresses. @@ -131,8 +122,6 @@ func SetupSandboxNetwork(ctx context.Context, cfg SandboxNetworkConfig) (_ *Sand ActorUID: actorUID, RuntimeNetNS: actorNS, GatewayNetNS: atunnelNS, - // Every actor holds the same address; the namespace is the identity. - PodSideIP: net.ParseIP(ActorVethIP), }, nil } @@ -162,9 +151,6 @@ func setupVethPair(ctx context.Context, cfg SandboxNetworkConfig, actorNS netns. // runs on the resume path. PeerNamespace: netlink.NsFd(int(actorNS)), } - if cfg.GatewayHWAddr != nil { - veth.LinkAttrs.HardwareAddr = cfg.GatewayHWAddr - } if err := netlink.LinkAdd(veth); err != nil { return fmt.Errorf("while creating the veth pair: %w", err) } @@ -345,85 +331,32 @@ func ListenInNetNS(ctx context.Context, ns netns.NsHandle, ports []uint16) (_ [] return listeners, nil } -// EgressServer serves one actor's captured connections. Satisfied by +// egressServer serves one actor's captured connections. Satisfied by // atunnel.Egress; an interface so this package does not depend on it. -type EgressServer interface { - ServeFor(ctx context.Context, actorKey string, listener net.Listener) error +type egressServer interface { + Serve(ctx context.Context, listener net.Listener) error } -// ServeSandboxEgress puts the egress server's sockets inside the actor's own -// namespace, where the local default route delivers everything it sends. The -// listener is the actor's identity: they all hold the same address, so nothing -// about a connection distinguishes them. -// -// Only ports gets captured. A port with no listener is refused rather than -// escaping, which is the fail-closed half of routing everything through the -// tunnel. Closing the returned listeners stops the actor's egress. -func ServeSandboxEgress(ctx context.Context, e EgressServer, actorKey string, ns netns.NsHandle, ports []uint16) ([]net.Listener, error) { +// ServeSandboxEgress serves redirected TCP in the gateway namespace. +// Closing the returned listeners stops accepting new connections. +func serveSandboxEgress(ctx context.Context, e egressServer, ns netns.NsHandle, ports []uint16) ([]io.Closer, []func(), error) { listeners, err := ListenInNetNS(ctx, ns, ports) if err != nil { - return nil, fmt.Errorf("while opening actor egress listeners: %w", err) + return nil, nil, fmt.Errorf("while opening actor egress listeners: %w", err) } + serve := make([]func(), 0, len(listeners)) + closers := make([]io.Closer, 0, len(listeners)) for _, l := range listeners { - go func(l net.Listener) { + closers = append(closers, l) + serve = append(serve, func() { // Background rather than the caller's context: these outlive the // activation and are stopped by closing the listener. - if err := e.ServeFor(context.Background(), actorKey, l); err != nil { - slog.WarnContext(ctx, "Actor egress listener stopped", - slog.String("actorUID", actorKey), slog.Any("err", err)) + if err := e.Serve(context.Background(), l); err != nil { + slog.WarnContext(ctx, "Sandbox egress listener stopped", slog.Any("err", err)) } - }(l) - } - return listeners, nil -} - -// DNSServer answers an actor's DNS. Satisfied by atunnel.DNSRelay; an interface -// so this package does not depend on it. -type DNSServer interface { - ServePacket(ctx context.Context, pc net.PacketConn) error - Serve(ctx context.Context, listener net.Listener) error -} - -// ServeSandboxDNS serves UDP and TCP DNS in the gateway namespace. -func ServeSandboxDNS(ctx context.Context, relay DNSServer, ns netns.NsHandle, port uint16) (_ []io.Closer, retErr error) { - // Bind the wildcard because the microVM tap's gateway address is added later. - address := net.JoinHostPort("0.0.0.0", strconv.Itoa(int(port))) - - var packet net.PacketConn - var stream net.Listener - if err := NetNSDo(ctx, ns, func(context.Context) error { - pc, err := net.ListenPacket("udp", address) - if err != nil { - return fmt.Errorf("while opening the actor DNS socket: %w", err) - } - packet = pc - l, err := net.Listen("tcp", address) - if err != nil { - _ = pc.Close() - return fmt.Errorf("while opening the actor DNS listener: %w", err) - } - stream = l - return nil - }); err != nil { - return nil, err + }) } - - // Detached from the activation RPC's context but cancelable: the relay's - // capacity is the worker's, so teardown must drop queries still in flight. - serveCtx, stopServing := context.WithCancel(context.WithoutCancel(ctx)) - go func() { - if err := relay.ServePacket(serveCtx, packet); err != nil { - slog.WarnContext(ctx, "Actor DNS socket stopped", slog.Any("err", err)) - } - }() - go func() { - if err := relay.Serve(serveCtx, stream); err != nil { - slog.WarnContext(ctx, "Actor DNS listener stopped", slog.Any("err", err)) - } - }() - // Cancel first: closing the sockets alone leaves the queries already being - // resolved holding the relay. - return []io.Closer{closerFunc(func() error { stopServing(); return nil }), packet, stream}, nil + return closers, serve, nil } // closerFunc adapts a cancel function to io.Closer, so a caller takes a @@ -536,3 +469,163 @@ func validateNetNSDialTarget(network, addr string) error { } return nil } + +// SandboxSession owns a sandbox's network and serving sockets. +type SandboxSession struct { + Network *SandboxNetwork + + mu sync.Mutex + sockets []io.Closer + // serving counts the goroutines serving this sandbox, so Close can wait + // for them rather than just closing their sockets. + serving sync.WaitGroup +} + +// ServeSandbox builds a sandbox's network and serves egress and DNS from its +// gateway namespace. A nil server leaves that unserved, which fails closed. +func ServeSandbox(ctx context.Context, cfg SandboxNetworkConfig, egress egressServer, dns dnsServer) (_ *SandboxSession, retErr error) { + network, err := SetupSandboxNetwork(ctx, cfg) + if err != nil { + return nil, err + } + session := &SandboxSession{Network: network} + defer func() { + if retErr != nil { + _ = session.Close(ctx) + } + }() + + var serve []func() + if egress != nil { + closers, serveEgress, err := serveSandboxEgress(ctx, egress, network.GatewayNetNS, []uint16{cfg.EgressPort}) + if err != nil { + return nil, err + } + session.sockets = append(session.sockets, closers...) + serve = append(serve, serveEgress...) + } + if dns != nil { + closers, serveDNS, err := serveSandboxDNS(ctx, dns, network.GatewayNetNS, cfg.DNSPort) + if err != nil { + return nil, err + } + session.sockets = append(session.sockets, closers...) + serve = append(serve, serveDNS...) + } + + // Started here rather than inside the helpers so the session owns them and + // Close can report when they have stopped. + for _, fn := range serve { + session.serving.Add(1) + go func() { + defer session.serving.Done() + fn() + }() + } + return session, nil +} + +// Close cancels the work in flight, closes the sockets, waits for the serving +// goroutines, then removes the namespaces. ctx bounds only the wait; the +// namespaces go either way, since leaving them wedges the next activation. +// Idempotent, and every step's error is returned. +func (s *SandboxSession) Close(ctx context.Context) error { + s.mu.Lock() + sockets, network := s.sockets, s.Network + s.sockets, s.Network = nil, nil + s.mu.Unlock() + + var errs error + for _, c := range sockets { + if err := c.Close(); err != nil && !errors.Is(err, net.ErrClosed) { + errs = errors.Join(errs, err) + } + } + + stopped := make(chan struct{}) + go func() { + s.serving.Wait() + close(stopped) + }() + select { + case <-stopped: + case <-ctx.Done(): + errs = errors.Join(errs, fmt.Errorf("while waiting for the sandbox's serving goroutines: %w", ctx.Err())) + } + + if network != nil { + errs = errors.Join(errs, CleanupSandboxNetwork(network)) + } + return errs +} + +// SessionHolder is the one sandbox session a worker is serving, for the ateoms +// to share what is otherwise the same locking, replacement and dialing in both. +type SessionHolder struct { + mu sync.Mutex + session *SandboxSession +} + +// Replace closes whatever session is held and installs next. A previous +// actor's session can still be here if its teardown never ran, and overwriting +// it would leak both namespaces, their /run/netns mounts and the goroutines +// serving them. +func (h *SessionHolder) Replace(ctx context.Context, next *SandboxSession) error { + if err := h.Close(ctx); err != nil { + return fmt.Errorf("while releasing the previous sandbox network: %w", err) + } + h.mu.Lock() + defer h.mu.Unlock() + h.session = next + return nil +} + +// Session is what is held, or nil between activations. +func (h *SessionHolder) Session() *SandboxSession { + h.mu.Lock() + defer h.mu.Unlock() + return h.session +} + +// Close releases the held session, if any. +func (h *SessionHolder) Close(ctx context.Context) error { + h.mu.Lock() + session := h.session + h.session = nil + h.mu.Unlock() + if session == nil { + return nil + } + return session.Close(ctx) +} + +// Dialer reaches whichever sandbox is held when the dial happens. +func (h *SessionHolder) Dialer() func(context.Context, string, string) (net.Conn, error) { + return func(ctx context.Context, network, address string) (net.Conn, error) { + session := h.Session() + if session == nil { + return nil, errors.New("no actor is active on this worker") + } + return session.Dialer()(ctx, network, address) + } +} + +// Dialer reaches the sandbox from the gateway namespace, the only place its +// address is routable. +func (s *SandboxSession) Dialer() func(context.Context, string, string) (net.Conn, error) { + return func(ctx context.Context, network, address string) (net.Conn, error) { + s.mu.Lock() + if s.Network == nil { + s.mu.Unlock() + return nil, net.ErrClosed + } + fd, err := unix.FcntlInt(uintptr(s.Network.GatewayNetNS), unix.F_DUPFD_CLOEXEC, 0) + s.mu.Unlock() + if err != nil { + return nil, fmt.Errorf("while retaining the sandbox namespace: %w", err) + } + ns := netns.NsHandle(fd) + defer ns.Close() + return NetNSDialer(ns)(ctx, network, address) + } +} diff --git a/internal/ateomnet/sandbox_linux_test.go b/internal/ateomnet/sandbox_linux_test.go index 1982a13f58..237932d448 100644 --- a/internal/ateomnet/sandbox_linux_test.go +++ b/internal/ateomnet/sandbox_linux_test.go @@ -20,8 +20,6 @@ import ( "context" "errors" "fmt" - "github.com/agent-substrate/substrate/internal/ateompath" - "github.com/vishvananda/netlink" "io" "net" "net/http" @@ -32,7 +30,11 @@ import ( "testing" "time" + "github.com/agent-substrate/substrate/internal/ateompath" + "github.com/vishvananda/netlink" + "github.com/agent-substrate/substrate/internal/roottest" + "github.com/vishvananda/netns" ) const testEgressPort = 15001 @@ -138,6 +140,53 @@ func TestNetNSDialerDoesNotPinPendingThreads(t *testing.T) { } } +func TestSandboxSessionDialerAfterClose(t *testing.T) { + session := &SandboxSession{} + dial := session.Dialer() + if err := session.Close(context.Background()); err != nil { + t.Fatal(err) + } + if _, err := dial(context.Background(), "tcp", "127.0.0.1:1"); !errors.Is(err, net.ErrClosed) { + t.Fatalf("dial after close: got %v, want closed", err) + } +} + +func TestSandboxSessionDialerConcurrentClose(t *testing.T) { + roottest.Require(t, "creates network namespaces") + session, err := ServeSandbox(context.Background(), SandboxNetworkConfig{ + ActorUID: "concurrent-close", EgressPort: testEgressPort, + }, nil, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = session.Close(context.Background()) }) + dial := session.Dialer() + start := make(chan struct{}) + var workers sync.WaitGroup + for range 32 { + workers.Add(1) + go func() { + defer workers.Done() + <-start + for range 32 { + conn, err := dial(context.Background(), "udp", "127.0.0.1:9") + if err != nil { + if !errors.Is(err, net.ErrClosed) { + t.Errorf("dial during close: %v", err) + } + return + } + _ = conn.Close() + } + }() + } + close(start) + if err := session.Close(context.Background()); err != nil { + t.Error(err) + } + workers.Wait() +} + func TestNetNSDialerCanceledContext(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() @@ -165,9 +214,6 @@ func TestSetupSandboxNetwork(t *testing.T) { t.Errorf("cleanup %s: %v", uid, err) } }) - if got := n.PodSideIP.String(); got != ActorVethIP { - t.Errorf("actor address = %s, want the same %s every actor holds", got, ActorVethIP) - } // The actor's app, bound where a real one binds, inside its namespace. var lis net.Listener @@ -318,41 +364,6 @@ func TestSetupSandboxNetworkWithoutVeth(t *testing.T) { } } -// A micro-VM snapshot freezes the guest's ARP entry for its gateway, so the -// gateway has to answer with the same MAC on every worker. -func TestGatewayHardwareAddressIsFixedWhenAsked(t *testing.T) { - roottest.Require(t, "creates network namespaces") - ctx := context.Background() - - want, err := net.ParseMAC("02:00:00:00:17:01") - if err != nil { - t.Fatal(err) - } - n, err := SetupSandboxNetwork(ctx, SandboxNetworkConfig{ - ActorUID: "88888888-8888-8888-8888-888888888888", - Veth: true, - EgressPort: testEgressPort, - GatewayHWAddr: want, - }) - if err != nil { - t.Fatalf("SetupSandboxNetwork: %v", err) - } - t.Cleanup(func() { CleanupSandboxNetwork(n) }) - - if err := NetNSDo(ctx, n.GatewayNetNS, func(context.Context) error { - l, err := netlink.LinkByName("atside") - if err != nil { - return err - } - if got := l.Attrs().HardwareAddr.String(); got != want.String() { - t.Errorf("gateway MAC is %s, want %s", got, want) - } - return nil - }); err != nil { - t.Fatalf("reading the gateway link: %v", err) - } -} - func TestSetupSucceedsOverALeftoverNamespace(t *testing.T) { roottest.Require(t, "creates network namespaces") ctx := context.Background() @@ -464,10 +475,13 @@ func TestClosingSandboxDNSStopsServing(t *testing.T) { defer func() { _ = CleanupSandboxNetwork(network) }() relay := &stoppableDNS{packet: make(chan struct{}), stream: make(chan struct{})} - closers, err := ServeSandboxDNS(context.Background(), relay, network.GatewayNetNS, 53) + closers, serve, err := serveSandboxDNS(context.Background(), relay, network.GatewayNetNS, 53) if err != nil { t.Fatal(err) } + for _, fn := range serve { + go fn() + } for _, c := range closers { _ = c.Close() } @@ -483,3 +497,75 @@ func TestClosingSandboxDNSStopsServing(t *testing.T) { } } } + +// slowDNS holds its serving goroutines open until released, so a test can tell +// whether Close waits for them or merely closes their sockets. +type slowDNS struct{ release chan struct{} } + +func (d *slowDNS) ServePacket(ctx context.Context, pc net.PacketConn) error { + <-ctx.Done() + <-d.release + return pc.Close() +} + +func (d *slowDNS) Serve(ctx context.Context, l net.Listener) error { + <-ctx.Done() + <-d.release + return l.Close() +} + +// Close's contract is that serving has stopped when it returns, not just that +// the sockets are shut: a caller tearing an actor down needs the relay's +// capacity back. +func TestSessionCloseWaitsForServingToStop(t *testing.T) { + roottest.Require(t, "creates network namespaces") + relay := &slowDNS{release: make(chan struct{})} + session, err := ServeSandbox(context.Background(), SandboxNetworkConfig{ + ActorUID: "close-waits", EgressPort: testEgressPort, DNSPort: 53, + }, nil, relay) + if err != nil { + t.Fatal(err) + } + + returned := make(chan error, 1) + go func() { returned <- session.Close(context.Background()) }() + select { + case <-returned: + t.Fatal("Close returned while the relay was still serving") + case <-time.After(250 * time.Millisecond): + } + + close(relay.release) + select { + case err := <-returned: + if err != nil { + t.Errorf("Close: %v", err) + } + case <-time.After(30 * time.Second): + t.Error("Close did not return after serving stopped") + } +} + +// A caller that cannot wait forever gets its deadline back as an error, and +// the namespaces are still removed -- leaving them would wedge the next +// activation of this actor. +func TestSessionCloseReportsAWaitItCouldNotFinish(t *testing.T) { + roottest.Require(t, "creates network namespaces") + relay := &slowDNS{release: make(chan struct{})} + defer close(relay.release) + session, err := ServeSandbox(context.Background(), SandboxNetworkConfig{ + ActorUID: "close-deadline", EgressPort: testEgressPort, DNSPort: 53, + }, nil, relay) + if err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + if err := session.Close(ctx); !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("Close = %v, want the deadline reported", err) + } + if _, err := netns.GetFromName(ateompath.ActorNetNSName("close-deadline")); err == nil { + t.Error("the namespace survived a Close whose wait timed out") + } +} diff --git a/internal/ateompath/ateompath.go b/internal/ateompath/ateompath.go index 277673b276..ea5ed4a50d 100644 --- a/internal/ateompath/ateompath.go +++ b/internal/ateompath/ateompath.go @@ -82,18 +82,6 @@ func AteletOTLPSocketPath() string { // AteomsDir is the parent of every per-ateom directory. Each ateom creates // AteomPath(podUID) under it when it boots, so listing this directory is how a -// ActorNetNSName is the named network namespace one actor's sandbox runs in. -// Per actor rather than per pod, so a worker can hold several sandboxes whose -// networks cannot see each other. -func ActorNetNSName(actorUID string) string { - return "ateom-actor:" + actorUID -} - -// ActorNetNSPath is where the kernel exposes that namespace. -func ActorNetNSPath(actorUID string) string { - return filepath.Join("/run/netns", ActorNetNSName(actorUID)) -} - // scraper with no prior knowledge discovers the node's ateoms. func AteomsDir() string { return filepath.Join(BasePath, "ateoms") @@ -110,15 +98,19 @@ func AteomSocketPath(podUID string) string { ) } -func AteomNetNSName(podUID string) string { - return "ateom:" + podUID +// ActorNetNSName names an actor's sandbox network namespace. +func ActorNetNSName(actorUID string) string { + return "ateom-actor:" + actorUID } -func AteomNetNSPath(podUID string) string { - return filepath.Join( - "/run/netns", - AteomNetNSName(podUID), - ) +// ActorNetNSPath is the mount path of the actor's named namespace. +func ActorNetNSPath(actorUID string) string { + return filepath.Join("/run/netns", ActorNetNSName(actorUID)) +} + +// ActorResolvConfPath is the resolver bind source outside the actor's rootfs. +func ActorResolvConfPath(actorUID string) string { + return filepath.Join(ActorPath(actorUID), "resolv.conf") } func ActorPath(actorUID string) string { diff --git a/internal/atunnel/egress.go b/internal/atunnel/egress.go index a31396c584..4173e3b9a1 100644 --- a/internal/atunnel/egress.go +++ b/internal/atunnel/egress.go @@ -303,3 +303,18 @@ func closeWrite(conn net.Conn) { _ = conn.CloseWrite() } } + +// EgressPort is the port from a listen address, which each sandbox's redirect +// aims at. The address itself is never bound: egress is served from inside the +// sandbox namespaces. +func EgressPort(listenAddress string) (uint16, error) { + _, port, err := net.SplitHostPort(listenAddress) + if err != nil { + return 0, fmt.Errorf("atunnel: egress listen address %q: %w", listenAddress, err) + } + p, ok := ParsePort(port) + if !ok { + return 0, fmt.Errorf("atunnel: egress listen address %q has no usable port", listenAddress) + } + return uint16(p), nil +} diff --git a/internal/atunnel/ingress.go b/internal/atunnel/ingress.go index 3eb5f8d125..8500cc479f 100644 --- a/internal/atunnel/ingress.go +++ b/internal/atunnel/ingress.go @@ -69,6 +69,10 @@ type Config struct { TrustBundlePath string AllowedClientID string Upstream *url.URL + // Dial reaches the sandbox, for both proxied requests and CONNECT tunnels. + // Needed when it is reachable only from inside its own network namespace; + // nil dials from the worker's. + Dial DialFunc } // Server is an HTTPS reverse proxy for the worker's active actors. @@ -76,7 +80,13 @@ type Server struct { credentialBundlePath string tlsConfig *tls.Config proxy *httputil.ReverseProxy - upstream *url.URL + // newTransport builds an activation's round tripper. A field so a test can + // substitute one without the production path branching on its type. + newTransport func(DialFunc) http.RoundTripper + upstream *url.URL + // dial reaches the sandbox, for CONNECT tunnels. The reverse proxy's + // transport is given the same dialer. + dial DialFunc mu sync.Mutex active *activation @@ -87,6 +97,8 @@ type activation struct { ctx context.Context cancel context.CancelFunc wg sync.WaitGroup + dial DialFunc + proxy *httputil.ReverseProxy } // NewServer creates a Server and validates its TLS material. @@ -119,7 +131,11 @@ func NewServer(cfg Config) (*Server, error) { return nil, fmt.Errorf("atunnel: trust bundle %q contains no certificates", cfg.TrustBundlePath) } - transport := newProtocolMirrorTransport() + dial := cfg.Dial + if dial == nil { + dial = (&net.Dialer{}).DialContext + } + transport := newProtocolMirrorTransport(dial) proxy := &httputil.ReverseProxy{ Rewrite: func(pr *httputil.ProxyRequest) { pr.SetURL(cfg.Upstream) @@ -146,6 +162,8 @@ func NewServer(cfg Config) (*Server, error) { credentialBundlePath: cfg.CredentialBundlePath, proxy: proxy, upstream: cfg.Upstream, + dial: dial, + newTransport: func(d DialFunc) http.RoundTripper { return newProtocolMirrorTransport(d) }, } s.tlsConfig = &tls.Config{ MinVersion: tls.VersionTLS12, @@ -206,9 +224,13 @@ type protocolMirrorTransport struct { h1, h2c *http.Transport } -func newProtocolMirrorTransport() protocolMirrorTransport { +func newProtocolMirrorTransport(dial DialFunc) protocolMirrorTransport { h1 := http.DefaultTransport.(*http.Transport).Clone() h2c := http.DefaultTransport.(*http.Transport).Clone() + if dial != nil { + h1.DialContext = dial + h2c.DialContext = dial + } protocols := new(http.Protocols) protocols.SetUnencryptedHTTP2(true) h2c.Protocols = protocols @@ -297,7 +319,7 @@ func (s *Server) ServeConnectHTTP(w http.ResponseWriter, r *http.Request) { http.Error(w, "CONNECT required", http.StatusMethodNotAllowed) return } - ref, ctx, release, ok := s.authorize(r) + active, ctx, release, ok := s.authorize(r) if !ok { s.reject(w) return @@ -314,10 +336,11 @@ func (s *Server) ServeConnectHTTP(w http.ResponseWriter, r *http.Request) { return } - dialer := &net.Dialer{Timeout: 5 * time.Second} - upstream, err := dialer.DialContext(ctx, "tcp", net.JoinHostPort(s.upstream.Hostname(), port)) + dialCtx, cancelDial := context.WithTimeout(ctx, 5*time.Second) + defer cancelDial() + upstream, err := active.dial(dialCtx, "tcp", net.JoinHostPort(s.upstream.Hostname(), port)) if err != nil { - slog.WarnContext(r.Context(), "atunnel CONNECT upstream failed", slog.Any("actor", ref), slog.Any("err", err)) + slog.WarnContext(r.Context(), "atunnel CONNECT upstream failed", slog.Any("actor", active.ref), slog.Any("err", err)) http.Error(w, "bad gateway", http.StatusBadGateway) return } @@ -415,14 +438,41 @@ func (s *Server) Activate(atespace, actorName string) error { return fmt.Errorf("atunnel: actor %s is already active", s.active.ref) } ctx, cancel := context.WithCancel(context.Background()) + dial := activationDialer(ctx, s.dial) + // Its own transport, so an activation's pooled connections cannot outlive + // it and hand the next actor a socket to its predecessor. + proxy := *s.proxy + proxy.Transport = s.newTransport(dial) s.active = &activation{ ref: resources.ActorRef{Atespace: atespace, Name: actorName}, ctx: ctx, cancel: cancel, + dial: dial, + proxy: &proxy, } return nil } +func activationDialer(activeCtx context.Context, dial DialFunc) DialFunc { + return func(ctx context.Context, network, address string) (net.Conn, error) { + if err := activeCtx.Err(); err != nil { + return nil, err + } + ctx, cancel := context.WithCancel(ctx) + defer cancel() + stop := context.AfterFunc(activeCtx, cancel) + defer stop() + conn, err := dial(ctx, network, address) + if canceled := errors.Join(activeCtx.Err(), ctx.Err()); canceled != nil { + if conn != nil { + _ = conn.Close() + } + return nil, canceled + } + return conn, err + } +} + // Deactivate rejects new requests, cancels requests for the active actor, and // waits for their handlers to exit before returning. func (s *Server) Deactivate(ctx context.Context) error { @@ -444,43 +494,43 @@ func (s *Server) Deactivate(ctx context.Context) error { }() select { case <-done: - s.closeIdleUpstreamConnections() + closeIdleUpstreamConnections(active) return nil case <-ctx.Done(): - s.closeIdleUpstreamConnections() + closeIdleUpstreamConnections(active) return fmt.Errorf("atunnel: waiting for active requests to stop: %w", ctx.Err()) } } -func (s *Server) closeIdleUpstreamConnections() { - if transport, ok := s.proxy.Transport.(interface{ CloseIdleConnections() }); ok { +func closeIdleUpstreamConnections(active *activation) { + if transport, ok := active.proxy.Transport.(interface{ CloseIdleConnections() }); ok { transport.CloseIdleConnections() } } // ServeHTTP validates the actor routing header on every request before proxying it. func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { - _, requestCtx, release, ok := s.authorize(r) + active, requestCtx, release, ok := s.authorize(r) if !ok { s.reject(w) return } defer release() - s.proxy.ServeHTTP(w, r.WithContext(requestCtx)) + active.proxy.ServeHTTP(w, r.WithContext(requestCtx)) } -func (s *Server) authorize(r *http.Request) (resources.ActorRef, context.Context, func(), bool) { +func (s *Server) authorize(r *http.Request) (*activation, context.Context, func(), bool) { ref, err := atenet.ParseTargetActor(r.Header.Get(atenet.TargetActorHeader)) if err != nil { - return resources.ActorRef{}, nil, nil, false + return nil, nil, nil, false } s.mu.Lock() active := s.active if active == nil || active.ref != ref { s.mu.Unlock() - return resources.ActorRef{}, nil, nil, false + return nil, nil, nil, false } active.wg.Add(1) s.mu.Unlock() @@ -491,7 +541,7 @@ func (s *Server) authorize(r *http.Request) (resources.ActorRef, context.Context stop() cancel() } - return ref, requestCtx, release, true + return active, requestCtx, release, true } func (s *Server) reject(w http.ResponseWriter) { diff --git a/internal/atunnel/ingress_test.go b/internal/atunnel/ingress_test.go index 5efd7afeb7..3344170baf 100644 --- a/internal/atunnel/ingress_test.go +++ b/internal/atunnel/ingress_test.go @@ -24,6 +24,7 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/pem" + "errors" "io" "math/big" "net" @@ -32,12 +33,99 @@ import ( "net/url" "os" "path/filepath" + "sync/atomic" "testing" "time" "github.com/agent-substrate/substrate/internal/atenet" ) +func TestActivationDialerClosesLateConnection(t *testing.T) { + client, peer := net.Pipe() + defer peer.Close() + defer client.Close() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dial := activationDialer(ctx, func(context.Context, string, string) (net.Conn, error) { + cancel() + return client, nil + }) + conn, err := dial(context.Background(), "tcp", "actor:80") + if conn != nil || !errors.Is(err, context.Canceled) { + t.Fatalf("late dial returned %v, %v", conn, err) + } + _ = peer.SetReadDeadline(time.Now().Add(time.Second)) + if _, err := peer.Read(make([]byte, 1)); !errors.Is(err, io.EOF) { + t.Fatalf("late connection was not closed: %v", err) + } +} + +func TestDeactivateIsolatesLateDials(t *testing.T) { + firstActor := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, "actor-1") + })) + defer firstActor.Close() + secondActor := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, "actor-2") + })) + defer secondActor.Close() + upstream, _ := url.Parse("http://127.0.0.1:80") + server := newTestServer(t, upstream) + firstReady := make(chan struct{}) + secondReady := make(chan struct{}) + releaseFirst := make(chan struct{}) + releaseSecond := make(chan struct{}) + var calls atomic.Int32 + server.dial = func(ctx context.Context, network, address string) (net.Conn, error) { + if calls.Add(1) == 1 { + conn, err := net.Dial("tcp", firstActor.Listener.Addr().String()) + close(firstReady) + <-releaseFirst + return conn, err + } + close(secondReady) + <-releaseSecond + return (&net.Dialer{}).DialContext(ctx, "tcp", secondActor.Listener.Addr().String()) + } + if err := server.Activate("team-a", "actor-1"); err != nil { + t.Fatal(err) + } + firstActivation := server.active + request := func(actorName string, done chan string) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + req := httptest.NewRequest(http.MethodGet, "http://worker/", nil).WithContext(ctx) + req.Header.Set(atenet.TargetActorHeader, "team-a/"+actorName) + recorder := httptest.NewRecorder() + server.ServeHTTP(recorder, req) + done <- recorder.Body.String() + } + firstDone := make(chan string, 1) + go request("actor-1", firstDone) + receiveWithin(t, firstReady, "first dial") + if err := server.Deactivate(context.Background()); err != nil { + t.Fatal(err) + } + receiveWithin(t, firstDone, "first request cancellation") + if err := server.Activate("team-a", "actor-2"); err != nil { + t.Fatal(err) + } + if firstActivation.proxy.Transport == server.active.proxy.Transport { + t.Error("activations share a transport") + } + secondDone := make(chan string, 1) + go request("actor-2", secondDone) + receiveWithin(t, secondReady, "second dial") + close(releaseFirst) + close(releaseSecond) + if body := receiveWithin(t, secondDone, "second response"); body != "actor-2" { + t.Errorf("second actor received %q", body) + } + if err := server.Deactivate(context.Background()); err != nil { + t.Fatal(err) + } +} + func TestRelayIngressWithHalfClose(t *testing.T) { listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { @@ -138,14 +226,14 @@ func TestServeHTTP(t *testing.T) { } s := newTestServer(t, upstreamURL) - s.proxy.Transport = roundTripFunc(func(r *http.Request) (*http.Response, error) { + setActivationTransport(s, roundTripFunc(func(r *http.Request) (*http.Response, error) { upstreamHosts = append(upstreamHosts, r.Host) return &http.Response{ StatusCode: http.StatusNoContent, Header: make(http.Header), Body: http.NoBody, }, nil - }) + })) if err := s.Activate("team-a", "actor-1"); err != nil { t.Fatal(err) } @@ -207,7 +295,7 @@ func TestServeHTTPHonorsTargetPortHeader(t *testing.T) { s := newTestServer(t, upstreamURL) var gotURLHost, gotHost http.Header - s.proxy.Transport = roundTripFunc(func(r *http.Request) (*http.Response, error) { + setActivationTransport(s, roundTripFunc(func(r *http.Request) (*http.Response, error) { gotURLHost = http.Header{"Host": []string{r.URL.Host}} gotHost = r.Header.Clone() gotHost.Set("Host", r.Host) @@ -216,7 +304,7 @@ func TestServeHTTPHonorsTargetPortHeader(t *testing.T) { Header: make(http.Header), Body: http.NoBody, }, nil - }) + })) if err := s.Activate("team-a", "actor-1"); err != nil { t.Fatal(err) } @@ -326,7 +414,7 @@ func TestDeactivateClosesIdleUpstreamConnections(t *testing.T) { } s := newTestServer(t, upstream) transport := &idleClosingRoundTripper{} - s.proxy.Transport = transport + setActivationTransport(s, transport) if err := s.Activate("team-a", "actor-1"); err != nil { t.Fatal(err) } @@ -559,15 +647,15 @@ func TestDeactivateCancelsInflightRequest(t *testing.T) { t.Fatal(err) } s := newTestServer(t, upstream) - if err := s.Activate("team-a", "actor-1"); err != nil { - t.Fatal(err) - } started := make(chan struct{}) - s.proxy.Transport = roundTripFunc(func(r *http.Request) (*http.Response, error) { + setActivationTransport(s, roundTripFunc(func(r *http.Request) (*http.Response, error) { close(started) <-r.Context().Done() return nil, r.Context().Err() - }) + })) + if err := s.Activate("team-a", "actor-1"); err != nil { + t.Fatal(err) + } done := make(chan struct{}) go func() { @@ -584,6 +672,13 @@ func TestDeactivateCancelsInflightRequest(t *testing.T) { receiveWithin(t, done, "canceled in-flight request") } +// setActivationTransport makes every activation use rt, standing in for the +// per-activation transport the server builds in production. +func setActivationTransport(s *Server, rt http.RoundTripper) { + s.newTransport = func(DialFunc) http.RoundTripper { return rt } + s.proxy.Transport = rt +} + func newTestServer(t *testing.T, upstream *url.URL) *Server { t.Helper() dir := t.TempDir() @@ -847,7 +942,7 @@ func TestProtocolMirrorTransport(t *testing.T) { } { t.Run(tt.name, func(t *testing.T) { addr, protoSeen := mirrorBackend(t, tt.backendH1, tt.backendH2C) - transport := newProtocolMirrorTransport() + transport := newProtocolMirrorTransport(nil) req, err := http.NewRequest(tt.method, "http://"+addr+"/", http.NoBody) if err != nil { t.Fatal(err) @@ -886,3 +981,62 @@ func receiveWithin[T any](t *testing.T, channel <-chan T, description string) T return zero } } + +// CONNECT must use the configured dialer independently of the proxy transport. +func TestServeConnectHTTPDialsTheSandbox(t *testing.T) { + actor, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer actor.Close() + go func() { + conn, err := actor.Accept() + if err != nil { + return + } + defer conn.Close() + _, _ = io.WriteString(conn, "hello from the sandbox") + }() + + upstreamURL, err := url.Parse("http://actor.internal:80") + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + bundle, trust := makeCertFiles(t, dir) + var dialed string + s, err := NewServer(Config{ + CredentialBundlePath: bundle, + TrustBundlePath: trust, + AllowedClientID: "spiffe://cluster.local/ns/ate-system/sa/atenet-router", + Upstream: upstreamURL, + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + dialed = address + return (&net.Dialer{}).DialContext(ctx, network, actor.Addr().String()) + }, + }) + if err != nil { + t.Fatal(err) + } + if err := s.Activate("team-a", "actor-1"); err != nil { + t.Fatal(err) + } + + // HTTP/2 CONNECT supports a recorder without socket hijacking. + req := httptest.NewRequest(http.MethodConnect, "https://worker/", http.NoBody) + req.ProtoMajor, req.ProtoMinor, req.Proto = 2, 0, "HTTP/2.0" + req.Host = "actor-1.team-a.actors.resources.substrate.ate.dev:9090" + req.Header.Set(atenet.TargetActorHeader, "team-a/actor-1") + rec := httptest.NewRecorder() + s.ServeConnectHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: the tunnel never reached the sandbox", rec.Code, http.StatusOK) + } + if got, want := rec.Body.String(), "hello from the sandbox"; got != want { + t.Errorf("tunnel carried %q, want %q", got, want) + } + if want := "actor.internal:9090"; dialed != want { + t.Errorf("dialed %q through the sandbox dialer, want %q", dialed, want) + } +} diff --git a/internal/proto/ateletpb/atelet.pb.go b/internal/proto/ateletpb/atelet.pb.go index 4c4047ae8c..54893861d0 100644 --- a/internal/proto/ateletpb/atelet.pb.go +++ b/internal/proto/ateletpb/atelet.pb.go @@ -546,7 +546,7 @@ type RunRequest struct { // fetches the relevant assets and records them with the actor's on-node state // so a later Checkpoint can pin the same version into the snapshot manifest. SandboxAssets *SandboxAssets `protobuf:"bytes,8,opt,name=sandbox_assets,json=sandboxAssets,proto3" json:"sandbox_assets,omitempty"` - // When absent, actor traffic uses direct egress instead of atunnel. + // When absent the actor has no egress: its TCP is captured and refused. EgressGateway *EgressGateway `protobuf:"bytes,9,opt,name=egress_gateway,json=egressGateway,proto3,oneof" json:"egress_gateway,omitempty"` // The actor's declared size, from the ActorTemplate's resource limits. atelet // passes these through to the sandbox so it is sized to the actor (not the @@ -2464,7 +2464,7 @@ type RestoreRequest struct { // of the `config` oneof: the actor's snapshot may be local (a pause // checkpoint) while the golden snapshot is always external. GoldenSnapshotUri string `protobuf:"bytes,12,opt,name=golden_snapshot_uri,json=goldenSnapshotUri,proto3" json:"golden_snapshot_uri,omitempty"` - // When absent, actor traffic uses direct egress instead of atunnel. + // When absent the actor has no egress: its TCP is captured and refused. EgressGateway *EgressGateway `protobuf:"bytes,13,opt,name=egress_gateway,json=egressGateway,proto3,oneof" json:"egress_gateway,omitempty"` // The actor's declared size, from the ActorTemplate's resource limits. For // gVisor and micro-VM DATA-scope restores the sandbox is (re)sized to these; diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index c4c79201dc..158cf7d7d1 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -125,7 +125,7 @@ message RunRequest { // so a later Checkpoint can pin the same version into the snapshot manifest. SandboxAssets sandbox_assets = 8; - // When absent, actor traffic uses direct egress instead of atunnel. + // When absent the actor has no egress: its TCP is captured and refused. optional EgressGateway egress_gateway = 9; // The actor's declared size, from the ActorTemplate's resource limits. atelet @@ -451,7 +451,7 @@ message RestoreRequest { // checkpoint) while the golden snapshot is always external. string golden_snapshot_uri = 12; - // When absent, actor traffic uses direct egress instead of atunnel. + // When absent the actor has no egress: its TCP is captured and refused. optional EgressGateway egress_gateway = 13; // The actor's declared size, from the ActorTemplate's resource limits. For diff --git a/internal/proto/ateompb/ateom.pb.go b/internal/proto/ateompb/ateom.pb.go index 59ce521b64..a6ae0f36e9 100644 --- a/internal/proto/ateompb/ateom.pb.go +++ b/internal/proto/ateompb/ateom.pb.go @@ -354,7 +354,7 @@ type RunWorkloadRequest struct { // to the local on-disk path atelet fetched it to (content-addressed, like // runsc_path). Empty for the gVisor runtime, which uses runsc_path. RuntimeAssetPaths map[string]string `protobuf:"bytes,8,rep,name=runtime_asset_paths,json=runtimeAssetPaths,proto3" json:"runtime_asset_paths,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - // When absent, actor traffic uses direct egress instead of atunnel. + // When absent the actor has no egress: its TCP is captured and refused. EgressGateway *EgressGateway `protobuf:"bytes,10,opt,name=egress_gateway,json=egressGateway,proto3,oneof" json:"egress_gateway,omitempty"` // The actor's declared size, from the ActorTemplate's resource limits. ateom // sizes the sandbox to these (cgroup caps via the OCI spec, and for the @@ -1220,7 +1220,7 @@ type RestoreWorkloadRequest struct { RuntimeAssetPaths map[string]string `protobuf:"bytes,9,rep,name=runtime_asset_paths,json=runtimeAssetPaths,proto3" json:"runtime_asset_paths,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // What content to restore from the snapshot. Scope SnapshotScope `protobuf:"varint,10,opt,name=scope,proto3,enum=ateom.SnapshotScope" json:"scope,omitempty"` - // When absent, actor traffic uses direct egress instead of atunnel. + // When absent the actor has no egress: its TCP is captured and refused. EgressGateway *EgressGateway `protobuf:"bytes,12,opt,name=egress_gateway,json=egressGateway,proto3,oneof" json:"egress_gateway,omitempty"` // The object storage URI of the ActorTemplate's golden snapshot. // Set only when scope is SNAPSHOT_SCOPE_DATA_ON_GOLDEN. Mirrors the diff --git a/internal/proto/ateompb/ateom.proto b/internal/proto/ateompb/ateom.proto index 15f57b09f1..ce11275cf1 100644 --- a/internal/proto/ateompb/ateom.proto +++ b/internal/proto/ateompb/ateom.proto @@ -136,7 +136,7 @@ message RunWorkloadRequest { // runsc_path). Empty for the gVisor runtime, which uses runsc_path. map runtime_asset_paths = 8; - // When absent, actor traffic uses direct egress instead of atunnel. + // When absent the actor has no egress: its TCP is captured and refused. optional EgressGateway egress_gateway = 10; // The actor's declared size, from the ActorTemplate's resource limits. ateom @@ -318,7 +318,7 @@ message RestoreWorkloadRequest { // What content to restore from the snapshot. SnapshotScope scope = 10; - // When absent, actor traffic uses direct egress instead of atunnel. + // When absent the actor has no egress: its TCP is captured and refused. optional EgressGateway egress_gateway = 12; // The object storage URI of the ActorTemplate's golden snapshot.