diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 6155e49aa0..6ab6138e2a 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -1449,7 +1449,7 @@ func (s *AteomHerder) prepareOCIBundles( []string{"/pause"}, nil, nil, - ateompath.AteomNetNSPath(targetAteomUid), + ateompath.ActorNetNSPath(actorUID), nil, // pause is sandbox infra; it mounts no volumes. nil, nil, // pause only reaps; it needs no capabilities. @@ -1477,7 +1477,7 @@ func (s *AteomHerder) prepareOCIBundles( ctr.GetCommand(), ctr.GetArgs(), envs, - ateompath.AteomNetNSPath(targetAteomUid), + ateompath.ActorNetNSPath(actorUID), spec.GetVolumes(), ctr.GetVolumeMounts(), resolveCapabilities(ctr.GetSecurityContext().GetCapabilities()), diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index 2b965f7217..722d11d150 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -54,7 +54,6 @@ import ( "github.com/agent-substrate/substrate/internal/sizing" "github.com/agent-substrate/substrate/internal/version" "github.com/spf13/pflag" - "github.com/vishvananda/netns" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "golang.org/x/sys/unix" "google.golang.org/grpc" @@ -195,26 +194,30 @@ func do(ctx context.Context) error { return fmt.Errorf("while opening unix socket: %w", err) } - // Create a new network namespace that we will pass to gVisor. gVisor will - // read the addresses and routes off of every link in the namespace, then - // remove all the addresses and handle injecting packets into the interfaces - // using AF_PACKET. - interiorNetNS, err := ateomnet.CreateNetNSWithoutSwitching(ateompath.AteomNetNSName(*podUID)) - if err != nil { - return fmt.Errorf("while creating ateom-interior netns: %w", err) - } - actorLogger := actorlog.NewActorLogger(syncedWriter, metadata.OnGCE()) upstream, err := url.Parse(actorHTTPUpstream) if err != nil { return fmt.Errorf("while parsing atunnel upstream: %w", err) } - atunnelIngress, atunnelEgress, atunnelEgressPort, err := runAtunnel(ctx, upstream) + // Use the pod's resolvers for cluster DNS access. + nameservers, err := atunnel.ResolvConfNameservers("/etc/resolv.conf") if err != nil { - return err + return fmt.Errorf("while reading the worker pod resolvers: %w", err) + } + dnsRelay, err := atunnel.NewDNSRelay(nameservers) + if err != nil { + return fmt.Errorf("while building the actor DNS relay: %w", err) } + slog.InfoContext(ctx, "Actor DNS relay ready", slog.Any("upstreams", nameservers)) - ateomService := NewService(interiorNetNS, actorLogger, atunnelIngress, atunnelEgress, atunnelEgressPort, *workerCredentialBundle, *podIdentityTrustBundle, *egressGatewayTrustBundle) + // Construct the service first so atunnel can use its namespace dialer. + ateomService := NewService(dnsRelay, actorLogger, *workerCredentialBundle, *podIdentityTrustBundle, *egressGatewayTrustBundle) + + atunnelIngress, atunnelEgress, atunnelEgressPort, err := runAtunnel(ctx, upstream, ateomService.sandbox.Dialer()) + if err != nil { + return err + } + ateomService.attachAtunnel(atunnelIngress, atunnelEgress, atunnelEgressPort) svr := grpc.NewServer( grpc.StatsHandler(otelgrpc.NewServerHandler()), @@ -266,12 +269,13 @@ func do(ctx context.Context) error { return nil } -func runAtunnel(ctx context.Context, upstream *url.URL) (*atunnel.Server, *atunnel.Egress, uint16, error) { +func runAtunnel(ctx context.Context, upstream *url.URL, dial atunnel.DialFunc) (*atunnel.Server, *atunnel.Egress, uint16, error) { atunnelIngress, err := atunnel.NewServer(atunnel.Config{ CredentialBundlePath: *workerCredentialBundle, TrustBundlePath: *podIdentityTrustBundle, AllowedClientID: *atunnelClientIdentity, Upstream: upstream, + Dial: dial, }) if err != nil { return nil, nil, 0, fmt.Errorf("while configuring atunnel: %w", err) @@ -301,22 +305,11 @@ func runAtunnel(ctx context.Context, upstream *url.URL) (*atunnel.Server, *atunn if err != nil { return nil, nil, 0, fmt.Errorf("while configuring atunnel egress: %w", err) } - egressListener, err := net.Listen("tcp", *atunnelEgressListenAddress) + // Bind egress only in sandbox namespaces. + atunnelEgressPort, err := atunnel.EgressPort(*atunnelEgressListenAddress) if err != nil { - return nil, nil, 0, fmt.Errorf("while opening atunnel egress listener: %w", err) - } - egressTCPAddr, ok := egressListener.Addr().(*net.TCPAddr) - if !ok || egressTCPAddr.Port < 1 || egressTCPAddr.Port > 65535 { - _ = egressListener.Close() - return nil, nil, 0, fmt.Errorf("atunnel egress listener has invalid address %q", egressListener.Addr()) + return nil, nil, 0, err } - atunnelEgressPort := uint16(egressTCPAddr.Port) - go func() { - if err := atunnelEgress.Serve(ctx, egressListener); err != nil { - serverboot.Fatal(ctx, "Failed to serve actor egress", err) - } - }() - slog.InfoContext(ctx, "atunnel egress serving", slog.String("address", *atunnelEgressListenAddress)) return atunnelIngress, atunnelEgress, atunnelEgressPort, nil } @@ -374,11 +367,16 @@ type AteomService struct { // subcommands are probably not safe to call concurrently. lock *cancelableMutex - interiorNetNS netns.NsHandle + // sandbox is the network of the actor this worker is serving. + sandbox ateomnet.SessionHolder + actorLogger *actorlog.ActorLogger atunnelIngress *atunnel.Server atunnelEgress *atunnel.Egress + // dnsRelay answers the sandbox's DNS from inside its own namespace. + dnsRelay *atunnel.DNSRelay + // atunnelEgressPort is the local atunnel listener used as the target of the // actor network's transparent TCP redirect. atunnelEgressPort uint16 @@ -440,14 +438,11 @@ type AteomService struct { var _ ateompb.AteomServer = (*AteomService)(nil) // NewService creates a new AteomService. -func NewService(interiorNetNS netns.NsHandle, actorLogger *actorlog.ActorLogger, atunnelIngress *atunnel.Server, atunnelEgress *atunnel.Egress, atunnelEgressPort uint16, workerCredentialBundlePath, podIdentityTrustBundlePath, egressGatewayTrustBundlePath string) *AteomService { +func NewService(dnsRelay *atunnel.DNSRelay, actorLogger *actorlog.ActorLogger, workerCredentialBundlePath, podIdentityTrustBundlePath, egressGatewayTrustBundlePath string) *AteomService { return &AteomService{ lock: newCancelableMutex(), - interiorNetNS: interiorNetNS, + dnsRelay: dnsRelay, actorLogger: actorLogger, - atunnelIngress: atunnelIngress, - atunnelEgress: atunnelEgress, - atunnelEgressPort: atunnelEgressPort, workerCredentialBundlePath: workerCredentialBundlePath, podIdentityTrustBundlePath: podIdentityTrustBundlePath, egressGatewayTrustBundlePath: egressGatewayTrustBundlePath, @@ -666,15 +661,11 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload if err != nil { return nil, err } - if err := ateomnet.SetupActorNetwork(ctx, ateomnet.NetworkConfig{ - InteriorNetNS: s.interiorNetNS, - DumpNetInfo: true, - EgressRedirectPort: s.egressRedirectPort(req.GetEgressGateway() != nil), - }); err != nil { + if err := s.prepareSandboxNetwork(ctx, req.GetActorUid()); err != nil { // Cleared here as well as in the deferred cleanup below, because that // defer is not registered until after this check. s.activeActor.Store(nil) - return nil, fmt.Errorf("while setting up actor network: %w", err) + return nil, err } rcmd := &runsc{ path: req.GetRunscPath(), @@ -700,7 +691,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload slog.WarnContext(ctx, "Failed to unmount bundle rootfs overlays after Run failure", "actorUID", req.GetActorUid(), "err", err) } - if err := ateomnet.CleanupActorNetwork(cleanupCtx, s.interiorNetNS); err != nil { + if err := s.releaseSandboxNetwork(cleanupCtx); err != nil { slog.WarnContext(cleanupCtx, "Failed to clean up actor network after Run failure", slog.Any("err", err)) } } @@ -742,7 +733,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload } // Block until every readyz-enabled container reports 200. - if err := readyz.WaitAll(ctx, req.GetSpec().GetContainers(), ateomnet.ActorVethIP); err != nil { + if err := readyz.WaitAll(ctx, req.GetSpec().GetContainers(), ateomnet.ActorVethIP, readyz.DialFunc(s.sandbox.Dialer())); err != nil { return nil, fmt.Errorf("while waiting for container readyz: %w", err) } if err := s.activateActorNetworking(req.GetAtespace(), req.GetActorName(), egress); err != nil { @@ -965,14 +956,10 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore if err != nil { return nil, err } - if err := ateomnet.SetupActorNetwork(ctx, ateomnet.NetworkConfig{ - InteriorNetNS: s.interiorNetNS, - DumpNetInfo: true, - EgressRedirectPort: s.egressRedirectPort(req.GetEgressGateway() != nil), - }); err != nil { + if err := s.prepareSandboxNetwork(ctx, req.GetActorUid()); err != nil { // Same as the Run path: the defer below is not registered yet. s.activeActor.Store(nil) - return nil, fmt.Errorf("while setting up actor network: %w", err) + return nil, err } rcmd := &runsc{ path: req.GetRunscPath(), @@ -995,7 +982,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore slog.WarnContext(ctx, "Failed to unmount bundle rootfs overlays after Restore failure", "actorUID", req.GetActorUid(), "err", err) } - if err := ateomnet.CleanupActorNetwork(cleanupCtx, s.interiorNetNS); err != nil { + if err := s.releaseSandboxNetwork(cleanupCtx); err != nil { slog.WarnContext(cleanupCtx, "Failed to clean up actor network after Restore failure", slog.Any("err", err)) } } @@ -1071,7 +1058,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore } // Block until every readyz-enabled container reports 200. - if err := readyz.WaitAll(ctx, req.GetSpec().GetContainers(), ateomnet.ActorVethIP); err != nil { + if err := readyz.WaitAll(ctx, req.GetSpec().GetContainers(), ateomnet.ActorVethIP, readyz.DialFunc(s.sandbox.Dialer())); err != nil { return nil, fmt.Errorf("while waiting for container readyz: %w", err) } if err := s.activateActorNetworking(req.GetAtespace(), req.GetActorName(), egress); err != nil { @@ -1182,7 +1169,7 @@ func (s *AteomService) terminateWorkload(ctx context.Context, actorRef resources errs = append(errs, fmt.Errorf("while unmounting bundle rootfs overlays: %w", err)) } - if err := ateomnet.CleanupActorNetwork(ctx, s.interiorNetNS); err != nil { + if err := s.releaseSandboxNetwork(ctx); err != nil { errs = append(errs, fmt.Errorf("while cleaning up actor network: %w", err)) } @@ -1221,16 +1208,6 @@ func (s *AteomService) deactivateActorNetworking(ctx context.Context) error { return nil } -// egressRedirectPort returns the local atunnel egress listener port when the -// activation arms tunneled egress, and zero otherwise, which leaves the -// prerouting redirect uninstalled and actor egress on the masquerade path. -func (s *AteomService) egressRedirectPort(redirectEgress bool) uint16 { - if !redirectEgress { - return 0 - } - return s.atunnelEgressPort -} - // setupCgroupDelegation prepares the worker pod's cgroup so runsc can create a // per-actor-container leaf under it with real cpu/memory/pids accounting. // diff --git a/cmd/ateom-gvisor/runsc.go b/cmd/ateom-gvisor/runsc.go index 4720b9c4e0..bbf9e13ed5 100644 --- a/cmd/ateom-gvisor/runsc.go +++ b/cmd/ateom-gvisor/runsc.go @@ -69,6 +69,7 @@ func (r *runsc) shapeSpec(containerName string) error { ContainerName: containerName, DurableVolumes: r.durableVolumes, Size: r.size, + ResolvConf: ateompath.ActorResolvConfPath(r.actorUID), }) return ocispec.Save(bundle, spec) } diff --git a/cmd/ateom-gvisor/sandboxnet.go b/cmd/ateom-gvisor/sandboxnet.go new file mode 100644 index 0000000000..00b4a8ee36 --- /dev/null +++ b/cmd/ateom-gvisor/sandboxnet.go @@ -0,0 +1,89 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + "fmt" + "io/fs" + "log/slog" + "os" + "path/filepath" + + "github.com/agent-substrate/substrate/internal/ateomnet" + "github.com/agent-substrate/substrate/internal/ateompath" + "github.com/agent-substrate/substrate/internal/atunnel" +) + +// prepareSandboxNetwork builds the actor's network and starts serving it. +func (s *AteomService) prepareSandboxNetwork(ctx context.Context, actorUID string) error { + if err := s.releaseSandboxNetwork(ctx); err != nil { + return err + } + session, err := ateomnet.ServeSandbox(ctx, ateomnet.SandboxNetworkConfig{ + ActorUID: actorUID, + Veth: true, + EgressPort: s.atunnelEgressPort, + DNSPort: atunnel.DNSPort, + }, s.atunnelEgress, s.dnsRelay) + if err != nil { + return fmt.Errorf("while setting up the sandbox network: %w", err) + } + + // Point the sandbox resolver at its gateway. + if _, err := actorResolvConf(actorUID); err != nil { + _ = session.Close(ctx) + return err + } + + return s.sandbox.Replace(ctx, session) +} + +// releaseSandboxNetwork stops serving the actor and takes its network down, +// along with the resolv.conf atelet's per-activation reset leaves behind. +func (s *AteomService) releaseSandboxNetwork(ctx context.Context) error { + if session := s.sandbox.Session(); session != nil { + if err := os.Remove(ateompath.ActorResolvConfPath(session.Network.ActorUID)); err != nil && !errors.Is(err, fs.ErrNotExist) { + slog.WarnContext(ctx, "Failed to remove the actor resolv.conf", slog.Any("err", err)) + } + } + return s.sandbox.Close(ctx) +} + +// actorResolvConf writes the resolver bind source outside the actor's rootfs. +func actorResolvConf(actorUID string) (string, error) { + pod, err := os.ReadFile("/etc/resolv.conf") + if err != nil { + return "", fmt.Errorf("reading the worker pod resolv.conf: %w", err) + } + path := ateompath.ActorResolvConfPath(actorUID) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return "", fmt.Errorf("creating the actor directory: %w", err) + } + if err := os.WriteFile(path, ateomnet.SandboxResolvConf(pod), 0o644); err != nil { + return "", fmt.Errorf("writing the actor resolv.conf: %w", err) + } + return path, nil +} + +// attachAtunnel completes setup after atunnel receives the service's dialer. +func (s *AteomService) attachAtunnel(ingress *atunnel.Server, egress *atunnel.Egress, egressPort uint16) { + s.atunnelIngress = ingress + s.atunnelEgress = egress + s.atunnelEgressPort = egressPort +} diff --git a/cmd/ateom-microvm/checkpoint.go b/cmd/ateom-microvm/checkpoint.go index 8a4c8c75a7..fa8c4c8e80 100644 --- a/cmd/ateom-microvm/checkpoint.go +++ b/cmd/ateom-microvm/checkpoint.go @@ -25,8 +25,6 @@ import ( "path/filepath" "time" - "github.com/agent-substrate/substrate/internal/ateomnet" - "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/ch" "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata" "github.com/agent-substrate/substrate/internal/ateompath" @@ -398,7 +396,7 @@ func (s *AteomService) terminateWorkload(ctx context.Context, actorUID string) e // the two views of "is an actor here" from disagreeing. s.activeActor.Store(nil) - if err := ateomnet.CleanupActorNetwork(ctx, s.interiorNetNS); err != nil { + if err := s.releaseSandboxNetwork(ctx); err != nil { errs = append(errs, fmt.Errorf("while cleaning up actor network: %w", err)) } return errors.Join(errs...) diff --git a/cmd/ateom-microvm/main.go b/cmd/ateom-microvm/main.go index ba76a18924..dff05e2174 100644 --- a/cmd/ateom-microvm/main.go +++ b/cmd/ateom-microvm/main.go @@ -51,7 +51,6 @@ import ( "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/internal/serverboot" "github.com/agent-substrate/substrate/internal/version" - "github.com/vishvananda/netns" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "golang.org/x/sys/unix" "google.golang.org/grpc" @@ -191,13 +190,6 @@ func do(ctx context.Context) error { return fmt.Errorf("while opening unix socket: %w", err) } - // Networking: create a named interior netns; each activation builds a fresh - // veth pair into it (see net.go) and points kata at it. - interiorNetNS, err := ateomnet.CreateNetNSWithoutSwitching(ateompath.AteomNetNSName(*podUID)) - if err != nil { - return fmt.Errorf("while creating interior netns: %w", err) - } - // Forward the actor container's stdout/stderr to the worker pod's stdout as // JSON with ate.dev/* labels (logging parity with ateom-gvisor). It shares // logWriter with the runtime logger so the two streams to os.Stdout are @@ -207,11 +199,28 @@ func do(ctx context.Context) error { if err != nil { return fmt.Errorf("while parsing atunnel upstream: %w", err) } + // The pod's own resolvers, so an actor resolves exactly what the worker + // resolves -- cluster DNS included. + nameservers, err := atunnel.ResolvConfNameservers("/etc/resolv.conf") + if err != nil { + return fmt.Errorf("while reading the worker pod resolvers: %w", err) + } + dnsRelay, err := atunnel.NewDNSRelay(nameservers) + if err != nil { + return fmt.Errorf("while building the actor DNS relay: %w", err) + } + slog.InfoContext(ctx, "Actor DNS relay ready", slog.Any("upstreams", nameservers)) + + // The service owns the actor's namespace, and atunnel reaches the actor + // through it, so it is built first and handed to atunnel as a dialer. + ateomService := NewService(*podUID, *chBinary, *kataConfig, *kataDebug, *vmmMemReserve, dnsRelay, actorLogger, *workerCredentialBundle, *podIdentityTrustBundle, *egressGatewayTrustBundle) + atunnelIngress, err := atunnel.NewServer(atunnel.Config{ CredentialBundlePath: *workerCredentialBundle, TrustBundlePath: *podIdentityTrustBundle, AllowedClientID: *atunnelClientIdentity, Upstream: upstream, + Dial: ateomService.sandbox.Dialer(), }) if err != nil { return fmt.Errorf("while configuring atunnel: %w", err) @@ -240,24 +249,13 @@ func do(ctx context.Context) error { if err != nil { return fmt.Errorf("while configuring atunnel egress: %w", err) } - egressListener, err := net.Listen("tcp", *atunnelEgressListenAddress) + // Bind egress only in sandbox namespaces. + atunnelEgressPort, err := atunnel.EgressPort(*atunnelEgressListenAddress) if err != nil { - return fmt.Errorf("while opening atunnel egress listener: %w", err) - } - egressTCPAddr, ok := egressListener.Addr().(*net.TCPAddr) - if !ok || egressTCPAddr.Port < 1 || egressTCPAddr.Port > 65535 { - _ = egressListener.Close() - return fmt.Errorf("atunnel egress listener has invalid address %q", egressListener.Addr()) + return err } - atunnelEgressPort := uint16(egressTCPAddr.Port) - go func() { - if err := atunnelEgress.Serve(ctx, egressListener); err != nil { - serverboot.Fatal(ctx, "Failed to serve actor egress", err) - } - }() - slog.InfoContext(ctx, "atunnel egress serving", slog.String("address", *atunnelEgressListenAddress)) - ateomService := NewService(*podUID, *chBinary, *kataConfig, *kataDebug, *vmmMemReserve, interiorNetNS, actorLogger, atunnelIngress, atunnelEgress, atunnelEgressPort, *workerCredentialBundle, *podIdentityTrustBundle, *egressGatewayTrustBundle) + ateomService.attachAtunnel(atunnelIngress, atunnelEgress, atunnelEgressPort) svr := grpc.NewServer( grpc.StatsHandler(otelgrpc.NewServerHandler()), @@ -419,9 +417,11 @@ type AteomService struct { // with the guest RAM). Set from --vmm-mem-reserve-mib. memReserveMiB int - // interiorNetNS hosts the per-activation actor veth peer (see net.go); - // kata is pointed at it. - interiorNetNS netns.NsHandle + // sandbox is the network of the actor this worker is serving. + sandbox ateomnet.SessionHolder + + // dnsRelay answers the actor's DNS from inside its own namespace. + dnsRelay *atunnel.DNSRelay // actorLogger forwards the actor container's stdout/stderr to the worker pod's // stdout as ate.dev/*-labeled JSON and emits actor lifecycle events (parity @@ -493,7 +493,7 @@ type AteomService struct { var _ ateompb.AteomServer = (*AteomService)(nil) // NewService creates a new AteomService. -func NewService(podUID, chBinary, kataConfig string, kataDebug bool, memReserveMiB int, interiorNetNS netns.NsHandle, actorLogger *actorlog.ActorLogger, atunnelIngress *atunnel.Server, atunnelEgress *atunnel.Egress, atunnelEgressPort uint16, workerCredentialBundlePath, podIdentityTrustBundlePath, egressGatewayTrustBundlePath string) *AteomService { +func NewService(podUID, chBinary, kataConfig string, kataDebug bool, memReserveMiB int, dnsRelay *atunnel.DNSRelay, actorLogger *actorlog.ActorLogger, workerCredentialBundlePath, podIdentityTrustBundlePath, egressGatewayTrustBundlePath string) *AteomService { return &AteomService{ lock: newCancelableMutex(), podUID: podUID, @@ -501,11 +501,8 @@ func NewService(podUID, chBinary, kataConfig string, kataDebug bool, memReserveM kataConfig: kataConfig, kataDebug: kataDebug, memReserveMiB: memReserveMiB, - interiorNetNS: interiorNetNS, + dnsRelay: dnsRelay, actorLogger: actorLogger, - atunnelIngress: atunnelIngress, - atunnelEgress: atunnelEgress, - atunnelEgressPort: atunnelEgressPort, workerCredentialBundlePath: workerCredentialBundlePath, podIdentityTrustBundlePath: podIdentityTrustBundlePath, egressGatewayTrustBundlePath: egressGatewayTrustBundlePath, @@ -585,16 +582,6 @@ func (s *AteomService) deactivateActorNetworking(ctx context.Context) error { return nil } -// egressRedirectPort returns the local atunnel egress listener port when the -// activation arms tunneled egress, and zero otherwise, which leaves the -// prerouting redirect uninstalled and actor egress on the masquerade path. -func (s *AteomService) egressRedirectPort(redirectEgress bool) uint16 { - if !redirectEgress { - return 0 - } - return s.atunnelEgressPort -} - // rejectIfDraining returns a codes.Unavailable error if ateom has begun graceful // shutdown, so the control plane reschedules the actor onto a live worker. func (s *AteomService) rejectIfDraining() error { diff --git a/cmd/ateom-microvm/net.go b/cmd/ateom-microvm/net.go index 8d627123c8..3e795c92eb 100644 --- a/cmd/ateom-microvm/net.go +++ b/cmd/ateom-microvm/net.go @@ -23,47 +23,37 @@ import ( "os" "github.com/vishvananda/netlink" - "golang.org/x/sys/unix" + "github.com/vishvananda/netns" "github.com/agent-substrate/substrate/internal/ateomnet" ) const ( - // hostVethMAC is deliberately FIXED (locally administered), unlike - // ateom-gvisor where the kernel's random veth MAC is fine. A CH snapshot - // freezes the guest kernel's ARP cache, including the entry for the - // gateway 169.254.17.1; restoring against a new veth pair with a random - // MAC would blackhole guest egress until that entry expires. A constant - // gateway MAC keeps the frozen entry valid on every pod. - hostVethMAC = "02:a8:1e:00:00:01" + // gatewayMAC is deliberately FIXED (locally administered), unlike + // ateom-gvisor where a random veth MAC is fine. A CH snapshot freezes the + // guest kernel's ARP cache, including the entry for the gateway + // 169.254.17.1; restoring against a tap with a fresh random MAC would + // blackhole guest egress until that entry expired. + gatewayMAC = "02:a8:1e:00:00:01" // actorGuestMAC is the FIXED MAC for the guest's eth0 (the CH virtio-net). - // Fixed for the same reason as hostVethMAC: a cold boot freezes this MAC into - // the guest+snapshot, and restore re-adds the - // virtio-net under the same MAC (SnapshotNetDevices reads it back), so the - // guest's frozen interface config stays valid across pods. Distinct from the - // gateway MAC (…:01). + // Fixed for the same reason as gatewayMAC: a cold boot freezes this MAC into + // the guest+snapshot, and restore re-adds the virtio-net under the same MAC + // (SnapshotNetDevices reads it back), so the guest's frozen interface config + // stays valid across pods. Distinct from the gateway MAC (…:01). actorGuestMAC = "02:a8:1e:00:00:02" -) -var ( - hostVethHWAddr = ateomnet.MustParseMAC(hostVethMAC) + // actorTapMTU is the kernel default the guest was snapshotted against. + actorTapMTU = 1500 ) -// setupRestoreTap recreates, in the interior netns, the tap + TC-mirror wiring -// kata's tcfilter network model builds at boot: a tap device cross-connected to -// eth0 (the actor veth peer) with mirred-redirect ingress filters in both -// directions. Returns the open tap FDs (one per queue pair) for -// cloud-hypervisor to adopt via vm.restore net_fds (the snapshot's virtio-net -// device is fd-backed, so CH requires fresh FDs on restore). Call after -// setupActorNetwork. -func (s *AteomService) setupRestoreTap(ctx context.Context, name string, queuePairs int) ([]*os.File, error) { +var gatewayHWAddr = ateomnet.MustParseMAC(gatewayMAC) + +// setupActorTap creates the guest's tap with a fixed gateway address and MAC. +// Returns the FDs cloud-hypervisor adopts on boot or restore. +func setupActorTap(ctx context.Context, actorNetNS netns.NsHandle, name string, queuePairs int) ([]*os.File, error) { var fds []*os.File - err := ateomnet.NetNSDo(ctx, s.interiorNetNS, func(ctx context.Context) error { - eth0, err := netlink.LinkByName(ateomnet.ActorVethName) - if err != nil { - return fmt.Errorf("acquiring actor veth in interior netns: %w", err) - } + err := ateomnet.NetNSDo(ctx, actorNetNS, func(ctx context.Context) error { if old, lerr := netlink.LinkByName(name); lerr == nil { _ = netlink.LinkDel(old) } @@ -72,7 +62,7 @@ func (s *AteomService) setupRestoreTap(ctx context.Context, name string, queuePa flags |= netlink.TUNTAP_MULTI_QUEUE } tap := &netlink.Tuntap{ - LinkAttrs: netlink.LinkAttrs{Name: name, MTU: eth0.Attrs().MTU}, + LinkAttrs: netlink.LinkAttrs{Name: name, MTU: actorTapMTU}, Mode: netlink.TUNTAP_MODE_TAP, Flags: flags, Queues: queuePairs, @@ -81,34 +71,20 @@ func (s *AteomService) setupRestoreTap(ctx context.Context, name string, queuePa return fmt.Errorf("creating tap %q: %w", name, err) } fds = tap.Fds - if err := netlink.LinkSetUp(tap); err != nil { - return fmt.Errorf("bringing up tap %q: %w", name, err) + // Set the MAC after LinkAdd; tuntap creation ignores LinkAttrs.HardwareAddr. + // It must match the guest's snapshotted ARP entry. + link, err := netlink.LinkByName(name) + if err != nil { + return err + } + if err := netlink.LinkSetHardwareAddr(link, gatewayHWAddr); err != nil { + return fmt.Errorf("setting the gateway MAC on tap %q: %w", name, err) } - // Cross-connect: everything arriving on the veth peer redirects out the - // tap and vice versa (kata's TCFilterModel: ingress qdisc + match-all u32 - // with a mirred egress-redirect action, here via U32.RedirIndex). - for _, pair := range [][2]netlink.Link{{eth0, tap}, {tap, eth0}} { - qdisc := &netlink.Ingress{QdiscAttrs: netlink.QdiscAttrs{ - LinkIndex: pair[0].Attrs().Index, - Parent: netlink.HANDLE_INGRESS, - Handle: netlink.MakeHandle(0xffff, 0), - }} - if err := netlink.QdiscReplace(qdisc); err != nil { - return fmt.Errorf("adding ingress qdisc to %q: %w", pair[0].Attrs().Name, err) - } - filter := &netlink.U32{ - FilterAttrs: netlink.FilterAttrs{ - LinkIndex: pair[0].Attrs().Index, - Parent: netlink.MakeHandle(0xffff, 0), - Priority: 1, - Protocol: unix.ETH_P_ALL, - }, - ClassId: netlink.MakeHandle(1, 1), - RedirIndex: pair[1].Attrs().Index, - } - if err := netlink.FilterAdd(filter); err != nil { - return fmt.Errorf("adding mirred filter %s -> %s: %w", pair[0].Attrs().Name, pair[1].Attrs().Name, err) - } + if err := netlink.AddrReplace(link, ateomnet.HostVethAddr); err != nil { + return fmt.Errorf("assigning the gateway address to tap %q: %w", name, err) + } + if err := netlink.LinkSetUp(link); err != nil { + return fmt.Errorf("bringing up tap %q: %w", name, err) } return nil }) @@ -121,17 +97,15 @@ func (s *AteomService) setupRestoreTap(ctx context.Context, name string, queuePa return fds, nil } -// actorVethMTU reads the MTU of the actor veth (eth0 in the interior netns) so -// ateom can configure the guest eth0 with a matching MTU via the agent -// (UpdateInterface). Defaults to 1500 if the link can't be read. -func (s *AteomService) actorVethMTU(ctx context.Context) int { - mtu := 1500 - _ = ateomnet.NetNSDo(ctx, s.interiorNetNS, func(ctx context.Context) error { - if l, err := netlink.LinkByName(ateomnet.ActorVethName); err == nil { +// actorTapMTUOf reads the tap MTU, falling back to actorTapMTU on error. +func actorTapMTUOf(ctx context.Context, actorNetNS netns.NsHandle, name string) int { + mtu := actorTapMTU + _ = ateomnet.NetNSDo(ctx, actorNetNS, func(ctx context.Context) error { + if l, err := netlink.LinkByName(name); err == nil { mtu = l.Attrs().MTU } else { - slog.WarnContext(ctx, "Failed to read actor veth MTU; using default", - slog.String("link", ateomnet.ActorVethName), slog.Int("default_mtu", mtu), slog.Any("err", err)) + slog.WarnContext(ctx, "Failed to read actor tap MTU; using default", + slog.String("link", name), slog.Int("default_mtu", mtu), slog.Any("err", err)) } return nil }) diff --git a/cmd/ateom-microvm/net_linux_test.go b/cmd/ateom-microvm/net_linux_test.go new file mode 100644 index 0000000000..b54184781f --- /dev/null +++ b/cmd/ateom-microvm/net_linux_test.go @@ -0,0 +1,166 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "testing" + + "github.com/vishvananda/netlink" + "github.com/vishvananda/netns" + + "github.com/agent-substrate/substrate/internal/ateomnet" + "github.com/agent-substrate/substrate/internal/ateompath" + "github.com/agent-substrate/substrate/internal/atunnel" + "github.com/agent-substrate/substrate/internal/roottest" +) + +func TestPrepareSandboxNetworkReplacesSameActor(t *testing.T) { + roottest.Require(t, "creates network namespaces") + ctx := context.Background() + egress, err := atunnel.NewEgress(atunnel.TCPOriginalDestination) + if err != nil { + t.Fatal(err) + } + dns, err := atunnel.NewDNSRelay([]string{"127.0.0.1:53"}) + if err != nil { + t.Fatal(err) + } + service := &AteomService{atunnelEgress: egress, atunnelEgressPort: 15001, dnsRelay: dns} + t.Cleanup(func() { + if err := service.releaseSandboxNetwork(ctx); err != nil { + t.Error(err) + } + }) + const actorUID = "microvm-network-replace" + for range 2 { + if err := service.prepareSandboxNetwork(ctx, actorUID); err != nil { + t.Fatal(err) + } + } + named, err := netns.GetFromName(ateompath.ActorNetNSName(actorUID)) + if err != nil { + t.Fatalf("opening replacement namespace by name: %v", err) + } + defer named.Close() + if !named.Equal(service.sandboxNetNS()) { + t.Fatal("namespace name does not refer to the replacement") + } +} + +// tapNetNS gives a test its own namespace to build a tap in. +func tapNetNS(t *testing.T, name string) netns.NsHandle { + t.Helper() + ns, err := ateomnet.CreateNetNSWithoutSwitching(name) + if err != nil { + t.Fatalf("creating namespace: %v", err) + } + t.Cleanup(func() { + ns.Close() + _ = netns.DeleteNamed(name) + }) + return ns +} + +// The tap is the micro-VM's whole boundary: cloud-hypervisor adopts the +// descriptors, and the kernel keeps the interface side, carrying the gateway +// address and the MAC the guest's snapshot froze into its ARP cache. +func TestSetupActorTap(t *testing.T) { + roottest.Require(t, "creates network namespaces and tap devices") + ctx := context.Background() + ns := tapNetNS(t, "microvm-tap-test") + + fds, err := setupActorTap(ctx, ns, "tap0_kata", 1) + if err != nil { + t.Fatalf("setupActorTap: %v", err) + } + t.Cleanup(func() { + for _, f := range fds { + _ = f.Close() + } + }) + if len(fds) != 1 { + t.Errorf("got %d descriptors, want one per queue pair", len(fds)) + } + + if err := ateomnet.NetNSDo(ctx, ns, func(context.Context) error { + link, err := netlink.LinkByName("tap0_kata") + if err != nil { + return err + } + if got := link.Attrs().HardwareAddr.String(); got != gatewayMAC { + t.Errorf("tap MAC = %s, want the fixed %s the guest's frozen ARP entry names", got, gatewayMAC) + } + if link.Attrs().Flags&1 == 0 { // net.FlagUp + t.Error("tap is down") + } + if got := link.Attrs().MTU; got != actorTapMTU { + t.Errorf("tap MTU = %d, want %d, the value the guest was snapshotted against", got, actorTapMTU) + } + addrs, err := netlink.AddrList(link, netlink.FAMILY_V4) + if err != nil { + return err + } + var found bool + for _, a := range addrs { + if a.IPNet.String() == ateomnet.HostVethAddr.IPNet.String() { + found = true + } + } + if !found { + t.Errorf("tap addresses = %v, want the gateway %s the guest routes to", addrs, ateomnet.HostVethAddr) + } + return nil + }); err != nil { + t.Fatalf("inspecting the tap: %v", err) + } + + if got := actorTapMTUOf(ctx, ns, "tap0_kata"); got != actorTapMTU { + t.Errorf("actorTapMTUOf = %d, want %d", got, actorTapMTU) + } +} + +// A restore rebuilds the tap in a namespace that may still hold the previous +// one, and cloud-hypervisor needs fresh descriptors for its fd-backed +// virtio-net either way. +func TestSetupActorTapReplacesALeftover(t *testing.T) { + roottest.Require(t, "creates network namespaces and tap devices") + ctx := context.Background() + ns := tapNetNS(t, "microvm-tap-replace") + + first, err := setupActorTap(ctx, ns, "tap0_kata", 1) + if err != nil { + t.Fatalf("first setupActorTap: %v", err) + } + for _, f := range first { + defer f.Close() + } + + second, err := setupActorTap(ctx, ns, "tap0_kata", 2) + if err != nil { + t.Fatalf("second setupActorTap over a leftover: %v", err) + } + for _, f := range second { + defer f.Close() + } + if len(second) != 2 { + t.Errorf("got %d descriptors, want one per queue pair", len(second)) + } + if first[0].Fd() == second[0].Fd() { + t.Error("the replacement reused the first tap's descriptor") + } +} diff --git a/cmd/ateom-microvm/restore.go b/cmd/ateom-microvm/restore.go index 186c9ca658..bbe36899ae 100644 --- a/cmd/ateom-microvm/restore.go +++ b/cmd/ateom-microvm/restore.go @@ -86,8 +86,6 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore return nil, err } - // Same as RunWorkload: a restore is a boot, and graceful shutdown cancels it - // rather than queueing behind it. ctx, cancel := context.WithCancel(ctx) defer cancel() s.setActiveRPC(rpcRestoreWorkload, cancel) @@ -264,15 +262,10 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, tLowers := time.Now() tDurable := tLowers - // Networking: rebuild the per-activation veth + tap; the snapshot's virtio-net - // is fd-backed, so CH needs fresh tap FDs (net_fds) on restore. - if err := ateomnet.SetupActorNetwork(ctx, ateomnet.NetworkConfig{ - InteriorNetNS: s.interiorNetNS, - HostVethHWAddr: hostVethHWAddr, - SweepInteriorLinks: true, - EgressRedirectPort: s.egressRedirectPort(p.egressGateway != nil), - }); err != nil { - return fmt.Errorf("while setting up actor network: %w", err) + // Networking: rebuild the actor's namespace; the snapshot's virtio-net is + // fd-backed, so CH needs fresh tap FDs (net_fds) on restore. + if err := s.prepareSandboxNetwork(ctx, actorUID); err != nil { + return err } defer func() { if retErr != nil { @@ -281,7 +274,7 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, if cleanupErr := s.deactivateActorNetworking(cleanupCtx); cleanupErr != nil { slog.WarnContext(cleanupCtx, "Failed to deactivate actor networking after Restore failure", slog.Any("err", cleanupErr)) } - if cleanupErr := ateomnet.CleanupActorNetwork(cleanupCtx, s.interiorNetNS); cleanupErr != nil { + if cleanupErr := s.releaseSandboxNetwork(cleanupCtx); cleanupErr != nil { slog.WarnContext(cleanupCtx, "Failed to clean up actor network after Restore failure", slog.Any("err", cleanupErr)) } // Detach any bundle rootfs overlays mounted by buildActorContainers @@ -303,7 +296,7 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, } }() for i, nd := range netDevs { - files, terr := s.setupRestoreTap(ctx, fmt.Sprintf("tap%d_kata", i), nd.QueuePairs) + files, terr := setupActorTap(ctx, s.sandboxNetNS(), fmt.Sprintf("tap%d_kata", i), nd.QueuePairs) if terr != nil { return fmt.Errorf("while building restore tap for %s: %w", nd.ID, terr) } @@ -354,7 +347,7 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, tResume := time.Now() // Block until every readyz-enabled container reports 200. - if err := readyz.WaitAll(ctx, containers, ateomnet.ActorVethIP); err != nil { + if err := readyz.WaitAll(ctx, containers, ateomnet.ActorVethIP, readyz.DialFunc(s.sandbox.Dialer())); err != nil { return fmt.Errorf("while waiting for container readyz: %w", err) } diff --git a/cmd/ateom-microvm/run.go b/cmd/ateom-microvm/run.go index 7422d15f69..004c65d5bb 100644 --- a/cmd/ateom-microvm/run.go +++ b/cmd/ateom-microvm/run.go @@ -213,47 +213,6 @@ func (s *AteomService) resolveRuntime(paths map[string]string) resolvedRuntime { } } -// writeGuestResolvConf copies the worker pod's /etc/resolv.conf into a container's -// bundle rootfs (the overlay RO lower) so the guest gets cluster DNS: ateom drops -// atelet's resolv.conf bind and sends no CreateSandbox.Dns, so the guest can -// otherwise reach IPs but not resolve names. -// -// The rootfs is untrusted, so the write goes through os.Root and unlinks rather -// than truncates: an image-planted /etc or /etc/resolv.conf symlink would -// otherwise be followed and clobber that path on the worker pod as root. -func writeGuestResolvConf(rootfs string) error { - content, err := os.ReadFile("/etc/resolv.conf") - if err != nil { - return fmt.Errorf("reading host resolv.conf: %w", err) - } - if len(content) == 0 { - return fmt.Errorf("host /etc/resolv.conf is empty") - } - root, err := os.OpenRoot(rootfs) - if err != nil { - return fmt.Errorf("opening rootfs %q: %w", rootfs, err) - } - defer root.Close() - if err := root.Mkdir("etc", 0o755); err != nil && !errors.Is(err, fs.ErrExist) { - return fmt.Errorf("creating %q: %w", filepath.Join(rootfs, "etc"), err) - } - if err := root.Remove("etc/resolv.conf"); err != nil && !errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("removing existing guest resolv.conf: %w", err) - } - f, err := root.OpenFile("etc/resolv.conf", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) - if err != nil { - return fmt.Errorf("creating guest resolv.conf: %w", err) - } - _, err = f.Write(content) - if closeErr := f.Close(); err == nil { - err = closeErr - } - if err != nil { - return fmt.Errorf("writing guest resolv.conf: %w", err) - } - return nil -} - // RunWorkload boots the actor as a cloud-hypervisor micro-VM and starts its containers. // // ateom boots cloud-hypervisor directly (no kata shim) and gives each container a @@ -409,15 +368,10 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re return err } - // Networking (host side): per-activation veth into the interior netns. The - // tap + TC mirror is built below (after the VM exists) so its FDs are fresh. - if err := ateomnet.SetupActorNetwork(ctx, ateomnet.NetworkConfig{ - InteriorNetNS: s.interiorNetNS, - HostVethHWAddr: hostVethHWAddr, - SweepInteriorLinks: true, - EgressRedirectPort: s.egressRedirectPort(p.egressGateway != nil), - }); err != nil { - return fmt.Errorf("while setting up actor network: %w", err) + // Networking (host side): the actor's own namespace. The tap is built below + // (after the VM exists) so its FDs are fresh. + if err := s.prepareSandboxNetwork(ctx, actorUID); err != nil { + return err } defer func() { if retErr != nil { @@ -426,7 +380,7 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re if cleanupErr := s.deactivateActorNetworking(cleanupCtx); cleanupErr != nil { slog.WarnContext(cleanupCtx, "Failed to deactivate actor networking after Run failure", slog.Any("err", cleanupErr)) } - if cleanupErr := ateomnet.CleanupActorNetwork(cleanupCtx, s.interiorNetNS); cleanupErr != nil { + if cleanupErr := s.releaseSandboxNetwork(cleanupCtx); cleanupErr != nil { slog.WarnContext(cleanupCtx, "Failed to clean up actor network after Run failure", slog.Any("err", cleanupErr)) } // Detach any bundle rootfs overlays mounted by buildActorContainers @@ -536,9 +490,9 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re return fmt.Errorf("while creating VM: %w", err) } - // Network device: build the tap + TC mirror against the actor veth and add a - // virtio-net to the created (pre-boot) VM with the tap FDs (SCM_RIGHTS). - tapFiles, err := s.setupRestoreTap(ctx, "tap0_kata", 1) + // Network device: build the actor's tap and add a virtio-net to the created + // (pre-boot) VM with its FDs (SCM_RIGHTS). + tapFiles, err := setupActorTap(ctx, s.sandboxNetNS(), "tap0_kata", 1) if err != nil { return fmt.Errorf("while building tap: %w", err) } @@ -594,7 +548,7 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re tContainers := time.Now() // Block until every readyz-enabled container reports 200. - if err := readyz.WaitAll(ctx, containers, ateomnet.ActorVethIP); err != nil { + if err := readyz.WaitAll(ctx, containers, ateomnet.ActorVethIP, readyz.DialFunc(s.sandbox.Dialer())); err != nil { return fmt.Errorf("while waiting for container readyz: %w", err) } @@ -661,11 +615,8 @@ func (s *AteomService) buildActorContainers(actorUID string, containers []*ateom return nil, fmt.Errorf("while composing rootfs for %q: %w", cn, err) } bundleRootfs := filepath.Join(bundle, "rootfs") - // Write cluster DNS into the lower before it's served over virtio-fs: ateom - // drops atelet's resolv.conf bind and sends no CreateSandbox.Dns, so without - // this the guest can reach IPs but not resolve names. Doing it here covers both - // run and restore (both reconstruct the lower from the bundle). - if err := writeGuestResolvConf(bundleRootfs); err != nil { + // Set guest DNS before serving the rootfs over virtio-fs, on boot and restore. + if err := writeActorResolvConf(bundleRootfs); err != nil { return nil, fmt.Errorf("while writing guest resolv.conf for %q: %w", cn, err) } ctrs[i] = actorContainer{ @@ -897,7 +848,7 @@ func (s *AteomService) startActorContainers(ctx context.Context, ac *kata.AgentC tSandbox := time.Now() // Configure guest networking (the shim's job): eth0 IP/MAC/MTU, routes, ARP. - mtu := uint64(s.actorVethMTU(ctx)) + mtu := uint64(actorTapMTUOf(ctx, s.sandboxNetNS(), "tap0_kata")) netCtx, netCancel := context.WithTimeout(ctx, 20*time.Second) err = s.configureGuestNetwork(netCtx, ac, mtu) netCancel() @@ -1082,7 +1033,7 @@ func (s *AteomService) configureGuestNetwork(ctx context.Context, ac *kata.Agent return ac.AddARPNeighbors(ctx, []*agentpb.ARPNeighbor{{ ToIPAddress: &agentpb.IPAddress{Family: agentpb.IPFamily_v4, Address: ateomnet.ActorVethGateway}, Device: ateomnet.ActorVethName, - Lladdr: hostVethMAC, + Lladdr: gatewayMAC, State: 0x80, // NUD_PERMANENT }}) } diff --git a/cmd/ateom-microvm/run_test.go b/cmd/ateom-microvm/run_test.go index c511d078a1..6639e38dc2 100644 --- a/cmd/ateom-microvm/run_test.go +++ b/cmd/ateom-microvm/run_test.go @@ -20,7 +20,6 @@ import ( "context" "errors" "net" - "os" "path/filepath" "slices" "strings" @@ -30,70 +29,6 @@ import ( "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata" ) -// A symlink planted in the image at /etc or /etc/resolv.conf must not be followed -// out of the rootfs, or the image picks what ateom overwrites as root. -func TestWriteGuestResolvConfSymlinkEscape(t *testing.T) { - for _, tc := range []struct { - name string - link string // rootfs-relative path planted as a symlink to the canary - // A planted resolv.conf is just unlinked; only an escaping directory fails. - wantErr bool - }{ - {name: "etc dir", link: "etc", wantErr: true}, - {name: "resolv.conf", link: "etc/resolv.conf"}, - } { - t.Run(tc.name, func(t *testing.T) { - dir := t.TempDir() - canary := filepath.Join(dir, "canary") - if err := os.WriteFile(canary, []byte("INITIAL_STATE"), 0o644); err != nil { - t.Fatal(err) - } - rootfs := filepath.Join(dir, "rootfs") - if err := os.MkdirAll(filepath.Join(rootfs, filepath.Dir(tc.link)), 0o755); err != nil { - t.Fatal(err) - } - if err := os.Symlink(canary, filepath.Join(rootfs, tc.link)); err != nil { - t.Fatal(err) - } - - if err := writeGuestResolvConf(rootfs); (err != nil) != tc.wantErr { - t.Errorf("writeGuestResolvConf(%q) error = %v, wantErr %v", rootfs, err, tc.wantErr) - } - got, err := os.ReadFile(canary) - if err != nil { - t.Fatal(err) - } - if string(got) != "INITIAL_STATE" { - t.Errorf("canary = %q, want it untouched: the symlink was followed out of the rootfs", got) - } - }) - } -} - -func TestWriteGuestResolvConf(t *testing.T) { - want, err := os.ReadFile("/etc/resolv.conf") - if err != nil || len(want) == 0 { - t.Skipf("no host /etc/resolv.conf to copy: %v", err) - } - rootfs := t.TempDir() - - if err := writeGuestResolvConf(rootfs); err != nil { - t.Fatalf("writeGuestResolvConf(%q) = %v", rootfs, err) - } - - got, err := os.ReadFile(filepath.Join(rootfs, "etc", "resolv.conf")) - if err != nil { - t.Fatal(err) - } - if string(got) != string(want) { - t.Errorf("guest resolv.conf = %q, want %q", got, want) - } - // A second boot of the same bundle must not fail on the file it just wrote. - if err := writeGuestResolvConf(rootfs); err != nil { - t.Errorf("writeGuestResolvConf(%q) second call = %v", rootfs, err) - } -} - // A vsock socket that has gone missing means cloud-hypervisor stopped the VM // (it unlinks the socket in the vsock device's shutdown), so the poll must give // up at once instead of spending the whole timeout on a guest that is gone. diff --git a/cmd/ateom-microvm/sandboxnet.go b/cmd/ateom-microvm/sandboxnet.go new file mode 100644 index 0000000000..75e342fce7 --- /dev/null +++ b/cmd/ateom-microvm/sandboxnet.go @@ -0,0 +1,76 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "os" + + "github.com/vishvananda/netns" + + "github.com/agent-substrate/substrate/internal/ateomnet" + "github.com/agent-substrate/substrate/internal/atunnel" +) + +// prepareSandboxNetwork builds the actor's network and starts serving it. +func (s *AteomService) prepareSandboxNetwork(ctx context.Context, actorUID string) error { + if err := s.releaseSandboxNetwork(ctx); err != nil { + return err + } + session, err := ateomnet.ServeSandbox(ctx, ateomnet.SandboxNetworkConfig{ + ActorUID: actorUID, + EgressPort: s.atunnelEgressPort, + DNSPort: atunnel.DNSPort, + }, s.atunnelEgress, s.dnsRelay) + if err != nil { + return fmt.Errorf("while setting up the sandbox network: %w", err) + } + + return s.sandbox.Replace(ctx, session) +} + +// releaseSandboxNetwork stops serving the actor and takes its network down. +func (s *AteomService) releaseSandboxNetwork(ctx context.Context) error { + return s.sandbox.Close(ctx) +} + +// sandboxNetNS is where the actor's tap and atunnel's sockets live, or -1 +// between activations. +func (s *AteomService) sandboxNetNS() netns.NsHandle { + session := s.sandbox.Session() + if session == nil { + return -1 + } + return session.Network.GatewayNetNS +} + +// writeActorResolvConf points the guest resolver at its fixed gateway address. +func writeActorResolvConf(rootfs string) error { + pod, err := os.ReadFile("/etc/resolv.conf") + if err != nil { + return fmt.Errorf("reading the worker pod resolv.conf: %w", err) + } + return ateomnet.WriteRootfsResolvConf(rootfs, ateomnet.SandboxResolvConf(pod)) +} + +// attachAtunnel completes setup after atunnel receives the service's dialer. +func (s *AteomService) attachAtunnel(ingress *atunnel.Server, egress *atunnel.Egress, egressPort uint16) { + s.atunnelIngress = ingress + s.atunnelEgress = egress + s.atunnelEgressPort = egressPort +} diff --git a/docs/egress-trust-bundle.md b/docs/egress-trust-bundle.md index 95e7a3cb57..7249dfe45e 100644 --- a/docs/egress-trust-bundle.md +++ b/docs/egress-trust-bundle.md @@ -10,6 +10,16 @@ certificate error. This guide covers how to project the gateway's CA into an actor's filesystem and how to point the actor's TLS client at it. +## DNS and egress policy + +Actors send DNS queries to a relay at their sandbox's default gateway. The +relay forwards UDP and TCP DNS to the worker pod's configured resolvers, +without passing through the external egress gateway or checking egress policy. +There is currently no per-actor setting to disable this relay or filter queries. + +DNS remains available when no egress gateway is configured. Other outbound TCP +connections are captured by atunnel and refused in that configuration. + ## When you need this You need it when **all** of the following hold: diff --git a/internal/ateomnet/dns.go b/internal/ateomnet/dns.go new file mode 100644 index 0000000000..b8be5210fa --- /dev/null +++ b/internal/ateomnet/dns.go @@ -0,0 +1,80 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ateomnet + +import ( + "context" + "fmt" + "io" + "log/slog" + "net" + "strconv" + + "github.com/vishvananda/netns" +) + +// dnsServer answers an actor's DNS. Satisfied by atunnel.DNSRelay; an interface +// so this package does not depend on it. +type dnsServer interface { + ServePacket(ctx context.Context, pc net.PacketConn) error + Serve(ctx context.Context, listener net.Listener) error +} + +// serveSandboxDNS serves UDP and TCP DNS in the sandbox's local gateway namespace. +func serveSandboxDNS(ctx context.Context, relay dnsServer, ns netns.NsHandle, port uint16) (_ []io.Closer, _ []func(), retErr error) { + // Bind the wildcard because the microVM tap's gateway address is added later. + address := net.JoinHostPort("0.0.0.0", strconv.Itoa(int(port))) + + var packet net.PacketConn + var stream net.Listener + if err := NetNSDo(ctx, ns, func(context.Context) error { + pc, err := net.ListenPacket("udp", address) + if err != nil { + return fmt.Errorf("while opening the actor DNS socket: %w", err) + } + packet = pc + l, err := net.Listen("tcp", address) + if err != nil { + _ = pc.Close() + return fmt.Errorf("while opening the actor DNS listener: %w", err) + } + stream = l + return nil + }); err != nil { + return nil, nil, err + } + + // Detached from the activation RPC's context but cancelable: the relay's + // capacity is the worker's, so teardown must drop queries still in flight. + serveCtx, stopServing := context.WithCancel(context.WithoutCancel(ctx)) + serve := []func(){ + func() { + if err := relay.ServePacket(serveCtx, packet); err != nil { + slog.WarnContext(ctx, "Actor DNS socket stopped", slog.Any("err", err)) + } + }, + func() { + if err := relay.Serve(serveCtx, stream); err != nil { + slog.WarnContext(ctx, "Actor DNS listener stopped", slog.Any("err", err)) + } + }, + } + // Cancel first: closing the sockets alone leaves the queries already being + // resolved holding the relay. + closers := []io.Closer{closerFunc(func() error { stopServing(); return nil }), packet, stream} + return closers, serve, nil +} diff --git a/internal/ateomnet/net.go b/internal/ateomnet/net.go index 831d4859e6..6800690db8 100644 --- a/internal/ateomnet/net.go +++ b/internal/ateomnet/net.go @@ -21,15 +21,12 @@ import ( "context" "errors" "fmt" - "log/slog" "net" "os" "path/filepath" "runtime" "strings" - "github.com/google/nftables" - "github.com/google/nftables/binaryutil" "github.com/google/nftables/expr" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" @@ -37,14 +34,11 @@ import ( ) const ( - HostVethName = "ateom0" - ActorVethName = "eth0" - HostVethCIDR = "169.254.17.1/30" - ActorVethCIDR = "169.254.17.2/30" - ActorVethGateway = "169.254.17.1" - ActorVethIP = "169.254.17.2" - ActorNftTableName = "ateom_actor" - dnsPort = 53 + ActorVethName = "eth0" + HostVethCIDR = "169.254.17.1/30" + ActorVethCIDR = "169.254.17.2/30" + ActorVethGateway = "169.254.17.1" + ActorVethIP = "169.254.17.2" // ActorVethSubnet is the point-to-point /30 the actor veth lives on. ActorVethSubnet = "169.254.17.0/30" @@ -83,248 +77,40 @@ func MustParseMAC(s string) net.HardwareAddr { return m } -// ConfigureActorVeth configures the actor veth inside the interior netns. -// It assumes it is already running inside the target network namespace. -func ConfigureActorVeth(ctx context.Context) error { - // Run inside the gVisor interior netns. SetupActorNetwork has already created - // the veth peer here, under its final name, so this only has to address it. - // gVisor reads link names, addresses, and routes from this namespace when the - // workload starts, so eth0 is configured like a normal container interface: - // - // * lo is brought up for localhost behavior. - // * eth0 receives the actor-side /30 address. - // * the default route points to the worker-side veth gateway. - loLink, err := netlink.LinkByName("lo") - if err != nil { - return fmt.Errorf("while acquiring lo in interior netns: %w", err) - } - if err := netlink.LinkSetUp(loLink); err != nil { - return fmt.Errorf("while bringing up lo in interior netns: %w", err) - } - - actorLink, err := netlink.LinkByName(ActorVethName) - if err != nil { - return fmt.Errorf("while acquiring actor veth in interior netns: %w", err) - } - - if err := netlink.AddrReplace(actorLink, ActorVethAddr); err != nil { - return fmt.Errorf("while assigning actor veth address: %w", err) - } - if err := netlink.LinkSetUp(actorLink); err != nil { - return fmt.Errorf("while bringing up actor veth: %w", err) - } - - if err := netlink.RouteReplace(&netlink.Route{ - LinkIndex: actorLink.Attrs().Index, - Gw: ActorVethGwIP, - }); err != nil { - return fmt.Errorf("while installing actor default route: %w", err) - } - - return nil +// AllowUnprivilegedPorts lets this namespace bind ports below 1024 without +// CAP_NET_BIND_SERVICE, which is how atunnel answers a sandbox's DNS on 53. +// The sysctl is per-namespace and grants nothing outside it. +func AllowUnprivilegedPorts() error { + return setNetSysctl("net/ipv4/ip_unprivileged_port_start", "0") } -// CleanupActorNetwork removes all per-activation network state owned by ateom. -// Intentionally idempotent. -func CleanupActorNetwork(ctx context.Context, interiorNetNS netns.NsHandle) error { - // Remove all per-activation network state owned by ateom. Deleting the - // worker-side veth also deletes its peer, but the pair is born with its peer - // already in the actor netns, so a setup that failed before the worker side - // was named can leave that peer behind on its own. For that reason cleanup - // also enters the interior netns and deletes the actor interface if present. - // - // This function is intentionally idempotent so it can run before setup, after - // checkpoint, and from setup failure cleanup without requiring the caller to - // know how far network initialization progressed. - var cleanupErr error - if err := RemoveActorNftablesRules(); err != nil { - cleanupErr = errors.Join(cleanupErr, fmt.Errorf("while removing actor nftables rules: %w", err)) - slog.WarnContext(ctx, "Failed to remove actor nftables rules; continuing actor netns cleanup", "err", err) - } - - if link, err := netlink.LinkByName(HostVethName); err == nil { - if err := netlink.LinkDel(link); err != nil { - cleanupErr = errors.Join(cleanupErr, fmt.Errorf("while deleting host veth: %w", err)) - slog.WarnContext(ctx, "Failed to delete host veth; continuing actor netns cleanup", "err", err) - } - } else if _, notFound := errors.AsType[netlink.LinkNotFoundError](err); !notFound { - cleanupErr = errors.Join(cleanupErr, fmt.Errorf("while looking up host veth: %w", err)) - slog.WarnContext(ctx, "Failed to look up host veth; continuing actor netns cleanup", "err", err) - } - - if err := NetNSDo(ctx, interiorNetNS, func(_ context.Context) error { - link, err := netlink.LinkByName(ActorVethName) - if err == nil { - if err := netlink.LinkDel(link); err != nil { - return fmt.Errorf("while deleting interior veth %q: %w", ActorVethName, err) - } - return nil - } - if _, notFound := errors.AsType[netlink.LinkNotFoundError](err); !notFound { - return fmt.Errorf("while looking up interior veth %q: %w", ActorVethName, err) - } - return nil - }); err != nil { - cleanupErr = errors.Join(cleanupErr, fmt.Errorf("while cleaning interior netns links: %w", err)) - } - - return cleanupErr -} - -// EnableIPv4Forwarding enables IPv4 forwarding in the current network namespace. -func EnableIPv4Forwarding() error { - // Forwarding is required because actor packets now enter the worker pod via - // the host-side veth and then leave through the pod's eth0. Without this, the - // kernel would not route traffic between those interfaces even though both - // live in the worker pod network namespace. - // - // Without privileged, the container runtime bind-mounts /proc/sys read-only. - // The worker holds CAP_SYS_ADMIN and uses no user namespace, so the ro flag - // is not locked: clear it, write the sysctl, restore ro. - const path = "/proc/sys/net/ipv4/ip_forward" - if b, err := os.ReadFile(path); err == nil && len(b) > 0 && b[0] == '1' { +// setNetSysctl writes value to the named sysctl in the current network +// namespace, remounting /proc/sys read-write when the runtime bind-mounted it +// read-only. A no-op when it already reads that way. +func setNetSysctl(key, value string) error { + path := "/proc/sys/" + key + if b, err := os.ReadFile(path); err == nil && strings.TrimSpace(string(b)) == value { return nil } - if err := os.WriteFile(path, []byte("1\n"), 0o644); err == nil { + // Only EROFS is worth remounting for; any other error is returned as is. + if err := os.WriteFile(path, []byte(value+"\n"), 0o644); !errors.Is(err, unix.EROFS) { + if err != nil { + return fmt.Errorf("while setting %s in worker pod netns: %w", key, err) + } return nil } if err := unix.Mount("none", "/proc/sys", "", unix.MS_BIND|unix.MS_REMOUNT, ""); err != nil { - return fmt.Errorf("while remounting /proc/sys read-write to enable IPv4 forwarding: %w", err) + return fmt.Errorf("while remounting /proc/sys read-write to set %s: %w", key, err) } defer func() { _ = unix.Mount("none", "/proc/sys", "", unix.MS_BIND|unix.MS_REMOUNT|unix.MS_RDONLY, "") }() - if err := os.WriteFile(path, []byte("1\n"), 0o644); err != nil { - return fmt.Errorf("while enabling IPv4 forwarding in worker pod netns: %w", err) - } - return nil -} - -// InstallActorNftablesRules configures the NAT and filtering rules for the -// actor. egressPort, when non-zero, is the local atunnel egress listener actor -// TCP egress is redirected to; zero leaves the redirect uninstalled. -func InstallActorNftablesRules(egressPort uint16) error { - // Install a dedicated nftables table for the active actor. Keeping all - // rules in an ateom-owned table makes cleanup simple and avoids mutating - // Kubernetes or CNI-managed chains directly. - // - // TODO: Add IPv6 veth addressing, forwarding, and nftables rules once actor - // networking supports dual-stack pods. The current actor network is IPv4-only. - // - // The rules do three things: - // - // * prerouting: redirect new actor TCP connections, other than traffic to - // destination port 53, to atunnel's local listener. REDIRECT preserves - // SO_ORIGINAL_DST for the CONNECT authority. - // * postrouting: masquerade traffic not handled by the TCP tunnel, notably - // traffic to TCP or UDP destination port 53, so hostname resolution - // continues to work. - // * forward: drop actor UDP egress to any port but DNS, and accept the rest - // of the packets forwarded between the actor veth and pod eth0. - if err := RemoveActorNftablesRules(); err != nil { - return err - } - - c := &nftables.Conn{} - table := &nftables.Table{ - Family: nftables.TableFamilyIPv4, - Name: ActorNftTableName, - } - c.AddTable(table) - - prerouting := c.AddChain(&nftables.Chain{ - Name: "prerouting", - Table: table, - Type: nftables.ChainTypeNAT, - Hooknum: nftables.ChainHookPrerouting, - Priority: nftables.ChainPriorityNATDest, - }) - if redirectRule := ActorEgressRedirectRule(table, prerouting, egressPort); redirectRule != nil { - c.AddRule(redirectRule) - } - - postrouting := c.AddChain(&nftables.Chain{ - Name: "postrouting", - Table: table, - Type: nftables.ChainTypeNAT, - Hooknum: nftables.ChainHookPostrouting, - Priority: nftables.ChainPriorityNATSource, - }) - c.AddRule(&nftables.Rule{ - Table: table, - Chain: postrouting, - Exprs: append(IPSourceEqual(ActorVethIP), &expr.Masq{}), - }) - - acceptPolicy := nftables.ChainPolicyAccept - forward := c.AddChain(&nftables.Chain{ - Name: "forward", - Table: table, - Type: nftables.ChainTypeFilter, - Hooknum: nftables.ChainHookForward, - Priority: nftables.ChainPriorityFilter, - Policy: &acceptPolicy, - }) - // Order matters: the accept below is a catch-all, so the drop has to precede - // it. - c.AddRule(actorNonDNSUDPDropRule(table, forward)) - c.AddRule(&nftables.Rule{ - Table: table, - Chain: forward, - Exprs: []expr.Any{ - &expr.Verdict{Kind: expr.VerdictAccept}, - }, - }) - - if err := c.Flush(); err != nil { - return fmt.Errorf("while installing actor nftables rules: %w", err) - } - return nil -} - -// RemoveActorNftablesRules removes the ateom nftables table. -func RemoveActorNftablesRules() error { - // Delete the whole ateom nftables table if it exists. The table is - // per-worker and currently per-active-actor because this worker path runs at - // most one actor at a time. Missing tables are treated as already clean. - c := &nftables.Conn{} - tables, err := c.ListTablesOfFamily(nftables.TableFamilyIPv4) - if err != nil { - return fmt.Errorf("while listing nftables tables: %w", err) - } - for _, table := range tables { - if table.Name != ActorNftTableName { - continue - } - c.DelTable(table) - if err := c.Flush(); err != nil { - return fmt.Errorf("while deleting actor nftables table: %w", err) - } - return nil + if err := os.WriteFile(path, []byte(value+"\n"), 0o644); err != nil { + return fmt.Errorf("while setting %s in worker pod netns: %w", key, err) } return nil } -func IPSourceEqual(ip string) []expr.Any { - return IPPayloadEqual(12, ip) -} - -func IPPayloadEqual(offset uint32, ip string) []expr.Any { - return []expr.Any{ - &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseNetworkHeader, - Offset: offset, - Len: 4, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: net.ParseIP(ip).To4(), - }, - } -} - func l4ProtocolEqual(proto byte) []expr.Any { return []expr.Any{ &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, @@ -336,65 +122,6 @@ func l4ProtocolEqual(proto byte) []expr.Any { } } -// ActorEgressRedirectRule returns the prerouting rule that redirects actor TCP -// egress, except traffic to [dnsPort], to the local atunnel egress listener on -// port, or nil when port is zero (tunneled egress disabled, so actor egress -// stays on the masquerade path). -func ActorEgressRedirectRule(table *nftables.Table, chain *nftables.Chain, port uint16) *nftables.Rule { - if port == 0 { - return nil - } - exprs := append(IPSourceEqual(ActorVethIP), l4ProtocolEqual(unix.IPPROTO_TCP)...) - exprs = append(exprs, - // Traffic to destination port 53 bypasses atunnel and follows the direct - // masquerade path. - &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseTransportHeader, - Offset: 2, - Len: 2, - }, - &expr.Cmp{ - Op: expr.CmpOpNeq, - Register: 1, - Data: binaryutil.BigEndian.PutUint16(dnsPort), - }, - &expr.Immediate{ - Register: 1, - Data: binaryutil.BigEndian.PutUint16(port), - }, - &expr.Redir{RegisterProtoMin: 1}, - ) - return &nftables.Rule{Table: table, Chain: chain, Exprs: exprs} -} - -// actorNonDNSUDPDropRule returns the forward-chain rule that drops actor UDP -// egress to every destination port but [dnsPort]. -// -// The rule counts what it drops: a workload that legitimately needs UDP shows -// up as a rising counter in `nft list table ip ateom_actor` rather than as an -// unexplained timeout. -func actorNonDNSUDPDropRule(table *nftables.Table, chain *nftables.Chain) *nftables.Rule { - exprs := append(IPSourceEqual(ActorVethIP), l4ProtocolEqual(unix.IPPROTO_UDP)...) - exprs = append(exprs, - // Destination port, at offset 2 of the UDP header. - &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseTransportHeader, - Offset: 2, - Len: 2, - }, - &expr.Cmp{ - Op: expr.CmpOpNeq, - Register: 1, - Data: binaryutil.BigEndian.PutUint16(dnsPort), - }, - &expr.Counter{}, - &expr.Verdict{Kind: expr.VerdictDrop}, - ) - return &nftables.Rule{Table: table, Chain: chain, Exprs: exprs} -} - // CreateNetNSWithoutSwitching creates a named netns and returns its handle, // restoring the caller's current netns before returning. // @@ -476,160 +203,3 @@ func NetNSDo(ctx context.Context, targetNS netns.NsHandle, do func(context.Conte } return nil } - -// DumpNetInfo dumps link and route information for debugging. -func DumpNetInfo(ctx context.Context, prefix string) error { - links, err := netlink.LinkList() - if err != nil { - return fmt.Errorf("in netlink.LinkList(): %w", err) - } - - for _, link := range links { - slog.InfoContext(ctx, prefix+"Link", slog.String("name", link.Attrs().Name), slog.String("type", link.Type()), slog.Any("attrs", link.Attrs())) - - addrs, err := netlink.AddrList(link, netlink.FAMILY_V4) - if err != nil { - return fmt.Errorf("while getting pod eth0 addresses: %w", err) - } - slog.InfoContext(ctx, prefix+"Link Addresses", slog.String("link", link.Attrs().Name), slog.Any("addrs", addrs)) - - rts, err := netlink.RouteList(link, netlink.FAMILY_V4) - if err != nil { - return fmt.Errorf("while getting routes off eth0: %w", err) - } - for _, rt := range rts { - slog.InfoContext(ctx, prefix+"Link Routes", slog.Any("link", link.Attrs().Name), slog.Any("route", rt), slog.Any("route-string", rt.String())) - } - } - - return nil -} - -type NetworkConfig struct { - // InteriorNetNS is the target network namespace for the actor's veth pair peer. - // Used by: Both gVisor and MicroVM. - InteriorNetNS netns.NsHandle - - // HostVethHWAddr is the hardware address to assign to the host veth interface. - // Used by: MicroVM (to ensure consistent MAC addresses across snapshot/restore). - HostVethHWAddr net.HardwareAddr - // SweepInteriorLinks indicates whether to delete existing links in the interior netns (excluding loopback). - // Used by: MicroVM (to clean up stale tap devices). - SweepInteriorLinks bool - - // DumpNetInfo indicates whether to dump network information to the logs for debugging purposes. - // Used by: gVisor. - DumpNetInfo bool - - // EgressRedirectPort is the local atunnel egress listener port actor TCP - // egress is redirected to. Zero installs no redirect, leaving actor egress - // on the masquerade path. - // Used by: Both gVisor and MicroVM. - EgressRedirectPort uint16 -} - -// SetupActorNetwork builds a fresh point-to-point network between the worker -// pod netns and the interior netns. -func SetupActorNetwork(ctx context.Context, cfg NetworkConfig) (retErr error) { - // Build a fresh point-to-point network between the worker pod netns and the - // gVisor interior netns. The worker side keeps the pod's real eth0 and creates - // ateom0 as the gateway; the pair's peer is born inside the actor netns as - // eth0, where it gets the actor-side address and a default route via the - // worker-side veth address. This replaces the old behavior of moving the - // Kubernetes-provided eth0 out of the worker pod. - // - // The nftables rules installed here redirect actor TCP egress to atunnel - // when configured, masquerade traffic the TCP tunnel does not handle - // (notably DNS over UDP), and drop actor UDP egress to any other port. - // - // Clean up stale state from a failed prior activation before creating the - // next actor-side network. The worker currently runs one actor at a time. - if err := CleanupActorNetwork(ctx, cfg.InteriorNetNS); err != nil { - return fmt.Errorf("failed to clean up stale actor network before setup: %w", err) - } - defer func() { - if retErr != nil { - if err := CleanupActorNetwork(ctx, cfg.InteriorNetNS); err != nil { - slog.WarnContext(ctx, "Failed to clean up partially configured actor network", slog.Any("err", err)) - } - } - }() - - if cfg.SweepInteriorLinks { - if err := NetNSDo(ctx, cfg.InteriorNetNS, func(ctx context.Context) error { - links, err := netlink.LinkList() - if err != nil { - return fmt.Errorf("while listing interior netns links: %w", err) - } - for _, l := range links { - if l.Attrs().Name == "lo" { - continue - } - if err := netlink.LinkDel(l); err != nil { - slog.WarnContext(ctx, "Failed to delete leftover interior link", slog.String("link", l.Attrs().Name), slog.Any("err", err)) - } - } - return nil - }); err != nil { - return err - } - } - - // The peer is born in the interior netns under its final name. Do not replace - // this with the obvious "create locally, LinkSetNsFd across, rename to eth0": - // moving and renaming a netdev each cost an RCU grace period under the global - // RTNL lock, which is ~18ms vs ~3ms for all of SetupActorNetwork, on the - // resume path. Naming the peer here is safe because its name is resolved in - // its own netns, so it never collides with the pod's eth0. - veth := &netlink.Veth{ - LinkAttrs: netlink.LinkAttrs{ - Name: HostVethName, - }, - PeerName: ActorVethName, - // netlink.NsFd, not netns.NsHandle: only the netlink type is recognized - // as IFLA_NET_NS_FD on the peer, though both are file descriptors. - PeerNamespace: netlink.NsFd(int(cfg.InteriorNetNS)), - } - if len(cfg.HostVethHWAddr) > 0 { - veth.LinkAttrs.HardwareAddr = cfg.HostVethHWAddr - } - - if err := netlink.LinkAdd(veth); err != nil { - return fmt.Errorf("while creating actor veth pair: %w", err) - } - - hostLink, err := netlink.LinkByName(HostVethName) - if err != nil { - return fmt.Errorf("while getting host veth: %w", err) - } - if err := netlink.AddrReplace(hostLink, HostVethAddr); err != nil { - return fmt.Errorf("while assigning host veth address: %w", err) - } - if err := netlink.LinkSetUp(hostLink); err != nil { - return fmt.Errorf("while bringing up host veth: %w", err) - } - - if err := NetNSDo(ctx, cfg.InteriorNetNS, ConfigureActorVeth); err != nil { - return fmt.Errorf("while configuring actor veth in interior netns: %w", err) - } - - if err := EnableIPv4Forwarding(); err != nil { - return err - } - if err := InstallActorNftablesRules(cfg.EgressRedirectPort); err != nil { - return err - } - - if cfg.DumpNetInfo { - if err := DumpNetInfo(ctx, "Pod NetNS "); err != nil { - return fmt.Errorf("while dumping pod netns links: %w", err) - } - if err := NetNSDo(ctx, cfg.InteriorNetNS, func(ctx context.Context) error { - return DumpNetInfo(ctx, "Interior NetNS ") - }); err != nil { - return fmt.Errorf("while dumping interior netns links: %w", err) - } - } - - return nil -} diff --git a/internal/ateomnet/net_linux_test.go b/internal/ateomnet/net_linux_test.go index 3ee8def3b7..2438b54edb 100644 --- a/internal/ateomnet/net_linux_test.go +++ b/internal/ateomnet/net_linux_test.go @@ -17,441 +17,16 @@ package ateomnet import ( - "bytes" "context" "errors" - "net" "os" - "runtime" "testing" "github.com/agent-substrate/substrate/internal/roottest" - "github.com/google/nftables" - "github.com/google/nftables/binaryutil" - "github.com/google/nftables/expr" "github.com/vishvananda/netlink" - "github.com/vishvananda/netns" "golang.org/x/sys/unix" ) -// withTestNetNS runs fn with the calling thread inside a throwaway netns -// standing in for the worker pod's, and hands it a second throwaway netns -// standing in for an actor's interior one. -// -// Both are anonymous (netns.New, not NewNamed) so the test leaves nothing behind -// in /run/netns, and every link, sysctl, and nftables table SetupActorNetwork -// touches is scoped to a namespace that disappears with the test rather than to -// the machine running it. -func withTestNetNS(t *testing.T, fn func(interior netns.NsHandle)) { - t.Helper() - - // Locked for the whole body: netns is a per-thread property, so an unlocked - // goroutine could be rescheduled onto a thread still in the original - // namespace midway through. It also means a t.Fatal inside fn tears the - // thread down instead of returning it to the pool mis-configured. - runtime.LockOSThread() - defer runtime.UnlockOSThread() - - orig, err := netns.Get() - if err != nil { - t.Fatalf("getting current netns: %v", err) - } - defer orig.Close() - // Registered before the namespaces below so it runs after they are closed. - defer func() { - if err := netns.Set(orig); err != nil { - t.Errorf("restoring original netns: %v", err) - } - }() - - pod, err := netns.New() // netns.New switches the thread into the new namespace - if err != nil { - t.Fatalf("creating pod netns: %v", err) - } - defer pod.Close() - interior, err := netns.New() - if err != nil { - t.Fatalf("creating interior netns: %v", err) - } - defer interior.Close() - if err := netns.Set(pod); err != nil { - t.Fatalf("entering pod netns: %v", err) - } - - fn(interior) -} - -// requireNftables skips when the kernel in this environment cannot serve the -// nftables netlink API at all, which SetupActorNetwork needs and which is a -// property of the machine rather than of the code under test. -func requireNftables(t *testing.T) { - t.Helper() - c := &nftables.Conn{} - if _, err := c.ListTablesOfFamily(nftables.TableFamilyIPv4); err != nil { - t.Skipf("nftables unavailable in this environment: %v", err) - } -} - -// linkByName returns the link, or nil when it does not exist. -func linkByName(t *testing.T, name string) netlink.Link { - t.Helper() - link, err := netlink.LinkByName(name) - if err == nil { - return link - } - if _, ok := errors.AsType[netlink.LinkNotFoundError](err); ok { - return nil - } - t.Fatalf("looking up link %q: %v", name, err) - return nil -} - -func hasAddr(t *testing.T, link netlink.Link, cidr string) bool { - t.Helper() - addrs, err := netlink.AddrList(link, netlink.FAMILY_V4) - if err != nil { - t.Fatalf("listing addresses of %q: %v", link.Attrs().Name, err) - } - want := MustParseAddr(cidr) - for _, addr := range addrs { - if addr.IPNet != nil && addr.IPNet.String() == want.IPNet.String() { - return true - } - } - return false -} - -// TestSetupActorNetworkFinalState pins the namespace state gVisor and the -// micro-VM guest read after an activation: what links exist, where, with which -// addresses and routes. It deliberately asserts the end state rather than the -// sequence of netlink calls that produced it, so the setup path stays free to -// get there differently (as it did when the veth peer stopped being created in -// the pod netns and moved across). -func TestSetupActorNetworkFinalState(t *testing.T) { - roottest.Require(t, "creating network namespaces, veth pairs, and nftables rules") - ctx := context.Background() - - withTestNetNS(t, func(interior netns.NsHandle) { - requireNftables(t) - - if err := SetupActorNetwork(ctx, NetworkConfig{InteriorNetNS: interior}); err != nil { - t.Fatalf("SetupActorNetwork: %v", err) - } - - // Worker pod side: the gateway end of the point-to-point link. - host := linkByName(t, HostVethName) - if host == nil { - t.Fatalf("host veth %q missing from the pod netns", HostVethName) - } - if !hasAddr(t, host, HostVethCIDR) { - t.Errorf("host veth %q does not carry %s", HostVethName, HostVethCIDR) - } - if host.Attrs().Flags&1 == 0 { // net.FlagUp - t.Errorf("host veth %q is not up", HostVethName) - } - - // The actor interface must exist ONLY in the interior netns. A peer left - // in the pod netns would mean the pair was built the old way, and worse, - // would collide with the pod's own eth0 on a real worker. - if stray := linkByName(t, ActorVethName); stray != nil { - t.Errorf("actor interface %q must not exist in the pod netns", ActorVethName) - } - - if err := NetNSDo(ctx, interior, func(context.Context) error { - actor := linkByName(t, ActorVethName) - if actor == nil { - t.Fatalf("actor veth %q missing from the interior netns", ActorVethName) - } - if !hasAddr(t, actor, ActorVethCIDR) { - t.Errorf("actor veth %q does not carry %s", ActorVethName, ActorVethCIDR) - } - if actor.Attrs().Flags&1 == 0 { - t.Errorf("actor veth %q is not up", ActorVethName) - } - - if lo := linkByName(t, "lo"); lo == nil { - t.Error("interior netns has no loopback") - } else if lo.Attrs().Flags&1 == 0 { - t.Error("interior loopback is not up") - } - - routes, err := netlink.RouteList(actor, netlink.FAMILY_V4) - if err != nil { - t.Fatalf("listing interior routes: %v", err) - } - // A default route reports its destination either as nil or as an - // explicit 0.0.0.0/0, depending on how the kernel rendered it. - isDefault := func(route netlink.Route) bool { - if route.Dst == nil { - return true - } - ones, _ := route.Dst.Mask.Size() - return ones == 0 - } - var haveDefault bool - for _, route := range routes { - if isDefault(route) && route.Gw.Equal(ActorVethGwIP) { - haveDefault = true - } - } - if !haveDefault { - t.Errorf("interior netns has no default route via %s, got %v", ActorVethGateway, routes) - } - return nil - }); err != nil { - t.Fatalf("inspecting interior netns: %v", err) - } - }) -} - -// TestSetupActorNetworkIsRepeatable covers the activation cycle a reused worker -// runs: set up, tear down, set up again. The second setup has to succeed against -// whatever the first one left behind. -func TestSetupActorNetworkIsRepeatable(t *testing.T) { - roottest.Require(t, "creating network namespaces, veth pairs, and nftables rules") - ctx := context.Background() - - withTestNetNS(t, func(interior netns.NsHandle) { - requireNftables(t) - - for i := range 3 { - if err := SetupActorNetwork(ctx, NetworkConfig{InteriorNetNS: interior}); err != nil { - t.Fatalf("SetupActorNetwork (activation %d): %v", i, err) - } - if linkByName(t, HostVethName) == nil { - t.Fatalf("host veth %q missing after activation %d", HostVethName, i) - } - if err := CleanupActorNetwork(ctx, interior); err != nil { - t.Fatalf("CleanupActorNetwork (activation %d): %v", i, err) - } - } - - // Cleanup is idempotent: the extra call after the loop's last one must - // still succeed, and both ends must be gone. - if err := CleanupActorNetwork(ctx, interior); err != nil { - t.Fatalf("CleanupActorNetwork on an already-clean network: %v", err) - } - if stray := linkByName(t, HostVethName); stray != nil { - t.Errorf("host veth %q survived cleanup", HostVethName) - } - if err := NetNSDo(ctx, interior, func(context.Context) error { - if stray := linkByName(t, ActorVethName); stray != nil { - t.Errorf("actor veth %q survived cleanup", ActorVethName) - } - return nil - }); err != nil { - t.Fatalf("inspecting interior netns: %v", err) - } - }) -} - -// addForwardingTarget gives the pod netns somewhere to forward actor packets -// to, standing in for the real pod's eth0 and default route. Without it a -// forwarded packet is dropped for want of a route before it ever reaches the -// forward hook the rules under test live on. The device is a dummy, so the -// packets go nowhere after that, which is all the assertions need. -func addForwardingTarget(t *testing.T, cidr string) { - t.Helper() - link := &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: "target0"}} - if err := netlink.LinkAdd(link); err != nil { - t.Fatalf("creating the forwarding target link: %v", err) - } - if err := netlink.AddrAdd(link, MustParseAddr(cidr)); err != nil { - t.Fatalf("addressing the forwarding target link: %v", err) - } - if err := netlink.LinkSetUp(link); err != nil { - t.Fatalf("bringing up the forwarding target link: %v", err) - } -} - -// droppedUDPPackets reads the packet count off the forward chain's counted -// rule, which [actorNonDNSUDPDropRule] is. -func droppedUDPPackets(t *testing.T) uint64 { - t.Helper() - c := &nftables.Conn{} - tables, err := c.ListTablesOfFamily(nftables.TableFamilyIPv4) - if err != nil { - t.Fatalf("listing nftables tables: %v", err) - } - for _, table := range tables { - if table.Name != ActorNftTableName { - continue - } - rules, err := c.GetRules(table, &nftables.Chain{Name: "forward", Table: table}) - if err != nil { - t.Fatalf("listing forward chain rules: %v", err) - } - for _, rule := range rules { - for _, e := range rule.Exprs { - if counter, ok := e.(*expr.Counter); ok { - return counter.Packets - } - } - } - t.Fatalf("forward chain has no counted rule, got %d rules", len(rules)) - } - t.Fatalf("nftables table %q is missing", ActorNftTableName) - return 0 -} - -// sendUDP sends one datagram to addr and reports whether the local send -// succeeded. UDP has no acknowledgement, so a successful send says nothing -// about delivery -- the drop is observed through the nftables counter instead. -func sendUDP(t *testing.T, addr string) { - t.Helper() - conn, err := net.Dial("udp4", addr) - if err != nil { - t.Fatalf("dialing %s: %v", addr, err) - } - defer conn.Close() - if _, err := conn.Write([]byte("probe")); err != nil { - t.Fatalf("sending a datagram to %s: %v", addr, err) - } -} - -func TestActorEgressRedirectRuleExcludesDNS(t *testing.T) { - table := &nftables.Table{} - chain := &nftables.Chain{Table: table} - - if rule := ActorEgressRedirectRule(table, chain, 0); rule != nil { - t.Fatal("ActorEgressRedirectRule returned a rule when tunneled egress is disabled") - } - - rule := ActorEgressRedirectRule(table, chain, 15001) - if rule == nil { - t.Fatal("ActorEgressRedirectRule returned nil when tunneled egress is enabled") - } - if len(rule.Exprs) != 8 { - t.Fatalf("redirect rule has %d expressions, want 8", len(rule.Exprs)) - } - payload, ok := rule.Exprs[4].(*expr.Payload) - if !ok { - t.Fatalf("redirect expression 4 is %T, want *expr.Payload", rule.Exprs[4]) - } - if payload.Base != expr.PayloadBaseTransportHeader || payload.Offset != 2 || payload.Len != 2 { - t.Errorf("redirect destination-port payload = %+v, want transport-header offset 2 length 2", payload) - } - cmp, ok := rule.Exprs[5].(*expr.Cmp) - if !ok { - t.Fatalf("redirect expression 5 is %T, want *expr.Cmp", rule.Exprs[5]) - } - if cmp.Op != expr.CmpOpNeq || !bytes.Equal(cmp.Data, binaryutil.BigEndian.PutUint16(dnsPort)) { - t.Errorf("redirect destination-port comparison = %+v, want destination port != %d", cmp, dnsPort) - } -} - -// TestActorNonDNSUDPIsDropped covers the forward-chain rule behaviorally: only -// TCP not destined for port 53 is redirected into atunnel, so UDP on any port -// but 53 must not reach the masquerade, and DNS must still get through or the -// sandbox cannot resolve anything. -func TestActorNonDNSUDPIsDropped(t *testing.T) { - roottest.Require(t, "creating network namespaces, veth pairs, and nftables rules") - ctx := context.Background() - - withTestNetNS(t, func(interior netns.NsHandle) { - requireNftables(t) - - const target = "192.0.2.1" - addForwardingTarget(t, "192.0.2.254/24") - if err := SetupActorNetwork(ctx, NetworkConfig{InteriorNetNS: interior}); err != nil { - t.Fatalf("SetupActorNetwork: %v", err) - } - - before := droppedUDPPackets(t) - if err := NetNSDo(ctx, interior, func(context.Context) error { - sendUDP(t, net.JoinHostPort(target, "53")) - return nil - }); err != nil { - t.Fatalf("sending DNS from the interior netns: %v", err) - } - if got := droppedUDPPackets(t); got != before { - t.Errorf("DNS datagram was dropped: counter went from %d to %d", before, got) - } - - if err := NetNSDo(ctx, interior, func(context.Context) error { - sendUDP(t, net.JoinHostPort(target, "443")) - sendUDP(t, net.JoinHostPort(target, "9999")) - return nil - }); err != nil { - t.Fatalf("sending non-DNS UDP from the interior netns: %v", err) - } - if got := droppedUDPPackets(t); got != before+2 { - t.Errorf("dropped packets = %d, want %d: non-DNS UDP reached the masquerade", got, before+2) - } - }) -} - -// TestSetupActorNetworkHostVethHWAddr covers the micro-VM requirement: a CH -// snapshot freezes the guest's ARP entry for the gateway, so the worker-side -// veth MAC has to be exactly the one the caller asked for, on every pod. -func TestSetupActorNetworkHostVethHWAddr(t *testing.T) { - roottest.Require(t, "creating network namespaces, veth pairs, and nftables rules") - ctx := context.Background() - - withTestNetNS(t, func(interior netns.NsHandle) { - requireNftables(t) - - want := MustParseMAC("02:a8:1e:00:00:01") - if err := SetupActorNetwork(ctx, NetworkConfig{ - InteriorNetNS: interior, - HostVethHWAddr: want, - SweepInteriorLinks: true, - }); err != nil { - t.Fatalf("SetupActorNetwork: %v", err) - } - - host := linkByName(t, HostVethName) - if host == nil { - t.Fatalf("host veth %q missing from the pod netns", HostVethName) - } - if got := host.Attrs().HardwareAddr.String(); got != want.String() { - t.Errorf("host veth MAC = %s, want %s", got, want) - } - }) -} - -// TestSetupActorNetworkSweepsInteriorLinks covers the other half of the micro-VM -// path: SweepInteriorLinks clears a previous activation's leftovers (kata's tap -// device) before the new pair is created, and must not take the loopback or the -// freshly created actor veth with it. -func TestSetupActorNetworkSweepsInteriorLinks(t *testing.T) { - roottest.Require(t, "creating network namespaces, veth pairs, and nftables rules") - ctx := context.Background() - - withTestNetNS(t, func(interior netns.NsHandle) { - requireNftables(t) - - const leftover = "stale-tap0" - if err := NetNSDo(ctx, interior, func(context.Context) error { - return netlink.LinkAdd(&netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: leftover}}) - }); err != nil { - t.Fatalf("planting a leftover interior link: %v", err) - } - - if err := SetupActorNetwork(ctx, NetworkConfig{ - InteriorNetNS: interior, - SweepInteriorLinks: true, - }); err != nil { - t.Fatalf("SetupActorNetwork: %v", err) - } - - if err := NetNSDo(ctx, interior, func(context.Context) error { - if stray := linkByName(t, leftover); stray != nil { - t.Errorf("leftover interior link %q was not swept", leftover) - } - if linkByName(t, ActorVethName) == nil { - t.Errorf("actor veth %q missing after a sweeping setup", ActorVethName) - } - if linkByName(t, "lo") == nil { - t.Error("sweep removed the interior loopback") - } - return nil - }); err != nil { - t.Fatalf("inspecting interior netns: %v", err) - } - }) -} - func TestNamedNetNSRejectsInvalidNames(t *testing.T) { for _, name := range []string{"", ".", "..", "/absolute", "../outside", "nested/name", "ateom-actor:uid/../../outside", "nul\x00name"} { t.Run(name, func(t *testing.T) { @@ -561,3 +136,20 @@ func TestCreateNetNSWithoutSwitchingReplacesALeftover(t *testing.T) { }) } } + +// Only EROFS takes the remount path: any other error is reported as it is, +// and /proc/sys is left as it was found. Remounting it read-only on the way +// out would break every later write. +func TestSetNetSysctlReportsAnUnrelatedError(t *testing.T) { + err := setNetSysctl("net/ipv4/ateomnet_no_such_sysctl", "0") + if !errors.Is(err, unix.ENOENT) { + t.Fatalf("setNetSysctl() on a missing key: got %v, want ENOENT", err) + } + var st unix.Statfs_t + if err := unix.Statfs("/proc/sys", &st); err != nil { + t.Fatalf("statfs /proc/sys: %v", err) + } + if st.Flags&unix.ST_RDONLY != 0 { + t.Error("/proc/sys was left read-only") + } +} diff --git a/internal/ateomnet/resolvconf.go b/internal/ateomnet/resolvconf.go new file mode 100644 index 0000000000..7fd0f08f79 --- /dev/null +++ b/internal/ateomnet/resolvconf.go @@ -0,0 +1,75 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ateomnet + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// SandboxResolvConf replaces nameservers with the sandbox gateway while +// preserving the pod's search domains and options for Kubernetes DNS. +func SandboxResolvConf(podResolvConf []byte) []byte { + var out strings.Builder + out.WriteString("nameserver " + ActorVethGateway + "\n") + for line := range strings.SplitSeq(string(podResolvConf), "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "nameserver") { + continue + } + if strings.TrimSpace(line) == "" { + continue + } + out.WriteString(line + "\n") + } + return []byte(out.String()) +} + +// WriteRootfsResolvConf installs content at /etc/resolv.conf inside rootfs. +// +// os.Root confines path traversal; unlinking prevents writes through existing links. +func WriteRootfsResolvConf(rootfs string, content []byte) error { + if len(content) == 0 { + return fmt.Errorf("actornet: refusing to write an empty resolv.conf") + } + root, err := os.OpenRoot(rootfs) + if err != nil { + return fmt.Errorf("opening rootfs %q: %w", rootfs, err) + } + defer root.Close() + if err := root.Mkdir("etc", 0o755); err != nil && !errors.Is(err, fs.ErrExist) { + return fmt.Errorf("creating %q: %w", filepath.Join(rootfs, "etc"), err) + } + if err := root.Remove("etc/resolv.conf"); err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("removing existing resolv.conf: %w", err) + } + f, err := root.OpenFile("etc/resolv.conf", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + return fmt.Errorf("creating resolv.conf: %w", err) + } + _, err = f.Write(content) + if closeErr := f.Close(); err == nil { + err = closeErr + } + if err != nil { + return fmt.Errorf("writing resolv.conf: %w", err) + } + return nil +} diff --git a/internal/ateomnet/resolvconf_linux_test.go b/internal/ateomnet/resolvconf_linux_test.go new file mode 100644 index 0000000000..18ef1f2ff1 --- /dev/null +++ b/internal/ateomnet/resolvconf_linux_test.go @@ -0,0 +1,112 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ateomnet + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestSandboxResolvConf(t *testing.T) { + // The shape kubelet writes into a pod. + pod := "nameserver 10.96.0.10\n" + + "search ate-demo.svc.cluster.local svc.cluster.local cluster.local\n" + + "options ndots:5\n" + + got := string(SandboxResolvConf([]byte(pod))) + + want := "nameserver 169.254.17.1\n" + + "search ate-demo.svc.cluster.local svc.cluster.local cluster.local\n" + + "options ndots:5\n" + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("actor resolv.conf mismatch (-want +got):\n%s", diff) + } +} + +// Short service names depend on the search list and on ndots, so an actor given +// only a nameserver line resolves public names but not cluster ones. +func TestSandboxResolvConfKeepsSearchAndOptions(t *testing.T) { + pod := "search svc.cluster.local\nnameserver 10.96.0.10\nnameserver 10.96.0.11\noptions ndots:5 timeout:1\n" + got := string(SandboxResolvConf([]byte(pod))) + + if strings.Contains(got, "10.96.0.10") || strings.Contains(got, "10.96.0.11") { + t.Errorf("a pod resolver survived into the actor's file, so its DNS would bypass atunnel:\n%s", got) + } + if !strings.Contains(got, "search svc.cluster.local") { + t.Errorf("search list lost; cluster short names would stop resolving:\n%s", got) + } + if !strings.Contains(got, "options ndots:5 timeout:1") { + t.Errorf("options lost:\n%s", got) + } + if n := strings.Count(got, "nameserver"); n != 1 { + t.Errorf("got %d nameserver lines, want exactly the gateway", n) + } +} + +func TestWriteRootfsResolvConf(t *testing.T) { + rootfs := t.TempDir() + if err := WriteRootfsResolvConf(rootfs, []byte("nameserver 169.254.17.1\n")); err != nil { + t.Fatalf("WriteRootfsResolvConf: %v", err) + } + got, err := os.ReadFile(filepath.Join(rootfs, "etc", "resolv.conf")) + if err != nil { + t.Fatalf("reading what was written: %v", err) + } + if string(got) != "nameserver 169.254.17.1\n" { + t.Errorf("wrote %q", got) + } + + // Replacing an existing file is the ordinary case: the image usually ships one. + if err := WriteRootfsResolvConf(rootfs, []byte("nameserver 169.254.17.1\nsearch x\n")); err != nil { + t.Fatalf("rewriting: %v", err) + } + + if err := WriteRootfsResolvConf(rootfs, nil); err == nil { + t.Error("an empty resolv.conf was accepted; the actor would resolve nothing") + } +} + +// The rootfs comes from an untrusted image, so a planted symlink must not be +// followed out of it and clobber the worker pod's own file. +func TestWriteRootfsResolvConfDoesNotFollowAPlantedSymlink(t *testing.T) { + rootfs := t.TempDir() + outside := filepath.Join(t.TempDir(), "victim") + if err := os.WriteFile(outside, []byte("original"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(rootfs, "etc"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(rootfs, "etc", "resolv.conf")); err != nil { + t.Fatal(err) + } + + if err := WriteRootfsResolvConf(rootfs, []byte("nameserver 169.254.17.1\n")); err != nil { + t.Fatalf("WriteRootfsResolvConf: %v", err) + } + victim, err := os.ReadFile(outside) + if err != nil { + t.Fatal(err) + } + if string(victim) != "original" { + t.Errorf("the symlink was followed and the file outside the rootfs was overwritten with %q", victim) + } +} diff --git a/internal/ateomnet/sandbox.go b/internal/ateomnet/sandbox.go new file mode 100644 index 0000000000..d235c5c2ac --- /dev/null +++ b/internal/ateomnet/sandbox.go @@ -0,0 +1,631 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ateomnet + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/netip" + "runtime" + "sync" + "syscall" + + "github.com/agent-substrate/substrate/internal/ateompath" + + "github.com/google/nftables" + "github.com/google/nftables/binaryutil" + "github.com/google/nftables/expr" + "github.com/vishvananda/netlink" + "github.com/vishvananda/netns" + "golang.org/x/sys/unix" +) + +// SandboxNetwork holds a sandbox's runtime and gateway namespaces. +type SandboxNetwork struct { + // ActorUID is the Actor resource's UID. It names the namespaces, so a + // teardown can find them again. + ActorUID string + // RuntimeNetNS is what the sandbox runs in, whichever runtime that is: the + // micro-VM's tap lives here, and gVisor claims every interface here and + // moves their addresses into its own stack. + RuntimeNetNS netns.NsHandle + // GatewayNetNS holds the sandbox's default gateway, DNS relay, and atunnel + // sockets. This is local to the sandbox, not the external egress gateway. + // For microVMs it shares RuntimeNetNS. + // + // TODO: we hope gVisor can take that same single-namespace shape soon, + // once runsc can be given one interface rather than claiming every + // interface in the namespace it runs in. + GatewayNetNS netns.NsHandle +} + +func (n *SandboxNetwork) holdsNetNS() bool { return n.RuntimeNetNS > 0 } + +// SandboxNetworkConfig describes one actor's private networking. +type SandboxNetworkConfig struct { + ActorUID string + + // Veth separates gVisor's interfaces from the kernel-owned gateway. + // MicroVMs use a tap in a single namespace instead. + Veth bool + + // EgressPort is where atunnel serves this actor. Every TCP connection the + // actor makes is redirected to it, whatever port it was aimed at. + EgressPort uint16 + + // DNSPort is where the DNS relay answers, on the sandbox's default gateway. + DNSPort uint16 +} + +// SetupSandboxNetwork creates isolated networking with fixed sandbox addresses. +// gVisor uses a veth pair across runtime and gateway namespaces because it takes +// over every interface in its namespace. MicroVMs use a tap in one namespace. +// nftables at the sandbox's default gateway redirect outbound TCP to atunnel, preserving +// SO_ORIGINAL_DST. Traffic to the gateway address is not redirected, so DNS +// over UDP and TCP reaches the relay's own sockets. +func SetupSandboxNetwork(ctx context.Context, cfg SandboxNetworkConfig) (_ *SandboxNetwork, retErr error) { + actorUID := cfg.ActorUID + if actorUID == "" { + return nil, fmt.Errorf("actornet: actor UID is required") + } + + actorNSName := ateompath.ActorNetNSName(actorUID) + actorNS, err := CreateNetNSWithoutSwitching(actorNSName) + if err != nil { + return nil, fmt.Errorf("while creating the actor netns %s: %w", actorNSName, err) + } + defer func() { + if retErr != nil { + actorNS.Close() + _ = removeNamedNetNS(actorNSName) + } + }() + + // Without a veth, atunnel shares the namespace with the runtime's tap. + atunnelNS := actorNS + if cfg.Veth { + outer, err := setupVethPair(ctx, cfg, actorNS) + if err != nil { + return nil, err + } + defer func() { + if retErr != nil { + outer.Close() + _ = removeNamedNetNS(SandboxGatewayNetNSName(cfg.ActorUID)) + } + }() + atunnelNS = outer + } + if err := setupGatewaySide(ctx, atunnelNS, cfg.EgressPort); err != nil { + return nil, err + } + + return &SandboxNetwork{ + ActorUID: actorUID, + RuntimeNetNS: actorNS, + GatewayNetNS: atunnelNS, + }, nil +} + +// setupVethPair creates the gateway namespace and the veth pair joining it to +// actorNS. The caller owns the returned handle and its name. +func setupVethPair(ctx context.Context, cfg SandboxNetworkConfig, actorNS netns.NsHandle) (_ netns.NsHandle, retErr error) { + gatewayNSName := SandboxGatewayNetNSName(cfg.ActorUID) + outer, err := CreateNetNSWithoutSwitching(gatewayNSName) + if err != nil { + return 0, fmt.Errorf("while creating the outer netns %s: %w", gatewayNSName, err) + } + defer func() { + if retErr != nil { + outer.Close() + _ = removeNamedNetNS(gatewayNSName) + } + }() + + // Keep the kernel-owned peer outside gVisor's namespace. + if err := NetNSDo(ctx, outer, func(context.Context) error { + veth := &netlink.Veth{ + LinkAttrs: netlink.LinkAttrs{Name: gatewayVethName}, + PeerName: ActorVethName, + // Create the peer directly in the actor's namespace. Moving a + // netdev across namespaces afterwards costs several times the + // whole setup, all of it under the global RTNL lock, and this + // runs on the resume path. + PeerNamespace: netlink.NsFd(int(actorNS)), + } + if err := netlink.LinkAdd(veth); err != nil { + return fmt.Errorf("while creating the veth pair: %w", err) + } + atSide, err := netlink.LinkByName(gatewayVethName) + if err != nil { + return err + } + if err := netlink.AddrReplace(atSide, HostVethAddr); err != nil { + return fmt.Errorf("while assigning the atunnel-side address: %w", err) + } + if err := netlink.LinkSetUp(atSide); err != nil { + return err + } + return nil + }); err != nil { + return 0, err + } + + // gVisor imports these addresses and routes into its network stack. + if err := NetNSDo(ctx, actorNS, func(context.Context) error { + // Loopback lets the actor reach its own address. + if err := linkUp("lo"); err != nil { + return err + } + eth0, err := netlink.LinkByName(ActorVethName) + if err != nil { + return err + } + if err := netlink.AddrReplace(eth0, ActorVethAddr); err != nil { + return fmt.Errorf("while assigning the actor address: %w", err) + } + if err := netlink.LinkSetUp(eth0); err != nil { + return err + } + return netlink.RouteReplace(&netlink.Route{ + LinkIndex: eth0.Attrs().Index, + Gw: ActorVethGwIP, + }) + }); err != nil { + return 0, err + } + return outer, nil +} + +func linkUp(name string) error { + link, err := netlink.LinkByName(name) + if err != nil { + return err + } + return netlink.LinkSetUp(link) +} + +// setupGatewaySide brings up lo and puts atunnel in front of the actor's TCP. +func setupGatewaySide(ctx context.Context, ns netns.NsHandle, egressPort uint16) error { + if err := NetNSDo(ctx, ns, func(context.Context) error { + // atunnel answers the actor's DNS on 53, and the worker holds no + // CAP_NET_BIND_SERVICE. + if err := AllowUnprivilegedPorts(); err != nil { + return err + } + return linkUp("lo") + }); err != nil { + return err + } + return installEgressRedirect(ns, egressPort) +} + +// installEgressRedirect redirects TCP egress to atunnel, excluding the sandbox's +// own /30: that keeps ingress replies and DNS over TCP to the gateway off the +// redirect, so the relay serves them on its own listener. +func installEgressRedirect(ns netns.NsHandle, egressPort uint16) error { + if egressPort == 0 { + return fmt.Errorf("actornet: atunnel egress port is required") + } + c, err := nftables.New(nftables.WithNetNSFd(int(ns))) + if err != nil { + return fmt.Errorf("while opening nftables in the actor namespace: %w", err) + } + defer func() { _ = c.CloseLasting() }() + + table := c.AddTable(&nftables.Table{Family: nftables.TableFamilyIPv4, Name: "ateom-actor"}) + prerouting := c.AddChain(&nftables.Chain{ + Name: "prerouting", Table: table, Type: nftables.ChainTypeNAT, + Hooknum: nftables.ChainHookPrerouting, Priority: nftables.ChainPriorityNATDest, + }) + + // Offset of the destination address in an IPv4 header. + const ipv4HeaderDst = 16 + exprs := []expr.Any{ + &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: ipv4HeaderDst, Len: 4}, + &expr.Bitwise{ + SourceRegister: 1, DestRegister: 1, Len: 4, + Mask: actorSubnetMask, Xor: []byte{0, 0, 0, 0}, + }, + &expr.Cmp{Op: expr.CmpOpNeq, Register: 1, Data: actorSubnetBase}, + } + exprs = append(exprs, l4ProtocolEqual(unix.IPPROTO_TCP)...) + exprs = append(exprs, + &expr.Immediate{Register: 1, Data: binaryutil.BigEndian.PutUint16(egressPort)}, + &expr.Redir{RegisterProtoMin: 1}, + ) + c.AddRule(&nftables.Rule{Table: table, Chain: prerouting, Exprs: exprs}) + + if err := c.Flush(); err != nil { + return fmt.Errorf("while installing the actor egress redirect: %w", err) + } + return nil +} + +// The actor subnet, in the form the nftables comparison takes. +var actorSubnetBase, actorSubnetMask = func() ([]byte, []byte) { + _, subnet, err := net.ParseCIDR(ActorVethSubnet) + if err != nil { + panic(fmt.Sprintf("parsing constant actor subnet %q: %v", ActorVethSubnet, err)) + } + return subnet.IP.To4(), subnet.Mask +}() + +// gatewayVethName is the veth peer in the gateway namespace. +const gatewayVethName = "atside" + +// SandboxGatewayNetNSName names the namespace holding the veth peer and atunnel's +// sockets for one actor. +func SandboxGatewayNetNSName(actorUID string) string { + return ateompath.ActorNetNSName(actorUID) + "-at" +} + +// CleanupSandboxNetwork closes namespace handles and removes their names. +func CleanupSandboxNetwork(network *SandboxNetwork) error { + if network == nil { + return nil + } + var errs error + // Compare before Close sets the handle to -1; microVMs share one descriptor. + separateNS := network.GatewayNetNS != network.RuntimeNetNS + if network.holdsNetNS() { + if err := network.RuntimeNetNS.Close(); err != nil { + errs = errors.Join(errs, fmt.Errorf("while closing the sandbox netns: %w", err)) + } + } + if separateNS && network.GatewayNetNS > 0 { + if err := network.GatewayNetNS.Close(); err != nil { + errs = errors.Join(errs, fmt.Errorf("while closing the gateway netns: %w", err)) + } + } + // Deleting the namespaces takes any veth pair with them. + for _, name := range []string{ateompath.ActorNetNSName(network.ActorUID), SandboxGatewayNetNSName(network.ActorUID)} { + if err := removeNamedNetNS(name); err != nil { + errs = errors.Join(errs, fmt.Errorf("while deleting netns %s: %w", name, err)) + } + } + return errs +} + +// ListenInNetNS opens wildcard TCP listeners inside ns. +// Sockets retain their namespace and can be served from another namespace. +func ListenInNetNS(ctx context.Context, ns netns.NsHandle, ports []uint16) (_ []net.Listener, retErr error) { + var listeners []net.Listener + defer func() { + if retErr != nil { + for _, l := range listeners { + _ = l.Close() + } + } + }() + if err := NetNSDo(ctx, ns, func(context.Context) error { + for _, port := range ports { + l, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", port)) + if err != nil { + return fmt.Errorf("while listening on port %d: %w", port, err) + } + listeners = append(listeners, l) + } + return nil + }); err != nil { + return nil, err + } + return listeners, nil +} + +// egressServer serves one actor's captured connections. Satisfied by +// atunnel.Egress; an interface so this package does not depend on it. +type egressServer interface { + Serve(ctx context.Context, listener net.Listener) error +} + +// ServeSandboxEgress serves redirected TCP in the gateway namespace. +// Closing the returned listeners stops accepting new connections. +func serveSandboxEgress(ctx context.Context, e egressServer, ns netns.NsHandle, ports []uint16) ([]io.Closer, []func(), error) { + listeners, err := ListenInNetNS(ctx, ns, ports) + if err != nil { + return nil, nil, fmt.Errorf("while opening actor egress listeners: %w", err) + } + serve := make([]func(), 0, len(listeners)) + closers := make([]io.Closer, 0, len(listeners)) + for _, l := range listeners { + closers = append(closers, l) + serve = append(serve, func() { + // Background rather than the caller's context: these outlive the + // activation and are stopped by closing the listener. + if err := e.Serve(context.Background(), l); err != nil { + slog.WarnContext(ctx, "Sandbox egress listener stopped", slog.Any("err", err)) + } + }) + } + return closers, serve, nil +} + +// closerFunc adapts a cancel function to io.Closer, so a caller takes a +// sandbox's sockets and the work behind them down as one list. +type closerFunc func() error + +func (f closerFunc) Close() error { return f() } + +// withNetNS switches to targetNS, calls run, then restores the original namespace. +// run can call restore to switch back and unlock the OS thread before returning. +// Calling restore again after it succeeds has no effect. +// +// A separate goroutine lets us leave the thread locked if restoration fails. +// Go then discards that thread when the goroutine exits. +func withNetNS(targetNS netns.NsHandle, run func(restore func() error) error) error { + var resultErr error + var done sync.WaitGroup + done.Add(1) + go func() { + defer done.Done() + runtime.LockOSThread() + originalNS, err := netns.Get() + if err != nil { + runtime.UnlockOSThread() + resultErr = fmt.Errorf("while reading the current netns: %w", err) + return + } + defer originalNS.Close() + if err := netns.Set(targetNS); err != nil { + runtime.UnlockOSThread() + resultErr = fmt.Errorf("while entering the actor netns: %w", err) + return + } + + restored := false + restore := func() error { + if restored { + return nil + } + if err := netns.Set(originalNS); err != nil { + return fmt.Errorf("while restoring the worker netns: %w", err) + } + runtime.UnlockOSThread() + restored = true + return nil + } + + resultErr = run(restore) + if err := restore(); err != nil { + resultErr = err + } + }() + done.Wait() + return resultErr +} + +// NetNSDialer dials TCP or UDP IP literals in ns, pinning a thread only until +// the socket is created. +func NetNSDialer(ns netns.NsHandle) func(context.Context, string, string) (net.Conn, error) { + return func(ctx context.Context, network, addr string) (net.Conn, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if err := validateNetNSDialTarget(network, addr); err != nil { + return nil, err + } + + var conn net.Conn + dialErr := withNetNS(ns, func(restore func() error) error { + // Only creating the socket needs the namespace, and + // ControlContext runs once it exists: restore there rather than + // holding the thread for the whole connect. + socketCreated := false + dialer := net.Dialer{ControlContext: func(context.Context, string, string, syscall.RawConn) error { + if socketCreated { + return errors.New("sandbox dial cannot recreate its socket outside the namespace") + } + socketCreated = true + return restore() + }} + var err error + conn, err = dialer.DialContext(ctx, network, addr) + return err + }) + if dialErr != nil || ctx.Err() != nil { + if conn != nil { + _ = conn.Close() + } + if dialErr != nil { + return nil, dialErr + } + return nil, ctx.Err() + } + return conn, nil + } +} + +func validateNetNSDialTarget(network, addr string) error { + switch network { + case "tcp", "tcp4", "tcp6", "udp", "udp4", "udp6": + default: + return net.UnknownNetworkError(network) + } + hostname, _, err := net.SplitHostPort(addr) + if err != nil { + return err + } + if _, err := netip.ParseAddr(hostname); err != nil { + return fmt.Errorf("NetNSDialer supports only IP literals (got %q): %w", hostname, err) + } + return nil +} + +// SandboxSession owns a sandbox's network and serving sockets. +type SandboxSession struct { + Network *SandboxNetwork + + mu sync.Mutex + sockets []io.Closer + // serving counts the goroutines serving this sandbox, so Close can wait + // for them rather than just closing their sockets. + serving sync.WaitGroup +} + +// ServeSandbox builds a sandbox's network and serves egress and DNS from its +// gateway namespace. A nil server leaves that unserved, which fails closed. +func ServeSandbox(ctx context.Context, cfg SandboxNetworkConfig, egress egressServer, dns dnsServer) (_ *SandboxSession, retErr error) { + network, err := SetupSandboxNetwork(ctx, cfg) + if err != nil { + return nil, err + } + session := &SandboxSession{Network: network} + defer func() { + if retErr != nil { + _ = session.Close(ctx) + } + }() + + var serve []func() + if egress != nil { + closers, serveEgress, err := serveSandboxEgress(ctx, egress, network.GatewayNetNS, []uint16{cfg.EgressPort}) + if err != nil { + return nil, err + } + session.sockets = append(session.sockets, closers...) + serve = append(serve, serveEgress...) + } + if dns != nil { + closers, serveDNS, err := serveSandboxDNS(ctx, dns, network.GatewayNetNS, cfg.DNSPort) + if err != nil { + return nil, err + } + session.sockets = append(session.sockets, closers...) + serve = append(serve, serveDNS...) + } + + // Started here rather than inside the helpers so the session owns them and + // Close can report when they have stopped. + for _, fn := range serve { + session.serving.Add(1) + go func() { + defer session.serving.Done() + fn() + }() + } + return session, nil +} + +// Close cancels the work in flight, closes the sockets, waits for the serving +// goroutines, then removes the namespaces. ctx bounds only the wait; the +// namespaces go either way, since leaving them wedges the next activation. +// Idempotent, and every step's error is returned. +func (s *SandboxSession) Close(ctx context.Context) error { + s.mu.Lock() + sockets, network := s.sockets, s.Network + s.sockets, s.Network = nil, nil + s.mu.Unlock() + + var errs error + for _, c := range sockets { + if err := c.Close(); err != nil && !errors.Is(err, net.ErrClosed) { + errs = errors.Join(errs, err) + } + } + + stopped := make(chan struct{}) + go func() { + s.serving.Wait() + close(stopped) + }() + select { + case <-stopped: + case <-ctx.Done(): + errs = errors.Join(errs, fmt.Errorf("while waiting for the sandbox's serving goroutines: %w", ctx.Err())) + } + + if network != nil { + errs = errors.Join(errs, CleanupSandboxNetwork(network)) + } + return errs +} + +// SessionHolder is the one sandbox session a worker is serving, for the ateoms +// to share what is otherwise the same locking, replacement and dialing in both. +type SessionHolder struct { + mu sync.Mutex + session *SandboxSession +} + +// Replace closes whatever session is held and installs next. A previous +// actor's session can still be here if its teardown never ran, and overwriting +// it would leak both namespaces, their /run/netns mounts and the goroutines +// serving them. +func (h *SessionHolder) Replace(ctx context.Context, next *SandboxSession) error { + if err := h.Close(ctx); err != nil { + return fmt.Errorf("while releasing the previous sandbox network: %w", err) + } + h.mu.Lock() + defer h.mu.Unlock() + h.session = next + return nil +} + +// Session is what is held, or nil between activations. +func (h *SessionHolder) Session() *SandboxSession { + h.mu.Lock() + defer h.mu.Unlock() + return h.session +} + +// Close releases the held session, if any. +func (h *SessionHolder) Close(ctx context.Context) error { + h.mu.Lock() + session := h.session + h.session = nil + h.mu.Unlock() + if session == nil { + return nil + } + return session.Close(ctx) +} + +// Dialer reaches whichever sandbox is held when the dial happens. +func (h *SessionHolder) Dialer() func(context.Context, string, string) (net.Conn, error) { + return func(ctx context.Context, network, address string) (net.Conn, error) { + session := h.Session() + if session == nil { + return nil, errors.New("no actor is active on this worker") + } + return session.Dialer()(ctx, network, address) + } +} + +// Dialer reaches the sandbox from the gateway namespace, the only place its +// address is routable. +func (s *SandboxSession) Dialer() func(context.Context, string, string) (net.Conn, error) { + return func(ctx context.Context, network, address string) (net.Conn, error) { + s.mu.Lock() + if s.Network == nil { + s.mu.Unlock() + return nil, net.ErrClosed + } + fd, err := unix.FcntlInt(uintptr(s.Network.GatewayNetNS), unix.F_DUPFD_CLOEXEC, 0) + s.mu.Unlock() + if err != nil { + return nil, fmt.Errorf("while retaining the sandbox namespace: %w", err) + } + ns := netns.NsHandle(fd) + defer ns.Close() + return NetNSDialer(ns)(ctx, network, address) + } +} diff --git a/internal/ateomnet/sandbox_atunnel_linux_test.go b/internal/ateomnet/sandbox_atunnel_linux_test.go new file mode 100644 index 0000000000..2fc166e6e8 --- /dev/null +++ b/internal/ateomnet/sandbox_atunnel_linux_test.go @@ -0,0 +1,97 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// These integration tests use an external package to avoid an import cycle: +// atunnel imports ateomnet. +package ateomnet_test + +import ( + "context" + "io" + "net" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/ateomnet" + "github.com/agent-substrate/substrate/internal/atunnel" + "github.com/agent-substrate/substrate/internal/roottest" +) + +// Verify redirection preserves the destination, including unconfigured ports. +func TestSandboxEgressReachesAtunnelOnAnyPort(t *testing.T) { + roottest.Require(t, "creates network namespaces") + ctx := context.Background() + const egressPort = 15001 + + n, err := ateomnet.SetupSandboxNetwork(ctx, ateomnet.SandboxNetworkConfig{ + ActorUID: "66666666-6666-6666-6666-666666666666", Veth: true, EgressPort: egressPort, + }) + if err != nil { + t.Fatalf("SetupSandboxNetwork: %v", err) + } + t.Cleanup(func() { ateomnet.CleanupSandboxNetwork(n) }) + + listeners, err := ateomnet.ListenInNetNS(ctx, n.GatewayNetNS, []uint16{egressPort}) + if err != nil { + t.Fatalf("ListenInNetNS: %v", err) + } + defer listeners[0].Close() + + type capture struct{ destination, payload string } + got := make(chan capture, 1) + accept := func() { + c, err := listeners[0].Accept() + if err != nil { + return + } + defer c.Close() + dst, err := atunnel.TCPOriginalDestination(c) + if err != nil { + t.Errorf("TCPOriginalDestination: %v", err) + return + } + buf := make([]byte, len("hello")) + io.ReadFull(c, buf) + got <- capture{destination: dst, payload: string(buf)} + } + go accept() + + for _, want := range []string{"93.184.216.34:443", "93.184.216.34:8080", "93.184.216.34:9999"} { + if err := ateomnet.NetNSDo(ctx, n.RuntimeNetNS, func(context.Context) error { + c, err := net.Dial("tcp", want) + if err != nil { + return err + } + defer c.Close() + _, err = io.WriteString(c, "hello") + return err + }); err != nil { + t.Fatalf("sandbox egress dial to %s: %v", want, err) + } + select { + case c := <-got: + if c.destination != want { + t.Errorf("atunnel saw destination %q, want %q", c.destination, want) + } + if c.payload != "hello" { + t.Errorf("atunnel read %q, want %q", c.payload, "hello") + } + case <-time.After(10 * time.Second): + t.Fatalf("the sandbox's connection to %s never reached atunnel", want) + } + go accept() + } +} diff --git a/internal/ateomnet/sandbox_linux_test.go b/internal/ateomnet/sandbox_linux_test.go new file mode 100644 index 0000000000..237932d448 --- /dev/null +++ b/internal/ateomnet/sandbox_linux_test.go @@ -0,0 +1,571 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ateomnet + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "runtime/pprof" + "sync" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/ateompath" + "github.com/vishvananda/netlink" + + "github.com/agent-substrate/substrate/internal/roottest" + "github.com/vishvananda/netns" +) + +const testEgressPort = 15001 + +func TestValidateNetNSDialTarget(t *testing.T) { + for _, target := range []struct { + network, address string + wantErr bool + }{ + {"tcp", "127.0.0.1:80", false}, + {"tcp4", "127.0.0.1:80", false}, + {"tcp6", "[::1]:80", false}, + {"udp", "127.0.0.1:53", false}, + {"udp4", "127.0.0.1:53", false}, + {"udp6", "[fe80::1%eth0]:53", false}, + {"tcp", "localhost:80", true}, + {"tcp", ":80", true}, + {"tcp", "127.0.0.1", true}, + {"unix", "/tmp/socket", true}, + {"ip", "127.0.0.1:80", true}, + } { + t.Run(target.network+"/"+target.address, func(t *testing.T) { + err := validateNetNSDialTarget(target.network, target.address) + if (err != nil) != target.wantErr { + t.Errorf("validateNetNSDialTarget = %v, want error: %t", err, target.wantErr) + } + }) + } + if err := validateNetNSDialTarget("unix", "/tmp/socket"); !errors.Is(err, net.UnknownNetworkError("unix")) { + t.Errorf("unsupported network: got %v, want UnknownNetworkError", err) + } +} + +func TestNetNSDialerRejectsNonIPTargets(t *testing.T) { + for _, target := range []struct{ network, address string }{ + {"tcp", "localhost:80"}, + {"tcp", ":80"}, + {"tcp", "127.0.0.1"}, + {"unix", "/tmp/socket"}, + } { + if conn, err := NetNSDialer(-1)(context.Background(), target.network, target.address); err == nil { + _ = conn.Close() + t.Errorf("accepted %s %s", target.network, target.address) + } + } +} + +func TestNetNSDialerDoesNotPinPendingThreads(t *testing.T) { + roottest.Require(t, "creates network namespaces") + network, err := SetupSandboxNetwork(context.Background(), SandboxNetworkConfig{ + ActorUID: "pending-dial-threads", EgressPort: testEgressPort, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = CleanupSandboxNetwork(network) }) + if err := NetNSDo(context.Background(), network.GatewayNetNS, func(context.Context) error { + link := &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: "silent"}} + if err := netlink.LinkAdd(link); err != nil { + return err + } + if err := netlink.AddrReplace(link, MustParseAddr("192.0.2.1/24")); err != nil { + return err + } + return netlink.LinkSetUp(link) + }); err != nil { + t.Fatal(err) + } + before := pprof.Lookup("threadcreate").Count() + ctx, cancel := context.WithCancel(context.Background()) + var workers sync.WaitGroup + defer func() { cancel(); workers.Wait() }() + finished := make(chan error, 64) + for range 64 { + workers.Add(1) + go func() { + defer workers.Done() + conn, err := NetNSDialer(network.GatewayNetNS)(ctx, "tcp", "192.0.2.2:80") + if conn != nil { + _ = conn.Close() + } + finished <- err + }() + } + select { + case err := <-finished: + t.Fatalf("dial did not remain pending: %v", err) + case <-time.After(250 * time.Millisecond): + } + if growth := pprof.Lookup("threadcreate").Count() - before; growth >= 48 { + t.Errorf("64 pending dials created %d native threads", growth) + } + cancel() + for range 64 { + select { + case err := <-finished: + if !errors.Is(err, context.Canceled) { + t.Errorf("canceled dial: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("dial did not stop after cancellation") + } + } +} + +func TestSandboxSessionDialerAfterClose(t *testing.T) { + session := &SandboxSession{} + dial := session.Dialer() + if err := session.Close(context.Background()); err != nil { + t.Fatal(err) + } + if _, err := dial(context.Background(), "tcp", "127.0.0.1:1"); !errors.Is(err, net.ErrClosed) { + t.Fatalf("dial after close: got %v, want closed", err) + } +} + +func TestSandboxSessionDialerConcurrentClose(t *testing.T) { + roottest.Require(t, "creates network namespaces") + session, err := ServeSandbox(context.Background(), SandboxNetworkConfig{ + ActorUID: "concurrent-close", EgressPort: testEgressPort, + }, nil, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = session.Close(context.Background()) }) + dial := session.Dialer() + start := make(chan struct{}) + var workers sync.WaitGroup + for range 32 { + workers.Add(1) + go func() { + defer workers.Done() + <-start + for range 32 { + conn, err := dial(context.Background(), "udp", "127.0.0.1:9") + if err != nil { + if !errors.Is(err, net.ErrClosed) { + t.Errorf("dial during close: %v", err) + } + return + } + _ = conn.Close() + } + }() + } + close(start) + if err := session.Close(context.Background()); err != nil { + t.Error(err) + } + workers.Wait() +} + +func TestNetNSDialerCanceledContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := NetNSDialer(-1)(ctx, "tcp", "127.0.0.1:1"); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled dial: got %v, want cancellation", err) + } +} + +func TestSetupSandboxNetwork(t *testing.T) { + roottest.Require(t, "creates network namespaces") + ctx := context.Background() + + type actor struct { + net *SandboxNetwork + body string + } + actors := map[string]*actor{} + for _, uid := range []string{"11111111-1111-1111-1111-111111111111", "22222222-2222-2222-2222-222222222222"} { + n, err := SetupSandboxNetwork(ctx, SandboxNetworkConfig{ActorUID: uid, Veth: true, EgressPort: testEgressPort}) + if err != nil { + t.Fatalf("SetupSandboxNetwork(%s): %v", uid, err) + } + t.Cleanup(func() { + if err := CleanupSandboxNetwork(n); err != nil { + t.Errorf("cleanup %s: %v", uid, err) + } + }) + + // The actor's app, bound where a real one binds, inside its namespace. + var lis net.Listener + if err := NetNSDo(ctx, n.RuntimeNetNS, func(context.Context) error { + l, err := net.Listen("tcp", net.JoinHostPort(ActorVethIP, "80")) + lis = l + return err + }); err != nil { + t.Fatalf("actor %s listen: %v", uid, err) + } + t.Cleanup(func() { lis.Close() }) + body := "i-am-" + uid[:8] + go http.Serve(lis, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, body) + })) + actors[uid] = &actor{net: n, body: body} + } + + // Reaching each actor is a matter of which namespace the dial is made from. + for uid, a := range actors { + client := &http.Client{Transport: &http.Transport{DialContext: NetNSDialer(a.net.RuntimeNetNS)}, Timeout: 5 * time.Second} + resp, err := client.Get((&url.URL{Scheme: "http", Host: net.JoinHostPort(ActorVethIP, "80")}).String()) + if err != nil { + t.Fatalf("reaching actor %s: %v", uid, err) + } + got, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if string(got) != a.body { + t.Errorf("actor %s answered %q, want %q", uid, got, a.body) + } + } + + // Sandbox addresses must not be reachable from the worker namespace. + direct := &http.Client{Timeout: 2 * time.Second} + if _, err := direct.Get("http://" + net.JoinHostPort(ActorVethIP, "80")); err == nil { + t.Error("the worker namespace reached an actor directly; addresses are not isolated") + } +} + +func TestActorEgressIsFailClosedWithoutAtunnel(t *testing.T) { + roottest.Require(t, "creates network namespaces") + ctx := context.Background() + + n, err := SetupSandboxNetwork(ctx, SandboxNetworkConfig{ + ActorUID: "33333333-3333-3333-3333-333333333333", Veth: true, EgressPort: testEgressPort, + }) + if err != nil { + t.Fatalf("SetupSandboxNetwork: %v", err) + } + t.Cleanup(func() { CleanupSandboxNetwork(n) }) + + for _, destination := range []string{"93.184.216.34:443", "93.184.216.34:8080"} { + if err := NetNSDo(ctx, n.RuntimeNetNS, func(context.Context) error { + c, err := net.DialTimeout("tcp", destination, 3*time.Second) + if err != nil { + return err + } + c.Close() + return nil + }); err == nil { + t.Errorf("the actor reached %s with no atunnel listening; egress is not fail-closed", destination) + } + } +} + +func TestIngressCrossesThePairWhileEgressIsCaptured(t *testing.T) { + roottest.Require(t, "creates network namespaces") + ctx := context.Background() + + n, err := SetupSandboxNetwork(ctx, SandboxNetworkConfig{ActorUID: "44444444-4444-4444-4444-444444444444", Veth: true, EgressPort: testEgressPort}) + if err != nil { + t.Fatalf("SetupSandboxNetwork: %v", err) + } + t.Cleanup(func() { CleanupSandboxNetwork(n) }) + + var app net.Listener + if err := NetNSDo(ctx, n.RuntimeNetNS, func(context.Context) error { + l, e := net.Listen("tcp", net.JoinHostPort(ActorVethIP, "80")) + app = l + return e + }); err != nil { + t.Fatalf("actor listen: %v", err) + } + defer app.Close() + go func() { + for { + c, e := app.Accept() + if e != nil { + return + } + io.WriteString(c, "the-actor") + c.Close() + } + }() + + dial := NetNSDialer(n.GatewayNetNS) + c, err := dial(ctx, "tcp", net.JoinHostPort(ActorVethIP, "80")) + if err != nil { + t.Fatalf("ingress dial: %v", err) + } + got, _ := io.ReadAll(c) + c.Close() + if string(got) != "the-actor" { + t.Errorf("ingress reached %q, want %q", got, "the-actor") + } + + // An unopened port must be refused, not redirected to atunnel. + cctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + if c2, e := dial(cctx, "tcp", net.JoinHostPort(ActorVethIP, "81")); e == nil { + c2.Close() + t.Error("a port with no listener was accepted, so ingress is not reaching the sandbox") + } +} + +// This tests namespace setup only; testing microVM egress requires a tap. +func TestSetupSandboxNetworkWithoutVeth(t *testing.T) { + roottest.Require(t, "creates network namespaces") + ctx := context.Background() + + n, err := SetupSandboxNetwork(ctx, SandboxNetworkConfig{ + ActorUID: "77777777-7777-7777-7777-777777777777", + EgressPort: testEgressPort, + }) + if err != nil { + t.Fatalf("SetupSandboxNetwork: %v", err) + } + t.Cleanup(func() { CleanupSandboxNetwork(n) }) + + if n.GatewayNetNS != n.RuntimeNetNS { + t.Errorf("got a second namespace (%v vs %v); this shape needs one", n.GatewayNetNS, n.RuntimeNetNS) + } + + // No veth was built, so nothing but lo is here until the tap arrives. + if err := NetNSDo(ctx, n.RuntimeNetNS, func(context.Context) error { + links, err := netlink.LinkList() + if err != nil { + return err + } + for _, l := range links { + if l.Attrs().Name != "lo" { + t.Errorf("unexpected interface %q in the actor namespace", l.Attrs().Name) + } + } + return nil + }); err != nil { + t.Fatalf("listing links: %v", err) + } +} + +func TestSetupSucceedsOverALeftoverNamespace(t *testing.T) { + roottest.Require(t, "creates network namespaces") + ctx := context.Background() + const uid = "aaaaaaaa-0000-0000-0000-00000000000a" + cfg := SandboxNetworkConfig{ActorUID: uid, Veth: true, EgressPort: testEgressPort} + + first, err := SetupSandboxNetwork(ctx, cfg) + if err != nil { + t.Fatalf("first SetupSandboxNetwork: %v", err) + } + // Simulate interrupted teardown by leaving the namespace names mounted. + first.RuntimeNetNS.Close() + if first.GatewayNetNS != first.RuntimeNetNS { + first.GatewayNetNS.Close() + } + for _, name := range []string{ateompath.ActorNetNSName(uid), SandboxGatewayNetNSName(uid)} { + if _, err := os.Stat("/var/run/netns/" + name); err != nil { + t.Fatalf("expected leftover netns %s: %v", name, err) + } + } + + second, err := SetupSandboxNetwork(ctx, cfg) + if err != nil { + t.Fatalf("the actor is wedged by its own leftover namespace: %v", err) + } + t.Cleanup(func() { CleanupSandboxNetwork(second) }) + + if err := NetNSDo(ctx, second.RuntimeNetNS, func(context.Context) error { + if _, err := netlink.LinkByName(ActorVethName); err != nil { + return fmt.Errorf("actor interface missing after reuse: %w", err) + } + return nil + }); err != nil { + t.Error(err) + } +} + +// Check from the gateway: a successful UDP send from the actor does not prove delivery. +func TestActorUDPHasNowhereToGoBeyondTheNamespacePair(t *testing.T) { + roottest.Require(t, "creates network namespaces") + ctx := context.Background() + + n, err := SetupSandboxNetwork(ctx, SandboxNetworkConfig{ + ActorUID: "bbbbbbbb-0000-0000-0000-00000000000b", Veth: true, EgressPort: testEgressPort, + }) + if err != nil { + t.Fatalf("SetupSandboxNetwork: %v", err) + } + t.Cleanup(func() { CleanupSandboxNetwork(n) }) + + for _, destination := range []string{"93.184.216.34:443", "93.184.216.34:53"} { + if err := NetNSDo(ctx, n.GatewayNetNS, func(context.Context) error { + c, err := net.Dial("udp", destination) + if err != nil { + return err + } + defer c.Close() + _, err = c.Write([]byte("probe")) + return err + }); err == nil { + t.Errorf("the atunnel namespace can reach %s over UDP; actor UDP could follow it out", destination) + } + } +} + +func TestCleanupClosesEachDescriptorOnce(t *testing.T) { + roottest.Require(t, "creates network namespaces") + network, err := SetupSandboxNetwork(context.Background(), SandboxNetworkConfig{ + ActorUID: "close-once", + EgressPort: 15001, + }) + if err != nil { + t.Fatal(err) + } + if network.RuntimeNetNS != network.GatewayNetNS { + t.Fatalf("expected one namespace without a veth, got %d and %d", network.RuntimeNetNS, network.GatewayNetNS) + } + + // A second close of the same descriptor reports EBADF. + if err := CleanupSandboxNetwork(network); err != nil { + t.Fatalf("CleanupSandboxNetwork closed a descriptor twice: %v", err) + } +} + +// stoppableDNS records that its serving contexts were canceled. +type stoppableDNS struct{ packet, stream chan struct{} } + +func (d *stoppableDNS) ServePacket(ctx context.Context, pc net.PacketConn) error { + <-ctx.Done() + close(d.packet) + return pc.Close() +} + +func (d *stoppableDNS) Serve(ctx context.Context, l net.Listener) error { + <-ctx.Done() + close(d.stream) + return l.Close() +} + +func TestClosingSandboxDNSStopsServing(t *testing.T) { + roottest.Require(t, "creates network namespaces") + network, err := SetupSandboxNetwork(context.Background(), SandboxNetworkConfig{ + ActorUID: "dns-teardown", + EgressPort: 15001, + }) + if err != nil { + t.Fatal(err) + } + defer func() { _ = CleanupSandboxNetwork(network) }() + + relay := &stoppableDNS{packet: make(chan struct{}), stream: make(chan struct{})} + closers, serve, err := serveSandboxDNS(context.Background(), relay, network.GatewayNetNS, 53) + if err != nil { + t.Fatal(err) + } + for _, fn := range serve { + go fn() + } + for _, c := range closers { + _ = c.Close() + } + + for _, tc := range []struct { + name string + stopped chan struct{} + }{{"UDP", relay.packet}, {"TCP", relay.stream}} { + select { + case <-tc.stopped: + case <-time.After(5 * time.Second): + t.Errorf("%s serving outlived the sandbox's sockets", tc.name) + } + } +} + +// slowDNS holds its serving goroutines open until released, so a test can tell +// whether Close waits for them or merely closes their sockets. +type slowDNS struct{ release chan struct{} } + +func (d *slowDNS) ServePacket(ctx context.Context, pc net.PacketConn) error { + <-ctx.Done() + <-d.release + return pc.Close() +} + +func (d *slowDNS) Serve(ctx context.Context, l net.Listener) error { + <-ctx.Done() + <-d.release + return l.Close() +} + +// Close's contract is that serving has stopped when it returns, not just that +// the sockets are shut: a caller tearing an actor down needs the relay's +// capacity back. +func TestSessionCloseWaitsForServingToStop(t *testing.T) { + roottest.Require(t, "creates network namespaces") + relay := &slowDNS{release: make(chan struct{})} + session, err := ServeSandbox(context.Background(), SandboxNetworkConfig{ + ActorUID: "close-waits", EgressPort: testEgressPort, DNSPort: 53, + }, nil, relay) + if err != nil { + t.Fatal(err) + } + + returned := make(chan error, 1) + go func() { returned <- session.Close(context.Background()) }() + select { + case <-returned: + t.Fatal("Close returned while the relay was still serving") + case <-time.After(250 * time.Millisecond): + } + + close(relay.release) + select { + case err := <-returned: + if err != nil { + t.Errorf("Close: %v", err) + } + case <-time.After(30 * time.Second): + t.Error("Close did not return after serving stopped") + } +} + +// A caller that cannot wait forever gets its deadline back as an error, and +// the namespaces are still removed -- leaving them would wedge the next +// activation of this actor. +func TestSessionCloseReportsAWaitItCouldNotFinish(t *testing.T) { + roottest.Require(t, "creates network namespaces") + relay := &slowDNS{release: make(chan struct{})} + defer close(relay.release) + session, err := ServeSandbox(context.Background(), SandboxNetworkConfig{ + ActorUID: "close-deadline", EgressPort: testEgressPort, DNSPort: 53, + }, nil, relay) + if err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + if err := session.Close(ctx); !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("Close = %v, want the deadline reported", err) + } + if _, err := netns.GetFromName(ateompath.ActorNetNSName("close-deadline")); err == nil { + t.Error("the namespace survived a Close whose wait timed out") + } +} diff --git a/internal/ateompath/ateompath.go b/internal/ateompath/ateompath.go index 2a18303c12..ea5ed4a50d 100644 --- a/internal/ateompath/ateompath.go +++ b/internal/ateompath/ateompath.go @@ -98,15 +98,19 @@ func AteomSocketPath(podUID string) string { ) } -func AteomNetNSName(podUID string) string { - return "ateom:" + podUID +// ActorNetNSName names an actor's sandbox network namespace. +func ActorNetNSName(actorUID string) string { + return "ateom-actor:" + actorUID } -func AteomNetNSPath(podUID string) string { - return filepath.Join( - "/run/netns", - AteomNetNSName(podUID), - ) +// ActorNetNSPath is the mount path of the actor's named namespace. +func ActorNetNSPath(actorUID string) string { + return filepath.Join("/run/netns", ActorNetNSName(actorUID)) +} + +// ActorResolvConfPath is the resolver bind source outside the actor's rootfs. +func ActorResolvConfPath(actorUID string) string { + return filepath.Join(ActorPath(actorUID), "resolv.conf") } func ActorPath(actorUID string) string { diff --git a/internal/atunnel/dns.go b/internal/atunnel/dns.go new file mode 100644 index 0000000000..24d6541b33 --- /dev/null +++ b/internal/atunnel/dns.go @@ -0,0 +1,348 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package atunnel + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "log/slog" + "net" + "os" + "strings" + "sync" + "time" +) + +const ( + // DNSPort is the relay port on the sandbox's gateway. + DNSPort = 53 + + // Read complete UDP datagrams without truncating EDNS responses. + maxDNSDatagram = 65535 + + // Drop excess UDP queries to bound goroutines and upstream sockets. + maxInFlightDNS = 64 + + // maxDNSConnections bounds open TCP connections. + maxDNSConnections = 16 + + // dnsTCPTimeout limits connection lifetime, including idle clients. + dnsTCPTimeout = 30 * time.Second + + // dnsExchangeTimeout bounds each upstream attempt. + dnsExchangeTimeout = 5 * time.Second +) + +// DNSRelay forwards UDP and TCP DNS unchanged to the worker pod's resolvers. +// It listens in the sandbox's gateway namespace and dials from the worker's. +// DNS bypasses the egress tunnel and is not checked against egress policy. +type DNSRelay struct { + upstreams []string + // dialer reaches upstream resolvers from the worker namespace. + dialer *net.Dialer + + // Limits are shared across all sandboxes using this relay. + inFlight chan struct{} + connections chan struct{} +} + +// NewDNSRelay forwards to upstreams, each "host:port". +func NewDNSRelay(upstreams []string) (*DNSRelay, error) { + if len(upstreams) == 0 { + return nil, fmt.Errorf("atunnel: at least one upstream resolver is required") + } + for _, u := range upstreams { + if _, _, err := net.SplitHostPort(u); err != nil { + return nil, fmt.Errorf("atunnel: invalid upstream resolver %q: %w", u, err) + } + } + return &DNSRelay{ + upstreams: upstreams, + dialer: &net.Dialer{Timeout: dnsExchangeTimeout}, + inFlight: make(chan struct{}, maxInFlightDNS), + connections: make(chan struct{}, maxDNSConnections), + }, nil +} + +// ResolvConfNameservers reads nameservers from resolv.conf as "host:53". +func ResolvConfNameservers(path string) ([]string, error) { + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("atunnel: reading resolv.conf: %w", err) + } + defer f.Close() + + var out []string + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if i := strings.IndexAny(line, "#;"); i >= 0 { + line = strings.TrimSpace(line[:i]) + } + rest, ok := strings.CutPrefix(line, "nameserver") + if !ok { + continue + } + address := strings.TrimSpace(rest) + if address == "" || net.ParseIP(address) == nil { + continue + } + out = append(out, net.JoinHostPort(address, "53")) + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("atunnel: reading resolv.conf: %w", err) + } + if len(out) == 0 { + return nil, fmt.Errorf("atunnel: %s names no usable nameserver", path) + } + return out, nil +} + +// ServePacket answers UDP queries until ctx is canceled or the socket fails. +func (r *DNSRelay) ServePacket(ctx context.Context, pc net.PacketConn) error { + done := make(chan struct{}) + go func() { + select { + case <-ctx.Done(): + _ = pc.Close() + case <-done: + } + }() + defer close(done) + + var wg sync.WaitGroup + defer wg.Wait() + + buf := make([]byte, maxDNSDatagram) + for { + n, from, err := pc.ReadFrom(buf) + if err != nil { + if ctx.Err() != nil || errors.Is(err, net.ErrClosed) { + return nil + } + return fmt.Errorf("atunnel: reading actor DNS query: %w", err) + } + // Copied: the buffer is reused by the next read. + query := make([]byte, n) + copy(query, buf[:n]) + + select { + case r.inFlight <- struct{}{}: + default: + slog.DebugContext(ctx, "atunnel dropped a DNS query; too many in flight") + continue + } + wg.Add(1) + go func() { + defer wg.Done() + defer func() { <-r.inFlight }() + answer, err := r.exchangeUDP(ctx, query) + if err != nil { + slog.WarnContext(ctx, "atunnel could not resolve an actor DNS query", slog.Any("err", err)) + return + } + if _, err := pc.WriteTo(answer, from); err != nil && ctx.Err() == nil { + slog.WarnContext(ctx, "atunnel could not return a DNS answer", slog.Any("err", err)) + } + }() + } +} + +// Serve relays TCP DNS connections until ctx is canceled or the listener closes. +func (r *DNSRelay) Serve(ctx context.Context, listener net.Listener) error { + done := make(chan struct{}) + go func() { + select { + case <-ctx.Done(): + _ = listener.Close() + case <-done: + } + }() + defer close(done) + + // Wait for the relays to drain before returning, so a closed listener + // leaves no goroutine still holding a connection slot. + var wg sync.WaitGroup + defer wg.Wait() + + for { + conn, err := listener.Accept() + if err != nil { + if ctx.Err() != nil || errors.Is(err, net.ErrClosed) { + return nil + } + return fmt.Errorf("atunnel: accepting actor DNS connection: %w", err) + } + select { + case r.connections <- struct{}{}: + default: + slog.DebugContext(ctx, "atunnel refused a DNS connection; too many open") + _ = conn.Close() + continue + } + wg.Add(1) + go func() { + defer wg.Done() + defer func() { <-r.connections }() + r.relayTCP(ctx, conn) + }() + } +} + +func (r *DNSRelay) exchangeUDP(ctx context.Context, query []byte) ([]byte, error) { + var errs error + // deferred holds a server-failure answer to fall back on, see below. + var deferred []byte + for _, upstream := range r.upstreams { + if err := ctx.Err(); err != nil { + return nil, err + } + conn, err := r.dialer.DialContext(ctx, "udp", upstream) + if err != nil { + errs = errors.Join(errs, err) + continue + } + answer, err := func() ([]byte, error) { + defer conn.Close() + stop := context.AfterFunc(ctx, func() { _ = conn.Close() }) + defer stop() + if err := conn.SetDeadline(time.Now().Add(dnsExchangeTimeout)); err != nil { + return nil, err + } + if _, err := conn.Write(query); err != nil { + return nil, err + } + buf := make([]byte, maxDNSDatagram) + n, err := conn.Read(buf) + if err != nil { + return nil, err + } + return buf[:n], nil + }() + if err != nil { + errs = errors.Join(errs, fmt.Errorf("upstream %s: %w", upstream, err)) + continue + } + // SERVFAIL is not an answer, and the sandbox sees only the gateway, so + // it cannot try the pod's other resolvers itself. Keep the last one to + // return if none does better: a real response beats a timeout. + if rcode, ok := failoverRcode(answer); ok { + errs = errors.Join(errs, fmt.Errorf("upstream %s: %w", upstream, rcodeError(rcode))) + deferred = answer + continue + } + return answer, nil + } + if deferred != nil { + return deferred, nil + } + return nil, fmt.Errorf("atunnel: no upstream resolver answered: %w", errs) +} + +// Response codes that say the resolver failed rather than answered. NXDOMAIN +// and NOERROR are answers and are passed back as they are. +const ( + rcodeServFail = 2 + rcodeNotImp = 4 + rcodeRefused = 5 +) + +// failoverRcode reports the response code when the relay should try the next +// upstream. Reads the 12-byte header only; anything shorter is passed through. +func failoverRcode(msg []byte) (byte, bool) { + if len(msg) < 12 { + return 0, false + } + rcode := msg[3] & 0x0f + switch rcode { + case rcodeServFail, rcodeNotImp, rcodeRefused: + return rcode, true + } + return 0, false +} + +func rcodeError(rcode byte) error { + switch rcode { + case rcodeServFail: + return errors.New("answered SERVFAIL") + case rcodeNotImp: + return errors.New("answered NOTIMP") + case rcodeRefused: + return errors.New("answered REFUSED") + } + return fmt.Errorf("answered rcode %d", rcode) +} + +// relayTCP copies a DNS stream without parsing its length-prefixed messages. +func (r *DNSRelay) relayTCP(ctx context.Context, downstream net.Conn) { + defer downstream.Close() + + var upstream net.Conn + var errs error + for _, address := range r.upstreams { + conn, err := r.dialer.DialContext(ctx, "tcp", address) + if err != nil { + errs = errors.Join(errs, err) + continue + } + upstream = conn + break + } + if upstream == nil { + slog.WarnContext(ctx, "atunnel could not reach any resolver for an actor DNS connection", slog.Any("err", errs)) + return + } + defer upstream.Close() + + // Cancel active copies on teardown to release the worker's connection slots. + relayDone := make(chan struct{}) + defer close(relayDone) + go func() { + select { + case <-ctx.Done(): + _ = downstream.Close() + _ = upstream.Close() + case <-relayDone: + } + }() + + deadline := time.Now().Add(dnsTCPTimeout) + if ctxDeadline, ok := ctx.Deadline(); ok && ctxDeadline.Before(deadline) { + deadline = ctxDeadline + } + _ = downstream.SetDeadline(deadline) + _ = upstream.SetDeadline(deadline) + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + _, _ = io.Copy(upstream, downstream) + if c, ok := upstream.(*net.TCPConn); ok { + _ = c.CloseWrite() + } + }() + go func() { + defer wg.Done() + _, _ = io.Copy(downstream, upstream) + if c, ok := downstream.(*net.TCPConn); ok { + _ = c.CloseWrite() + } + }() + wg.Wait() +} diff --git a/internal/atunnel/dns_test.go b/internal/atunnel/dns_test.go new file mode 100644 index 0000000000..67cc3deb9f --- /dev/null +++ b/internal/atunnel/dns_test.go @@ -0,0 +1,538 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package atunnel + +import ( + "bytes" + "context" + "errors" + "io" + "net" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/google/go-cmp/cmp" +) + +func TestResolvConfNameservers(t *testing.T) { + for _, tc := range []struct { + name string + content string + want []string + wantErr bool + }{ + { + name: "cluster resolv.conf", + content: "search ate-system.svc.cluster.local svc.cluster.local\nnameserver 10.96.0.10\noptions ndots:5\n", + want: []string{"10.96.0.10:53"}, + }, + { + name: "several, in order", + content: "nameserver 10.96.0.10\nnameserver 8.8.8.8\n", + want: []string{"10.96.0.10:53", "8.8.8.8:53"}, + }, + { + name: "comments and blanks", + content: "# generated\n\n nameserver 10.96.0.10 # cluster\n;nameserver 1.1.1.1\n", + want: []string{"10.96.0.10:53"}, + }, + {name: "no nameserver", content: "search cluster.local\n", wantErr: true}, + {name: "unparsable address", content: "nameserver not-an-ip\n", wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "resolv.conf") + if err := os.WriteFile(path, []byte(tc.content), 0o644); err != nil { + t.Fatal(err) + } + got, err := ResolvConfNameservers(path) + if tc.wantErr { + if err == nil { + t.Fatalf("ResolvConfNameservers() = %v, want an error", got) + } + return + } + if err != nil { + t.Fatalf("ResolvConfNameservers: %v", err) + } + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("nameservers mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestDNSRelayCancelsUDPExchange(t *testing.T) { + // A resolver that receives the query and never answers, so the exchange is + // blocked on the read when the context is canceled. + silent, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer silent.Close() + asked := make(chan struct{}, 1) + go func() { + buf := make([]byte, maxDNSDatagram) + for { + if _, _, err := silent.ReadFrom(buf); err != nil { + return + } + select { + case asked <- struct{}{}: + default: + } + } + }() + + // Two upstreams: a canceled exchange must not move on to the second. + second := newFakeResolver(t, func(query []byte) []byte { return query }) + relay, err := NewDNSRelay([]string{silent.LocalAddr().String(), second}) + if err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { + _, err := relay.exchangeUDP(ctx, dnsQuery(0x1234)) + done <- err + }() + select { + case <-asked: + case <-time.After(5 * time.Second): + t.Fatal("the resolver never saw the query") + } + cancel() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("exchange returned %v, want cancellation", err) + } + case <-time.After(5 * time.Second): + t.Fatal("canceled exchange remained blocked reading upstream") + } +} + +func TestDNSRelayForwardsUDPVerbatim(t *testing.T) { + upstream := newFakeResolver(t, func(query []byte) []byte { + return append([]byte{0xff}, query...) + }) + + relay, err := NewDNSRelay([]string{upstream}) + if err != nil { + t.Fatal(err) + } + client := serveRelayUDP(t, relay) + + query := []byte{0xab, 0xcd, 0x01, 0x00, 0x00, 0x01} + if _, err := client.Write(query); err != nil { + t.Fatal(err) + } + answer := readWithin(t, client) + if diff := cmp.Diff(append([]byte{0xff}, query...), answer); diff != "" { + t.Errorf("answer mismatch (-want +got):\n%s", diff) + } +} + +func TestDNSRelayFallsBackToTheNextResolver(t *testing.T) { + dead, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + deadAddress := dead.LocalAddr().String() + // Closed, so the exchange fails rather than hanging to its deadline. + dead.Close() + + live := newFakeResolver(t, func(query []byte) []byte { return []byte("answered") }) + relay, err := NewDNSRelay([]string{deadAddress, live}) + if err != nil { + t.Fatal(err) + } + client := serveRelayUDP(t, relay) + + if _, err := client.Write([]byte("query")); err != nil { + t.Fatal(err) + } + if got := string(readWithin(t, client)); got != "answered" { + t.Errorf("answer = %q, want %q", got, "answered") + } +} + +func TestNewDNSRelayRejects(t *testing.T) { + if _, err := NewDNSRelay(nil); err == nil { + t.Error("NewDNSRelay(nil) succeeded; a relay with no upstream can answer nothing") + } + if _, err := NewDNSRelay([]string{"10.96.0.10"}); err == nil { + t.Error("NewDNSRelay accepted an address with no port") + } +} + +// newFakeResolver answers UDP with respond(query), and returns its address. +func newFakeResolver(t *testing.T, respond func([]byte) []byte) string { + t.Helper() + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { pc.Close() }) + go func() { + buf := make([]byte, maxDNSDatagram) + for { + n, from, err := pc.ReadFrom(buf) + if err != nil { + return + } + query := make([]byte, n) + copy(query, buf[:n]) + _, _ = pc.WriteTo(respond(query), from) + } + }() + return pc.LocalAddr().String() +} + +// serveRelayUDP runs the relay on a loopback socket and returns a connection to it. +func serveRelayUDP(t *testing.T, relay *DNSRelay) net.Conn { + t.Helper() + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + go func() { _ = relay.ServePacket(ctx, pc) }() + + client, err := net.Dial("udp", pc.LocalAddr().String()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { client.Close() }) + return client +} + +func readWithin(t *testing.T, conn net.Conn) []byte { + t.Helper() + if err := conn.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil { + t.Fatal(err) + } + buf := make([]byte, maxDNSDatagram) + n, err := conn.Read(buf) + if err != nil { + t.Fatalf("reading the relay's answer: %v", err) + } + return buf[:n] +} + +func TestDNSRelayForwardsAnswersLargerThanTheCommonBuffer(t *testing.T) { + const size = 9000 + answer := make([]byte, size) + for i := range answer { + answer[i] = byte(i) + } + upstream := newFakeResolver(t, func([]byte) []byte { return answer }) + + relay, err := NewDNSRelay([]string{upstream}) + if err != nil { + t.Fatal(err) + } + client := serveRelayUDP(t, relay) + if _, err := client.Write([]byte("query")); err != nil { + t.Fatal(err) + } + + got := readWithin(t, client) + if len(got) != size { + t.Fatalf("answer is %d bytes, want %d: it was cut down in the relay", len(got), size) + } + if !bytes.Equal(got, answer) { + t.Error("answer differs from what the resolver sent") + } +} + +func TestDNSRelayDropsQueriesBeyondItsInFlightLimit(t *testing.T) { + // Hold concurrent queries to exercise the relay's limit. + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { pc.Close() }) + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + var inFlight atomic.Int64 + go func() { + buf := make([]byte, maxDNSDatagram) + for { + n, from, err := pc.ReadFrom(buf) + if err != nil { + return + } + answer := make([]byte, n) + copy(answer, buf[:n]) + go func() { + inFlight.Add(1) + <-release + _, _ = pc.WriteTo(answer, from) + }() + } + }() + + relay, err := NewDNSRelay([]string{pc.LocalAddr().String()}) + if err != nil { + t.Fatal(err) + } + client := serveRelayUDP(t, relay) + + for range maxInFlightDNS * 4 { + if _, err := client.Write([]byte("query")); err != nil { + t.Fatal(err) + } + } + // Wait for the accepted-query count to stabilize. + deadline := time.Now().Add(10 * time.Second) + last := int64(-1) + for time.Now().Before(deadline) { + time.Sleep(100 * time.Millisecond) + if n := inFlight.Load(); n == last { + break + } else { + last = n + } + } + if got := inFlight.Load(); got > maxInFlightDNS { + t.Errorf("the relay had %d queries in flight, want at most %d", got, maxInFlightDNS) + } +} + +// newHeldTCPResolver accepts DNS connections and answers none, holding each +// until the test ends. It reports how many it is holding. +func newHeldTCPResolver(t *testing.T) (address string, accepted *atomic.Int64) { + t.Helper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { lis.Close() }) + + var held atomic.Int64 + var mu sync.Mutex + var conns []net.Conn + t.Cleanup(func() { + mu.Lock() + defer mu.Unlock() + for _, c := range conns { + c.Close() + } + }) + go func() { + for { + conn, err := lis.Accept() + if err != nil { + return + } + mu.Lock() + conns = append(conns, conn) + mu.Unlock() + held.Add(1) + } + }() + return lis.Addr().String(), &held +} + +// serveRelayTCP runs the relay's TCP side on a loopback listener. +func serveRelayTCP(t *testing.T, relay *DNSRelay, ctx context.Context) net.Addr { + t.Helper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { lis.Close() }) + go func() { _ = relay.Serve(ctx, lis) }() + return lis.Addr() +} + +// waitFor polls until cond holds, failing the test if it never does. +func waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatalf("timed out waiting for %s", what) +} + +func TestDNSRelayRefusesTCPConnectionsBeyondItsLimit(t *testing.T) { + upstream, held := newHeldTCPResolver(t) + relay, err := NewDNSRelay([]string{upstream}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + relayAddr := serveRelayTCP(t, relay, ctx) + + for range maxDNSConnections { + conn, err := net.Dial("tcp", relayAddr.String()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close() }) + } + waitFor(t, "the relay to fill up", func() bool { return held.Load() == maxDNSConnections }) + + // The relay should close a connection accepted beyond its limit. + extra, err := net.Dial("tcp", relayAddr.String()) + if err != nil { + t.Fatal(err) + } + defer extra.Close() + if err := extra.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil { + t.Fatal(err) + } + if _, err := extra.Read(make([]byte, 1)); !errors.Is(err, io.EOF) { + t.Errorf("reading the refused connection = %v, want %v", err, io.EOF) + } + if got := held.Load(); got != maxDNSConnections { + t.Errorf("the relay holds %d upstream connections, want %d", got, maxDNSConnections) + } +} + +func TestDNSRelayClosesTCPConnectionsWhenServingEnds(t *testing.T) { + upstream, held := newHeldTCPResolver(t) + relay, err := NewDNSRelay([]string{upstream}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + relayAddr := serveRelayTCP(t, relay, ctx) + + conn, err := net.Dial("tcp", relayAddr.String()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + waitFor(t, "the connection to reach an upstream", func() bool { return held.Load() == 1 }) + + cancel() + + // Cancellation must close the connection before dnsTCPTimeout. + if err := conn.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil { + t.Fatal(err) + } + if _, err := conn.Read(make([]byte, 1)); !errors.Is(err, io.EOF) { + t.Errorf("reading after teardown = %v, want %v: the connection outlived the actor", err, io.EOF) + } +} + +// dnsQuery builds a real query for example.com A, so tests exercise messages a +// resolver would actually send rather than an arbitrary string. +func dnsQuery(id uint16) []byte { + msg := []byte{byte(id >> 8), byte(id), 0x01, 0x00, 0, 1, 0, 0, 0, 0, 0, 0} + for _, label := range []string{"example", "com"} { + msg = append(msg, byte(len(label))) + msg = append(msg, label...) + } + return append(msg, 0, 0, 1, 0, 1) // root label, QTYPE A, QCLASS IN +} + +// dnsAnswer echoes a query back as a response carrying rcode. +func dnsAnswer(query []byte, rcode byte) []byte { + resp := make([]byte, len(query)) + copy(resp, query) + resp[2] |= 0x80 // QR: this is a response + resp[3] = resp[3]&0xf0 | rcode + return resp +} + +// A resolver that answers SERVFAIL has not answered the question, and the +// sandbox can no longer consult the pod's other resolvers itself: it is given +// the gateway as its only nameserver. The relay has to fail over for it. +func TestDNSRelayFailsOverOnServerFailure(t *testing.T) { + var sickCalls, healthyCalls atomic.Int32 + sick := newFakeResolver(t, func(query []byte) []byte { + sickCalls.Add(1) + return dnsAnswer(query, rcodeServFail) + }) + healthy := newFakeResolver(t, func(query []byte) []byte { + healthyCalls.Add(1) + return dnsAnswer(query, 0) + }) + + relay, err := NewDNSRelay([]string{sick, healthy}) + if err != nil { + t.Fatal(err) + } + answer, err := relay.exchangeUDP(context.Background(), dnsQuery(0x1234)) + if err != nil { + t.Fatalf("exchange: %v", err) + } + if got := answer[3] & 0x0f; got != 0 { + t.Errorf("answer rcode = %d, want 0: the relay returned the failing resolver's answer", got) + } + if healthyCalls.Load() != 1 { + t.Errorf("the healthy resolver was asked %d times, want 1", healthyCalls.Load()) + } +} + +// With every resolver failing there is nothing better to return, and a real +// SERVFAIL beats a timeout: the sandbox's resolver can act on it. +func TestDNSRelayReturnsServerFailureWhenAllFail(t *testing.T) { + first := newFakeResolver(t, func(query []byte) []byte { return dnsAnswer(query, rcodeServFail) }) + second := newFakeResolver(t, func(query []byte) []byte { return dnsAnswer(query, rcodeRefused) }) + + relay, err := NewDNSRelay([]string{first, second}) + if err != nil { + t.Fatal(err) + } + answer, err := relay.exchangeUDP(context.Background(), dnsQuery(0x2345)) + if err != nil { + t.Fatalf("exchange: %v", err) + } + if got := answer[3] & 0x0f; got != rcodeRefused { + t.Errorf("answer rcode = %d, want the last resolver's %d", got, rcodeRefused) + } +} + +// NXDOMAIN is an answer, not a failure: failing over would ask every resolver +// about a name that does not exist. +func TestDNSRelayReturnsNXDomainWithoutFailover(t *testing.T) { + const rcodeNXDomain = 3 + var secondCalls atomic.Int32 + first := newFakeResolver(t, func(query []byte) []byte { return dnsAnswer(query, rcodeNXDomain) }) + second := newFakeResolver(t, func(query []byte) []byte { + secondCalls.Add(1) + return dnsAnswer(query, 0) + }) + + relay, err := NewDNSRelay([]string{first, second}) + if err != nil { + t.Fatal(err) + } + answer, err := relay.exchangeUDP(context.Background(), dnsQuery(0x3456)) + if err != nil { + t.Fatalf("exchange: %v", err) + } + if got := answer[3] & 0x0f; got != rcodeNXDomain { + t.Errorf("answer rcode = %d, want NXDOMAIN %d", got, rcodeNXDomain) + } + if secondCalls.Load() != 0 { + t.Error("the relay failed over on NXDOMAIN, which is a valid answer") + } +} diff --git a/internal/atunnel/egress.go b/internal/atunnel/egress.go index a31396c584..4173e3b9a1 100644 --- a/internal/atunnel/egress.go +++ b/internal/atunnel/egress.go @@ -303,3 +303,18 @@ func closeWrite(conn net.Conn) { _ = conn.CloseWrite() } } + +// EgressPort is the port from a listen address, which each sandbox's redirect +// aims at. The address itself is never bound: egress is served from inside the +// sandbox namespaces. +func EgressPort(listenAddress string) (uint16, error) { + _, port, err := net.SplitHostPort(listenAddress) + if err != nil { + return 0, fmt.Errorf("atunnel: egress listen address %q: %w", listenAddress, err) + } + p, ok := ParsePort(port) + if !ok { + return 0, fmt.Errorf("atunnel: egress listen address %q has no usable port", listenAddress) + } + return uint16(p), nil +} diff --git a/internal/atunnel/ingress.go b/internal/atunnel/ingress.go index a0c167f5e7..8500cc479f 100644 --- a/internal/atunnel/ingress.go +++ b/internal/atunnel/ingress.go @@ -69,16 +69,24 @@ type Config struct { TrustBundlePath string AllowedClientID string Upstream *url.URL + // Dial reaches the sandbox, for both proxied requests and CONNECT tunnels. + // Needed when it is reachable only from inside its own network namespace; + // nil dials from the worker's. + Dial DialFunc } -// Server is an activation-aware HTTPS reverse proxy. It is long-lived across -// actor activations, but only routes requests for the actor currently assigned -// to its worker. +// Server is an HTTPS reverse proxy for the worker's active actors. type Server struct { credentialBundlePath string tlsConfig *tls.Config proxy *httputil.ReverseProxy - upstream *url.URL + // newTransport builds an activation's round tripper. A field so a test can + // substitute one without the production path branching on its type. + newTransport func(DialFunc) http.RoundTripper + upstream *url.URL + // dial reaches the sandbox, for CONNECT tunnels. The reverse proxy's + // transport is given the same dialer. + dial DialFunc mu sync.Mutex active *activation @@ -89,6 +97,8 @@ type activation struct { ctx context.Context cancel context.CancelFunc wg sync.WaitGroup + dial DialFunc + proxy *httputil.ReverseProxy } // NewServer creates a Server and validates its TLS material. @@ -121,7 +131,11 @@ func NewServer(cfg Config) (*Server, error) { return nil, fmt.Errorf("atunnel: trust bundle %q contains no certificates", cfg.TrustBundlePath) } - transport := newProtocolMirrorTransport() + dial := cfg.Dial + if dial == nil { + dial = (&net.Dialer{}).DialContext + } + transport := newProtocolMirrorTransport(dial) proxy := &httputil.ReverseProxy{ Rewrite: func(pr *httputil.ProxyRequest) { pr.SetURL(cfg.Upstream) @@ -148,6 +162,8 @@ func NewServer(cfg Config) (*Server, error) { credentialBundlePath: cfg.CredentialBundlePath, proxy: proxy, upstream: cfg.Upstream, + dial: dial, + newTransport: func(d DialFunc) http.RoundTripper { return newProtocolMirrorTransport(d) }, } s.tlsConfig = &tls.Config{ MinVersion: tls.VersionTLS12, @@ -208,9 +224,13 @@ type protocolMirrorTransport struct { h1, h2c *http.Transport } -func newProtocolMirrorTransport() protocolMirrorTransport { +func newProtocolMirrorTransport(dial DialFunc) protocolMirrorTransport { h1 := http.DefaultTransport.(*http.Transport).Clone() h2c := http.DefaultTransport.(*http.Transport).Clone() + if dial != nil { + h1.DialContext = dial + h2c.DialContext = dial + } protocols := new(http.Protocols) protocols.SetUnencryptedHTTP2(true) h2c.Protocols = protocols @@ -299,7 +319,7 @@ func (s *Server) ServeConnectHTTP(w http.ResponseWriter, r *http.Request) { http.Error(w, "CONNECT required", http.StatusMethodNotAllowed) return } - ref, ctx, release, ok := s.authorize(r) + active, ctx, release, ok := s.authorize(r) if !ok { s.reject(w) return @@ -316,10 +336,11 @@ func (s *Server) ServeConnectHTTP(w http.ResponseWriter, r *http.Request) { return } - dialer := &net.Dialer{Timeout: 5 * time.Second} - upstream, err := dialer.DialContext(ctx, "tcp", net.JoinHostPort(s.upstream.Hostname(), port)) + dialCtx, cancelDial := context.WithTimeout(ctx, 5*time.Second) + defer cancelDial() + upstream, err := active.dial(dialCtx, "tcp", net.JoinHostPort(s.upstream.Hostname(), port)) if err != nil { - slog.WarnContext(r.Context(), "atunnel CONNECT upstream failed", slog.Any("actor", ref), slog.Any("err", err)) + slog.WarnContext(r.Context(), "atunnel CONNECT upstream failed", slog.Any("actor", active.ref), slog.Any("err", err)) http.Error(w, "bad gateway", http.StatusBadGateway) return } @@ -417,14 +438,41 @@ func (s *Server) Activate(atespace, actorName string) error { return fmt.Errorf("atunnel: actor %s is already active", s.active.ref) } ctx, cancel := context.WithCancel(context.Background()) + dial := activationDialer(ctx, s.dial) + // Its own transport, so an activation's pooled connections cannot outlive + // it and hand the next actor a socket to its predecessor. + proxy := *s.proxy + proxy.Transport = s.newTransport(dial) s.active = &activation{ ref: resources.ActorRef{Atespace: atespace, Name: actorName}, ctx: ctx, cancel: cancel, + dial: dial, + proxy: &proxy, } return nil } +func activationDialer(activeCtx context.Context, dial DialFunc) DialFunc { + return func(ctx context.Context, network, address string) (net.Conn, error) { + if err := activeCtx.Err(); err != nil { + return nil, err + } + ctx, cancel := context.WithCancel(ctx) + defer cancel() + stop := context.AfterFunc(activeCtx, cancel) + defer stop() + conn, err := dial(ctx, network, address) + if canceled := errors.Join(activeCtx.Err(), ctx.Err()); canceled != nil { + if conn != nil { + _ = conn.Close() + } + return nil, canceled + } + return conn, err + } +} + // Deactivate rejects new requests, cancels requests for the active actor, and // waits for their handlers to exit before returning. func (s *Server) Deactivate(ctx context.Context) error { @@ -446,43 +494,43 @@ func (s *Server) Deactivate(ctx context.Context) error { }() select { case <-done: - s.closeIdleUpstreamConnections() + closeIdleUpstreamConnections(active) return nil case <-ctx.Done(): - s.closeIdleUpstreamConnections() + closeIdleUpstreamConnections(active) return fmt.Errorf("atunnel: waiting for active requests to stop: %w", ctx.Err()) } } -func (s *Server) closeIdleUpstreamConnections() { - if transport, ok := s.proxy.Transport.(interface{ CloseIdleConnections() }); ok { +func closeIdleUpstreamConnections(active *activation) { + if transport, ok := active.proxy.Transport.(interface{ CloseIdleConnections() }); ok { transport.CloseIdleConnections() } } // ServeHTTP validates the actor routing header on every request before proxying it. func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { - _, requestCtx, release, ok := s.authorize(r) + active, requestCtx, release, ok := s.authorize(r) if !ok { s.reject(w) return } defer release() - s.proxy.ServeHTTP(w, r.WithContext(requestCtx)) + active.proxy.ServeHTTP(w, r.WithContext(requestCtx)) } -func (s *Server) authorize(r *http.Request) (resources.ActorRef, context.Context, func(), bool) { +func (s *Server) authorize(r *http.Request) (*activation, context.Context, func(), bool) { ref, err := atenet.ParseTargetActor(r.Header.Get(atenet.TargetActorHeader)) if err != nil { - return resources.ActorRef{}, nil, nil, false + return nil, nil, nil, false } s.mu.Lock() active := s.active if active == nil || active.ref != ref { s.mu.Unlock() - return resources.ActorRef{}, nil, nil, false + return nil, nil, nil, false } active.wg.Add(1) s.mu.Unlock() @@ -493,7 +541,7 @@ func (s *Server) authorize(r *http.Request) (resources.ActorRef, context.Context stop() cancel() } - return ref, requestCtx, release, true + return active, requestCtx, release, true } func (s *Server) reject(w http.ResponseWriter) { diff --git a/internal/atunnel/ingress_test.go b/internal/atunnel/ingress_test.go index 5efd7afeb7..3344170baf 100644 --- a/internal/atunnel/ingress_test.go +++ b/internal/atunnel/ingress_test.go @@ -24,6 +24,7 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/pem" + "errors" "io" "math/big" "net" @@ -32,12 +33,99 @@ import ( "net/url" "os" "path/filepath" + "sync/atomic" "testing" "time" "github.com/agent-substrate/substrate/internal/atenet" ) +func TestActivationDialerClosesLateConnection(t *testing.T) { + client, peer := net.Pipe() + defer peer.Close() + defer client.Close() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dial := activationDialer(ctx, func(context.Context, string, string) (net.Conn, error) { + cancel() + return client, nil + }) + conn, err := dial(context.Background(), "tcp", "actor:80") + if conn != nil || !errors.Is(err, context.Canceled) { + t.Fatalf("late dial returned %v, %v", conn, err) + } + _ = peer.SetReadDeadline(time.Now().Add(time.Second)) + if _, err := peer.Read(make([]byte, 1)); !errors.Is(err, io.EOF) { + t.Fatalf("late connection was not closed: %v", err) + } +} + +func TestDeactivateIsolatesLateDials(t *testing.T) { + firstActor := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, "actor-1") + })) + defer firstActor.Close() + secondActor := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, "actor-2") + })) + defer secondActor.Close() + upstream, _ := url.Parse("http://127.0.0.1:80") + server := newTestServer(t, upstream) + firstReady := make(chan struct{}) + secondReady := make(chan struct{}) + releaseFirst := make(chan struct{}) + releaseSecond := make(chan struct{}) + var calls atomic.Int32 + server.dial = func(ctx context.Context, network, address string) (net.Conn, error) { + if calls.Add(1) == 1 { + conn, err := net.Dial("tcp", firstActor.Listener.Addr().String()) + close(firstReady) + <-releaseFirst + return conn, err + } + close(secondReady) + <-releaseSecond + return (&net.Dialer{}).DialContext(ctx, "tcp", secondActor.Listener.Addr().String()) + } + if err := server.Activate("team-a", "actor-1"); err != nil { + t.Fatal(err) + } + firstActivation := server.active + request := func(actorName string, done chan string) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + req := httptest.NewRequest(http.MethodGet, "http://worker/", nil).WithContext(ctx) + req.Header.Set(atenet.TargetActorHeader, "team-a/"+actorName) + recorder := httptest.NewRecorder() + server.ServeHTTP(recorder, req) + done <- recorder.Body.String() + } + firstDone := make(chan string, 1) + go request("actor-1", firstDone) + receiveWithin(t, firstReady, "first dial") + if err := server.Deactivate(context.Background()); err != nil { + t.Fatal(err) + } + receiveWithin(t, firstDone, "first request cancellation") + if err := server.Activate("team-a", "actor-2"); err != nil { + t.Fatal(err) + } + if firstActivation.proxy.Transport == server.active.proxy.Transport { + t.Error("activations share a transport") + } + secondDone := make(chan string, 1) + go request("actor-2", secondDone) + receiveWithin(t, secondReady, "second dial") + close(releaseFirst) + close(releaseSecond) + if body := receiveWithin(t, secondDone, "second response"); body != "actor-2" { + t.Errorf("second actor received %q", body) + } + if err := server.Deactivate(context.Background()); err != nil { + t.Fatal(err) + } +} + func TestRelayIngressWithHalfClose(t *testing.T) { listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { @@ -138,14 +226,14 @@ func TestServeHTTP(t *testing.T) { } s := newTestServer(t, upstreamURL) - s.proxy.Transport = roundTripFunc(func(r *http.Request) (*http.Response, error) { + setActivationTransport(s, roundTripFunc(func(r *http.Request) (*http.Response, error) { upstreamHosts = append(upstreamHosts, r.Host) return &http.Response{ StatusCode: http.StatusNoContent, Header: make(http.Header), Body: http.NoBody, }, nil - }) + })) if err := s.Activate("team-a", "actor-1"); err != nil { t.Fatal(err) } @@ -207,7 +295,7 @@ func TestServeHTTPHonorsTargetPortHeader(t *testing.T) { s := newTestServer(t, upstreamURL) var gotURLHost, gotHost http.Header - s.proxy.Transport = roundTripFunc(func(r *http.Request) (*http.Response, error) { + setActivationTransport(s, roundTripFunc(func(r *http.Request) (*http.Response, error) { gotURLHost = http.Header{"Host": []string{r.URL.Host}} gotHost = r.Header.Clone() gotHost.Set("Host", r.Host) @@ -216,7 +304,7 @@ func TestServeHTTPHonorsTargetPortHeader(t *testing.T) { Header: make(http.Header), Body: http.NoBody, }, nil - }) + })) if err := s.Activate("team-a", "actor-1"); err != nil { t.Fatal(err) } @@ -326,7 +414,7 @@ func TestDeactivateClosesIdleUpstreamConnections(t *testing.T) { } s := newTestServer(t, upstream) transport := &idleClosingRoundTripper{} - s.proxy.Transport = transport + setActivationTransport(s, transport) if err := s.Activate("team-a", "actor-1"); err != nil { t.Fatal(err) } @@ -559,15 +647,15 @@ func TestDeactivateCancelsInflightRequest(t *testing.T) { t.Fatal(err) } s := newTestServer(t, upstream) - if err := s.Activate("team-a", "actor-1"); err != nil { - t.Fatal(err) - } started := make(chan struct{}) - s.proxy.Transport = roundTripFunc(func(r *http.Request) (*http.Response, error) { + setActivationTransport(s, roundTripFunc(func(r *http.Request) (*http.Response, error) { close(started) <-r.Context().Done() return nil, r.Context().Err() - }) + })) + if err := s.Activate("team-a", "actor-1"); err != nil { + t.Fatal(err) + } done := make(chan struct{}) go func() { @@ -584,6 +672,13 @@ func TestDeactivateCancelsInflightRequest(t *testing.T) { receiveWithin(t, done, "canceled in-flight request") } +// setActivationTransport makes every activation use rt, standing in for the +// per-activation transport the server builds in production. +func setActivationTransport(s *Server, rt http.RoundTripper) { + s.newTransport = func(DialFunc) http.RoundTripper { return rt } + s.proxy.Transport = rt +} + func newTestServer(t *testing.T, upstream *url.URL) *Server { t.Helper() dir := t.TempDir() @@ -847,7 +942,7 @@ func TestProtocolMirrorTransport(t *testing.T) { } { t.Run(tt.name, func(t *testing.T) { addr, protoSeen := mirrorBackend(t, tt.backendH1, tt.backendH2C) - transport := newProtocolMirrorTransport() + transport := newProtocolMirrorTransport(nil) req, err := http.NewRequest(tt.method, "http://"+addr+"/", http.NoBody) if err != nil { t.Fatal(err) @@ -886,3 +981,62 @@ func receiveWithin[T any](t *testing.T, channel <-chan T, description string) T return zero } } + +// CONNECT must use the configured dialer independently of the proxy transport. +func TestServeConnectHTTPDialsTheSandbox(t *testing.T) { + actor, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer actor.Close() + go func() { + conn, err := actor.Accept() + if err != nil { + return + } + defer conn.Close() + _, _ = io.WriteString(conn, "hello from the sandbox") + }() + + upstreamURL, err := url.Parse("http://actor.internal:80") + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + bundle, trust := makeCertFiles(t, dir) + var dialed string + s, err := NewServer(Config{ + CredentialBundlePath: bundle, + TrustBundlePath: trust, + AllowedClientID: "spiffe://cluster.local/ns/ate-system/sa/atenet-router", + Upstream: upstreamURL, + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + dialed = address + return (&net.Dialer{}).DialContext(ctx, network, actor.Addr().String()) + }, + }) + if err != nil { + t.Fatal(err) + } + if err := s.Activate("team-a", "actor-1"); err != nil { + t.Fatal(err) + } + + // HTTP/2 CONNECT supports a recorder without socket hijacking. + req := httptest.NewRequest(http.MethodConnect, "https://worker/", http.NoBody) + req.ProtoMajor, req.ProtoMinor, req.Proto = 2, 0, "HTTP/2.0" + req.Host = "actor-1.team-a.actors.resources.substrate.ate.dev:9090" + req.Header.Set(atenet.TargetActorHeader, "team-a/actor-1") + rec := httptest.NewRecorder() + s.ServeConnectHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: the tunnel never reached the sandbox", rec.Code, http.StatusOK) + } + if got, want := rec.Body.String(), "hello from the sandbox"; got != want { + t.Errorf("tunnel carried %q, want %q", got, want) + } + if want := "actor.internal:9090"; dialed != want { + t.Errorf("dialed %q through the sandbox dialer, want %q", dialed, want) + } +} diff --git a/internal/ocispec/gvisor.go b/internal/ocispec/gvisor.go index 059cfb71b3..ae47f5ad2a 100644 --- a/internal/ocispec/gvisor.go +++ b/internal/ocispec/gvisor.go @@ -26,7 +26,7 @@ import ( // name is drawn from, so no actor container can collide with it. const PauseContainer = "_pause" -// resolvConf is the host resolver config bound into the sandbox. +// resolvConf is the sandbox resolver path and default bind source. const resolvConf = "/etc/resolv.conf" // GVisorOptions describes the gVisor-specific context of one actor container. @@ -35,6 +35,8 @@ type GVisorOptions struct { ContainerName string // DurableVolumes are declared on the sandbox (pause) spec only. DurableVolumes []string + // ResolvConf is the bind source for /etc/resolv.conf; empty uses the pod's file. + ResolvConf string // Size sizes the container's cgroup leaf. Only gVisor applies it; a micro-VM // container's limits come from its own declared resources (see sizing). Size sizing.SandboxSize @@ -65,10 +67,14 @@ func ShapeGVisor(spec *specs.Spec, o GVisorOptions) { if i < 0 { i = len(spec.Mounts) } + source := o.ResolvConf + if source == "" { + source = resolvConf + } spec.Mounts = slices.Insert(spec.Mounts, i, specs.Mount{ Destination: resolvConf, Type: "bind", - Source: resolvConf, + Source: source, Options: []string{"ro"}, }) } diff --git a/internal/ocispec/gvisor_test.go b/internal/ocispec/gvisor_test.go index 6d0ff33566..50f8ca15b9 100644 --- a/internal/ocispec/gvisor_test.go +++ b/internal/ocispec/gvisor_test.go @@ -53,3 +53,34 @@ func TestGVisorCgroupLeafMatchesTheShapedPath(t *testing.T) { t.Errorf("shaped cgroupsPath = %q, want %q", spec.Linux.CgroupsPath, want) } } + +func TestShapeGVisorBindsTheNamedResolvConf(t *testing.T) { + for _, tc := range []struct { + name string + resolvConf string + wantSource string + }{ + {name: "default is the worker pod's", wantSource: "/etc/resolv.conf"}, + { + name: "a sandbox may name its own", + resolvConf: "/var/lib/ateom-gvisor/actors/a/resolv.conf", + wantSource: "/var/lib/ateom-gvisor/actors/a/resolv.conf", + }, + } { + t.Run(tc.name, func(t *testing.T) { + spec := &specs.Spec{} + ShapeGVisor(spec, GVisorOptions{ + ActorUID: "a", ContainerName: "app", ResolvConf: tc.resolvConf, + }) + var got string + for _, m := range spec.Mounts { + if m.Destination == "/etc/resolv.conf" { + got = m.Source + } + } + if got != tc.wantSource { + t.Errorf("resolv.conf bound from %q, want %q", got, tc.wantSource) + } + }) + } +} diff --git a/internal/proto/ateletpb/atelet.pb.go b/internal/proto/ateletpb/atelet.pb.go index 4c4047ae8c..54893861d0 100644 --- a/internal/proto/ateletpb/atelet.pb.go +++ b/internal/proto/ateletpb/atelet.pb.go @@ -546,7 +546,7 @@ type RunRequest struct { // fetches the relevant assets and records them with the actor's on-node state // so a later Checkpoint can pin the same version into the snapshot manifest. SandboxAssets *SandboxAssets `protobuf:"bytes,8,opt,name=sandbox_assets,json=sandboxAssets,proto3" json:"sandbox_assets,omitempty"` - // When absent, actor traffic uses direct egress instead of atunnel. + // When absent the actor has no egress: its TCP is captured and refused. EgressGateway *EgressGateway `protobuf:"bytes,9,opt,name=egress_gateway,json=egressGateway,proto3,oneof" json:"egress_gateway,omitempty"` // The actor's declared size, from the ActorTemplate's resource limits. atelet // passes these through to the sandbox so it is sized to the actor (not the @@ -2464,7 +2464,7 @@ type RestoreRequest struct { // of the `config` oneof: the actor's snapshot may be local (a pause // checkpoint) while the golden snapshot is always external. GoldenSnapshotUri string `protobuf:"bytes,12,opt,name=golden_snapshot_uri,json=goldenSnapshotUri,proto3" json:"golden_snapshot_uri,omitempty"` - // When absent, actor traffic uses direct egress instead of atunnel. + // When absent the actor has no egress: its TCP is captured and refused. EgressGateway *EgressGateway `protobuf:"bytes,13,opt,name=egress_gateway,json=egressGateway,proto3,oneof" json:"egress_gateway,omitempty"` // The actor's declared size, from the ActorTemplate's resource limits. For // gVisor and micro-VM DATA-scope restores the sandbox is (re)sized to these; diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index c4c79201dc..158cf7d7d1 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -125,7 +125,7 @@ message RunRequest { // so a later Checkpoint can pin the same version into the snapshot manifest. SandboxAssets sandbox_assets = 8; - // When absent, actor traffic uses direct egress instead of atunnel. + // When absent the actor has no egress: its TCP is captured and refused. optional EgressGateway egress_gateway = 9; // The actor's declared size, from the ActorTemplate's resource limits. atelet @@ -451,7 +451,7 @@ message RestoreRequest { // checkpoint) while the golden snapshot is always external. string golden_snapshot_uri = 12; - // When absent, actor traffic uses direct egress instead of atunnel. + // When absent the actor has no egress: its TCP is captured and refused. optional EgressGateway egress_gateway = 13; // The actor's declared size, from the ActorTemplate's resource limits. For diff --git a/internal/proto/ateompb/ateom.pb.go b/internal/proto/ateompb/ateom.pb.go index 59ce521b64..a6ae0f36e9 100644 --- a/internal/proto/ateompb/ateom.pb.go +++ b/internal/proto/ateompb/ateom.pb.go @@ -354,7 +354,7 @@ type RunWorkloadRequest struct { // to the local on-disk path atelet fetched it to (content-addressed, like // runsc_path). Empty for the gVisor runtime, which uses runsc_path. RuntimeAssetPaths map[string]string `protobuf:"bytes,8,rep,name=runtime_asset_paths,json=runtimeAssetPaths,proto3" json:"runtime_asset_paths,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - // When absent, actor traffic uses direct egress instead of atunnel. + // When absent the actor has no egress: its TCP is captured and refused. EgressGateway *EgressGateway `protobuf:"bytes,10,opt,name=egress_gateway,json=egressGateway,proto3,oneof" json:"egress_gateway,omitempty"` // The actor's declared size, from the ActorTemplate's resource limits. ateom // sizes the sandbox to these (cgroup caps via the OCI spec, and for the @@ -1220,7 +1220,7 @@ type RestoreWorkloadRequest struct { RuntimeAssetPaths map[string]string `protobuf:"bytes,9,rep,name=runtime_asset_paths,json=runtimeAssetPaths,proto3" json:"runtime_asset_paths,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // What content to restore from the snapshot. Scope SnapshotScope `protobuf:"varint,10,opt,name=scope,proto3,enum=ateom.SnapshotScope" json:"scope,omitempty"` - // When absent, actor traffic uses direct egress instead of atunnel. + // When absent the actor has no egress: its TCP is captured and refused. EgressGateway *EgressGateway `protobuf:"bytes,12,opt,name=egress_gateway,json=egressGateway,proto3,oneof" json:"egress_gateway,omitempty"` // The object storage URI of the ActorTemplate's golden snapshot. // Set only when scope is SNAPSHOT_SCOPE_DATA_ON_GOLDEN. Mirrors the diff --git a/internal/proto/ateompb/ateom.proto b/internal/proto/ateompb/ateom.proto index 15f57b09f1..ce11275cf1 100644 --- a/internal/proto/ateompb/ateom.proto +++ b/internal/proto/ateompb/ateom.proto @@ -136,7 +136,7 @@ message RunWorkloadRequest { // runsc_path). Empty for the gVisor runtime, which uses runsc_path. map runtime_asset_paths = 8; - // When absent, actor traffic uses direct egress instead of atunnel. + // When absent the actor has no egress: its TCP is captured and refused. optional EgressGateway egress_gateway = 10; // The actor's declared size, from the ActorTemplate's resource limits. ateom @@ -318,7 +318,7 @@ message RestoreWorkloadRequest { // What content to restore from the snapshot. SnapshotScope scope = 10; - // When absent, actor traffic uses direct egress instead of atunnel. + // When absent the actor has no egress: its TCP is captured and refused. optional EgressGateway egress_gateway = 12; // The object storage URI of the ActorTemplate's golden snapshot. diff --git a/internal/readyz/readyz.go b/internal/readyz/readyz.go index cbe8c4ea6f..6c84632e9e 100644 --- a/internal/readyz/readyz.go +++ b/internal/readyz/readyz.go @@ -52,20 +52,31 @@ const ( maxIdleConnsHost = 1 ) +// DialFunc reaches the actor, which lives in its own network namespace and is +// not addressable from the caller's. Nil dials from the caller's namespace. +type DialFunc func(ctx context.Context, network, addr string) (net.Conn, error) + // HTTPClient builds a keep-alive HTTP client tuned for fast, repeated // probing of a single endpoint. Exposed as a var so tests can substitute a // transport that targets a test server's loopback address. -var HTTPClient = func() *http.Client { +var HTTPClient = func() *http.Client { return newClient(nil) } + +// newClient probes through dial, or from the caller's namespace when nil. +func newClient(dial DialFunc) *http.Client { + if dial == nil { + dial = (&net.Dialer{Timeout: RequestTimeout}).DialContext + } tr := &http.Transport{ DisableCompression: true, MaxIdleConnsPerHost: maxIdleConnsHost, - DialContext: (&net.Dialer{Timeout: RequestTimeout}).DialContext, + DialContext: dial, ResponseHeaderTimeout: RequestTimeout, } return &http.Client{Transport: tr, Timeout: RequestTimeout} } -// WaitAll blocks until every container with a readyz probe set reports 200, +// WaitAll blocks until every container with a readyz probe set reports 200 +// through dial, // or returns the first error. Containers without a probe are skipped (their // absence means "no readiness gate"). // @@ -73,7 +84,7 @@ var HTTPClient = func() *http.Client { // errors.As cannot cross a process, and the interceptor would flatten it to a // bare codes.Internal, leaving atelet reading UNKNOWN. The ErrorInfo detail is // what carries it. Internal and no crash directive both match today's behavior. -func WaitAll(ctx context.Context, containers []*ateompb.Container, actorIP string) error { +func WaitAll(ctx context.Context, containers []*ateompb.Container, actorIP string, dial DialFunc) error { g, gctx := errgroup.WithContext(ctx) for _, ac := range containers { if ac.GetReadyz() == nil { @@ -81,7 +92,7 @@ func WaitAll(ctx context.Context, containers []*ateompb.Container, actorIP strin } ac := ac g.Go(func() error { - return Wait(gctx, ac.GetName(), ac.GetReadyz(), actorIP) + return Wait(gctx, ac.GetName(), ac.GetReadyz(), actorIP, dial) }) } err := g.Wait() @@ -91,15 +102,18 @@ func WaitAll(ctx context.Context, containers []*ateompb.Container, actorIP strin return err } -// Wait polls the configured HTTP endpoint until it returns 200, the context -// is cancelled, or the overall deadline is exceeded. -func Wait(ctx context.Context, containerName string, probe *ateompb.Readyz, actorIP string) error { +// Wait polls the configured HTTP endpoint through dial until it returns 200, +// the context is cancelled, or the overall deadline is exceeded. +func Wait(ctx context.Context, containerName string, probe *ateompb.Readyz, actorIP string, dial DialFunc) error { url, err := URL(probe, actorIP) if err != nil { return fmt.Errorf("invalid readyz config for %q: %w", containerName, err) } client := HTTPClient() + if dial != nil { + client = newClient(dial) + } defer client.CloseIdleConnections() timeout := overallTimeout(probe) diff --git a/internal/readyz/readyz_test.go b/internal/readyz/readyz_test.go index e1d76655fb..d95106fca2 100644 --- a/internal/readyz/readyz_test.go +++ b/internal/readyz/readyz_test.go @@ -107,7 +107,7 @@ func TestWait_ReturnsOnFirst200(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - if err := Wait(ctx, "main", probe, ip); err != nil { + if err := Wait(ctx, "main", probe, ip, nil); err != nil { t.Fatalf("Wait returned error: %v", err) } } @@ -135,7 +135,7 @@ func TestWait_WaitsForServerToBecomeReady(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() start := time.Now() - if err := Wait(ctx, "main", probe, ip); err != nil { + if err := Wait(ctx, "main", probe, ip, nil); err != nil { t.Fatalf("Wait returned error: %v", err) } elapsed := time.Since(start) @@ -159,7 +159,7 @@ func TestWait_ContextCancellation(t *testing.T) { cancel() }() - err := Wait(ctx, "main", probe, "127.0.0.1") + err := Wait(ctx, "main", probe, "127.0.0.1", nil) if err == nil { t.Fatalf("Wait returned nil, expected cancellation error") } @@ -215,7 +215,7 @@ func TestWait_GivesUpAtProbeTimeout(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() start := time.Now() - err := Wait(ctx, "main", probe, "127.0.0.1") + err := Wait(ctx, "main", probe, "127.0.0.1", nil) if err == nil { t.Fatalf("Wait returned nil, expected a timeout error") } @@ -239,7 +239,7 @@ func TestWaitAll_SkipsContainersWithoutProbe(t *testing.T) { } ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() - if err := WaitAll(ctx, containers, "127.0.0.1"); err != nil { + if err := WaitAll(ctx, containers, "127.0.0.1", nil); err != nil { t.Fatalf("WaitAll with no probes returned error: %v", err) } } @@ -285,7 +285,7 @@ func TestWaitAll_ReasonSurvivesTheRPCBoundary(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - err := WaitAll(ctx, containers, "127.0.0.1") + err := WaitAll(ctx, containers, "127.0.0.1", nil) if err == nil { t.Fatal("WaitAll returned nil, expected a timeout error") } diff --git a/internal/resources/validate.go b/internal/resources/validate.go index 103c944ed7..85cd37519f 100644 --- a/internal/resources/validate.go +++ b/internal/resources/validate.go @@ -77,10 +77,9 @@ func ValidateGlobalObjectRef(ref *ateapipb.ObjectRef, fldPath *field.Path) field } // ValidateAteomUID rejects a target ateom pod UID that could escape the host -// paths built from it: the netns path (/run/netns/ateom:) and the ateom -// control socket (.../ateoms//ateom.sock). Kubernetes pod UIDs are UUIDs, -// which are valid DNS-1123 labels, so a label check accepts every legitimate -// value while rejecting separators and "..". +// path built from it: the ateom control socket (.../ateoms//ateom.sock). +// Kubernetes pod UIDs are UUIDs, which are valid DNS-1123 labels, so a label +// check accepts every legitimate value while rejecting separators and "..". func ValidateAteomUID(targetAteomUID string) error { if errs := content.IsDNS1123Label(targetAteomUID); len(errs) > 0 { return fmt.Errorf("invalid target ateom UID %q: %s", targetAteomUID, strings.Join(errs, "; "))