diff --git a/packages/envd/internal/services/process/handler/handler.go b/packages/envd/internal/services/process/handler/handler.go index e30320944d..2ccc65c2fc 100644 --- a/packages/envd/internal/services/process/handler/handler.go +++ b/packages/envd/internal/services/process/handler/handler.go @@ -89,6 +89,25 @@ func currentNice() int { return 20 - prio } +// wrapperPrefix builds the priority-tool part of the process wrapper from +// whatever the image actually ships (FEAT-145 W3). ionice/nice are util-linux/ +// coreutils conveniences that minimal and busybox-based images (Alpine, UBI) +// may lack or keep elsewhere than /usr/bin — a missing helper must degrade to +// running the command without that priority adjustment, never to a failed +// spawn (exit 127 killed every process on such images). lookPath is injected +// for testability; production passes exec.LookPath. +func wrapperPrefix(niceDelta int, lookPath func(string) (string, error)) string { + prefix := "" + if p, err := lookPath("ionice"); err == nil { + prefix += p + " -c 2 -n 4 " + } + if p, err := lookPath("nice"); err == nil { + prefix += fmt.Sprintf("%s -n %d ", p, niceDelta) + } + + return prefix +} + func New( ctx context.Context, user *user.User, @@ -101,9 +120,11 @@ func New( // User command string for logging (without the internal wrapper details). userCmd := strings.Join(append([]string{req.GetProcess().GetCmd()}, req.GetProcess().GetArgs()...), " ") - // Wrap in a shell that resets oom_score_adj, ioprio (ionice best-effort/4), and nice. + // Wrap in a shell that resets oom_score_adj, ioprio (ionice best-effort/4), + // and nice. The oom_score_adj write is pure /proc and always applied; the + // priority helpers are used only where the image provides them. niceDelta := defaultNice - currentNice() - oomWrapperScript := fmt.Sprintf(`echo %d > /proc/$$/oom_score_adj && exec /usr/bin/ionice -c 2 -n 4 /usr/bin/nice -n %d "${@}"`, defaultOomScore, niceDelta) + oomWrapperScript := fmt.Sprintf(`echo %d > /proc/$$/oom_score_adj && exec %s"${@}"`, defaultOomScore, wrapperPrefix(niceDelta, exec.LookPath)) wrapperArgs := append([]string{"-c", oomWrapperScript, "--", req.GetProcess().GetCmd()}, req.GetProcess().GetArgs()...) cmd := exec.CommandContext(ctx, "/bin/sh", wrapperArgs...) diff --git a/packages/envd/internal/services/process/handler/handler_test.go b/packages/envd/internal/services/process/handler/handler_test.go new file mode 100644 index 0000000000..9bb14506c2 --- /dev/null +++ b/packages/envd/internal/services/process/handler/handler_test.go @@ -0,0 +1,47 @@ +package handler + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" +) + +// TT-4 (FEAT-145 W3): the process wrapper must degrade cleanly when the +// priority helpers are absent — the user command still runs; a present helper +// is applied with its resolved path. +func TestWrapperPrefix(t *testing.T) { + t.Parallel() + + notFound := func(string) (string, error) { return "", errors.New("not found") } + all := func(name string) (string, error) { return "/usr/bin/" + name, nil } + only := func(want string) func(string) (string, error) { + return func(name string) (string, error) { + if name == want { + return "/bin/" + name, nil + } + + return "", errors.New("not found") + } + } + + t.Run("both present", func(t *testing.T) { + t.Parallel() + assert.Equal(t, "/usr/bin/ionice -c 2 -n 4 /usr/bin/nice -n 5 ", wrapperPrefix(5, all)) + }) + + t.Run("both absent degrades to bare exec", func(t *testing.T) { + t.Parallel() + assert.Empty(t, wrapperPrefix(5, notFound)) + }) + + t.Run("only nice", func(t *testing.T) { + t.Parallel() + assert.Equal(t, "/bin/nice -n -3 ", wrapperPrefix(-3, only("nice"))) + }) + + t.Run("only ionice", func(t *testing.T) { + t.Parallel() + assert.Equal(t, "/bin/ionice -c 2 -n 4 ", wrapperPrefix(0, only("ionice"))) + }) +} diff --git a/packages/orchestrator/README.md b/packages/orchestrator/README.md index 47bd5b72d6..b42cb42892 100644 --- a/packages/orchestrator/README.md +++ b/packages/orchestrator/README.md @@ -303,4 +303,4 @@ Automatically set in local mode. Set before running to override: ## Limitations -- Custom template builds require Debian/Ubuntu-based base images (images that provide the `apt` package manager). Non-Debian images such as Alpine, CentOS/RHEL, or other distributions without `apt` are not supported and will fail during the template build/provisioning process. The provisioning scripts used during template build call `apt` and expect Debian-specific package names and file locations. \ No newline at end of file +- Custom template builds support base images from the declared distro families (FEAT-145): **Debian/Ubuntu** (apt), the **RHEL family** — Fedora, RHEL, CentOS Stream, Rocky, Alma, Oracle Linux, Amazon Linux — (dnf/microdnf), **Arch** (pacman), and **Alpine** (apk, OpenRC). The distro is resolved from the image's `/etc/os-release` `ID` — never by probing for package managers (ADR-010). Images without an os-release identity (pure-Nix, distroless, scratch) or from an undeclared family are rejected fast, with a build-log error naming the reason and the supported families (see `pkg/template/build/phases/base/distro`). Minimal/restricted-repo images (e.g. RHEL UBI) may fail provisioning if their repos don't carry the required packages; the failure is surfaced in the build log. \ No newline at end of file diff --git a/packages/orchestrator/pkg/template/build/commands/user.go b/packages/orchestrator/pkg/template/build/commands/user.go index 0b1f0cb94e..ad32e7c0e0 100644 --- a/packages/orchestrator/pkg/template/build/commands/user.go +++ b/packages/orchestrator/pkg/template/build/commands/user.go @@ -60,7 +60,11 @@ func (u *User) Execute( lvl, prefix, sandboxID, - fmt.Sprintf("adduser --disabled-password --gecos \"\" %s", userArg), + // useradd is part of shadow(-utils) and present on every supported + // distro family, unlike Debian's adduser wrapper (FEAT-145). The + // created user has no password (locked), matching the previous + // adduser --disabled-password behavior. + fmt.Sprintf("useradd --create-home --shell /bin/bash %s", userArg), metadata.Context{ User: "root", EnvVars: cmdMetadata.EnvVars, @@ -99,7 +103,11 @@ func addToSudoers( lvl, prefix, sandboxID, - fmt.Sprintf("usermod -aG sudo %s", userArg), + // Admin group differs by distro (sudo on Debian/Ubuntu, wheel on + // RHEL/Arch/Alpine); the NOPASSWD sudoers entry below is what + // actually grants privileges (FEAT-145). Neither group existing is a + // real error, not something to swallow. + fmt.Sprintf(`if getent group sudo >/dev/null; then usermod -aG sudo %[1]s; elif getent group wheel >/dev/null; then usermod -aG wheel %[1]s; else echo "neither the sudo nor the wheel group exists on this image" >&2; exit 1; fi`, userArg), metadata.Context{ User: "root", EnvVars: cmdMetadata.EnvVars, diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.openrc.tpl b/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.openrc.tpl new file mode 100644 index 0000000000..110381066f --- /dev/null +++ b/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.openrc.tpl @@ -0,0 +1,41 @@ +{{- /*gotype:github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs.templateModel*/ -}} +{{ .WriteFile "/usr/local/share/e2b/envd.openrc" 0o755 }} + +#!/sbin/openrc-run +# E2B env daemon — the OpenRC counterpart of envd.service (see envd.service.tpl +# for the full rationale on each step; this mirrors it for the Alpine/OpenRC +# family). Baked at a neutral path and installed to /etc/init.d/envd by the +# OpenRC e2b_init_setup only: if it lived in /etc/init.d on every image, +# Debian's `systemctl enable envd` would hand it to update-rc.d, which aborts +# on a non-LSB script and fails the whole provisioning. +# +# /tmp-wipe ordering note: OpenRC's bootmisc (boot runlevel) wipes /tmp before +# the default runlevel starts, so envd — in default — can never answer an +# update-envd upload before the wipe. The race envd.service needs an explicit +# After=systemd-tmpfiles-setup.service for cannot happen here. + +description="E2B env daemon" + +supervisor=supervise-daemon +command=/usr/bin/envd +supervise_daemon_args="--env GOTRACEBACK=all --env GOMEMLIMIT={{ .MemoryLimit }}MiB --stdout /var/log/envd.log --stderr /var/log/envd.log" +# Retry forever (envd.service uses Restart=always + StartLimitIntervalSec=0). +respawn_delay=1 +respawn_max=0 + +depend() { + need localmount + after bootmisc + use net +} + +start_pre() { + # Shared with envd.service's ExecStartPre; warns-and-continues by design. + /usr/local/bin/e2b-seed-certs + # systemd-tmpfiles applies the fuse.conf tmpfiles.d rule on the systemd + # family; OpenRC has no tmpfiles pass, so set the mode here. + if [ -e /dev/fuse ]; then + chmod 666 /dev/fuse + fi + return 0 +} diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.service.tpl b/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.service.tpl index b66b7c7166..36e67e7a4b 100644 --- a/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.service.tpl +++ b/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.service.tpl @@ -46,7 +46,9 @@ LimitCORE=infinity # guaranteed present for the sandbox's routable lifetime. The only gap is guest # units that auto-start and egress over TLS before /init; that is accepted # (revisit if a template needs boot-time egress). -ExecStartPre=/bin/sh -c 'mountpoint -q /etc/ssl/certs || { mkdir -p /run/e2b/certs && { tar -C /run/e2b/certs -xf /usr/local/share/e2b/ssl-certs.tar 2>/dev/null || cp -a /etc/ssl/certs/. /run/e2b/certs/ 2>/dev/null; }; mount --bind /run/e2b/certs /etc/ssl/certs; } && ([ -s /etc/ssl/certs/ca-certificates.crt ] || update-ca-certificates)' +# The seeding logic is shared with the OpenRC service (Alpine) and never fails +# the unit — a degraded trust store is recoverable, a dead envd is not. +ExecStartPre=/usr/local/bin/e2b-seed-certs ExecStart=/usr/bin/envd Nice=-20 IOSchedulingClass=realtime diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/files/inittab.tpl b/packages/orchestrator/pkg/template/build/core/rootfs/files/inittab.tpl index 88d981c2ed..acf9c58493 100644 --- a/packages/orchestrator/pkg/template/build/core/rootfs/files/inittab.tpl +++ b/packages/orchestrator/pkg/template/build/core/rootfs/files/inittab.tpl @@ -1,18 +1,16 @@ {{- /*gotype:github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs.templateModel*/ -}} {{ .WriteFile "/etc/inittab" 0o777 }} -# Run system init -::sysinit:/etc/init.d/rcS - -# Run the provision script, prefix the output with a log prefix -::wait:/bin/sh -c '/usr/local/bin/provision.sh 2>&1 | sed "s/^/{{ .ProvisionLogPrefix }}/"' +# Provisioning-boot inittab (busybox init). Every entry is a plain exec with +# NO shell metacharacters: busybox init hands metachar lines to /bin/sh, and +# bare images (premade NixOS, distroless) have no /bin/sh — the pipeline +# logic lives in e2b-provision-runner instead (FEAT-145). -# Flush filesystem changes to disk -::wait:/usr/bin/busybox sync -::wait:fsfreeze --freeze / +# Run system init (mounts /proc /sys /dev /tmp /run through the baked busybox) +::sysinit:/etc/init.d/rcS -# Report the exit code of the provisioning script -::wait:/bin/sh -c 'echo "{{ .ProvisionExitPrefix }}$(cat {{ .ProvisionResultPath }} || printf 1)"' +# Run the provisioning pipeline and report its exit code +::wait:/usr/bin/busybox ash /usr/local/bin/e2b-provision-runner # Wait forever to prevent the VM from exiting until the sandbox is paused and snapshot is taken -::wait:/usr/bin/busybox sleep infinity \ No newline at end of file +::wait:/usr/bin/busybox sleep infinity diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/files/provision-runner.sh.tpl b/packages/orchestrator/pkg/template/build/core/rootfs/files/provision-runner.sh.tpl new file mode 100644 index 0000000000..081628dbb3 --- /dev/null +++ b/packages/orchestrator/pkg/template/build/core/rootfs/files/provision-runner.sh.tpl @@ -0,0 +1,33 @@ +{{- /*gotype:github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs.templateModel*/ -}} +{{ .WriteFile "usr/local/bin/e2b-provision-runner" 0o755 }} + +#!/usr/bin/busybox ash +# Drives the provisioning pipeline for the busybox-init boot. This logic lives +# in a script — NOT in /etc/inittab — because busybox init hands any inittab +# line containing shell metacharacters to /bin/sh, and bare images (premade +# NixOS, distroless) have no /bin/sh; a plain-exec inittab line running this +# script through the baked busybox works on every image (FEAT-145). +BB=/usr/bin/busybox + +# Run the provision script, prefix its output with the log prefix the +# orchestrator forwards to the customer's build logs. +$BB sh /usr/local/bin/provision.sh 2>&1 | $BB sed "s/^/{{ .ProvisionLogPrefix }}/" + +# Flush filesystem changes to disk before the snapshot. +$BB sync +if command -v fsfreeze >/dev/null 2>&1; then + fsfreeze --freeze / +else + # No util-linux on this image; the double sync flushes the ext4 journal + # and the VM is paused before the snapshot is taken. + echo "fsfreeze not available on this image; using sync-only flush" + $BB sync +fi + +# Report the provisioning exit code: provision.sh writes "0" on success and +# (running under set -e) leaves no file behind on failure. +if result=$($BB cat {{ .ProvisionResultPath }} 2>/dev/null); then + echo "{{ .ProvisionExitPrefix }}${result}" +else + echo "{{ .ProvisionExitPrefix }}1" +fi diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/files/rcS.sh.tpl b/packages/orchestrator/pkg/template/build/core/rootfs/files/rcS.sh.tpl index 08bbb99e93..5dd4a30a52 100644 --- a/packages/orchestrator/pkg/template/build/core/rootfs/files/rcS.sh.tpl +++ b/packages/orchestrator/pkg/template/build/core/rootfs/files/rcS.sh.tpl @@ -2,15 +2,20 @@ {{ .WriteFile "etc/init.d/rcS" 0o777 }} #!/usr/bin/busybox ash +# Every command goes through the baked busybox: this runs before provisioning +# on the raw base image, and bare images (premade NixOS, distroless) have no +# mkdir/mount on PATH at all (FEAT-145). +BB=/usr/bin/busybox + echo "Mounting essential filesystems" # Ensure necessary mount points exist -mkdir -p /proc /sys /dev /tmp /run +$BB mkdir -p /proc /sys /dev /tmp /run # Mount essential filesystems -mount -t proc proc /proc -mount -t sysfs sysfs /sys -mount -t devtmpfs devtmpfs /dev -mount -t tmpfs tmpfs /tmp -mount -t tmpfs tmpfs /run +$BB mount -t proc proc /proc +$BB mount -t sysfs sysfs /sys +$BB mount -t devtmpfs devtmpfs /dev +$BB mount -t tmpfs tmpfs /tmp +$BB mount -t tmpfs tmpfs /run -echo "System Init" \ No newline at end of file +echo "System Init" diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/files/seed-certs.sh.tpl b/packages/orchestrator/pkg/template/build/core/rootfs/files/seed-certs.sh.tpl new file mode 100644 index 0000000000..86fd282e55 --- /dev/null +++ b/packages/orchestrator/pkg/template/build/core/rootfs/files/seed-certs.sh.tpl @@ -0,0 +1,40 @@ +{{- /*gotype:github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs.templateModel*/ -}} +{{ .WriteFile "usr/local/bin/e2b-seed-certs" 0o755 }} + +#!/bin/sh +# Seeds the tmpfs-backed /etc/ssl/certs before envd starts — shared by +# envd.service (systemd) and /etc/init.d/envd (OpenRC). See envd.service.tpl +# for the full rationale (why a tar, why a bind mount, the egress-CA contract). +# +# Every failure path here WARNS and continues deliberately: a sandbox with a +# degraded trust store is recoverable (envd's POST /init reinstalls the egress +# CA), a sandbox whose envd never starts is not. + +if ! mountpoint -q /etc/ssl/certs; then + mkdir -p /run/e2b/certs + if [ -f /usr/local/share/e2b/ssl-certs.tar ]; then + if ! tar -C /run/e2b/certs -xf /usr/local/share/e2b/ssl-certs.tar; then + echo "e2b-seed-certs: ssl-certs.tar extraction failed; seeding from the live cert dir instead" >&2 + cp -a /etc/ssl/certs/. /run/e2b/certs/ + fi + else + # Only expected during the base-layer boot, before finalize packs the tar. + echo "e2b-seed-certs: ssl-certs.tar not packed yet; seeding from the live cert dir" + cp -a /etc/ssl/certs/. /run/e2b/certs/ + fi + if ! mount -o bind /run/e2b/certs /etc/ssl/certs; then + echo "e2b-seed-certs: bind mount failed; envd runs with the image's certs as-is" >&2 + fi +fi + +if [ ! -s /etc/ssl/certs/ca-certificates.crt ]; then + if command -v update-ca-certificates >/dev/null 2>&1; then + update-ca-certificates + else + # Provisioning guarantees the bundle on every supported family; + # reaching this means the image diverged after the build. + echo "e2b-seed-certs: CA bundle missing and no update-ca-certificates on this image; TLS trust will be degraded" >&2 + fi +fi + +exit 0 diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/rootfs.go b/packages/orchestrator/pkg/template/build/core/rootfs/rootfs.go index ca859899f0..8d92793f26 100644 --- a/packages/orchestrator/pkg/template/build/core/rootfs/rootfs.go +++ b/packages/orchestrator/pkg/template/build/core/rootfs/rootfs.go @@ -14,6 +14,7 @@ import ( "os" "path/filepath" "runtime" + "slices" "text/template" "github.com/dustin/go-humanize" @@ -41,9 +42,25 @@ var tracer = otel.Tracer("github.com/e2b-dev/infra/packages/orchestrator/pkg/tem var files embed.FS var fileTemplates = template.Must(template.ParseFS(files, "files/*")) -// filesHash is a stable hash of the embedded rootfs file templates. It is used -// only as part of the fallback provision version; explicit provision versions -// remain the rollout control. +// enableSymlinks is the content of the baked symlink layer. Package-level so +// it feeds filesHash: a change here must rotate the fallback provision +// version like any other baked-layer change (qa.md QA11). +var enableSymlinks = map[string]string{ + // Enable envd service autostart. The target MUST be absolute: the link + // lives in multi-user.target.wants/, so a relative target would resolve + // inside that directory and dangle — and provision.sh's offline + // `systemctl enable $E2B_TIMESYNC_UNIT` prunes dangling .wants symlinks, + // silently disabling envd on distros where the link dangles (FEAT-145). + "etc/systemd/system/multi-user.target.wants/envd.service": "/etc/systemd/system/envd.service", + // NOTE: chrony autostart is enabled by provision.sh via `systemctl enable + // $E2B_TIMESYNC_UNIT`, which picks the distro-correct unit name (chrony on + // Debian, chronyd on RHEL/Arch). A static chrony.service symlink here would + // dangle on non-Debian images where the unit is chronyd.service (FEAT-145). +} + +// filesHash is a stable hash of the embedded rootfs file templates plus the +// baked symlink layer. It is used only as part of the fallback provision +// version; explicit provision versions remain the rollout control. var filesHash = func() string { entries, _ := fs.ReadDir(files, "files") h := sha256.New() @@ -51,6 +68,14 @@ var filesHash = func() string { data, _ := files.ReadFile("files/" + e.Name()) fmt.Fprintf(h, "%s\x00%x\x00", e.Name(), data) } + links := make([]string, 0, len(enableSymlinks)) + for name := range enableSymlinks { + links = append(links, name) + } + slices.Sort(links) + for _, name := range links { + fmt.Fprintf(h, "%s\x00%s\x00", name, enableSymlinks[name]) + } return hex.EncodeToString(h.Sum(nil)) }() @@ -233,6 +258,15 @@ func additionalOCILayers( filesMap := map[string]oci.File{ storage.GuestEnvdPath: {Bytes: envdFileData, Mode: 0o777}, + // Systemd preset policy for envd. provision.sh removes /etc/machine-id, + // so the template's next boot is a systemd FIRST boot — and on first + // boot PID1 applies the distro preset policy to all units. On the + // RHEL family that policy ends with "disable *" (and the systemd RPM + // scriptlet's preset-all does the same during provisioning), which + // deletes envd's autostart symlink no matter how it was created. + // A 00- preset sorts before every distro policy file and wins. + "etc/systemd/system-preset/00-e2b.preset": {Bytes: []byte("enable envd.service\n"), Mode: 0o644}, + // Provision script "usr/local/bin/provision.sh": {Bytes: []byte(provisionScript), Mode: 0o777}, // Setup init system @@ -263,14 +297,7 @@ func additionalOCILayers( return nil, fmt.Errorf("error creating layer from files: %w", err) } - symlinkLayer, err := oci.LayerSymlink( - map[string]string{ - // Enable envd service autostart - "etc/systemd/system/multi-user.target.wants/envd.service": "etc/systemd/system/envd.service", - // Enable chrony service autostart - "etc/systemd/system/multi-user.target.wants/chrony.service": "etc/systemd/system/chrony.service", - }, - ) + symlinkLayer, err := oci.LayerSymlink(enableSymlinks) if err != nil { return nil, fmt.Errorf("error creating layer from symlinks: %w", err) } diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/rootfs_test.go b/packages/orchestrator/pkg/template/build/core/rootfs/rootfs_test.go index 963368d997..8618ef90ad 100644 --- a/packages/orchestrator/pkg/template/build/core/rootfs/rootfs_test.go +++ b/packages/orchestrator/pkg/template/build/core/rootfs/rootfs_test.go @@ -90,7 +90,45 @@ func TestAdditionalOCILayers(t *testing.T) { keysIter := maps.Keys(actualFiles) keys := slices.Collect(keysIter) - assert.Len(t, keys, 14) + assert.Len(t, keys, 18) + + // The provisioning boot must be self-contained on the baked busybox: + // bare images (premade NixOS, distroless) have no /bin/sh, and + // busybox init hands any inittab line with shell metacharacters to + // /bin/sh — so the pipeline lives in the runner script and every + // inittab entry is a plain exec. + inittab := actualFiles["etc/inittab"] + require.NotEmpty(t, inittab) + for line := range strings.SplitSeq(inittab, "\n") { + if !strings.HasPrefix(line, "::") { + continue + } + assert.NotContains(t, line, "|", "inittab entries must not need /bin/sh: %s", line) + assert.NotContains(t, line, "$", "inittab entries must not need /bin/sh: %s", line) + } + runner := actualFiles["usr/local/bin/e2b-provision-runner"] + require.NotEmpty(t, runner, "provision runner must be baked") + assert.Contains(t, runner, "#!/usr/bin/busybox ash") + + // Both init families' envd services seed certs via the shared script. + seedCerts := actualFiles["usr/local/bin/e2b-seed-certs"] + require.NotEmpty(t, seedCerts, "cert seeding script must be baked") + assert.Contains(t, actualFiles["etc/systemd/system/envd.service"], "ExecStartPre=/usr/local/bin/e2b-seed-certs") + + // envd must be preset-enabled: first boot (machine-id is removed by + // provisioning) applies the distro preset policy, and the RHEL + // family's "disable *" would otherwise delete envd's autostart link. + assert.Equal(t, "enable envd.service\n", actualFiles["etc/systemd/system-preset/00-e2b.preset"]) + + // The OpenRC counterpart (Alpine, IMPL-145 W5) ships alongside the + // systemd unit; it must supervise envd and honor the memory limit. + // It lives OUTSIDE /etc/init.d — Debian's update-rc.d aborts on a + // non-LSB script there — and is installed by the OpenRC init setup. + openrcEnvd := actualFiles["usr/local/share/e2b/envd.openrc"] + require.NotEmpty(t, openrcEnvd, "OpenRC envd service must be baked") + assert.Contains(t, openrcEnvd, "#!/sbin/openrc-run") + assert.Contains(t, openrcEnvd, "supervisor=supervise-daemon") + assert.Contains(t, openrcEnvd, "GOMEMLIMIT=50MiB") assert.Equal(t, "e2b.local", actualFiles["etc/hostname"]) assert.Equal(t, "nameserver 8.8.8.8", actualFiles["etc/resolv.conf"]) @@ -124,5 +162,36 @@ func TestAdditionalOCILayers(t *testing.T) { WatchdogSec=0`) assert.Equal(t, disabledContent, actualFiles["etc/systemd/system/systemd-journald.service.d/override.conf"]) assert.Equal(t, disabledContent, actualFiles["etc/systemd/system/systemd-networkd.service.d/override.conf"]) + + // Regression guard (FEAT-145): the envd autostart symlink must not dangle. + // A relative target resolves inside multi-user.target.wants/ and dangles, + // and provision.sh's offline `systemctl enable` prunes dangling .wants + // links — silently disabling envd autostart on e.g. Fedora. + symlinksLayer, err := layers[1].Uncompressed() + require.NoError(t, err) + t.Cleanup(func() { + err = symlinksLayer.Close() + assert.NoError(t, err) + }) + + actualSymlinks := map[string]string{} + symlinksTarReader := tar.NewReader(symlinksLayer) + for { + header, err := symlinksTarReader.Next() + if errors.Is(err, io.EOF) { + break + } + require.NoError(t, err) + + if header.Typeflag != tar.TypeSymlink { + continue + } + actualSymlinks[header.Name] = header.Linkname + } + + envdWants := actualSymlinks["etc/systemd/system/multi-user.target.wants/envd.service"] + require.NotEmpty(t, envdWants, "envd autostart symlink must be present") + assert.Equal(t, "/etc/systemd/system/envd.service", envdWants, + "envd autostart symlink target must be absolute so it never dangles") }) } diff --git a/packages/orchestrator/pkg/template/build/layer/layer_executor.go b/packages/orchestrator/pkg/template/build/layer/layer_executor.go index 5a738b0322..65b25bee68 100644 --- a/packages/orchestrator/pkg/template/build/layer/layer_executor.go +++ b/packages/orchestrator/pkg/template/build/layer/layer_executor.go @@ -212,13 +212,20 @@ func (lb *LayerExecutor) updateEnvdInSandbox( return fmt.Errorf("failed to replace envd binary: %w", err) } - // Step 3: Restart the systemd envd service - // Error is ignored because it's expected the envd connection will be lost + // Step 3: Restart the envd service. The error is ignored because the + // restart kills the very envd this command runs through — the connection + // loss is expected. systemctl hands the restart to PID1, which survives + // that. On OpenRC there is no PID1 handoff: rc-service would run in this + // very shell and die with envd between its stop and start halves + // (observed on Alpine — envd never came back), so instead envd is simply + // killed and supervise-daemon's respawn starts the replaced binary; the + // immediate death also means the post-update wait below can never mistake + // the old instance for the new one. _ = sandboxtools.RunCommand( ctx, lb.proxy, sbx.Runtime.SandboxID, - "systemctl restart envd", + "if command -v systemctl >/dev/null 2>&1; then systemctl restart envd; else kill -TERM $(pidof envd); fi", metadata.Context{User: "root"}, ) diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go b/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go new file mode 100644 index 0000000000..3c4d1cd043 --- /dev/null +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go @@ -0,0 +1,237 @@ +// Package distro makes template-build provisioning distro-aware (ADR-010 / +// FEAT-145 / IMPL-145 W1). It keys on the base image's DECLARED identity — its +// /etc/os-release ID — rather than probing which package-manager binary happens +// to exist. Each supported distribution is a declared Profile; the base-phase +// provisioning script selects the right profile by os-release ID in-guest, so +// the whole divergence between distros lives in one data table here instead of +// as scattered runtime detection. +// +// Scope: the systemd family (Debian/Ubuntu, Fedora/RHEL/CentOS/Rocky/Alma, +// Arch) plus Alpine on the OpenRC track (IMPL-145 W5). Distros with no +// os-release identity or no declared profile are rejected with a clear error. +package distro + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" +) + +// Version is the explicit provisioning-contract version, folded into +// Fingerprint. Bump it to force a base-layer rebuild for changes the +// generated selector text cannot capture. +const Version = "1" + +// Fingerprint is a stable hash of the whole generated provisioning contract +// (profiles, init-system blocks, selector, Version). It feeds the base-layer +// cache key: any profile or init-setup change MUST rotate the key, or already +// provisioned bases built from the old contract get silently reused +// (IMPL-145 qa.md QA11 — this exact staleness poisoned dev twice). +func Fingerprint() string { + sum := sha256.Sum256([]byte(Version + "\x00" + ShellSelector())) + + return hex.EncodeToString(sum[:]) +} + +// Profile is the declared, per-family provisioning contract. Everything that +// differs across distributions is data here — never discovered at runtime. +type Profile struct { + // Key is the canonical family key. + Key string + // Init is the init family the guest boots with; it selects the + // e2b_init_setup() body rendered into the selector (see init.go). + Init InitSystem + // IDs are the /etc/os-release ID values that map to this family. + IDs []string + // Packages is the required package set, in this family's package names. + Packages []string + // PkgQueryBody is the body of a shell function testing whether "$1" is installed. + PkgQueryBody string + // PkgInstall installs the packages passed as "$@". + PkgInstall string + // InitBinary is symlinked to /usr/sbin/init. + InitBinary string + // TimeSyncUnit is the chrony systemd unit name (differs: chrony vs chronyd). + TimeSyncUnit string + // AdminGroup is the passwordless-sudo group (sudo on Debian, wheel elsewhere). + AdminGroup string + // CABundle is the trust-store path envd expects. + CABundle string + // CARefresh regenerates the trust store (differs: update-ca-certificates vs update-ca-trust). + CARefresh string + // Bootstrap, when set, runs FIRST in the profile's selector arm — before + // any shared provisioning step. Premade images with no FHS userland + // (NixOS: nothing in /bin//usr/bin before the first activation) use it to + // put the baked busybox's applets on PATH so the shared body's external + // commands (mkdir, tee, passwd, ...) resolve. + Bootstrap string +} + +// Profiles is the declared registry (systemd family, v1). The package names, +// unit names and CA paths reuse the mapping validated in infra #2941; the +// *structure* (declared profile keyed on distro, not detected package manager) +// is deliberately different — see ADR-010. +var Profiles = []Profile{ + { + Key: "debian", + Init: InitSystemd, + IDs: []string{"debian", "ubuntu"}, + Packages: []string{ + "systemd", "systemd-sysv", "openssh-server", "sudo", "chrony", "socat", + "curl", "ca-certificates", "fuse3", "iptables", "git", "nfs-common", + "less", "nftables", "iputils-ping", "jq", + }, + PkgQueryBody: `dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -q "install ok installed"`, + PkgInstall: "apt-get -q update\n DEBIAN_FRONTEND=noninteractive DEBCONF_NOWARNINGS=yes apt-get -qq -o=Dpkg::Use-Pty=0 install -y --no-install-recommends \"$@\"", + InitBinary: "/lib/systemd/systemd", + TimeSyncUnit: "chrony", + AdminGroup: "sudo", + CABundle: "/etc/ssl/certs/ca-certificates.crt", + CARefresh: "update-ca-certificates", + }, + { + Key: "rhel", + Init: InitSystemd, + // Fedora, RHEL, CentOS Stream, Rocky, Alma, Oracle Linux, Amazon Linux. + IDs: []string{"fedora", "rhel", "centos", "rocky", "almalinux", "ol", "amzn"}, + Packages: []string{ + "systemd", "shadow-utils", "passwd", "openssh-server", "sudo", "chrony", + "socat", "curl", "ca-certificates", "fuse3", "iptables-nft", "git", + "nfs-utils", "less", "nftables", "iputils", "jq", "bash", + }, + PkgQueryBody: `rpm -q "$1" >/dev/null 2>&1`, + PkgInstall: `dnf -y --allowerasing install "$@" 2>/dev/null || microdnf -y install "$@"`, + InitBinary: "/usr/lib/systemd/systemd", + TimeSyncUnit: "chronyd", + AdminGroup: "wheel", + CABundle: "/etc/ssl/certs/ca-certificates.crt", + // update-ca-trust regenerates /etc/pki/ca-trust/extracted/* but never + // creates a file named ca-certificates.crt (that name is Debian's), so + // the bundle envd expects must be linked to the extracted PEM bundle + // explicitly — otherwise envd.service's ExecStartPre finds no bundle + // and its update-ca-certificates fallback doesn't exist on this family. + CARefresh: `update-ca-trust extract && ln -sf /etc/pki/tls/certs/ca-bundle.crt "$E2B_CA_BUNDLE"`, + }, + { + Key: "arch", + Init: InitSystemd, + IDs: []string{"arch", "archarm"}, + Packages: []string{ + "systemd", "shadow", "openssh", "sudo", "chrony", "socat", "curl", + "ca-certificates", "fuse3", "iptables", "git", "nfs-utils", "less", + "nftables", "iputils", "jq", "bash", + }, + PkgQueryBody: `pacman -Q "$1" >/dev/null 2>&1`, + PkgInstall: "pacman -Sy --noconfirm\n pacman -S --noconfirm --needed \"$@\"", + InitBinary: "/usr/lib/systemd/systemd", + TimeSyncUnit: "chronyd", + AdminGroup: "wheel", + CABundle: "/etc/ssl/certs/ca-certificates.crt", + // Arch ships p11-kit's update-ca-trust (no update-ca-certificates); + // its extract step does emit /etc/ssl/certs/ca-certificates.crt. + CARefresh: "update-ca-trust extract", + }, + { + Key: "alpine", + Init: InitOpenRC, + IDs: []string{"alpine"}, + // No systemd packages: Alpine boots busybox init → OpenRC. shadow + // provides the useradd/usermod the build steps use (busybox's + // adduser takes different flags). + Packages: []string{ + "openrc", "shadow", "openssh", "sudo", "chrony", "socat", "curl", + "ca-certificates", "fuse3", "iptables", "git", "nfs-utils", "less", + "nftables", "iputils", "jq", "bash", + }, + PkgQueryBody: `apk info -e "$1" >/dev/null 2>&1`, + PkgInstall: `apk add --no-cache "$@"`, + // busybox init is Alpine's standard PID1; it hands off to OpenRC via + // the inittab installed by e2b_init_setup (init.go). + InitBinary: "/bin/busybox", + TimeSyncUnit: "chronyd", + AdminGroup: "wheel", + CABundle: "/etc/ssl/certs/ca-certificates.crt", + // Alpine's ca-certificates ships a Debian-compatible + // update-ca-certificates that writes the bundle at CABundle. + CARefresh: "update-ca-certificates", + }, + { + Key: "nixos", + Init: InitNixOS, + IDs: []string{"nixos"}, + // NixOS images are PREMADE: built from the E2B NixOS configuration + // that declares everything provision.sh installs imperatively + // elsewhere (envd unit, chrony, sshd, default user, sudoers). There + // is no imperative package manager to drive — an image missing its + // declared parts is a broken premade image, not something to repair + // here (qa.md QA13). + Packages: nil, + PkgQueryBody: "true", + PkgInstall: `echo "[provision] ERROR: NixOS images are premade — packages must be declared in the image's NixOS configuration" >&2; exit 1`, + // Stage-2 init of the system closure, reachable via the profile + // symlink baked into the premade image. + InitBinary: "/nix/var/nix/profiles/system/init", + TimeSyncUnit: "chronyd", + AdminGroup: "wheel", + CABundle: "/etc/ssl/certs/ca-certificates.crt", + // The bundle appears at first activation (environment.etc); nothing + // can regenerate it pre-activation and envd's unit comes from the + // image's own configuration — an explicit, stated code path. + CARefresh: `echo "NixOS: the CA bundle is provided by the image configuration at first activation; nothing to refresh at provision time"`, + // No FHS userland before the first activation — put the baked + // busybox's applets on PATH for the shared provisioning body. + Bootstrap: `E2B_BB_DIR=/run/e2b-tools + /usr/bin/busybox mkdir -p "$E2B_BB_DIR" + /usr/bin/busybox --install -s "$E2B_BB_DIR" + export PATH="$E2B_BB_DIR:$PATH"`, + }, +} + +// SupportedIDs returns every os-release ID the v1 selector accepts. +func SupportedIDs() []string { + var ids []string + for _, p := range Profiles { + ids = append(ids, p.IDs...) + } + + return ids +} + +// ShellSelector generates the POSIX-sh block that provision.sh sources: it +// selects the profile by the guest's own $E2B_DISTRO_ID (set from /etc/os-release) +// and defines the profile's packages, package functions, init path, time-sync +// unit, admin group and CA handling. An unrecognized distro exits 1 with a clear, +// customer-visible error (FEAT-145 AC4) — never a silent best-effort. +// +// Selection is by DECLARED distro identity, not by `command -v ` — that +// is the whole point of ADR-010 and the reason this is not infra #2941. +func ShellSelector() string { + var b strings.Builder + b.WriteString(`case "$E2B_DISTRO_ID" in` + "\n") + for _, p := range Profiles { + fmt.Fprintf(&b, " %s)\n", strings.Join(p.IDs, "|")) + if p.Bootstrap != "" { + fmt.Fprintf(&b, " %s\n", p.Bootstrap) + } + fmt.Fprintf(&b, " E2B_PACKAGES=%q\n", strings.Join(p.Packages, " ")) + fmt.Fprintf(&b, " e2b_pkg_query() { %s; }\n", p.PkgQueryBody) + fmt.Fprintf(&b, " e2b_pkg_install() { %s; }\n", p.PkgInstall) + fmt.Fprintf(&b, " E2B_INIT_BIN=%q\n", p.InitBinary) + fmt.Fprintf(&b, " E2B_TIMESYNC_UNIT=%q\n", p.TimeSyncUnit) + fmt.Fprintf(&b, " E2B_ADMIN_GROUP=%q\n", p.AdminGroup) + fmt.Fprintf(&b, " E2B_CA_BUNDLE=%q\n", p.CABundle) + fmt.Fprintf(&b, " e2b_ca_refresh() { %s; }\n", p.CARefresh) + fmt.Fprintf(&b, " E2B_INIT_SYSTEM=%q\n", p.Init) + fmt.Fprintf(&b, " e2b_init_setup() {\n%s\n }\n", indentBlock(initSetup[p.Init], " ")) + fmt.Fprintf(&b, " ;;\n") + } + fmt.Fprintf(&b, " *)\n") + fmt.Fprintf(&b, " echo \"[provision] ERROR: unsupported base image distribution: ID='${E2B_DISTRO_ID:-unknown}'.\" >&2\n") + fmt.Fprintf(&b, " echo \"[provision] E2B template builds support: %s.\" >&2\n", strings.Join(SupportedIDs(), ", ")) + fmt.Fprintf(&b, " exit 1\n") + fmt.Fprintf(&b, " ;;\n") + b.WriteString("esac\n") + + return b.String() +} diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/distro_test.go b/packages/orchestrator/pkg/template/build/phases/base/distro/distro_test.go new file mode 100644 index 0000000000..d691ce8b5b --- /dev/null +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/distro_test.go @@ -0,0 +1,178 @@ +package distro + +import ( + "crypto/sha256" + "encoding/hex" + "strings" + "testing" +) + +// Golden lines lifted VERBATIM from the pre-change provision.sh +// (packages/orchestrator/pkg/template/build/phases/base/provision.sh @ infra main). +// The debian profile must reproduce these so Debian/Ubuntu behaviour is preserved +// (FEAT-145 AC2). +const ( + goldenDebianPackages = "systemd systemd-sysv openssh-server sudo chrony socat curl ca-certificates fuse3 iptables git nfs-common less nftables iputils-ping jq" + goldenDebianQuery = `dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -q "install ok installed"` + goldenDebianInit = "/lib/systemd/systemd" +) + +func profileByKey(t *testing.T, key string) Profile { + t.Helper() + for _, p := range Profiles { + if p.Key == key { + return p + } + } + t.Fatalf("no profile with key %q", key) + + return Profile{} +} + +// AC2: the debian profile preserves today's Debian package set / query / init path. +func TestDebianPreserved(t *testing.T) { + t.Parallel() + p := profileByKey(t, "debian") + if got := strings.Join(p.Packages, " "); got != goldenDebianPackages { + t.Errorf("debian packages drifted:\n got: %s\nwant: %s", got, goldenDebianPackages) + } + if p.PkgQueryBody != goldenDebianQuery { + t.Errorf("debian query drifted:\n got: %s\nwant: %s", p.PkgQueryBody, goldenDebianQuery) + } + if p.InitBinary != goldenDebianInit { + t.Errorf("debian init drifted: got %s want %s", p.InitBinary, goldenDebianInit) + } + if p.TimeSyncUnit != "chrony" || p.AdminGroup != "sudo" { + t.Errorf("debian unit/group drifted: %s / %s", p.TimeSyncUnit, p.AdminGroup) + } +} + +// The families genuinely diverge on the axes that matter. +func TestFamiliesDiffer(t *testing.T) { + t.Parallel() + rhel := profileByKey(t, "rhel") + if rhel.TimeSyncUnit != "chronyd" || rhel.AdminGroup != "wheel" { + t.Errorf("rhel unit/group wrong: %s / %s", rhel.TimeSyncUnit, rhel.AdminGroup) + } + // Must both regenerate the trust store AND materialize the bundle at the + // Debian-named path envd expects — update-ca-trust alone never creates + // ca-certificates.crt, which left envd.service unable to start on Fedora. + if !strings.Contains(rhel.CARefresh, "update-ca-trust extract") || + !strings.Contains(rhel.CARefresh, `ln -sf /etc/pki/tls/certs/ca-bundle.crt "$E2B_CA_BUNDLE"`) { + t.Errorf("rhel CA refresh wrong: %s", rhel.CARefresh) + } + if rhel.InitBinary != "/usr/lib/systemd/systemd" { + t.Errorf("rhel init path wrong: %s", rhel.InitBinary) + } + arch := profileByKey(t, "arch") + if !strings.Contains(arch.PkgInstall, "pacman") { + t.Errorf("arch install should use pacman: %s", arch.PkgInstall) + } +} + +// The generated selector keys on the DECLARED distro id, never on which +// package-manager binary exists (the anti-#2941 invariant, TT-2). +func TestSelectorNoPackageManagerProbing(t *testing.T) { + t.Parallel() + sel := ShellSelector() + for _, bad := range []string{ + "command -v apt-get", "command -v dnf", "command -v yum", + "command -v microdnf", "command -v pacman", "PKG_FAMILY", + } { + if strings.Contains(sel, bad) { + t.Errorf("selector leaked package-manager probing: %q", bad) + } + } + if !strings.Contains(sel, `case "$E2B_DISTRO_ID" in`) { + t.Error("selector must switch on $E2B_DISTRO_ID (declared distro identity)") + } +} + +// Every supported id gets a case arm; an unknown id hits the failing default (AC4). +func TestSelectorCoversIDsAndRejects(t *testing.T) { + t.Parallel() + sel := ShellSelector() + for _, id := range SupportedIDs() { + if !strings.Contains(sel, id) { + t.Errorf("selector missing arm for supported id %q", id) + } + } + for _, want := range []string{"*)", "unsupported base image", "exit 1"} { + if !strings.Contains(sel, want) { + t.Errorf("selector missing fast-reject piece %q", want) + } + } + // Alpine is now supported via the OpenRC track (W5) — and it must be the + // OpenRC profile, never folded into a systemd family. + alpine := profileByKey(t, "alpine") + if alpine.Init != InitOpenRC { + t.Errorf("alpine must be the OpenRC profile, got init %q", alpine.Init) + } +} + +// Every profile declares a known init system with a rendered setup body, and +// no body leaks another init system's tooling (systemctl in OpenRC or +// rc-update in systemd would fail at provisioning time). +func TestInitSystemsDeclaredAndCoherent(t *testing.T) { + t.Parallel() + for _, p := range Profiles { + setup, ok := initSetup[p.Init] + if !ok { + t.Errorf("profile %q declares init %q with no setup body", p.Key, p.Init) + + continue + } + switch p.Init { + case InitSystemd: + if strings.Contains(setup, "rc-update") { + t.Errorf("systemd init setup leaks rc-update (profile %q)", p.Key) + } + case InitOpenRC: + if strings.Contains(setup, "systemctl") { + t.Errorf("openrc init setup leaks systemctl (profile %q)", p.Key) + } + } + } + sel := ShellSelector() + if !strings.Contains(sel, "e2b_init_setup() {") { + t.Error("selector must define e2b_init_setup()") + } + // The OpenRC boot chain pieces the alpine arm must carry. + for _, want := range []string{"/etc/inittab", "rc-update add envd default", "openrc sysinit"} { + if !strings.Contains(sel, want) { + t.Errorf("selector missing OpenRC boot piece %q", want) + } + } +} + +// The cache fingerprint must cover the whole generated provisioning contract: +// stable across calls, and carrying both the selector text and the explicit +// Version (W1 T5 — a profile change must rotate the base-layer cache key). +func TestFingerprintStableAndVersioned(t *testing.T) { + t.Parallel() + a, b := Fingerprint(), Fingerprint() + if a != b || len(a) != 64 { + t.Errorf("fingerprint must be a stable sha256 hex: %q vs %q", a, b) + } + want := sha256.Sum256([]byte(Version + "\x00" + ShellSelector())) + if a != hex.EncodeToString(want[:]) { + t.Error("fingerprint must hash Version + selector text") + } +} + +// Sanity: RHEL-family aliases (rocky/alma/oracle/amazon) all resolve to one arm. +func TestRHELFamilyAliases(t *testing.T) { + t.Parallel() + rhel := profileByKey(t, "rhel") + for _, want := range []string{"fedora", "rhel", "centos", "rocky", "almalinux", "ol", "amzn"} { + found := false + for _, id := range rhel.IDs { + if id == want { + found = true + } + } + if !found { + t.Errorf("rhel family missing alias %q", want) + } + } +} diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/init.go b/packages/orchestrator/pkg/template/build/phases/base/distro/init.go new file mode 100644 index 0000000000..2ab320fed9 --- /dev/null +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/init.go @@ -0,0 +1,165 @@ +// Init-system axis of the distro profiles (ADR-010 / IMPL-145 W5). +// +// A Profile declares WHAT differs per distribution (packages, paths, units); +// the init system declares HOW the guest is arranged to boot: which services +// autostart, how boot noise is silenced, and — for OpenRC — how the image +// transitions from the one-shot provisioning inittab to a real boot sequence. +// Everything init-specific that runs at provisioning time lives here as one +// declared shell block per init system, rendered into the profile selector as +// `e2b_init_setup()`; provision.sh itself stays init-agnostic. +package distro + +import "strings" + +// InitSystem is the guest init family a profile boots with. +type InitSystem string + +const ( + // InitSystemd — Debian/Ubuntu, RHEL/Fedora, Arch. The guest boots + // /sbin/init → systemd; envd autostarts via the baked envd.service + + // 00-e2b.preset (see core/rootfs). + InitSystemd InitSystem = "systemd" + + // InitOpenRC — Alpine. The guest boots /sbin/init → busybox init → + // /etc/inittab → OpenRC runlevels; envd autostarts via the baked + // /etc/init.d/envd (envd.openrc.tpl) added to the default runlevel. + InitOpenRC InitSystem = "openrc" + + // InitNixOS — premade NixOS images. Stage-2 init runs the system + // activation and execs the closure's systemd; every service (envd, + // chrony, sshd) is wired declaratively by the image's own NixOS + // configuration, so provisioning neither enables nor masks anything — + // offline systemctl couldn't resolve store-path units pre-activation + // anyway. + InitNixOS InitSystem = "nixos" +) + +// initSetup is the provisioning-time shell block per init system, exposed to +// provision.sh as `e2b_init_setup()`. Bodies may reference the selector's +// profile variables (e.g. $E2B_TIMESYNC_UNIT) — they are defined in the same +// case arm before the function is called. +var initSetup = map[InitSystem]string{ + // The systemd body is the block that historically lived inline in + // provision.sh — command-for-command, so the Debian/Ubuntu render stays + // behaviorally identical (FEAT-145 AC2). + InitSystemd: `echo "Don't wait for ttyS0 (serial console kernel logs)" +# This is required when the Firecracker kernel args has specified console=ttyS0 +systemctl mask serial-getty@ttyS0.service + +echo "Disable network online wait" +systemctl mask systemd-networkd-wait-online.service + +echo "Disable system first boot wizard" +# This was problem with Ubuntu 24.04, that differently calculate wizard should be called +# and Linux boot was stuck in wizard until envd wait timeout +systemctl mask systemd-firstboot.service + +echo "Enable time synchronization ($E2B_TIMESYNC_UNIT)" +# Distro-correct chrony unit (chrony on Debian, chronyd on RHEL/Arch). +systemctl enable "$E2B_TIMESYNC_UNIT" + +echo "Enable envd autostart" +# Belt-and-suspenders with the baked 00-e2b.preset: on the RHEL family the +# package transaction above runs systemd's RPM scriptlet 'systemctl preset-all' +# (policy 'disable *'), which deletes the baked wants-symlink (qa.md QA11). +systemctl enable envd.service + +echo "Disable chrony-wait" +# chrony-wait blocks multi-user.target until the first clock sync (~8s); +# chrony still syncs in the background, nothing needs to wait for it. +# masking a unit that doesn't exist on this distro still succeeds (systemctl +# mask just writes the /dev/null symlink), so a failure here is real. +systemctl mask chrony-wait.service + +echo "Disable slow boot units not needed in the sandbox" +# binfmt registrations (foreign-arch exec) take ~1s of CPU early in boot and +# compete with envd start; e2scrub is for LVM-backed ext4 only. +systemctl mask systemd-binfmt.service +systemctl mask e2scrub_reap.service`, + + // OpenRC (Alpine). The image is still running the one-shot PROVISIONING + // inittab (it is what launched this script); replace it with the real + // boot sequence and wire the runlevels a container image ships without. + InitOpenRC: `echo "Installing boot inittab (busybox init -> OpenRC runlevels)" +printf '%s\n' \ + '::sysinit:/sbin/openrc sysinit' \ + '::sysinit:/sbin/openrc boot' \ + '::wait:/sbin/openrc default' \ + '::shutdown:/sbin/openrc shutdown' \ + '::ctrlaltdel:/sbin/reboot' \ + > /etc/inittab + +echo "Registering base OpenRC services" +# Container images carry no runlevel wiring at all (setup-alpine does this on +# real installs): kernel filesystems in sysinit, system prep in boot. bootmisc +# also wipes /tmp in the boot runlevel, so envd (default runlevel) can never +# race the wipe — the ordering systemd needs After= for is inherent here. +# Which scripts exist varies by image (mdev vs udev, procfs presence) — check +# and say so instead of swallowing rc-update errors. +for svc in devfs sysfs procfs dmesg mdev; do + if [ -e "/etc/init.d/$svc" ]; then + rc-update add "$svc" sysinit + else + echo "OpenRC service $svc not present on this image; skipping" + fi +done +for svc in localmount sysctl hostname bootmisc; do + if [ -e "/etc/init.d/$svc" ]; then + rc-update add "$svc" boot + else + echo "OpenRC service $svc not present on this image; skipping" + fi +done + +# The FC guest's eth0 is configured by the kernel (ip=), but OpenRC services +# declaring a "need net" dependency (chronyd) trigger the networking service, +# which errors out on a missing /etc/network/interfaces and takes chronyd +# down with it. A loopback-only interfaces file lets networking start (and +# provide "net") without touching the kernel-managed eth0. +printf 'auto lo\niface lo inet loopback\n' > /etc/network/interfaces +if [ -e /etc/init.d/networking ]; then + rc-update add networking boot +else + echo "OpenRC networking service not present on this image; services needing 'net' must not be enabled" +fi + +echo "Enable time synchronization ($E2B_TIMESYNC_UNIT)" +rc-update add "$E2B_TIMESYNC_UNIT" default + +echo "Enable envd autostart" +# The service script is baked at a neutral path (envd.openrc.tpl) so the +# Debian family's update-rc.d never sees it; install it for OpenRC here. +cp /usr/local/share/e2b/envd.openrc /etc/init.d/envd +chmod 0755 /etc/init.d/envd +rc-update add envd default + +echo "Enable sshd" +if [ -e /etc/init.d/sshd ]; then + rc-update add sshd default +else + echo "sshd service not present on this image; skipping" +fi`, + + // Premade NixOS: the image's declarative configuration owns everything + // this block does imperatively on other families — including the envd + // unit the OCI layer bakes for the systemd family. + InitNixOS: `echo "NixOS is declaratively configured; removing the baked systemd drop-ins" +# NixOS activation manages /etc/systemd/system as a symlink into the store; +# with foreign files in the way, setup-etc refuses to create it and systemd +# boots with NO units at all ("Unit default.target not found", observed on +# the real console). The premade image's configuration declares envd itself. +rm -rf /etc/systemd/system`, +} + +// indentBlock indents every non-empty line of a shell block for embedding +// inside the generated selector function bodies. +func indentBlock(s, prefix string) string { + lines := strings.Split(s, "\n") + for i, l := range lines { + if l != "" { + lines[i] = prefix + l + } + } + + return strings.Join(lines, "\n") +} diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/README.md b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/README.md new file mode 100644 index 0000000000..3d020efb4c --- /dev/null +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/README.md @@ -0,0 +1,43 @@ +# E2B premade NixOS base image + +NixOS templates work the inverse of every other family: instead of the +orchestrator provisioning the image imperatively, the image is **premade** from +`configuration.nix`, which declares everything `provision.sh` installs +elsewhere — the envd systemd unit, chrony, sshd, the default `user` (with a +matching `user` group and the exact sudoers line the build steps check for), +`/bin/bash` (build steps invoke it explicitly), and the journald watchdog +override. The orchestrator's `nixos` profile then only verifies and boots +(see `../distro.go` and the `InitNixOS` block in `../init.go`). + +## Building and publishing + +`build.sh` (run on a Linux host with docker): + +1. evaluates the NixOS system closure with `nix` inside a `nixos/nix` + container (`nixpkgs` channel pinned in the script), +2. packs the closure into a single-layer OCI rootfs tar, adding the three + pieces of glue the boot path needs: + - `/sbin/init -> /nix/var/nix/profiles/system/init` (the stage-2 init the + `nixos` profile points the kernel at), + - `/nix/var/nix/profiles/system -> `, + - a static `/etc/os-release` with `ID=nixos` so the distro selector can + identify the image *before* the first activation generates the real one, +3. `docker import`s and pushes the tar. + +**Push every rebuild under a NEW TAG.** The base-layer cache key includes the +image reference as written in the Dockerfile — republishing under the same tag +silently reuses the previously cached base layer (observed; same "default tag" +ambiguity called out in `phases/base/hash.go`). + +## Boot-path notes (all observed on real KVM, IMPL-145 qa.md QA14) + +- Before the first activation the image has **no FHS userland** — no + `/bin/sh`, no `mkdir`. The provisioning boot runs entirely through the baked + busybox (see `core/rootfs/files/rcS.sh.tpl`, `inittab.tpl`, + `provision-runner.sh.tpl`), and the `nixos` profile's `Bootstrap` puts + busybox applets on `PATH` for the shared provisioning body. +- NixOS activation manages `/etc/systemd/system` as a symlink into the store; + the baked systemd drop-ins must be removed at provisioning (the `InitNixOS` + setup does this) or `setup-etc` refuses the symlink and systemd boots with + no units at all ("Unit default.target not found"). +- The sandbox gets the nix toolchain natively (`nix-env` on PATH for `user`). diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/build.sh b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/build.sh new file mode 100644 index 0000000000..4d5b7fe0db --- /dev/null +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/build.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -e +cd /root/nixos-e2b +# Build the toplevel closure with nix inside the nixos/nix container. +docker run --rm -v /root/nixos-e2b:/build nixos/nix:latest sh -c " +set -e +nix-build -I nixpkgs=channel:nixos-24.05 -I nixos-config=/build/configuration.nix \ + '' -A config.system.build.toplevel -o /build/result +top=\$(readlink /build/result) +echo \"TOPLEVEL=\$top\" +# Pack the full closure + the boot/identity glue into one rootfs tar. +nix-store -qR /build/result > /build/closure.txt +tar -cf /build/nixos-rootfs.tar \$(cat /build/closure.txt) +staging=/tmp/extra +mkdir -p \$staging/sbin \$staging/etc \$staging/nix/var/nix/profiles +ln -s \$top \$staging/nix/var/nix/profiles/system +ln -s /nix/var/nix/profiles/system/init \$staging/sbin/init +cat > \$staging/etc/os-release < 0 { + return fmt.Errorf("exit status: %s; provisioning output tail:\n%s", exitStatus, strings.Join(tail, "\n")) + } + return fmt.Errorf("exit status: %s", exitStatus) } + if after, ok := strings.CutPrefix(line, logExternalPrefix); ok { + if trimmed := strings.TrimSpace(after); trimmed != "" { + tail = append(tail, trimmed) + if len(tail) > failureTailLines { + tail = tail[1:] + } + } + } } if err := scanner.Err(); err != nil && !errors.Is(err, io.EOF) { diff --git a/packages/orchestrator/pkg/template/build/phases/base/provision.sh b/packages/orchestrator/pkg/template/build/phases/base/provision.sh index 56ecd6d176..6d120c8ece 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/provision.sh +++ b/packages/orchestrator/pkg/template/build/phases/base/provision.sh @@ -13,13 +13,35 @@ echo "Starting provisioning script" echo "Making configuration immutable" $BUSYBOX chattr +i /etc/resolv.conf -# Helper function to check if a package is installed +# Identify the base image by its DECLARED /etc/os-release ID (FEAT-145 / ADR-010) +# — not by probing which package manager exists. The selector below is generated +# from the distro profile registry (packages/.../phases/base/distro); it sets +# E2B_PACKAGES, e2b_pkg_query(), e2b_pkg_install(), E2B_INIT_BIN, E2B_TIMESYNC_UNIT, +# E2B_ADMIN_GROUP, E2B_CA_BUNDLE, e2b_ca_refresh() — or exits 1 with a clear error +# on an unsupported distribution. +echo "Detecting base image distribution" +# os-release is the image's DECLARED identity (ADR-010) — never probe for +# package managers. Images without it (pure-Nix, distroless, scratch) are +# rejected with a message naming the real problem; supporting them needs the +# explicit distro-declaration override (qa.md QA3), not guessing. +if [ -r /etc/os-release ]; then + . /etc/os-release + E2B_DISTRO_ID="${ID:-unknown}" +else + E2B_DISTRO_ID="unknown (image has no /etc/os-release)" +fi + +{{ .DistroSelector }} + +echo "Provisioning for distro '$E2B_DISTRO_ID' (init=$E2B_INIT_BIN, timesync=$E2B_TIMESYNC_UNIT, admin-group=$E2B_ADMIN_GROUP)" + +# Helper function to check if a package is installed (distro-specific query) is_package_installed() { - dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -q "install ok installed" + e2b_pkg_query "$1" } # Install required packages if not already installed -PACKAGES="systemd systemd-sysv openssh-server sudo chrony socat curl ca-certificates fuse3 iptables git nfs-common less nftables iputils-ping jq" +PACKAGES="$E2B_PACKAGES" echo "Checking presence of the following packages: $PACKAGES" MISSING="" @@ -32,18 +54,33 @@ done if [ -n "$MISSING" ]; then echo "Missing packages detected, installing:$MISSING" - apt-get -q update - DEBIAN_FRONTEND=noninteractive DEBCONF_NOWARNINGS=yes apt-get -qq -o=Dpkg::Use-Pty=0 install -y --no-install-recommends $MISSING + # shellcheck disable=SC2086 + e2b_pkg_install $MISSING else echo "All required packages are already installed." fi +# Ensure the system CA trust bundle exists at the path envd expects. On Debian +# the ca-certificates package creates it; on RHEL it is generated under /etc/pki +# by update-ca-trust, so e2b_ca_refresh regenerates/exposes it (FEAT-145). +# A refresh failure fails provisioning (set -e) — a sandbox with silently +# broken TLS trust is worse than a legible build error. Profiles where the +# bundle legitimately appears later (NixOS: at first activation) say so in +# their e2b_ca_refresh instead of pretending to regenerate. +if [ ! -s "$E2B_CA_BUNDLE" ]; then + echo "CA trust bundle missing at $E2B_CA_BUNDLE — running the profile's refresh" + e2b_ca_refresh +fi + # Set /dev/fuse permissions to 666 for non-root access # Use systemd-tmpfiles to set permissions at boot mkdir -p /etc/tmpfiles.d echo 'z /dev/fuse 0666 root root -' > /etc/tmpfiles.d/fuse.conf echo "Setting up shell" +# Premade images (NixOS) generate /etc/profile.d and /root at first +# activation; create them so the drop-ins below always have a home. +mkdir -p /etc/profile.d /root echo "export SHELL='/bin/bash'" >/etc/profile.d/shell.sh echo "export PS1='\w \$ '" >/etc/profile.d/prompt.sh echo "export PS1='\w \$ '" >>"/etc/profile" @@ -53,19 +90,38 @@ echo "Use .bashrc and .profile" echo "if [ -f ~/.bashrc ]; then source ~/.bashrc; fi; if [ -f ~/.profile ]; then source ~/.profile; fi" >>/etc/profile echo "Remove root password" -passwd -d root +# Premade images (NixOS) manage accounts declaratively and have no /etc/passwd +# before their first activation — nothing to remove there, and that is a +# deliberate code path, not a swallowed failure. +if [ -f /etc/passwd ]; then + passwd -d root +else + echo "No /etc/passwd yet (declaratively managed image); root password is the image configuration's responsibility" +fi echo "Setting up chrony" mkdir -p /etc/chrony -cat </etc/chrony/chrony.conf -refclock PHC /dev/ptp0 poll 2 dpoll 2 -# Step (jump) the clock instead of slewing when the offset exceeds 1s, but only -# for the first 3 updates after chronyd starts. chronyd restarts on every cold -# boot/reboot, so this corrects a large boot-time offset fast (TLS needs a -# correct clock) without risking a backward jump under a running workload. -# Needed because chrony-wait is masked, so boot no longer blocks on first sync. -makestep 1.0 3 -EOF +{ + # Prefer the hypervisor's PTP clock (kvm-ptp): no network dependency and + # it tracks the host directly. It is missing where nested virtualization + # can't expose it (e.g. dev slots) — chronyd treats a missing PHC as a + # FATAL error, so only reference it when the device exists and fall back + # to NTP otherwise; a running chronyd without PHC beats a dead one. + # Device presence is probed in the provisioning VM, which runs on the same + # host/KVM as the runtime sandboxes. + if [ -e /dev/ptp0 ]; then + echo "refclock PHC /dev/ptp0 poll 2 dpoll 2" + else + echo "pool pool.ntp.org iburst maxsources 3" + fi + # Step (jump) the clock instead of slewing when the offset exceeds 1s, but + # only for the first 3 updates after chronyd starts. chronyd restarts on + # every cold boot/reboot, so this corrects a large boot-time offset fast + # (TLS needs a correct clock) without risking a backward jump under a + # running workload. Needed because chrony-wait is masked, so boot no + # longer blocks on first sync. + echo "makestep 1.0 3" +} >/etc/chrony/chrony.conf # Add a proxy config, as some environments expects it there (e.g. timemaster in Node Dockerimage) echo "include /etc/chrony/chrony.conf" >/etc/chrony.conf @@ -89,34 +145,22 @@ echo 'fs.inotify.max_user_watches=65536' | tee -a /etc/sysctl.conf echo "Disabling proactive memory compaction" echo 'vm.compaction_proactiveness=0' | tee -a /etc/sysctl.conf -echo "Don't wait for ttyS0 (serial console kernel logs)" -# This is required when the Firecracker kernel args has specified console=ttyS0 -systemctl mask serial-getty@ttyS0.service - -echo "Disable network online wait" -systemctl mask systemd-networkd-wait-online.service - -echo "Disable system first boot wizard" -# This was problem with Ubuntu 24.04, that differently calculate wizard should be called -# and Linux boot was stuck in wizard until envd wait timeout -systemctl mask systemd-firstboot.service - -echo "Disable chrony-wait" -# chrony-wait blocks multi-user.target until the first clock sync (~8s); -# chrony still syncs in the background, nothing needs to wait for it. -systemctl mask chrony-wait.service - -echo "Disable slow boot units not needed in the sandbox" -# binfmt registrations (foreign-arch exec) take ~1s of CPU early in boot and -# compete with envd start; e2scrub is for LVM-backed ext4 only. -systemctl mask systemd-binfmt.service -systemctl mask e2scrub_reap.service +# Init-system-specific boot arrangement: service autostarts (envd, time sync), +# boot-noise silencing, and — on OpenRC — replacing the one-shot provisioning +# inittab with the real boot sequence. The body is rendered per profile family +# by the distro selector (see distro/init.go); everything below stays shared. +e2b_init_setup # Clean machine-id from Docker rm -rf /etc/machine-id -echo "Linking systemd to init" -ln -sf /lib/systemd/systemd /usr/sbin/init +echo "Linking $E2B_INIT_BIN to init" +# Bare/premade images may not carry /usr/sbin at all. +mkdir -p /usr/sbin +ln -sf "$E2B_INIT_BIN" /usr/sbin/init +# /sbin is a real directory on non-usr-merged distros (Alpine) where the line +# above doesn't reach the /sbin/init the kernel is pointed at; link it too. +[ -L /sbin ] || ln -sf "$E2B_INIT_BIN" /sbin/init echo "Unlocking immutable configuration" $BUSYBOX chattr -i /etc/resolv.conf diff --git a/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh b/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh index 76202a81a5..6971507a85 100644 --- a/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh +++ b/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh @@ -10,23 +10,54 @@ TEMPLATE_ID={{ .TemplateID }} BUILD_ID={{ .BuildID }} EOF -# Create default user. -# if the /home/user directory exists, we copy the skeleton files to it because the adduser command -# will ignore the directory if it exists, but we want to include the skeleton files in the home directory -# in our case. +# Create default user. useradd is part of shadow(-utils) and present on every +# supported distro family (Debian/Ubuntu, RHEL/Fedora, Arch, Alpine), unlike +# Debian's adduser wrapper (FEAT-145). -m creates the home dir; -s the shell. +# A creation failure is a real error — a template whose default user silently +# doesn't exist fails much more confusingly later. echo "Create default user 'user' (if doesn't exist yet)" -ADDUSER_OUTPUT=$(adduser -disabled-password --gecos "" user 2>&1 || true) -echo "$ADDUSER_OUTPUT" -if echo "$ADDUSER_OUTPUT" | grep -q "The home directory \`/home/user' already exists"; then - # Copy skeleton files if they don't exist in the home directory - echo "Copy skeleton files to /home/user" - cp -rn /etc/skel/. /home/user/ +if ! id -u user >/dev/null 2>&1; then + useradd -m -s /bin/bash user +fi +# useradd -m skips skeleton files when /home/user already exists, so copy them +# explicitly (no-clobber) to match the previous adduser behaviour. Not every +# image ships /etc/skel — say so instead of hiding it. Walked file-by-file +# because `cp -n` exits non-zero when it skips an existing file on +# coreutils >= 9.2 (Fedora 40+), which is a skip, not a failure. +if [ -d /home/user ]; then + if [ -d /etc/skel ]; then + echo "Copy skeleton files to /home/user (keeping existing files)" + (cd /etc/skel && find . -type f -print | while IFS= read -r f; do + if [ ! -e "/home/user/$f" ]; then + mkdir -p "/home/user/$(dirname "$f")" + cp -a "/etc/skel/$f" "/home/user/$f" + fi + done) + else + echo "No /etc/skel on this image; skipping skeleton copy" + fi fi echo "Add sudo to 'user' with no password" -usermod -aG sudo user +# Admin group differs by distro (sudo on Debian/Ubuntu, wheel elsewhere); the +# NOPASSWD sudoers entry below is what actually grants privileges. Neither +# group existing is a real error. +if getent group sudo >/dev/null; then + usermod -aG sudo user +elif getent group wheel >/dev/null; then + usermod -aG wheel user +else + echo "ERROR: neither the sudo nor the wheel group exists on this image" >&2 + exit 1 +fi passwd -d user -echo "user ALL=(ALL:ALL) NOPASSWD: ALL" >>/etc/sudoers +# NixOS generates /etc/sudoers read-only from its configuration — the premade +# image declares this exact line, so the append is correctly skipped there. +if grep -q '^user ALL=(ALL:ALL) NOPASSWD: ALL' /etc/sudoers; then + echo "sudoers entry already present" +else + echo "user ALL=(ALL:ALL) NOPASSWD: ALL" >>/etc/sudoers +fi echo "Give 'user' ownership to /home/user" mkdir -p /home/user