diff --git a/packages/envd/internal/services/process/handler/handler.go b/packages/envd/internal/services/process/handler/handler.go index d7122f99ab..8c135c333e 100644 --- a/packages/envd/internal/services/process/handler/handler.go +++ b/packages/envd/internal/services/process/handler/handler.go @@ -29,6 +29,8 @@ import ( const ( defaultNice = 0 defaultOomScore = 100 + defaultIoClass = 2 // ionice best-effort + defaultIoPrio = 4 outputBufferSize = 64 systemTag = "_system" stdChunkSize = 32 << 10 // 32 KiB @@ -161,6 +163,25 @@ func currentNice() int { return 20 - prio } +// ioniceNicePrefix builds the ionice/nice part of the process wrapper from +// whatever the image actually ships. Both are util-linux/coreutils +// conveniences that minimal and busybox-based images (Alpine, UBI) may lack or +// keep elsewhere than /usr/bin — a missing helper must degrade to running the +// command without that priority adjustment, never to a failed spawn (exit 127 +// killed every process on such images). lookPath is injected for testability; +// production passes exec.LookPath. +func ioniceNicePrefix(ioClass, ioPrio, niceDelta int, lookPath func(string) (string, error)) string { + prefix := "" + if p, err := lookPath("ionice"); err == nil { + prefix += fmt.Sprintf("%s -c %d -n %d ", p, ioClass, ioPrio) + } + if p, err := lookPath("nice"); err == nil { + prefix += fmt.Sprintf("%s -n %d ", p, niceDelta) + } + + return prefix +} + func New( ctx context.Context, user *user.User, @@ -173,9 +194,11 @@ func New( // User command string for logging (without the internal wrapper details). userCmd := strings.Join(append([]string{req.GetProcess().GetCmd()}, req.GetProcess().GetArgs()...), " ") - // Wrap in a shell that resets oom_score_adj, ioprio (ionice best-effort/4), and nice. + // Wrap in a shell that resets oom_score_adj, ioprio (ionice best-effort/4), + // and nice. The oom_score_adj write is pure /proc and always applied; the + // priority helpers are used only where the image provides them. niceDelta := defaultNice - currentNice() - oomWrapperScript := fmt.Sprintf(`echo %d > /proc/$$/oom_score_adj && exec /usr/bin/ionice -c 2 -n 4 /usr/bin/nice -n %d "${@}"`, defaultOomScore, niceDelta) + oomWrapperScript := fmt.Sprintf(`echo %d > /proc/$$/oom_score_adj && exec %s"${@}"`, defaultOomScore, ioniceNicePrefix(defaultIoClass, defaultIoPrio, niceDelta, exec.LookPath)) wrapperArgs := append([]string{"-c", oomWrapperScript, "--", req.GetProcess().GetCmd()}, req.GetProcess().GetArgs()...) cmd := exec.CommandContext(ctx, "/bin/sh", wrapperArgs...) diff --git a/packages/envd/internal/services/process/handler/handler_test.go b/packages/envd/internal/services/process/handler/handler_test.go new file mode 100644 index 0000000000..d11fc49bf1 --- /dev/null +++ b/packages/envd/internal/services/process/handler/handler_test.go @@ -0,0 +1,52 @@ +package handler + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" +) + +// The process wrapper must degrade cleanly when the priority helpers are absent +// — the user command still runs; a present helper is applied with its resolved +// path. +func TestWrapperPrefix(t *testing.T) { + t.Parallel() + + notFound := func(string) (string, error) { return "", errors.New("not found") } + all := func(name string) (string, error) { return "/usr/bin/" + name, nil } + only := func(want string) func(string) (string, error) { + return func(name string) (string, error) { + if name == want { + return "/bin/" + name, nil + } + + return "", errors.New("not found") + } + } + + t.Run("both present", func(t *testing.T) { + t.Parallel() + assert.Equal(t, "/usr/bin/ionice -c 2 -n 4 /usr/bin/nice -n 5 ", ioniceNicePrefix(2, 4, 5, all)) + }) + + t.Run("both absent degrades to bare exec", func(t *testing.T) { + t.Parallel() + assert.Empty(t, ioniceNicePrefix(2, 4, 5, notFound)) + }) + + t.Run("only nice", func(t *testing.T) { + t.Parallel() + assert.Equal(t, "/bin/nice -n -3 ", ioniceNicePrefix(2, 4, -3, only("nice"))) + }) + + t.Run("only ionice", func(t *testing.T) { + t.Parallel() + assert.Equal(t, "/bin/ionice -c 2 -n 4 ", ioniceNicePrefix(2, 4, 0, only("ionice"))) + }) + + t.Run("class and priority are caller-controlled", func(t *testing.T) { + t.Parallel() + assert.Equal(t, "/bin/ionice -c 1 -n 6 ", ioniceNicePrefix(1, 6, 0, only("ionice"))) + }) +} diff --git a/packages/orchestrator/README.md b/packages/orchestrator/README.md index 47bd5b72d6..490274e28a 100644 --- a/packages/orchestrator/README.md +++ b/packages/orchestrator/README.md @@ -303,4 +303,5 @@ Automatically set in local mode. Set before running to override: ## Limitations -- Custom template builds require Debian/Ubuntu-based base images (images that provide the `apt` package manager). Non-Debian images such as Alpine, CentOS/RHEL, or other distributions without `apt` are not supported and will fail during the template build/provisioning process. The provisioning scripts used during template build call `apt` and expect Debian-specific package names and file locations. \ No newline at end of file +- Custom template builds support base images from the declared distro families: **Debian/Ubuntu** (apt), the **RHEL family** — Fedora, CentOS Stream, Rocky, Alma — (dnf/microdnf/yum), **Arch** (pacman), and **Alpine** (apk, OpenRC). The distro is resolved from the image's `/etc/os-release` `ID` — never by probing for package managers. Images without an os-release identity (distroless, scratch) or from an undeclared family are rejected fast, with a build-log error naming the reason and the supported families (see `pkg/template/build/phases/base/distro`). Minimal/restricted-repo images may fail provisioning if their repos don't carry the required packages; the failure is surfaced in the build log. +- Sandboxes always boot the kernel E2B supplies, never one from the base image: `/lib/modules` is empty, no kernel modules can be loaded, and SELinux is off. `kernel-devel` from a distro's repos resolves against a kernel that isn't running. **RHEL** (`rhel`, incl. UBI), **Oracle Linux** (`ol`) and **Amazon Linux** (`amzn`) are therefore not accepted even though they are RPM/dnf images — they are chosen for kABI, signed kernel modules and UEK, which this cannot honour. The community rebuilds above are supported because they are chosen for the userland. \ No newline at end of file diff --git a/packages/orchestrator/pkg/template/build/commands/user.go b/packages/orchestrator/pkg/template/build/commands/user.go index 0b1f0cb94e..3afdae39d6 100644 --- a/packages/orchestrator/pkg/template/build/commands/user.go +++ b/packages/orchestrator/pkg/template/build/commands/user.go @@ -60,7 +60,9 @@ func (u *User) Execute( lvl, prefix, sandboxID, - fmt.Sprintf("adduser --disabled-password --gecos \"\" %s", userArg), + // useradd is in shadow(-utils) on every supported distro family; the + // created user has no password (locked). + fmt.Sprintf("useradd --create-home --shell /bin/bash %s", userArg), metadata.Context{ User: "root", EnvVars: cmdMetadata.EnvVars, @@ -99,7 +101,27 @@ func addToSudoers( lvl, prefix, sandboxID, - fmt.Sprintf("usermod -aG sudo %s", userArg), + // The admin group comes from the distro profile, persisted by + // provisioning; the NOPASSWD sudoers entry below is what actually + // grants privileges. Templates built before distro.env existed + // (FROM-template parents keep their rootfs) fall back to probing. + fmt.Sprintf(`if [ -f /usr/local/share/e2b/distro.env ]; then + . /usr/local/share/e2b/distro.env +fi +if [ -n "${E2B_ADMIN_GROUP:-}" ]; then + if ! getent group "$E2B_ADMIN_GROUP" >/dev/null; then + echo "the '$E2B_ADMIN_GROUP' group from the distro profile does not exist on this image" >&2 + exit 1 + fi + usermod -aG "$E2B_ADMIN_GROUP" %[1]s +elif getent group sudo >/dev/null; then + usermod -aG sudo %[1]s +elif getent group wheel >/dev/null; then + usermod -aG wheel %[1]s +else + echo "neither the sudo nor the wheel group exists on this image" >&2 + exit 1 +fi`, userArg), metadata.Context{ User: "root", EnvVars: cmdMetadata.EnvVars, diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.openrc.tpl b/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.openrc.tpl new file mode 100644 index 0000000000..a5f9bbf16a --- /dev/null +++ b/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.openrc.tpl @@ -0,0 +1,46 @@ +{{- /*gotype:github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs.templateModel*/ -}} +{{ .WriteFile "/usr/local/share/e2b/envd.openrc" 0o755 }} + +#!/sbin/openrc-run +# E2B env daemon — the OpenRC counterpart of envd.service (see envd.service.tpl +# for the full rationale on each step; this mirrors it for the Alpine/OpenRC +# family). Baked at a neutral path and installed to /etc/init.d/envd by the +# OpenRC e2b_init_setup only: if it lived in /etc/init.d on every image, +# Debian's `systemctl enable envd` would hand it to update-rc.d, which aborts +# on a non-LSB script and fails the whole provisioning. +# +# /tmp-wipe ordering note: OpenRC's bootmisc (boot runlevel) wipes /tmp before +# the default runlevel starts, so envd — in default — can never answer an +# update-envd upload before the wipe. The race envd.service needs an explicit +# After=systemd-tmpfiles-setup.service for cannot happen here. + +description="E2B env daemon" + +supervisor=supervise-daemon +command=/usr/bin/envd +# --nicelevel/--ionice/--oom-score-adj mirror envd.service's Nice=-20, +# IOSchedulingClass=realtime + IOSchedulingPriority=4 (ionice class 1, data 4) +# and OOMScoreAdjust=-1000 — without them envd runs at default priority and is +# the first OOM-kill candidate on Alpine. (The unit's cgroup weights — +# MemoryMin/CPUWeight/IOWeight — have no supervise-daemon equivalent.) +supervise_daemon_args="--nicelevel -20 --ionice 1:4 --oom-score-adj -1000 --env GOTRACEBACK=all --env GOMEMLIMIT={{ .MemoryLimit }}MiB --stdout /var/log/envd.log --stderr /var/log/envd.log" +# Retry forever (envd.service uses Restart=always + StartLimitIntervalSec=0). +respawn_delay=1 +respawn_max=0 + +depend() { + need localmount + after bootmisc + use net +} + +start_pre() { + # Shared with envd.service's ExecStartPre; warns-and-continues by design. + /usr/local/bin/e2b-seed-certs + # systemd-tmpfiles applies the fuse.conf tmpfiles.d rule on the systemd + # family; OpenRC has no tmpfiles pass, so set the mode here. + if [ -e /dev/fuse ]; then + chmod 666 /dev/fuse + fi + return 0 +} diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.service.tpl b/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.service.tpl index b66b7c7166..36e67e7a4b 100644 --- a/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.service.tpl +++ b/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.service.tpl @@ -46,7 +46,9 @@ LimitCORE=infinity # guaranteed present for the sandbox's routable lifetime. The only gap is guest # units that auto-start and egress over TLS before /init; that is accepted # (revisit if a template needs boot-time egress). -ExecStartPre=/bin/sh -c 'mountpoint -q /etc/ssl/certs || { mkdir -p /run/e2b/certs && { tar -C /run/e2b/certs -xf /usr/local/share/e2b/ssl-certs.tar 2>/dev/null || cp -a /etc/ssl/certs/. /run/e2b/certs/ 2>/dev/null; }; mount --bind /run/e2b/certs /etc/ssl/certs; } && ([ -s /etc/ssl/certs/ca-certificates.crt ] || update-ca-certificates)' +# The seeding logic is shared with the OpenRC service (Alpine) and never fails +# the unit — a degraded trust store is recoverable, a dead envd is not. +ExecStartPre=/usr/local/bin/e2b-seed-certs ExecStart=/usr/bin/envd Nice=-20 IOSchedulingClass=realtime diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/files/inittab.tpl b/packages/orchestrator/pkg/template/build/core/rootfs/files/inittab.tpl index 88d981c2ed..9194829518 100644 --- a/packages/orchestrator/pkg/template/build/core/rootfs/files/inittab.tpl +++ b/packages/orchestrator/pkg/template/build/core/rootfs/files/inittab.tpl @@ -1,18 +1,16 @@ {{- /*gotype:github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs.templateModel*/ -}} {{ .WriteFile "/etc/inittab" 0o777 }} -# Run system init -::sysinit:/etc/init.d/rcS - -# Run the provision script, prefix the output with a log prefix -::wait:/bin/sh -c '/usr/local/bin/provision.sh 2>&1 | sed "s/^/{{ .ProvisionLogPrefix }}/"' +# Provisioning-boot inittab (busybox init). Every entry is a plain exec with +# NO shell metacharacters: busybox init hands metachar lines to /bin/sh, and +# minimal images (distroless) may have no /bin/sh — the pipeline logic lives in +# e2b-provision-runner instead. -# Flush filesystem changes to disk -::wait:/usr/bin/busybox sync -::wait:fsfreeze --freeze / +# Run system init (mounts /proc /sys /dev /tmp /run through the baked busybox) +::sysinit:/etc/init.d/rcS -# Report the exit code of the provisioning script -::wait:/bin/sh -c 'echo "{{ .ProvisionExitPrefix }}$(cat {{ .ProvisionResultPath }} || printf 1)"' +# Run the provisioning pipeline and report its exit code +::wait:/usr/bin/busybox ash /usr/local/bin/e2b-provision-runner # Wait forever to prevent the VM from exiting until the sandbox is paused and snapshot is taken -::wait:/usr/bin/busybox sleep infinity \ No newline at end of file +::wait:/usr/bin/busybox sleep infinity diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/files/provision-runner.sh.tpl b/packages/orchestrator/pkg/template/build/core/rootfs/files/provision-runner.sh.tpl new file mode 100644 index 0000000000..22135a9a63 --- /dev/null +++ b/packages/orchestrator/pkg/template/build/core/rootfs/files/provision-runner.sh.tpl @@ -0,0 +1,34 @@ +{{- /*gotype:github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs.templateModel*/ -}} +{{ .WriteFile "usr/local/bin/e2b-provision-runner" 0o755 }} + +#!/usr/bin/busybox ash +# Drives the provisioning pipeline for the busybox-init boot. This logic lives +# in a script — NOT in /etc/inittab — because busybox init hands any inittab +# line containing shell metacharacters to /bin/sh, and minimal images +# (distroless) may have no /bin/sh; a plain-exec inittab line running this +# script through the baked busybox works on every image. +BB=/usr/bin/busybox + +# Run the provision script, prefix its output with the log prefix the +# orchestrator forwards to the customer's build logs. +$BB sh /usr/local/bin/provision.sh 2>&1 | $BB sed "s/^/{{ .ProvisionLogPrefix }}/" + +# Flush filesystem changes to disk before the snapshot. +$BB sync +if command -v fsfreeze >/dev/null 2>&1; then + fsfreeze --freeze / +else + # No util-linux on this image; the double sync flushes the ext4 journal + # and the VM is paused before the snapshot is taken. Prefixed so the + # fallback shows up in the customer's build logs. + echo "{{ .ProvisionLogPrefix }}fsfreeze not available on this image; using sync-only flush" + $BB sync +fi + +# Report the provisioning exit code: provision.sh writes "0" on success and +# (running under set -e) leaves no file behind on failure. +if result=$($BB cat {{ .ProvisionResultPath }} 2>/dev/null); then + echo "{{ .ProvisionExitPrefix }}${result}" +else + echo "{{ .ProvisionExitPrefix }}1" +fi diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/files/rcS.sh.tpl b/packages/orchestrator/pkg/template/build/core/rootfs/files/rcS.sh.tpl index 08bbb99e93..3a3c3f29ab 100644 --- a/packages/orchestrator/pkg/template/build/core/rootfs/files/rcS.sh.tpl +++ b/packages/orchestrator/pkg/template/build/core/rootfs/files/rcS.sh.tpl @@ -2,15 +2,20 @@ {{ .WriteFile "etc/init.d/rcS" 0o777 }} #!/usr/bin/busybox ash +# Every command goes through the baked busybox: this runs before provisioning +# on the raw base image, and minimal images (distroless) may have no mkdir/mount +# on PATH at all. +BB=/usr/bin/busybox + echo "Mounting essential filesystems" # Ensure necessary mount points exist -mkdir -p /proc /sys /dev /tmp /run +$BB mkdir -p /proc /sys /dev /tmp /run # Mount essential filesystems -mount -t proc proc /proc -mount -t sysfs sysfs /sys -mount -t devtmpfs devtmpfs /dev -mount -t tmpfs tmpfs /tmp -mount -t tmpfs tmpfs /run +$BB mount -t proc proc /proc +$BB mount -t sysfs sysfs /sys +$BB mount -t devtmpfs devtmpfs /dev +$BB mount -t tmpfs tmpfs /tmp +$BB mount -t tmpfs tmpfs /run -echo "System Init" \ No newline at end of file +echo "System Init" diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/files/seed-certs.sh.tpl b/packages/orchestrator/pkg/template/build/core/rootfs/files/seed-certs.sh.tpl new file mode 100644 index 0000000000..5ae34599d3 --- /dev/null +++ b/packages/orchestrator/pkg/template/build/core/rootfs/files/seed-certs.sh.tpl @@ -0,0 +1,53 @@ +{{- /*gotype:github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs.templateModel*/ -}} +{{ .WriteFile "usr/local/bin/e2b-seed-certs" 0o755 }} + +#!/bin/sh +# Seeds the tmpfs-backed /etc/ssl/certs before envd starts — shared by +# envd.service (systemd) and /etc/init.d/envd (OpenRC). See envd.service.tpl +# for the full rationale (why a tar, why a bind mount, the egress-CA contract). +# +# Every failure path here WARNS and continues deliberately: a sandbox with a +# degraded trust store is recoverable (envd's POST /init reinstalls the egress +# CA), a sandbox whose envd never starts is not. + +# The copies DEREFERENCE symlinks (-L; the build-time tar already packs with +# -h): some images ship the /etc/ssl/certs bundle as symlinks into a read-only +# store, and envd's egress-proxy CA install APPENDS to +# /etc/ssl/certs/ca-certificates.crt — that only works if the tmpfs copy is a +# real file, not a copied symlink to an immutable target. +if ! mountpoint -q /etc/ssl/certs; then + mkdir -p /run/e2b/certs + if [ -f /usr/local/share/e2b/ssl-certs.tar ]; then + if ! tar -C /run/e2b/certs -xf /usr/local/share/e2b/ssl-certs.tar; then + echo "e2b-seed-certs: ssl-certs.tar extraction failed; seeding from the live cert dir instead" >&2 + cp -aL /etc/ssl/certs/. /run/e2b/certs/ + fi + else + # Only expected during the base-layer boot, before finalize packs the tar. + echo "e2b-seed-certs: ssl-certs.tar not packed yet; seeding from the live cert dir" + cp -aL /etc/ssl/certs/. /run/e2b/certs/ + fi + if ! mount -o bind /run/e2b/certs /etc/ssl/certs; then + echo "e2b-seed-certs: bind mount failed; envd runs with the image's certs as-is" >&2 + fi +fi + +if [ ! -s /etc/ssl/certs/ca-certificates.crt ]; then + if command -v update-ca-certificates >/dev/null 2>&1; then + update-ca-certificates + elif command -v update-ca-trust >/dev/null 2>&1; then + # RHEL family and Arch refresh with update-ca-trust. Arch's extract emits + # the Debian-named bundle itself; RHEL keeps it under /etc/pki, and the + # symlink provisioning made was dereferenced into this tmpfs, so copy it. + update-ca-trust extract + if [ ! -s /etc/ssl/certs/ca-certificates.crt ] && [ -s /etc/pki/tls/certs/ca-bundle.crt ]; then + cp -L /etc/pki/tls/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt + fi + else + # Provisioning guarantees the bundle on every supported family; + # reaching this means the image diverged after the build. + echo "e2b-seed-certs: CA bundle missing and no CA refresh tool on this image; TLS trust will be degraded" >&2 + fi +fi + +exit 0 diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/rootfs.go b/packages/orchestrator/pkg/template/build/core/rootfs/rootfs.go index ca859899f0..3cc6d43d75 100644 --- a/packages/orchestrator/pkg/template/build/core/rootfs/rootfs.go +++ b/packages/orchestrator/pkg/template/build/core/rootfs/rootfs.go @@ -14,6 +14,7 @@ import ( "os" "path/filepath" "runtime" + "slices" "text/template" "github.com/dustin/go-humanize" @@ -41,9 +42,25 @@ var tracer = otel.Tracer("github.com/e2b-dev/infra/packages/orchestrator/pkg/tem var files embed.FS var fileTemplates = template.Must(template.ParseFS(files, "files/*")) -// filesHash is a stable hash of the embedded rootfs file templates. It is used -// only as part of the fallback provision version; explicit provision versions -// remain the rollout control. +// enableSymlinks is the content of the baked symlink layer. Package-level so +// it feeds filesHash: a change here must rotate the fallback provision +// version like any other baked-layer change. +var enableSymlinks = map[string]string{ + // Enable envd service autostart. The target MUST be absolute: the link + // lives in multi-user.target.wants/, so a relative target would resolve + // inside that directory and dangle — and provision.sh's offline + // `systemctl enable $E2B_TIMESYNC_UNIT` prunes dangling .wants symlinks, + // silently disabling envd on distros where the link dangles. + "etc/systemd/system/multi-user.target.wants/envd.service": "/etc/systemd/system/envd.service", + // NOTE: chrony autostart is enabled by provision.sh via `systemctl enable + // $E2B_TIMESYNC_UNIT`, which picks the distro-correct unit name (chrony on + // Debian, chronyd on RHEL/Arch). A static chrony.service symlink here would + // dangle on non-Debian images where the unit is chronyd.service. +} + +// filesHash is a stable hash of the embedded rootfs file templates plus the +// baked symlink layer. It is used only as part of the fallback provision +// version; explicit provision versions remain the rollout control. var filesHash = func() string { entries, _ := fs.ReadDir(files, "files") h := sha256.New() @@ -51,6 +68,14 @@ var filesHash = func() string { data, _ := files.ReadFile("files/" + e.Name()) fmt.Fprintf(h, "%s\x00%x\x00", e.Name(), data) } + links := make([]string, 0, len(enableSymlinks)) + for name := range enableSymlinks { + links = append(links, name) + } + slices.Sort(links) + for _, name := range links { + fmt.Fprintf(h, "%s\x00%s\x00", name, enableSymlinks[name]) + } return hex.EncodeToString(h.Sum(nil)) }() @@ -233,6 +258,15 @@ func additionalOCILayers( filesMap := map[string]oci.File{ storage.GuestEnvdPath: {Bytes: envdFileData, Mode: 0o777}, + // Systemd preset policy for envd. provision.sh removes /etc/machine-id, + // so the template's next boot is a systemd FIRST boot — and on first + // boot PID1 applies the distro preset policy to all units. On the + // RHEL family that policy ends with "disable *" (and the systemd RPM + // scriptlet's preset-all does the same during provisioning), which + // deletes envd's autostart symlink no matter how it was created. + // A 00- preset sorts before every distro policy file and wins. + "etc/systemd/system-preset/00-e2b.preset": {Bytes: []byte("enable envd.service\n"), Mode: 0o644}, + // Provision script "usr/local/bin/provision.sh": {Bytes: []byte(provisionScript), Mode: 0o777}, // Setup init system @@ -263,14 +297,7 @@ func additionalOCILayers( return nil, fmt.Errorf("error creating layer from files: %w", err) } - symlinkLayer, err := oci.LayerSymlink( - map[string]string{ - // Enable envd service autostart - "etc/systemd/system/multi-user.target.wants/envd.service": "etc/systemd/system/envd.service", - // Enable chrony service autostart - "etc/systemd/system/multi-user.target.wants/chrony.service": "etc/systemd/system/chrony.service", - }, - ) + symlinkLayer, err := oci.LayerSymlink(enableSymlinks) if err != nil { return nil, fmt.Errorf("error creating layer from symlinks: %w", err) } diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/rootfs_test.go b/packages/orchestrator/pkg/template/build/core/rootfs/rootfs_test.go index 963368d997..747fcde460 100644 --- a/packages/orchestrator/pkg/template/build/core/rootfs/rootfs_test.go +++ b/packages/orchestrator/pkg/template/build/core/rootfs/rootfs_test.go @@ -90,7 +90,45 @@ func TestAdditionalOCILayers(t *testing.T) { keysIter := maps.Keys(actualFiles) keys := slices.Collect(keysIter) - assert.Len(t, keys, 14) + assert.Len(t, keys, 18) + + // The provisioning boot must be self-contained on the baked busybox: + // minimal images (distroless) may have no /bin/sh, and busybox init + // hands any inittab line with shell metacharacters to /bin/sh — so the + // pipeline lives in the runner script and every inittab entry is a + // plain exec. + inittab := actualFiles["etc/inittab"] + require.NotEmpty(t, inittab) + for line := range strings.SplitSeq(inittab, "\n") { + if !strings.HasPrefix(line, "::") { + continue + } + assert.NotContains(t, line, "|", "inittab entries must not need /bin/sh: %s", line) + assert.NotContains(t, line, "$", "inittab entries must not need /bin/sh: %s", line) + } + runner := actualFiles["usr/local/bin/e2b-provision-runner"] + require.NotEmpty(t, runner, "provision runner must be baked") + assert.Contains(t, runner, "#!/usr/bin/busybox ash") + + // Both init families' envd services seed certs via the shared script. + seedCerts := actualFiles["usr/local/bin/e2b-seed-certs"] + require.NotEmpty(t, seedCerts, "cert seeding script must be baked") + assert.Contains(t, actualFiles["etc/systemd/system/envd.service"], "ExecStartPre=/usr/local/bin/e2b-seed-certs") + + // envd must be preset-enabled: first boot (machine-id is removed by + // provisioning) applies the distro preset policy, and the RHEL + // family's "disable *" would otherwise delete envd's autostart link. + assert.Equal(t, "enable envd.service\n", actualFiles["etc/systemd/system-preset/00-e2b.preset"]) + + // The OpenRC counterpart (Alpine) ships alongside the systemd unit; it + // must supervise envd and honor the memory limit. + // It lives OUTSIDE /etc/init.d — Debian's update-rc.d aborts on a + // non-LSB script there — and is installed by the OpenRC init setup. + openrcEnvd := actualFiles["usr/local/share/e2b/envd.openrc"] + require.NotEmpty(t, openrcEnvd, "OpenRC envd service must be baked") + assert.Contains(t, openrcEnvd, "#!/sbin/openrc-run") + assert.Contains(t, openrcEnvd, "supervisor=supervise-daemon") + assert.Contains(t, openrcEnvd, "GOMEMLIMIT=50MiB") assert.Equal(t, "e2b.local", actualFiles["etc/hostname"]) assert.Equal(t, "nameserver 8.8.8.8", actualFiles["etc/resolv.conf"]) @@ -124,5 +162,36 @@ func TestAdditionalOCILayers(t *testing.T) { WatchdogSec=0`) assert.Equal(t, disabledContent, actualFiles["etc/systemd/system/systemd-journald.service.d/override.conf"]) assert.Equal(t, disabledContent, actualFiles["etc/systemd/system/systemd-networkd.service.d/override.conf"]) + + // Regression guard: the envd autostart symlink must not dangle. + // A relative target resolves inside multi-user.target.wants/ and dangles, + // and provision.sh's offline `systemctl enable` prunes dangling .wants + // links — silently disabling envd autostart on e.g. Fedora. + symlinksLayer, err := layers[1].Uncompressed() + require.NoError(t, err) + t.Cleanup(func() { + err = symlinksLayer.Close() + assert.NoError(t, err) + }) + + actualSymlinks := map[string]string{} + symlinksTarReader := tar.NewReader(symlinksLayer) + for { + header, err := symlinksTarReader.Next() + if errors.Is(err, io.EOF) { + break + } + require.NoError(t, err) + + if header.Typeflag != tar.TypeSymlink { + continue + } + actualSymlinks[header.Name] = header.Linkname + } + + envdWants := actualSymlinks["etc/systemd/system/multi-user.target.wants/envd.service"] + require.NotEmpty(t, envdWants, "envd autostart symlink must be present") + assert.Equal(t, "/etc/systemd/system/envd.service", envdWants, + "envd autostart symlink target must be absolute so it never dangles") }) } diff --git a/packages/orchestrator/pkg/template/build/layer/layer_executor.go b/packages/orchestrator/pkg/template/build/layer/layer_executor.go index 5a738b0322..2784811bd1 100644 --- a/packages/orchestrator/pkg/template/build/layer/layer_executor.go +++ b/packages/orchestrator/pkg/template/build/layer/layer_executor.go @@ -212,13 +212,24 @@ func (lb *LayerExecutor) updateEnvdInSandbox( return fmt.Errorf("failed to replace envd binary: %w", err) } - // Step 3: Restart the systemd envd service - // Error is ignored because it's expected the envd connection will be lost + // Step 3: Restart the envd service. The error is ignored because the + // restart kills the very envd this command runs through — the connection + // loss is expected. systemctl hands the restart to PID1, which survives + // that. On OpenRC there is no PID1 handoff: rc-service would run in this + // very shell and die with envd between its stop and start halves + // (observed on Alpine — envd never came back), so instead envd is simply + // killed and supervise-daemon's respawn starts the replaced binary; the + // immediate death also means the post-update wait below can never mistake + // the old instance for the new one. _ = sandboxtools.RunCommand( ctx, lb.proxy, sbx.Runtime.SandboxID, - "systemctl restart envd", + `if command -v systemctl >/dev/null 2>&1; then + systemctl restart envd +else + kill -TERM "$(pidof envd)" +fi`, metadata.Context{User: "root"}, ) diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go b/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go new file mode 100644 index 0000000000..8bac7f4480 --- /dev/null +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go @@ -0,0 +1,184 @@ +// Package distro makes template-build provisioning distro-aware: it selects a +// declared per-family Profile by the base image's /etc/os-release ID rather than +// probing for a package manager. Supported: the systemd family (Debian/Ubuntu, +// Fedora/RHEL/CentOS/Rocky/Alma, Arch) and Alpine on OpenRC; anything else is +// rejected with a clear error. +package distro + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" +) + +// Version forces a base-layer rebuild for provisioning changes the generated +// selector text can't otherwise capture; bump it when the contract changes. +const Version = "1" + +// Fingerprint hashes the whole generated provisioning contract into the +// base-layer cache key, so any profile or init-setup change rebuilds the base. +func Fingerprint() string { + sum := sha256.Sum256([]byte(Version + "\x00" + ShellSelector())) + + return hex.EncodeToString(sum[:]) +} + +// Profile is the declared, per-family provisioning contract. IDs are the +// /etc/os-release values that map to the family; PkgQueryBody, PkgInstall and +// CARefresh are shell fragments spliced into the generated selector. +type Profile struct { + Key string + Init InitSystem + IDs []string + Packages []string + PkgQueryBody string + PkgInstall string + InitBinary string + TimeSyncUnit string + SSHUnit string + AdminGroup string + CABundle string + CARefresh string +} + +var Profiles = []Profile{ + { + Key: "debian", + Init: InitSystemd, + IDs: []string{"debian", "ubuntu"}, + Packages: []string{ + "systemd", "systemd-sysv", "openssh-server", "sudo", "chrony", "socat", + "curl", "ca-certificates", "fuse3", "iptables", "git", "nfs-common", + "less", "nftables", "iputils-ping", "jq", + }, + PkgQueryBody: `dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -q "install ok installed"`, + PkgInstall: `apt-get -q update && DEBIAN_FRONTEND=noninteractive DEBCONF_NOWARNINGS=yes apt-get -qq -o=Dpkg::Use-Pty=0 install -y --no-install-recommends "$@"`, + InitBinary: "/lib/systemd/systemd", + TimeSyncUnit: "chrony", + SSHUnit: "ssh", + AdminGroup: "sudo", + CABundle: "/etc/ssl/certs/ca-certificates.crt", + CARefresh: "update-ca-certificates", + }, + { + Key: "rhel", // Fedora, CentOS Stream, Rocky, Alma + Init: InitSystemd, + // Deliberately not rhel/ol/amzn. Sandboxes always boot E2B's kernel, so + // /lib/modules is empty and SELinux is off — which is most of what RHEL + // and Oracle Linux are chosen for (kABI, signed kmods, UEK), and + // kernel-devel would resolve against a kernel that isn't running. RHEL's + // UBI repos also carry neither chrony nor nfs-utils. Accepting those IDs + // promised a fidelity this can't deliver; they now fail fast instead. + IDs: []string{"fedora", "centos", "rocky", "almalinux"}, + // "iptables" (not iptables-nft) and "tar" cover yum-era CentOS 7, which + // lacks the nft package and doesn't ship tar. + Packages: []string{ + "systemd", "shadow-utils", "passwd", "openssh-server", "sudo", "chrony", + "socat", "curl", "ca-certificates", "fuse3", "iptables", "git", + "nfs-utils", "less", "nftables", "iputils", "jq", "bash", "tar", + }, + PkgQueryBody: `rpm -q "$1" >/dev/null 2>&1`, + // dnf → microdnf → yum spans the family (yum for CentOS 7); errors reach + // the build log. --allowerasing goes AFTER the subcommand: dnf5 (Fedora + // 41+, where /usr/bin/dnf IS dnf5) rejects it before one with "Unknown + // argument" and exits 2, which put two errors in every modern Fedora + // build log and quietly demoted the install to the flagless + // microdnf/yum fallback, so the flag never applied. dnf4 accepts either + // position. + PkgInstall: `dnf -y install --allowerasing "$@" || microdnf -y install "$@" || yum -y install "$@"`, + InitBinary: "/usr/lib/systemd/systemd", + TimeSyncUnit: "chronyd", + SSHUnit: "sshd", + AdminGroup: "wheel", + CABundle: "/etc/ssl/certs/ca-certificates.crt", + // update-ca-trust never writes ca-certificates.crt (a Debian name), so + // link the bundle envd's ExecStartPre expects to the extracted PEM. + CARefresh: `update-ca-trust extract && ln -sf /etc/pki/tls/certs/ca-bundle.crt "$E2B_CA_BUNDLE"`, + }, + { + Key: "arch", + Init: InitSystemd, + IDs: []string{"arch", "archarm"}, + Packages: []string{ + "systemd", "shadow", "openssh", "sudo", "chrony", "socat", "curl", + "ca-certificates", "fuse3", "iptables", "git", "nfs-utils", "less", + "nftables", "iputils", "jq", "bash", + }, + PkgQueryBody: `pacman -Q "$1" >/dev/null 2>&1`, + // -Syu (not -Sy then -S): Arch documents partial upgrades as unsupported. + PkgInstall: `pacman -Syu --noconfirm --needed "$@"`, + InitBinary: "/usr/lib/systemd/systemd", + TimeSyncUnit: "chronyd", + SSHUnit: "sshd", + AdminGroup: "wheel", + CABundle: "/etc/ssl/certs/ca-certificates.crt", + // Arch ships /etc/ssl/certs/ca-certificates.crt as a symlink to the p11-kit + // bundle update-ca-trust extract regenerates — no manual link, unlike RHEL. + CARefresh: "update-ca-trust extract", + }, + { + Key: "alpine", + Init: InitOpenRC, + IDs: []string{"alpine"}, + // shadow provides useradd/usermod (busybox's adduser takes different flags). + // util-linux-misc provides ionice, which busybox does not ship: envd runs at + // realtime IO (supervise-daemon --ionice 1:4) and resets each user process to + // best-effort with it, so without the binary they'd inherit realtime IO. + Packages: []string{ + "openrc", "shadow", "openssh", "sudo", "chrony", "socat", "curl", + "ca-certificates", "fuse3", "iptables", "git", "nfs-utils", "less", + "nftables", "iputils", "jq", "bash", "util-linux-misc", + }, + PkgQueryBody: `apk info -e "$1" >/dev/null 2>&1`, + PkgInstall: `apk add --no-cache "$@"`, + InitBinary: "/bin/busybox", + TimeSyncUnit: "chronyd", + SSHUnit: "sshd", + AdminGroup: "wheel", + CABundle: "/etc/ssl/certs/ca-certificates.crt", + CARefresh: "update-ca-certificates", + }, +} + +// SupportedIDs returns every os-release ID the selector accepts. +func SupportedIDs() []string { + var ids []string + for _, p := range Profiles { + ids = append(ids, p.IDs...) + } + + return ids +} + +// ShellSelector generates the POSIX-sh block provision.sh sources: it switches +// on the guest's $E2B_DISTRO_ID and defines the profile's packages, shell +// functions, init path, time-sync unit, admin group and CA handling. An +// unrecognized id exits 1 with a customer-visible error. +func ShellSelector() string { + var b strings.Builder + b.WriteString(`case "$E2B_DISTRO_ID" in` + "\n") + for _, p := range Profiles { + fmt.Fprintf(&b, " %s)\n", strings.Join(p.IDs, "|")) + fmt.Fprintf(&b, " E2B_PACKAGES=%q\n", strings.Join(p.Packages, " ")) + fmt.Fprintf(&b, " e2b_pkg_query() { %s; }\n", p.PkgQueryBody) + fmt.Fprintf(&b, " e2b_pkg_install() { %s; }\n", p.PkgInstall) + fmt.Fprintf(&b, " E2B_INIT_BIN=%q\n", p.InitBinary) + fmt.Fprintf(&b, " E2B_TIMESYNC_UNIT=%q\n", p.TimeSyncUnit) + fmt.Fprintf(&b, " E2B_SSH_UNIT=%q\n", p.SSHUnit) + fmt.Fprintf(&b, " E2B_ADMIN_GROUP=%q\n", p.AdminGroup) + fmt.Fprintf(&b, " E2B_CA_BUNDLE=%q\n", p.CABundle) + fmt.Fprintf(&b, " e2b_ca_refresh() { %s; }\n", p.CARefresh) + fmt.Fprintf(&b, " E2B_INIT_SYSTEM=%q\n", p.Init) + fmt.Fprintf(&b, " e2b_init_setup() {\n%s\n }\n", indentBlock(initSetup[p.Init], " ")) + fmt.Fprintf(&b, " ;;\n") + } + fmt.Fprintf(&b, " *)\n") + fmt.Fprintf(&b, " echo \"[provision] ERROR: unsupported base image distribution: ID='${E2B_DISTRO_ID:-unknown}'.\" >&2\n") + fmt.Fprintf(&b, " echo \"[provision] E2B template builds support: %s.\" >&2\n", strings.Join(SupportedIDs(), ", ")) + fmt.Fprintf(&b, " exit 1\n") + fmt.Fprintf(&b, " ;;\n") + b.WriteString("esac\n") + + return b.String() +} diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/distro_test.go b/packages/orchestrator/pkg/template/build/phases/base/distro/distro_test.go new file mode 100644 index 0000000000..7f7096cb71 --- /dev/null +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/distro_test.go @@ -0,0 +1,204 @@ +package distro + +import ( + "crypto/sha256" + "encoding/hex" + "strings" + "testing" +) + +// Golden lines lifted VERBATIM from the pre-change provision.sh so the debian +// profile reproduces them and Debian/Ubuntu behaviour is preserved. +const ( + goldenDebianPackages = "systemd systemd-sysv openssh-server sudo chrony socat curl ca-certificates fuse3 iptables git nfs-common less nftables iputils-ping jq" + goldenDebianQuery = `dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -q "install ok installed"` + goldenDebianInit = "/lib/systemd/systemd" +) + +func profileByKey(t *testing.T, key string) Profile { + t.Helper() + for _, p := range Profiles { + if p.Key == key { + return p + } + } + t.Fatalf("no profile with key %q", key) + + return Profile{} +} + +// The debian profile preserves today's Debian package set / query / init path. +func TestDebianPreserved(t *testing.T) { + t.Parallel() + p := profileByKey(t, "debian") + if got := strings.Join(p.Packages, " "); got != goldenDebianPackages { + t.Errorf("debian packages drifted:\n got: %s\nwant: %s", got, goldenDebianPackages) + } + if p.PkgQueryBody != goldenDebianQuery { + t.Errorf("debian query drifted:\n got: %s\nwant: %s", p.PkgQueryBody, goldenDebianQuery) + } + if p.InitBinary != goldenDebianInit { + t.Errorf("debian init drifted: got %s want %s", p.InitBinary, goldenDebianInit) + } + if p.TimeSyncUnit != "chrony" || p.AdminGroup != "sudo" { + t.Errorf("debian unit/group drifted: %s / %s", p.TimeSyncUnit, p.AdminGroup) + } +} + +// The families genuinely diverge on the axes that matter. +func TestFamiliesDiffer(t *testing.T) { + t.Parallel() + rhel := profileByKey(t, "rhel") + if rhel.TimeSyncUnit != "chronyd" || rhel.AdminGroup != "wheel" { + t.Errorf("rhel unit/group wrong: %s / %s", rhel.TimeSyncUnit, rhel.AdminGroup) + } + // Must both regenerate the trust store AND materialize the bundle at the + // Debian-named path envd expects — update-ca-trust alone never creates + // ca-certificates.crt, which left envd.service unable to start on Fedora. + if !strings.Contains(rhel.CARefresh, "update-ca-trust extract") || + !strings.Contains(rhel.CARefresh, `ln -sf /etc/pki/tls/certs/ca-bundle.crt "$E2B_CA_BUNDLE"`) { + t.Errorf("rhel CA refresh wrong: %s", rhel.CARefresh) + } + if rhel.InitBinary != "/usr/lib/systemd/systemd" { + t.Errorf("rhel init path wrong: %s", rhel.InitBinary) + } + arch := profileByKey(t, "arch") + if !strings.Contains(arch.PkgInstall, "pacman") { + t.Errorf("arch install should use pacman: %s", arch.PkgInstall) + } +} + +// dnf5 (Fedora 41+) only accepts --allowerasing after the subcommand and exits 2 +// with "Unknown argument" when it comes first, which silently demoted every +// modern Fedora build to the flagless microdnf/yum fallback. +func TestRhelAllowErasingFollowsSubcommand(t *testing.T) { + t.Parallel() + rhel := profileByKey(t, "rhel") + if !strings.Contains(rhel.PkgInstall, "install --allowerasing") { + t.Errorf("rhel install must place --allowerasing after the subcommand: %s", rhel.PkgInstall) + } + if strings.Contains(rhel.PkgInstall, "--allowerasing install") { + t.Errorf("rhel install has --allowerasing before the subcommand, which dnf5 rejects: %s", rhel.PkgInstall) + } +} + +// The generated selector keys on the DECLARED distro id, never on which +// package-manager binary happens to exist. +func TestSelectorNoPackageManagerProbing(t *testing.T) { + t.Parallel() + sel := ShellSelector() + for _, bad := range []string{ + "command -v apt-get", "command -v dnf", "command -v yum", + "command -v microdnf", "command -v pacman", "PKG_FAMILY", + } { + if strings.Contains(sel, bad) { + t.Errorf("selector leaked package-manager probing: %q", bad) + } + } + if !strings.Contains(sel, `case "$E2B_DISTRO_ID" in`) { + t.Error("selector must switch on $E2B_DISTRO_ID (declared distro identity)") + } +} + +// Every supported id gets a case arm; an unknown id hits the failing default. +func TestSelectorCoversIDsAndRejects(t *testing.T) { + t.Parallel() + sel := ShellSelector() + for _, id := range SupportedIDs() { + if !strings.Contains(sel, id) { + t.Errorf("selector missing arm for supported id %q", id) + } + } + for _, want := range []string{"*)", "unsupported base image", "exit 1"} { + if !strings.Contains(sel, want) { + t.Errorf("selector missing fast-reject piece %q", want) + } + } + // Alpine is supported via the OpenRC track — and it must be the OpenRC + // profile, never folded into a systemd family. + alpine := profileByKey(t, "alpine") + if alpine.Init != InitOpenRC { + t.Errorf("alpine must be the OpenRC profile, got init %q", alpine.Init) + } +} + +// Every profile declares a known init system with a rendered setup body, and +// no body leaks another init system's tooling (systemctl in OpenRC or +// rc-update in systemd would fail at provisioning time). +func TestInitSystemsDeclaredAndCoherent(t *testing.T) { + t.Parallel() + for _, p := range Profiles { + setup, ok := initSetup[p.Init] + if !ok { + t.Errorf("profile %q declares init %q with no setup body", p.Key, p.Init) + + continue + } + switch p.Init { + case InitSystemd: + if strings.Contains(setup, "rc-update") { + t.Errorf("systemd init setup leaks rc-update (profile %q)", p.Key) + } + case InitOpenRC: + if strings.Contains(setup, "systemctl") { + t.Errorf("openrc init setup leaks systemctl (profile %q)", p.Key) + } + } + } + sel := ShellSelector() + if !strings.Contains(sel, "e2b_init_setup() {") { + t.Error("selector must define e2b_init_setup()") + } + // The OpenRC boot chain pieces the alpine arm must carry. + for _, want := range []string{"/etc/inittab", "rc-update add envd default", "openrc sysinit"} { + if !strings.Contains(sel, want) { + t.Errorf("selector missing OpenRC boot piece %q", want) + } + } +} + +// The cache fingerprint must cover the whole generated provisioning contract: +// stable across calls, and carrying both the selector text and the explicit +// Version (a profile change must rotate the base-layer cache key). +func TestFingerprintStableAndVersioned(t *testing.T) { + t.Parallel() + a, b := Fingerprint(), Fingerprint() + if a != b || len(a) != 64 { + t.Errorf("fingerprint must be a stable sha256 hex: %q vs %q", a, b) + } + want := sha256.Sum256([]byte(Version + "\x00" + ShellSelector())) + if a != hex.EncodeToString(want[:]) { + t.Error("fingerprint must hash Version + selector text") + } +} + +// Sanity: the RHEL-family rebuilds (centos/rocky/alma) all resolve to one arm. +func TestRHELFamilyAliases(t *testing.T) { + t.Parallel() + rhel := profileByKey(t, "rhel") + for _, want := range []string{"fedora", "centos", "rocky", "almalinux"} { + found := false + for _, id := range rhel.IDs { + if id == want { + found = true + } + } + if !found { + t.Errorf("rhel family missing alias %q", want) + } + } +} + +// RHEL, Oracle Linux and Amazon Linux are deliberately NOT accepted: sandboxes +// boot E2B's kernel, so the kernel-level fidelity those images are chosen for +// isn't there. Rejecting them up front beats building something subtly wrong. +func TestKernelDependentIDsAreRejected(t *testing.T) { + t.Parallel() + for _, id := range []string{"rhel", "ol", "amzn"} { + for _, got := range SupportedIDs() { + if got == id { + t.Errorf("id %q must not be accepted: E2B supplies the guest kernel", id) + } + } + } +} diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/init.go b/packages/orchestrator/pkg/template/build/phases/base/distro/init.go new file mode 100644 index 0000000000..99b898c601 --- /dev/null +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/init.go @@ -0,0 +1,143 @@ +// Init-system axis of the distro profiles: one provisioning-time shell block +// per init system, rendered into the selector as e2b_init_setup() so provision.sh +// stays init-agnostic. +package distro + +import "strings" + +// InitSystem is the guest init family a profile boots with. +type InitSystem string + +const ( + // InitSystemd — Debian/Ubuntu, RHEL/Fedora, Arch (systemd via envd.service). + InitSystemd InitSystem = "systemd" + // InitOpenRC — Alpine (busybox init → OpenRC via the baked /etc/init.d/envd). + InitOpenRC InitSystem = "openrc" +) + +// initSetup is the provisioning-time shell block per init system. Bodies may +// reference the selector's profile variables (e.g. $E2B_TIMESYNC_UNIT), defined +// in the same case arm before the function is called. +var initSetup = map[InitSystem]string{ + InitSystemd: `echo "Don't wait for ttyS0 (serial console kernel logs)" +# This is required when the Firecracker kernel args has specified console=ttyS0 +systemctl mask serial-getty@ttyS0.service + +echo "Disable network online wait" +systemctl mask systemd-networkd-wait-online.service + +echo "Disable system first boot wizard" +# This was problem with Ubuntu 24.04, that differently calculate wizard should be called +# and Linux boot was stuck in wizard until envd wait timeout +systemctl mask systemd-firstboot.service + +echo "Enable time synchronization ($E2B_TIMESYNC_UNIT)" +# Distro-correct chrony unit (chrony on Debian, chronyd on RHEL/Arch). +systemctl enable "$E2B_TIMESYNC_UNIT" + +echo "Enable SSH ($E2B_SSH_UNIT)" +# provision.sh writes the sandbox sshd_config on every family, but nothing was +# turning the unit on: Debian's postinst and the RHEL RPM scriptlet enable it +# themselves, Arch does not, so Arch sandboxes shipped with SSH configured and +# dead. Enabling is idempotent where the packaging already did it. +systemctl enable "$E2B_SSH_UNIT.service" + +echo "Enable envd autostart" +# Belt-and-suspenders with the baked 00-e2b.preset: on the RHEL family the +# package transaction above runs systemd's RPM scriptlet 'systemctl preset-all' +# (policy 'disable *'), which deletes the baked wants-symlink. +systemctl enable envd.service + +echo "Disable chrony-wait" +# chrony-wait blocks multi-user.target until the first clock sync (~8s); +# chrony still syncs in the background, nothing needs to wait for it. +# masking a unit that doesn't exist on this distro still succeeds (systemctl +# mask just writes the /dev/null symlink), so a failure here is real. +systemctl mask chrony-wait.service + +echo "Disable slow boot units not needed in the sandbox" +# binfmt registrations (foreign-arch exec) take ~1s of CPU early in boot and +# compete with envd start; e2scrub is for LVM-backed ext4 only. +systemctl mask systemd-binfmt.service +systemctl mask e2scrub_reap.service`, + + // OpenRC (Alpine). The image is still running the one-shot PROVISIONING + // inittab (it is what launched this script); replace it with the real + // boot sequence and wire the runlevels a container image ships without. + InitOpenRC: `echo "Installing boot inittab (busybox init -> OpenRC runlevels)" +printf '%s\n' \ + '::sysinit:/sbin/openrc sysinit' \ + '::sysinit:/sbin/openrc boot' \ + '::wait:/sbin/openrc default' \ + '::shutdown:/sbin/openrc shutdown' \ + '::ctrlaltdel:/sbin/reboot' \ + > /etc/inittab + +echo "Registering base OpenRC services" +# Container images carry no runlevel wiring at all (setup-alpine does this on +# real installs): kernel filesystems in sysinit, system prep in boot. bootmisc +# also wipes /tmp in the boot runlevel, so envd (default runlevel) can never +# race the wipe — the ordering systemd needs After= for is inherent here. +# Which scripts exist varies by image (mdev vs udev, procfs presence) — check +# and say so instead of swallowing rc-update errors. +for svc in devfs sysfs procfs dmesg mdev; do + if [ -e "/etc/init.d/$svc" ]; then + rc-update add "$svc" sysinit + else + echo "OpenRC service $svc not present on this image; skipping" + fi +done +for svc in localmount sysctl hostname bootmisc; do + if [ -e "/etc/init.d/$svc" ]; then + rc-update add "$svc" boot + else + echo "OpenRC service $svc not present on this image; skipping" + fi +done + +# The FC guest's eth0 is configured by the kernel (ip=), but OpenRC services +# declaring a "need net" dependency (chronyd) trigger the networking service, +# which errors out on a missing /etc/network/interfaces and takes chronyd +# down with it. A loopback-only interfaces file lets networking start (and +# provide "net") without touching the kernel-managed eth0. +printf 'auto lo\niface lo inet loopback\n' > /etc/network/interfaces +if [ -e /etc/init.d/networking ]; then + rc-update add networking boot +else + # openrc ships this script and the profile installs openrc, so this is + # unreachable on the images we support. If it ever fires, nothing provides + # "net": chronyd is still enabled below so the failure shows up in the boot + # log rather than the sandbox silently running without time sync. + echo "OpenRC networking service not present on this image; nothing provides 'net', so time sync will fail to start" +fi + +echo "Enable time synchronization ($E2B_TIMESYNC_UNIT)" +rc-update add "$E2B_TIMESYNC_UNIT" default + +echo "Enable envd autostart" +# The service script is baked at a neutral path (envd.openrc.tpl) so the +# Debian family's update-rc.d never sees it; install it for OpenRC here. +cp /usr/local/share/e2b/envd.openrc /etc/init.d/envd +chmod 0755 /etc/init.d/envd +rc-update add envd default + +echo "Enable sshd" +if [ -e /etc/init.d/sshd ]; then + rc-update add sshd default +else + echo "sshd service not present on this image; skipping" +fi`, +} + +// indentBlock indents every non-empty line of a shell block for embedding +// inside the generated selector function bodies. +func indentBlock(s, prefix string) string { + lines := strings.Split(s, "\n") + for i, l := range lines { + if l != "" { + lines[i] = prefix + l + } + } + + return strings.Join(lines, "\n") +} diff --git a/packages/orchestrator/pkg/template/build/phases/base/files.go b/packages/orchestrator/pkg/template/build/phases/base/files.go index afced98712..1563226a92 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/files.go +++ b/packages/orchestrator/pkg/template/build/phases/base/files.go @@ -15,6 +15,7 @@ import ( "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/config" "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs" "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/phases" + "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/phases/base/distro" artifactsregistry "github.com/e2b-dev/infra/packages/shared/pkg/artifacts-registry" "github.com/e2b-dev/infra/packages/shared/pkg/dockerhub" "github.com/e2b-dev/infra/packages/shared/pkg/featureflags" @@ -45,9 +46,10 @@ func constructLayerFilesFromOCI( featureFlags, ) provisionScript, err := getProvisionScript(ctx, ProvisionScriptParams{ - BusyBox: rootfs.SandboxBusyBoxPath, - ResultPath: provisionScriptResultPath, - Provider: buildContext.BuilderConfig.Provider, + BusyBox: rootfs.SandboxBusyBoxPath, + ResultPath: provisionScriptResultPath, + Provider: buildContext.BuilderConfig.Provider, + DistroSelector: distro.ShellSelector(), }) if err != nil { return nil, nil, containerregistry.Config{}, fmt.Errorf("error getting provision script: %w", err) diff --git a/packages/orchestrator/pkg/template/build/phases/base/hash.go b/packages/orchestrator/pkg/template/build/phases/base/hash.go index c5bc670322..39235ea5bf 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/hash.go +++ b/packages/orchestrator/pkg/template/build/phases/base/hash.go @@ -11,6 +11,7 @@ import ( "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs" "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/phases" + "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/phases/base/distro" "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/storage/cache" "github.com/e2b-dev/infra/packages/shared/pkg/featureflags" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" @@ -34,10 +35,12 @@ func (bb *BaseBuilder) Hash(ctx context.Context, _ phases.LayerResult) (string, baseSource = bb.Config.FromImage } - // For fallback/dev environments, include baked rootfs file contents in the - // provision version. In production, BuildProvisionVersion controls rollout - // invalidation explicitly. - provisionVersion := cache.HashKeys(provisionScriptFile, rootfs.FilesHash()) + // For fallback/dev environments, include baked rootfs file contents and + // the distro provisioning contract (profiles + init blocks — the rendered + // selector is part of the script but not of the raw provisionScriptFile + // hashed here) in the provision version. In production, + // BuildProvisionVersion controls rollout invalidation explicitly. + provisionVersion := cache.HashKeys(provisionScriptFile, rootfs.FilesHash(), distro.Fingerprint()) if val := bb.featureFlags.IntFlag( ctx, featureflags.BuildProvisionVersion, diff --git a/packages/orchestrator/pkg/template/build/phases/base/provision.go b/packages/orchestrator/pkg/template/build/phases/base/provision.go index 1ece2e91c2..1e94f6ce5e 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/provision.go +++ b/packages/orchestrator/pkg/template/build/phases/base/provision.go @@ -53,6 +53,9 @@ type ProvisionScriptParams struct { BusyBox string ResultPath string Provider string + // DistroSelector is the generated POSIX-sh block that selects the base + // image's distro profile by its /etc/os-release ID. + DistroSelector string } func getProvisionScript( @@ -96,6 +99,18 @@ func (bb *BaseBuilder) provisionSandbox( done.SetError(e) }() + // Rolling tail of the guest's own provisioning output (the + // prefix-marked lines only — kernel logs are unmarked). The full + // stream is logged at debug, invisible in customer build logs; on + // failure the tail is attached to the error, which IS user-visible, + // so rejections name their real reason ("unsupported base image + // distribution: …") instead of a bare exit status. + // 30 lines, because package managers emit per-package error spam that + // rotated the ROOT error ("No space left on device") out of an + // 8-line window (observed with yum on Amazon Linux 2). + const failureTailLines = 30 + var tail []string + scanner := bufio.NewScanner(exitCodeReader) for scanner.Scan() { line := scanner.Text() @@ -106,8 +121,20 @@ func (bb *BaseBuilder) provisionSandbox( return nil } + if len(tail) > 0 { + return fmt.Errorf("exit status: %s; provisioning output tail:\n%s", exitStatus, strings.Join(tail, "\n")) + } + return fmt.Errorf("exit status: %s", exitStatus) } + if after, ok := strings.CutPrefix(line, logExternalPrefix); ok { + if trimmed := strings.TrimSpace(after); trimmed != "" { + tail = append(tail, trimmed) + if len(tail) > failureTailLines { + tail = tail[1:] + } + } + } } if err := scanner.Err(); err != nil && !errors.Is(err, io.EOF) { diff --git a/packages/orchestrator/pkg/template/build/phases/base/provision.sh b/packages/orchestrator/pkg/template/build/phases/base/provision.sh index 56ecd6d176..bdd18c8252 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/provision.sh +++ b/packages/orchestrator/pkg/template/build/phases/base/provision.sh @@ -13,13 +13,39 @@ echo "Starting provisioning script" echo "Making configuration immutable" $BUSYBOX chattr +i /etc/resolv.conf -# Helper function to check if a package is installed +# Identify the base image by its DECLARED /etc/os-release ID, never by probing +# for a package manager. The selector (generated from the distro profile +# registry) defines the E2B_* vars and pkg/CA shell functions, or exits 1 on an +# unsupported distribution. Images with no os-release (distroless, scratch) are +# rejected by name rather than guessed. +echo "Detecting base image distribution" +if [ -r /etc/os-release ]; then + . /etc/os-release + E2B_DISTRO_ID="${ID:-unknown}" +else + E2B_DISTRO_ID="unknown (image has no /etc/os-release)" +fi + +{{ .DistroSelector }} + +echo "Provisioning for distro '$E2B_DISTRO_ID' (init=$E2B_INIT_BIN, timesync=$E2B_TIMESYNC_UNIT, admin-group=$E2B_ADMIN_GROUP)" + +# Persist the resolved identity for later build phases (finalize sources it), +# so the profile's values are defined once, here. +mkdir -p /usr/local/share/e2b +{ + echo "E2B_DISTRO_ID='$E2B_DISTRO_ID'" + echo "E2B_INIT_SYSTEM='$E2B_INIT_SYSTEM'" + echo "E2B_ADMIN_GROUP='$E2B_ADMIN_GROUP'" +} > /usr/local/share/e2b/distro.env + +# Helper function to check if a package is installed (distro-specific query) is_package_installed() { - dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -q "install ok installed" + e2b_pkg_query "$1" } # Install required packages if not already installed -PACKAGES="systemd systemd-sysv openssh-server sudo chrony socat curl ca-certificates fuse3 iptables git nfs-common less nftables iputils-ping jq" +PACKAGES="$E2B_PACKAGES" echo "Checking presence of the following packages: $PACKAGES" MISSING="" @@ -32,18 +58,29 @@ done if [ -n "$MISSING" ]; then echo "Missing packages detected, installing:$MISSING" - apt-get -q update - DEBIAN_FRONTEND=noninteractive DEBCONF_NOWARNINGS=yes apt-get -qq -o=Dpkg::Use-Pty=0 install -y --no-install-recommends $MISSING + # shellcheck disable=SC2086 + e2b_pkg_install $MISSING else echo "All required packages are already installed." fi +# Ensure the CA trust bundle exists where envd expects it; e2b_ca_refresh +# regenerates it per family. A refresh failure fails provisioning (set -e) — a +# silently broken trust store is worse than a legible build error. +if [ ! -s "$E2B_CA_BUNDLE" ]; then + echo "CA trust bundle missing at $E2B_CA_BUNDLE — running the profile's refresh" + e2b_ca_refresh +fi + # Set /dev/fuse permissions to 666 for non-root access # Use systemd-tmpfiles to set permissions at boot mkdir -p /etc/tmpfiles.d echo 'z /dev/fuse 0666 root root -' > /etc/tmpfiles.d/fuse.conf echo "Setting up shell" +# Not every base image ships /etc/profile.d or /root; create them so the +# drop-ins below always have a home. +mkdir -p /etc/profile.d /root echo "export SHELL='/bin/bash'" >/etc/profile.d/shell.sh echo "export PS1='\w \$ '" >/etc/profile.d/prompt.sh echo "export PS1='\w \$ '" >>"/etc/profile" @@ -53,19 +90,37 @@ echo "Use .bashrc and .profile" echo "if [ -f ~/.bashrc ]; then source ~/.bashrc; fi; if [ -f ~/.profile ]; then source ~/.profile; fi" >>/etc/profile echo "Remove root password" -passwd -d root +# A minimal image may not ship /etc/passwd; only clear the root password when it +# exists rather than failing provisioning. +if [ -f /etc/passwd ]; then + passwd -d root +else + echo "No /etc/passwd on this image; skipping root password removal" +fi echo "Setting up chrony" mkdir -p /etc/chrony -cat </etc/chrony/chrony.conf -refclock PHC /dev/ptp0 poll 2 dpoll 2 -# Step (jump) the clock instead of slewing when the offset exceeds 1s, but only -# for the first 3 updates after chronyd starts. chronyd restarts on every cold -# boot/reboot, so this corrects a large boot-time offset fast (TLS needs a -# correct clock) without risking a backward jump under a running workload. -# Needed because chrony-wait is masked, so boot no longer blocks on first sync. -makestep 1.0 3 -EOF +{ + # Prefer the hypervisor's PTP clock (kvm-ptp): no network dependency and + # it tracks the host directly. It is missing where nested virtualization + # can't expose it (e.g. dev slots) — chronyd treats a missing PHC as a + # FATAL error, so only reference it when the device exists and fall back + # to NTP otherwise; a running chronyd without PHC beats a dead one. + # Device presence is probed in the provisioning VM, which runs on the same + # host/KVM as the runtime sandboxes. + if [ -e /dev/ptp0 ]; then + echo "refclock PHC /dev/ptp0 poll 2 dpoll 2" + else + echo "pool pool.ntp.org iburst maxsources 3" + fi + # Step (jump) the clock instead of slewing when the offset exceeds 1s, but + # only for the first 3 updates after chronyd starts. chronyd restarts on + # every cold boot/reboot, so this corrects a large boot-time offset fast + # (TLS needs a correct clock) without risking a backward jump under a + # running workload. Needed because chrony-wait is masked, so boot no + # longer blocks on first sync. + echo "makestep 1.0 3" +} >/etc/chrony/chrony.conf # Add a proxy config, as some environments expects it there (e.g. timemaster in Node Dockerimage) echo "include /etc/chrony/chrony.conf" >/etc/chrony.conf @@ -89,34 +144,20 @@ echo 'fs.inotify.max_user_watches=65536' | tee -a /etc/sysctl.conf echo "Disabling proactive memory compaction" echo 'vm.compaction_proactiveness=0' | tee -a /etc/sysctl.conf -echo "Don't wait for ttyS0 (serial console kernel logs)" -# This is required when the Firecracker kernel args has specified console=ttyS0 -systemctl mask serial-getty@ttyS0.service - -echo "Disable network online wait" -systemctl mask systemd-networkd-wait-online.service - -echo "Disable system first boot wizard" -# This was problem with Ubuntu 24.04, that differently calculate wizard should be called -# and Linux boot was stuck in wizard until envd wait timeout -systemctl mask systemd-firstboot.service - -echo "Disable chrony-wait" -# chrony-wait blocks multi-user.target until the first clock sync (~8s); -# chrony still syncs in the background, nothing needs to wait for it. -systemctl mask chrony-wait.service - -echo "Disable slow boot units not needed in the sandbox" -# binfmt registrations (foreign-arch exec) take ~1s of CPU early in boot and -# compete with envd start; e2scrub is for LVM-backed ext4 only. -systemctl mask systemd-binfmt.service -systemctl mask e2scrub_reap.service +# Init-system-specific boot arrangement (autostarts, boot-noise silencing, the +# OpenRC inittab swap), rendered per family by the distro selector (distro/init.go). +e2b_init_setup # Clean machine-id from Docker rm -rf /etc/machine-id -echo "Linking systemd to init" -ln -sf /lib/systemd/systemd /usr/sbin/init +echo "Linking $E2B_INIT_BIN to init" +# Not every image ships /usr/sbin; create it before linking init. +mkdir -p /usr/sbin +ln -sf "$E2B_INIT_BIN" /usr/sbin/init +# /sbin is a real directory on non-usr-merged distros (Alpine) where the line +# above doesn't reach the /sbin/init the kernel is pointed at; link it too. +[ -L /sbin ] || ln -sf "$E2B_INIT_BIN" /sbin/init echo "Unlocking immutable configuration" $BUSYBOX chattr -i /etc/resolv.conf diff --git a/packages/orchestrator/pkg/template/build/phases/finalize/configure.go b/packages/orchestrator/pkg/template/build/phases/finalize/configure.go index 78059df3ed..2e521592e0 100644 --- a/packages/orchestrator/pkg/template/build/phases/finalize/configure.go +++ b/packages/orchestrator/pkg/template/build/phases/finalize/configure.go @@ -34,6 +34,14 @@ var ConfigureScriptTemplate = tt.Must(tt.New("provisioning-finish-script").Parse // on cold boot, trading the regen's scattered rootfs reads for one sequential // read. -h dereferences the hash-named symlinks so the real cert contents are // packed instead of links that would still fault the lazily-fetched rootfs. +// Deliberately Debian/Alpine-only. Adding an update-ca-trust branch for the RHEL +// family regresses it: extract regenerates the extracted/pem/directory-hash tree +// that /etc/ssl/certs points at, replacing the absolute ca-certificates.crt +// symlink provisioning created with a relative one that tar -h then packs as a +// link instead of dereferencing — the packed bundle drops from ~226 KB of PEM to +// a 20-byte symlink, and envd's egress-CA append needs a real file. Provisioning +// already refreshes the store per family; this step only has to merge CAs that +// later build layers dropped in, which is a Debian/Alpine convention anyway. const packCertBundleCmd = `set -e if command -v update-ca-certificates >/dev/null 2>&1; then update-ca-certificates diff --git a/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh b/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh index 76202a81a5..185f4047f7 100644 --- a/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh +++ b/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh @@ -10,34 +10,74 @@ TEMPLATE_ID={{ .TemplateID }} BUILD_ID={{ .BuildID }} EOF -# Create default user. -# if the /home/user directory exists, we copy the skeleton files to it because the adduser command -# will ignore the directory if it exists, but we want to include the skeleton files in the home directory -# in our case. +# Create default user. useradd is in shadow(-utils) on every supported distro +# family; -m creates the home dir, -s the shell. A creation failure is a real +# error — a template whose default user silently doesn't exist fails much more +# confusingly later. echo "Create default user 'user' (if doesn't exist yet)" -ADDUSER_OUTPUT=$(adduser -disabled-password --gecos "" user 2>&1 || true) -echo "$ADDUSER_OUTPUT" -if echo "$ADDUSER_OUTPUT" | grep -q "The home directory \`/home/user' already exists"; then - # Copy skeleton files if they don't exist in the home directory - echo "Copy skeleton files to /home/user" - cp -rn /etc/skel/. /home/user/ +if ! id -u user >/dev/null 2>&1; then + useradd -m -s /bin/bash user +fi +# useradd -m skips skeleton files when /home/user already exists, so copy them +# explicitly (no-clobber). Not every image ships /etc/skel — say so instead of +# hiding it. Walked file-by-file because `cp -n` exits non-zero when it skips an +# existing file on coreutils >= 9.2 (Fedora 40+), which is a skip, not a failure. +if [ -d /home/user ]; then + if [ -d /etc/skel ]; then + echo "Copy skeleton files to /home/user (keeping existing files)" + (cd /etc/skel && find . -type f -print | while IFS= read -r f; do + if [ ! -e "/home/user/$f" ]; then + mkdir -p "/home/user/$(dirname "$f")" + cp -a "/etc/skel/$f" "/home/user/$f" + fi + done) + else + echo "No /etc/skel on this image; skipping skeleton copy" + fi fi echo "Add sudo to 'user' with no password" -usermod -aG sudo user +# Admin group differs by distro (sudo on Debian/Ubuntu, wheel elsewhere); the +# NOPASSWD sudoers entry below is what actually grants privileges. The group +# comes from the distro profile, resolved and persisted by provisioning — +# defined once in distro.go, not re-probed here. Templates built before +# distro.env existed (FROM-template parents reuse their rootfs without +# re-provisioning) fall back to probing the two known groups. +if [ -f /usr/local/share/e2b/distro.env ]; then + . /usr/local/share/e2b/distro.env +fi +if [ -n "${E2B_ADMIN_GROUP:-}" ]; then + if ! getent group "$E2B_ADMIN_GROUP" >/dev/null; then + echo "ERROR: the '$E2B_ADMIN_GROUP' group from the distro profile does not exist on this image" >&2 + exit 1 + fi + usermod -aG "$E2B_ADMIN_GROUP" user +elif getent group sudo >/dev/null; then + usermod -aG sudo user +elif getent group wheel >/dev/null; then + usermod -aG wheel user +else + echo "ERROR: neither the sudo nor the wheel group exists on this image" >&2 + exit 1 +fi passwd -d user -echo "user ALL=(ALL:ALL) NOPASSWD: ALL" >>/etc/sudoers +# Skip the append when the entry is already present. +if grep -q '^user ALL=(ALL:ALL) NOPASSWD: ALL' /etc/sudoers; then + echo "sudoers entry already present" +else + echo "user ALL=(ALL:ALL) NOPASSWD: ALL" >>/etc/sudoers +fi echo "Give 'user' ownership to /home/user" mkdir -p /home/user chown -R user:user /home/user echo "Give 777 permission to /usr/local" -chmod 777 -R /usr/local +chmod -R 777 /usr/local echo "Create /code directory" mkdir -p /code echo "Give 777 permission to /code" -chmod 777 -R /code +chmod -R 777 /code echo "Finished configuration script"