From 89ea91b2b29c110e5761fff2d5f65928c29b2da0 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Mon, 27 Jul 2026 11:59:41 +0200 Subject: [PATCH 01/17] feat(orch): distro-aware template base-image provisioning Resolve the base image's distro from its /etc/os-release ID and drive provisioning from a declared per-family profile (Debian/Ubuntu, the RHEL family, Arch, and Alpine on OpenRC) instead of probing for a package manager. Unsupported or identity-less images are rejected with a clear build-log error. --- packages/orchestrator/README.md | 2 +- .../pkg/template/build/commands/user.go | 16 +- .../build/core/rootfs/files/envd.openrc.tpl | 46 +++++ .../build/core/rootfs/files/envd.service.tpl | 4 +- .../build/core/rootfs/files/inittab.tpl | 20 +- .../core/rootfs/files/provision-runner.sh.tpl | 33 ++++ .../build/core/rootfs/files/rcS.sh.tpl | 19 +- .../build/core/rootfs/files/seed-certs.sh.tpl | 45 +++++ .../pkg/template/build/core/rootfs/rootfs.go | 49 +++-- .../template/build/core/rootfs/rootfs_test.go | 71 ++++++- .../template/build/layer/layer_executor.go | 17 +- .../build/phases/base/distro/distro.go | 161 ++++++++++++++++ .../build/phases/base/distro/distro_test.go | 176 ++++++++++++++++++ .../template/build/phases/base/distro/init.go | 132 +++++++++++++ .../pkg/template/build/phases/base/files.go | 8 +- .../pkg/template/build/phases/base/hash.go | 11 +- .../template/build/phases/base/provision.go | 27 +++ .../template/build/phases/base/provision.sh | 110 +++++++---- .../build/phases/finalize/configure.sh | 52 ++++-- 19 files changed, 904 insertions(+), 95 deletions(-) create mode 100644 packages/orchestrator/pkg/template/build/core/rootfs/files/envd.openrc.tpl create mode 100644 packages/orchestrator/pkg/template/build/core/rootfs/files/provision-runner.sh.tpl create mode 100644 packages/orchestrator/pkg/template/build/core/rootfs/files/seed-certs.sh.tpl create mode 100644 packages/orchestrator/pkg/template/build/phases/base/distro/distro.go create mode 100644 packages/orchestrator/pkg/template/build/phases/base/distro/distro_test.go create mode 100644 packages/orchestrator/pkg/template/build/phases/base/distro/init.go diff --git a/packages/orchestrator/README.md b/packages/orchestrator/README.md index 47bd5b72d6..161fd75505 100644 --- a/packages/orchestrator/README.md +++ b/packages/orchestrator/README.md @@ -303,4 +303,4 @@ Automatically set in local mode. Set before running to override: ## Limitations -- Custom template builds require Debian/Ubuntu-based base images (images that provide the `apt` package manager). Non-Debian images such as Alpine, CentOS/RHEL, or other distributions without `apt` are not supported and will fail during the template build/provisioning process. The provisioning scripts used during template build call `apt` and expect Debian-specific package names and file locations. \ No newline at end of file +- Custom template builds support base images from the declared distro families: **Debian/Ubuntu** (apt), the **RHEL family** — Fedora, RHEL, CentOS Stream, Rocky, Alma, Oracle Linux, Amazon Linux — (dnf/microdnf/yum), **Arch** (pacman), and **Alpine** (apk, OpenRC). The distro is resolved from the image's `/etc/os-release` `ID` — never by probing for package managers. Images without an os-release identity (distroless, scratch) or from an undeclared family are rejected fast, with a build-log error naming the reason and the supported families (see `pkg/template/build/phases/base/distro`). Minimal/restricted-repo images (e.g. RHEL UBI) may fail provisioning if their repos don't carry the required packages; the failure is surfaced in the build log. \ No newline at end of file diff --git a/packages/orchestrator/pkg/template/build/commands/user.go b/packages/orchestrator/pkg/template/build/commands/user.go index 0b1f0cb94e..207c6a75e0 100644 --- a/packages/orchestrator/pkg/template/build/commands/user.go +++ b/packages/orchestrator/pkg/template/build/commands/user.go @@ -60,7 +60,9 @@ func (u *User) Execute( lvl, prefix, sandboxID, - fmt.Sprintf("adduser --disabled-password --gecos \"\" %s", userArg), + // useradd is in shadow(-utils) on every supported distro family; the + // created user has no password (locked). + fmt.Sprintf("useradd --create-home --shell /bin/bash %s", userArg), metadata.Context{ User: "root", EnvVars: cmdMetadata.EnvVars, @@ -99,7 +101,17 @@ func addToSudoers( lvl, prefix, sandboxID, - fmt.Sprintf("usermod -aG sudo %s", userArg), + // Admin group differs by distro (sudo on Debian/Ubuntu, wheel on + // RHEL/Arch/Alpine); the NOPASSWD sudoers entry below is what actually + // grants privileges. Neither group existing is a real error. + fmt.Sprintf(`if getent group sudo >/dev/null; then + usermod -aG sudo %[1]s +elif getent group wheel >/dev/null; then + usermod -aG wheel %[1]s +else + echo "neither the sudo nor the wheel group exists on this image" >&2 + exit 1 +fi`, userArg), metadata.Context{ User: "root", EnvVars: cmdMetadata.EnvVars, diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.openrc.tpl b/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.openrc.tpl new file mode 100644 index 0000000000..a5f9bbf16a --- /dev/null +++ b/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.openrc.tpl @@ -0,0 +1,46 @@ +{{- /*gotype:github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs.templateModel*/ -}} +{{ .WriteFile "/usr/local/share/e2b/envd.openrc" 0o755 }} + +#!/sbin/openrc-run +# E2B env daemon — the OpenRC counterpart of envd.service (see envd.service.tpl +# for the full rationale on each step; this mirrors it for the Alpine/OpenRC +# family). Baked at a neutral path and installed to /etc/init.d/envd by the +# OpenRC e2b_init_setup only: if it lived in /etc/init.d on every image, +# Debian's `systemctl enable envd` would hand it to update-rc.d, which aborts +# on a non-LSB script and fails the whole provisioning. +# +# /tmp-wipe ordering note: OpenRC's bootmisc (boot runlevel) wipes /tmp before +# the default runlevel starts, so envd — in default — can never answer an +# update-envd upload before the wipe. The race envd.service needs an explicit +# After=systemd-tmpfiles-setup.service for cannot happen here. + +description="E2B env daemon" + +supervisor=supervise-daemon +command=/usr/bin/envd +# --nicelevel/--ionice/--oom-score-adj mirror envd.service's Nice=-20, +# IOSchedulingClass=realtime + IOSchedulingPriority=4 (ionice class 1, data 4) +# and OOMScoreAdjust=-1000 — without them envd runs at default priority and is +# the first OOM-kill candidate on Alpine. (The unit's cgroup weights — +# MemoryMin/CPUWeight/IOWeight — have no supervise-daemon equivalent.) +supervise_daemon_args="--nicelevel -20 --ionice 1:4 --oom-score-adj -1000 --env GOTRACEBACK=all --env GOMEMLIMIT={{ .MemoryLimit }}MiB --stdout /var/log/envd.log --stderr /var/log/envd.log" +# Retry forever (envd.service uses Restart=always + StartLimitIntervalSec=0). +respawn_delay=1 +respawn_max=0 + +depend() { + need localmount + after bootmisc + use net +} + +start_pre() { + # Shared with envd.service's ExecStartPre; warns-and-continues by design. + /usr/local/bin/e2b-seed-certs + # systemd-tmpfiles applies the fuse.conf tmpfiles.d rule on the systemd + # family; OpenRC has no tmpfiles pass, so set the mode here. + if [ -e /dev/fuse ]; then + chmod 666 /dev/fuse + fi + return 0 +} diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.service.tpl b/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.service.tpl index b66b7c7166..36e67e7a4b 100644 --- a/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.service.tpl +++ b/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.service.tpl @@ -46,7 +46,9 @@ LimitCORE=infinity # guaranteed present for the sandbox's routable lifetime. The only gap is guest # units that auto-start and egress over TLS before /init; that is accepted # (revisit if a template needs boot-time egress). -ExecStartPre=/bin/sh -c 'mountpoint -q /etc/ssl/certs || { mkdir -p /run/e2b/certs && { tar -C /run/e2b/certs -xf /usr/local/share/e2b/ssl-certs.tar 2>/dev/null || cp -a /etc/ssl/certs/. /run/e2b/certs/ 2>/dev/null; }; mount --bind /run/e2b/certs /etc/ssl/certs; } && ([ -s /etc/ssl/certs/ca-certificates.crt ] || update-ca-certificates)' +# The seeding logic is shared with the OpenRC service (Alpine) and never fails +# the unit — a degraded trust store is recoverable, a dead envd is not. +ExecStartPre=/usr/local/bin/e2b-seed-certs ExecStart=/usr/bin/envd Nice=-20 IOSchedulingClass=realtime diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/files/inittab.tpl b/packages/orchestrator/pkg/template/build/core/rootfs/files/inittab.tpl index 88d981c2ed..9194829518 100644 --- a/packages/orchestrator/pkg/template/build/core/rootfs/files/inittab.tpl +++ b/packages/orchestrator/pkg/template/build/core/rootfs/files/inittab.tpl @@ -1,18 +1,16 @@ {{- /*gotype:github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs.templateModel*/ -}} {{ .WriteFile "/etc/inittab" 0o777 }} -# Run system init -::sysinit:/etc/init.d/rcS - -# Run the provision script, prefix the output with a log prefix -::wait:/bin/sh -c '/usr/local/bin/provision.sh 2>&1 | sed "s/^/{{ .ProvisionLogPrefix }}/"' +# Provisioning-boot inittab (busybox init). Every entry is a plain exec with +# NO shell metacharacters: busybox init hands metachar lines to /bin/sh, and +# minimal images (distroless) may have no /bin/sh — the pipeline logic lives in +# e2b-provision-runner instead. -# Flush filesystem changes to disk -::wait:/usr/bin/busybox sync -::wait:fsfreeze --freeze / +# Run system init (mounts /proc /sys /dev /tmp /run through the baked busybox) +::sysinit:/etc/init.d/rcS -# Report the exit code of the provisioning script -::wait:/bin/sh -c 'echo "{{ .ProvisionExitPrefix }}$(cat {{ .ProvisionResultPath }} || printf 1)"' +# Run the provisioning pipeline and report its exit code +::wait:/usr/bin/busybox ash /usr/local/bin/e2b-provision-runner # Wait forever to prevent the VM from exiting until the sandbox is paused and snapshot is taken -::wait:/usr/bin/busybox sleep infinity \ No newline at end of file +::wait:/usr/bin/busybox sleep infinity diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/files/provision-runner.sh.tpl b/packages/orchestrator/pkg/template/build/core/rootfs/files/provision-runner.sh.tpl new file mode 100644 index 0000000000..62827ac3ed --- /dev/null +++ b/packages/orchestrator/pkg/template/build/core/rootfs/files/provision-runner.sh.tpl @@ -0,0 +1,33 @@ +{{- /*gotype:github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs.templateModel*/ -}} +{{ .WriteFile "usr/local/bin/e2b-provision-runner" 0o755 }} + +#!/usr/bin/busybox ash +# Drives the provisioning pipeline for the busybox-init boot. This logic lives +# in a script — NOT in /etc/inittab — because busybox init hands any inittab +# line containing shell metacharacters to /bin/sh, and minimal images +# (distroless) may have no /bin/sh; a plain-exec inittab line running this +# script through the baked busybox works on every image. +BB=/usr/bin/busybox + +# Run the provision script, prefix its output with the log prefix the +# orchestrator forwards to the customer's build logs. +$BB sh /usr/local/bin/provision.sh 2>&1 | $BB sed "s/^/{{ .ProvisionLogPrefix }}/" + +# Flush filesystem changes to disk before the snapshot. +$BB sync +if command -v fsfreeze >/dev/null 2>&1; then + fsfreeze --freeze / +else + # No util-linux on this image; the double sync flushes the ext4 journal + # and the VM is paused before the snapshot is taken. + echo "fsfreeze not available on this image; using sync-only flush" + $BB sync +fi + +# Report the provisioning exit code: provision.sh writes "0" on success and +# (running under set -e) leaves no file behind on failure. +if result=$($BB cat {{ .ProvisionResultPath }} 2>/dev/null); then + echo "{{ .ProvisionExitPrefix }}${result}" +else + echo "{{ .ProvisionExitPrefix }}1" +fi diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/files/rcS.sh.tpl b/packages/orchestrator/pkg/template/build/core/rootfs/files/rcS.sh.tpl index 08bbb99e93..3a3c3f29ab 100644 --- a/packages/orchestrator/pkg/template/build/core/rootfs/files/rcS.sh.tpl +++ b/packages/orchestrator/pkg/template/build/core/rootfs/files/rcS.sh.tpl @@ -2,15 +2,20 @@ {{ .WriteFile "etc/init.d/rcS" 0o777 }} #!/usr/bin/busybox ash +# Every command goes through the baked busybox: this runs before provisioning +# on the raw base image, and minimal images (distroless) may have no mkdir/mount +# on PATH at all. +BB=/usr/bin/busybox + echo "Mounting essential filesystems" # Ensure necessary mount points exist -mkdir -p /proc /sys /dev /tmp /run +$BB mkdir -p /proc /sys /dev /tmp /run # Mount essential filesystems -mount -t proc proc /proc -mount -t sysfs sysfs /sys -mount -t devtmpfs devtmpfs /dev -mount -t tmpfs tmpfs /tmp -mount -t tmpfs tmpfs /run +$BB mount -t proc proc /proc +$BB mount -t sysfs sysfs /sys +$BB mount -t devtmpfs devtmpfs /dev +$BB mount -t tmpfs tmpfs /tmp +$BB mount -t tmpfs tmpfs /run -echo "System Init" \ No newline at end of file +echo "System Init" diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/files/seed-certs.sh.tpl b/packages/orchestrator/pkg/template/build/core/rootfs/files/seed-certs.sh.tpl new file mode 100644 index 0000000000..b078a49be2 --- /dev/null +++ b/packages/orchestrator/pkg/template/build/core/rootfs/files/seed-certs.sh.tpl @@ -0,0 +1,45 @@ +{{- /*gotype:github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs.templateModel*/ -}} +{{ .WriteFile "usr/local/bin/e2b-seed-certs" 0o755 }} + +#!/bin/sh +# Seeds the tmpfs-backed /etc/ssl/certs before envd starts — shared by +# envd.service (systemd) and /etc/init.d/envd (OpenRC). See envd.service.tpl +# for the full rationale (why a tar, why a bind mount, the egress-CA contract). +# +# Every failure path here WARNS and continues deliberately: a sandbox with a +# degraded trust store is recoverable (envd's POST /init reinstalls the egress +# CA), a sandbox whose envd never starts is not. + +# The copies DEREFERENCE symlinks (-L; the build-time tar already packs with +# -h): some images ship the /etc/ssl/certs bundle as symlinks into a read-only +# store, and envd's egress-proxy CA install APPENDS to +# /etc/ssl/certs/ca-certificates.crt — that only works if the tmpfs copy is a +# real file, not a copied symlink to an immutable target. +if ! mountpoint -q /etc/ssl/certs; then + mkdir -p /run/e2b/certs + if [ -f /usr/local/share/e2b/ssl-certs.tar ]; then + if ! tar -C /run/e2b/certs -xf /usr/local/share/e2b/ssl-certs.tar; then + echo "e2b-seed-certs: ssl-certs.tar extraction failed; seeding from the live cert dir instead" >&2 + cp -aL /etc/ssl/certs/. /run/e2b/certs/ + fi + else + # Only expected during the base-layer boot, before finalize packs the tar. + echo "e2b-seed-certs: ssl-certs.tar not packed yet; seeding from the live cert dir" + cp -aL /etc/ssl/certs/. /run/e2b/certs/ + fi + if ! mount -o bind /run/e2b/certs /etc/ssl/certs; then + echo "e2b-seed-certs: bind mount failed; envd runs with the image's certs as-is" >&2 + fi +fi + +if [ ! -s /etc/ssl/certs/ca-certificates.crt ]; then + if command -v update-ca-certificates >/dev/null 2>&1; then + update-ca-certificates + else + # Provisioning guarantees the bundle on every supported family; + # reaching this means the image diverged after the build. + echo "e2b-seed-certs: CA bundle missing and no update-ca-certificates on this image; TLS trust will be degraded" >&2 + fi +fi + +exit 0 diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/rootfs.go b/packages/orchestrator/pkg/template/build/core/rootfs/rootfs.go index ca859899f0..3cc6d43d75 100644 --- a/packages/orchestrator/pkg/template/build/core/rootfs/rootfs.go +++ b/packages/orchestrator/pkg/template/build/core/rootfs/rootfs.go @@ -14,6 +14,7 @@ import ( "os" "path/filepath" "runtime" + "slices" "text/template" "github.com/dustin/go-humanize" @@ -41,9 +42,25 @@ var tracer = otel.Tracer("github.com/e2b-dev/infra/packages/orchestrator/pkg/tem var files embed.FS var fileTemplates = template.Must(template.ParseFS(files, "files/*")) -// filesHash is a stable hash of the embedded rootfs file templates. It is used -// only as part of the fallback provision version; explicit provision versions -// remain the rollout control. +// enableSymlinks is the content of the baked symlink layer. Package-level so +// it feeds filesHash: a change here must rotate the fallback provision +// version like any other baked-layer change. +var enableSymlinks = map[string]string{ + // Enable envd service autostart. The target MUST be absolute: the link + // lives in multi-user.target.wants/, so a relative target would resolve + // inside that directory and dangle — and provision.sh's offline + // `systemctl enable $E2B_TIMESYNC_UNIT` prunes dangling .wants symlinks, + // silently disabling envd on distros where the link dangles. + "etc/systemd/system/multi-user.target.wants/envd.service": "/etc/systemd/system/envd.service", + // NOTE: chrony autostart is enabled by provision.sh via `systemctl enable + // $E2B_TIMESYNC_UNIT`, which picks the distro-correct unit name (chrony on + // Debian, chronyd on RHEL/Arch). A static chrony.service symlink here would + // dangle on non-Debian images where the unit is chronyd.service. +} + +// filesHash is a stable hash of the embedded rootfs file templates plus the +// baked symlink layer. It is used only as part of the fallback provision +// version; explicit provision versions remain the rollout control. var filesHash = func() string { entries, _ := fs.ReadDir(files, "files") h := sha256.New() @@ -51,6 +68,14 @@ var filesHash = func() string { data, _ := files.ReadFile("files/" + e.Name()) fmt.Fprintf(h, "%s\x00%x\x00", e.Name(), data) } + links := make([]string, 0, len(enableSymlinks)) + for name := range enableSymlinks { + links = append(links, name) + } + slices.Sort(links) + for _, name := range links { + fmt.Fprintf(h, "%s\x00%s\x00", name, enableSymlinks[name]) + } return hex.EncodeToString(h.Sum(nil)) }() @@ -233,6 +258,15 @@ func additionalOCILayers( filesMap := map[string]oci.File{ storage.GuestEnvdPath: {Bytes: envdFileData, Mode: 0o777}, + // Systemd preset policy for envd. provision.sh removes /etc/machine-id, + // so the template's next boot is a systemd FIRST boot — and on first + // boot PID1 applies the distro preset policy to all units. On the + // RHEL family that policy ends with "disable *" (and the systemd RPM + // scriptlet's preset-all does the same during provisioning), which + // deletes envd's autostart symlink no matter how it was created. + // A 00- preset sorts before every distro policy file and wins. + "etc/systemd/system-preset/00-e2b.preset": {Bytes: []byte("enable envd.service\n"), Mode: 0o644}, + // Provision script "usr/local/bin/provision.sh": {Bytes: []byte(provisionScript), Mode: 0o777}, // Setup init system @@ -263,14 +297,7 @@ func additionalOCILayers( return nil, fmt.Errorf("error creating layer from files: %w", err) } - symlinkLayer, err := oci.LayerSymlink( - map[string]string{ - // Enable envd service autostart - "etc/systemd/system/multi-user.target.wants/envd.service": "etc/systemd/system/envd.service", - // Enable chrony service autostart - "etc/systemd/system/multi-user.target.wants/chrony.service": "etc/systemd/system/chrony.service", - }, - ) + symlinkLayer, err := oci.LayerSymlink(enableSymlinks) if err != nil { return nil, fmt.Errorf("error creating layer from symlinks: %w", err) } diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/rootfs_test.go b/packages/orchestrator/pkg/template/build/core/rootfs/rootfs_test.go index 963368d997..747fcde460 100644 --- a/packages/orchestrator/pkg/template/build/core/rootfs/rootfs_test.go +++ b/packages/orchestrator/pkg/template/build/core/rootfs/rootfs_test.go @@ -90,7 +90,45 @@ func TestAdditionalOCILayers(t *testing.T) { keysIter := maps.Keys(actualFiles) keys := slices.Collect(keysIter) - assert.Len(t, keys, 14) + assert.Len(t, keys, 18) + + // The provisioning boot must be self-contained on the baked busybox: + // minimal images (distroless) may have no /bin/sh, and busybox init + // hands any inittab line with shell metacharacters to /bin/sh — so the + // pipeline lives in the runner script and every inittab entry is a + // plain exec. + inittab := actualFiles["etc/inittab"] + require.NotEmpty(t, inittab) + for line := range strings.SplitSeq(inittab, "\n") { + if !strings.HasPrefix(line, "::") { + continue + } + assert.NotContains(t, line, "|", "inittab entries must not need /bin/sh: %s", line) + assert.NotContains(t, line, "$", "inittab entries must not need /bin/sh: %s", line) + } + runner := actualFiles["usr/local/bin/e2b-provision-runner"] + require.NotEmpty(t, runner, "provision runner must be baked") + assert.Contains(t, runner, "#!/usr/bin/busybox ash") + + // Both init families' envd services seed certs via the shared script. + seedCerts := actualFiles["usr/local/bin/e2b-seed-certs"] + require.NotEmpty(t, seedCerts, "cert seeding script must be baked") + assert.Contains(t, actualFiles["etc/systemd/system/envd.service"], "ExecStartPre=/usr/local/bin/e2b-seed-certs") + + // envd must be preset-enabled: first boot (machine-id is removed by + // provisioning) applies the distro preset policy, and the RHEL + // family's "disable *" would otherwise delete envd's autostart link. + assert.Equal(t, "enable envd.service\n", actualFiles["etc/systemd/system-preset/00-e2b.preset"]) + + // The OpenRC counterpart (Alpine) ships alongside the systemd unit; it + // must supervise envd and honor the memory limit. + // It lives OUTSIDE /etc/init.d — Debian's update-rc.d aborts on a + // non-LSB script there — and is installed by the OpenRC init setup. + openrcEnvd := actualFiles["usr/local/share/e2b/envd.openrc"] + require.NotEmpty(t, openrcEnvd, "OpenRC envd service must be baked") + assert.Contains(t, openrcEnvd, "#!/sbin/openrc-run") + assert.Contains(t, openrcEnvd, "supervisor=supervise-daemon") + assert.Contains(t, openrcEnvd, "GOMEMLIMIT=50MiB") assert.Equal(t, "e2b.local", actualFiles["etc/hostname"]) assert.Equal(t, "nameserver 8.8.8.8", actualFiles["etc/resolv.conf"]) @@ -124,5 +162,36 @@ func TestAdditionalOCILayers(t *testing.T) { WatchdogSec=0`) assert.Equal(t, disabledContent, actualFiles["etc/systemd/system/systemd-journald.service.d/override.conf"]) assert.Equal(t, disabledContent, actualFiles["etc/systemd/system/systemd-networkd.service.d/override.conf"]) + + // Regression guard: the envd autostart symlink must not dangle. + // A relative target resolves inside multi-user.target.wants/ and dangles, + // and provision.sh's offline `systemctl enable` prunes dangling .wants + // links — silently disabling envd autostart on e.g. Fedora. + symlinksLayer, err := layers[1].Uncompressed() + require.NoError(t, err) + t.Cleanup(func() { + err = symlinksLayer.Close() + assert.NoError(t, err) + }) + + actualSymlinks := map[string]string{} + symlinksTarReader := tar.NewReader(symlinksLayer) + for { + header, err := symlinksTarReader.Next() + if errors.Is(err, io.EOF) { + break + } + require.NoError(t, err) + + if header.Typeflag != tar.TypeSymlink { + continue + } + actualSymlinks[header.Name] = header.Linkname + } + + envdWants := actualSymlinks["etc/systemd/system/multi-user.target.wants/envd.service"] + require.NotEmpty(t, envdWants, "envd autostart symlink must be present") + assert.Equal(t, "/etc/systemd/system/envd.service", envdWants, + "envd autostart symlink target must be absolute so it never dangles") }) } diff --git a/packages/orchestrator/pkg/template/build/layer/layer_executor.go b/packages/orchestrator/pkg/template/build/layer/layer_executor.go index 5a738b0322..2784811bd1 100644 --- a/packages/orchestrator/pkg/template/build/layer/layer_executor.go +++ b/packages/orchestrator/pkg/template/build/layer/layer_executor.go @@ -212,13 +212,24 @@ func (lb *LayerExecutor) updateEnvdInSandbox( return fmt.Errorf("failed to replace envd binary: %w", err) } - // Step 3: Restart the systemd envd service - // Error is ignored because it's expected the envd connection will be lost + // Step 3: Restart the envd service. The error is ignored because the + // restart kills the very envd this command runs through — the connection + // loss is expected. systemctl hands the restart to PID1, which survives + // that. On OpenRC there is no PID1 handoff: rc-service would run in this + // very shell and die with envd between its stop and start halves + // (observed on Alpine — envd never came back), so instead envd is simply + // killed and supervise-daemon's respawn starts the replaced binary; the + // immediate death also means the post-update wait below can never mistake + // the old instance for the new one. _ = sandboxtools.RunCommand( ctx, lb.proxy, sbx.Runtime.SandboxID, - "systemctl restart envd", + `if command -v systemctl >/dev/null 2>&1; then + systemctl restart envd +else + kill -TERM "$(pidof envd)" +fi`, metadata.Context{User: "root"}, ) diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go b/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go new file mode 100644 index 0000000000..4654444177 --- /dev/null +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go @@ -0,0 +1,161 @@ +// Package distro makes template-build provisioning distro-aware: it selects a +// declared per-family Profile by the base image's /etc/os-release ID rather than +// probing for a package manager. Supported: the systemd family (Debian/Ubuntu, +// Fedora/RHEL/CentOS/Rocky/Alma, Arch) and Alpine on OpenRC; anything else is +// rejected with a clear error. +package distro + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" +) + +// Version forces a base-layer rebuild for provisioning changes the generated +// selector text can't otherwise capture; bump it when the contract changes. +const Version = "1" + +// Fingerprint hashes the whole generated provisioning contract into the +// base-layer cache key, so any profile or init-setup change rebuilds the base. +func Fingerprint() string { + sum := sha256.Sum256([]byte(Version + "\x00" + ShellSelector())) + + return hex.EncodeToString(sum[:]) +} + +// Profile is the declared, per-family provisioning contract. IDs are the +// /etc/os-release values that map to the family; PkgQueryBody, PkgInstall and +// CARefresh are shell fragments spliced into the generated selector. +type Profile struct { + Key string + Init InitSystem + IDs []string + Packages []string + PkgQueryBody string + PkgInstall string + InitBinary string + TimeSyncUnit string + AdminGroup string + CABundle string + CARefresh string +} + +var Profiles = []Profile{ + { + Key: "debian", + Init: InitSystemd, + IDs: []string{"debian", "ubuntu"}, + Packages: []string{ + "systemd", "systemd-sysv", "openssh-server", "sudo", "chrony", "socat", + "curl", "ca-certificates", "fuse3", "iptables", "git", "nfs-common", + "less", "nftables", "iputils-ping", "jq", + }, + PkgQueryBody: `dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -q "install ok installed"`, + PkgInstall: "apt-get -q update\n DEBIAN_FRONTEND=noninteractive DEBCONF_NOWARNINGS=yes apt-get -qq -o=Dpkg::Use-Pty=0 install -y --no-install-recommends \"$@\"", + InitBinary: "/lib/systemd/systemd", + TimeSyncUnit: "chrony", + AdminGroup: "sudo", + CABundle: "/etc/ssl/certs/ca-certificates.crt", + CARefresh: "update-ca-certificates", + }, + { + Key: "rhel", // Fedora, RHEL, CentOS Stream, Rocky, Alma, Oracle Linux, Amazon Linux + Init: InitSystemd, + IDs: []string{"fedora", "rhel", "centos", "rocky", "almalinux", "ol", "amzn"}, + // "iptables" (not iptables-nft) and "tar" cover the yum-era members + // (CentOS 7, Amazon Linux 2) that lack the nft package or don't ship tar. + Packages: []string{ + "systemd", "shadow-utils", "passwd", "openssh-server", "sudo", "chrony", + "socat", "curl", "ca-certificates", "fuse3", "iptables", "git", + "nfs-utils", "less", "nftables", "iputils", "jq", "bash", "tar", + }, + PkgQueryBody: `rpm -q "$1" >/dev/null 2>&1`, + // dnf → microdnf → yum spans the whole family; errors reach the build log. + PkgInstall: `dnf -y --allowerasing install "$@" || microdnf -y install "$@" || yum -y install "$@"`, + InitBinary: "/usr/lib/systemd/systemd", + TimeSyncUnit: "chronyd", + AdminGroup: "wheel", + CABundle: "/etc/ssl/certs/ca-certificates.crt", + // update-ca-trust never writes ca-certificates.crt (a Debian name), so + // link the bundle envd's ExecStartPre expects to the extracted PEM. + CARefresh: `update-ca-trust extract && ln -sf /etc/pki/tls/certs/ca-bundle.crt "$E2B_CA_BUNDLE"`, + }, + { + Key: "arch", + Init: InitSystemd, + IDs: []string{"arch", "archarm"}, + Packages: []string{ + "systemd", "shadow", "openssh", "sudo", "chrony", "socat", "curl", + "ca-certificates", "fuse3", "iptables", "git", "nfs-utils", "less", + "nftables", "iputils", "jq", "bash", + }, + PkgQueryBody: `pacman -Q "$1" >/dev/null 2>&1`, + // -Syu (not -Sy then -S): Arch documents partial upgrades as unsupported. + PkgInstall: `pacman -Syu --noconfirm --needed "$@"`, + InitBinary: "/usr/lib/systemd/systemd", + TimeSyncUnit: "chronyd", + AdminGroup: "wheel", + CABundle: "/etc/ssl/certs/ca-certificates.crt", + CARefresh: "update-ca-trust extract", + }, + { + Key: "alpine", + Init: InitOpenRC, + IDs: []string{"alpine"}, + // shadow provides useradd/usermod (busybox's adduser takes different flags). + Packages: []string{ + "openrc", "shadow", "openssh", "sudo", "chrony", "socat", "curl", + "ca-certificates", "fuse3", "iptables", "git", "nfs-utils", "less", + "nftables", "iputils", "jq", "bash", + }, + PkgQueryBody: `apk info -e "$1" >/dev/null 2>&1`, + PkgInstall: `apk add --no-cache "$@"`, + InitBinary: "/bin/busybox", + TimeSyncUnit: "chronyd", + AdminGroup: "wheel", + CABundle: "/etc/ssl/certs/ca-certificates.crt", + CARefresh: "update-ca-certificates", + }, +} + +// SupportedIDs returns every os-release ID the selector accepts. +func SupportedIDs() []string { + var ids []string + for _, p := range Profiles { + ids = append(ids, p.IDs...) + } + + return ids +} + +// ShellSelector generates the POSIX-sh block provision.sh sources: it switches +// on the guest's $E2B_DISTRO_ID and defines the profile's packages, shell +// functions, init path, time-sync unit, admin group and CA handling. An +// unrecognized id exits 1 with a customer-visible error. +func ShellSelector() string { + var b strings.Builder + b.WriteString(`case "$E2B_DISTRO_ID" in` + "\n") + for _, p := range Profiles { + fmt.Fprintf(&b, " %s)\n", strings.Join(p.IDs, "|")) + fmt.Fprintf(&b, " E2B_PACKAGES=%q\n", strings.Join(p.Packages, " ")) + fmt.Fprintf(&b, " e2b_pkg_query() { %s; }\n", p.PkgQueryBody) + fmt.Fprintf(&b, " e2b_pkg_install() { %s; }\n", p.PkgInstall) + fmt.Fprintf(&b, " E2B_INIT_BIN=%q\n", p.InitBinary) + fmt.Fprintf(&b, " E2B_TIMESYNC_UNIT=%q\n", p.TimeSyncUnit) + fmt.Fprintf(&b, " E2B_ADMIN_GROUP=%q\n", p.AdminGroup) + fmt.Fprintf(&b, " E2B_CA_BUNDLE=%q\n", p.CABundle) + fmt.Fprintf(&b, " e2b_ca_refresh() { %s; }\n", p.CARefresh) + fmt.Fprintf(&b, " E2B_INIT_SYSTEM=%q\n", p.Init) + fmt.Fprintf(&b, " e2b_init_setup() {\n%s\n }\n", indentBlock(initSetup[p.Init], " ")) + fmt.Fprintf(&b, " ;;\n") + } + fmt.Fprintf(&b, " *)\n") + fmt.Fprintf(&b, " echo \"[provision] ERROR: unsupported base image distribution: ID='${E2B_DISTRO_ID:-unknown}'.\" >&2\n") + fmt.Fprintf(&b, " echo \"[provision] E2B template builds support: %s.\" >&2\n", strings.Join(SupportedIDs(), ", ")) + fmt.Fprintf(&b, " exit 1\n") + fmt.Fprintf(&b, " ;;\n") + b.WriteString("esac\n") + + return b.String() +} diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/distro_test.go b/packages/orchestrator/pkg/template/build/phases/base/distro/distro_test.go new file mode 100644 index 0000000000..5aa9f70641 --- /dev/null +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/distro_test.go @@ -0,0 +1,176 @@ +package distro + +import ( + "crypto/sha256" + "encoding/hex" + "strings" + "testing" +) + +// Golden lines lifted VERBATIM from the pre-change provision.sh so the debian +// profile reproduces them and Debian/Ubuntu behaviour is preserved. +const ( + goldenDebianPackages = "systemd systemd-sysv openssh-server sudo chrony socat curl ca-certificates fuse3 iptables git nfs-common less nftables iputils-ping jq" + goldenDebianQuery = `dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -q "install ok installed"` + goldenDebianInit = "/lib/systemd/systemd" +) + +func profileByKey(t *testing.T, key string) Profile { + t.Helper() + for _, p := range Profiles { + if p.Key == key { + return p + } + } + t.Fatalf("no profile with key %q", key) + + return Profile{} +} + +// The debian profile preserves today's Debian package set / query / init path. +func TestDebianPreserved(t *testing.T) { + t.Parallel() + p := profileByKey(t, "debian") + if got := strings.Join(p.Packages, " "); got != goldenDebianPackages { + t.Errorf("debian packages drifted:\n got: %s\nwant: %s", got, goldenDebianPackages) + } + if p.PkgQueryBody != goldenDebianQuery { + t.Errorf("debian query drifted:\n got: %s\nwant: %s", p.PkgQueryBody, goldenDebianQuery) + } + if p.InitBinary != goldenDebianInit { + t.Errorf("debian init drifted: got %s want %s", p.InitBinary, goldenDebianInit) + } + if p.TimeSyncUnit != "chrony" || p.AdminGroup != "sudo" { + t.Errorf("debian unit/group drifted: %s / %s", p.TimeSyncUnit, p.AdminGroup) + } +} + +// The families genuinely diverge on the axes that matter. +func TestFamiliesDiffer(t *testing.T) { + t.Parallel() + rhel := profileByKey(t, "rhel") + if rhel.TimeSyncUnit != "chronyd" || rhel.AdminGroup != "wheel" { + t.Errorf("rhel unit/group wrong: %s / %s", rhel.TimeSyncUnit, rhel.AdminGroup) + } + // Must both regenerate the trust store AND materialize the bundle at the + // Debian-named path envd expects — update-ca-trust alone never creates + // ca-certificates.crt, which left envd.service unable to start on Fedora. + if !strings.Contains(rhel.CARefresh, "update-ca-trust extract") || + !strings.Contains(rhel.CARefresh, `ln -sf /etc/pki/tls/certs/ca-bundle.crt "$E2B_CA_BUNDLE"`) { + t.Errorf("rhel CA refresh wrong: %s", rhel.CARefresh) + } + if rhel.InitBinary != "/usr/lib/systemd/systemd" { + t.Errorf("rhel init path wrong: %s", rhel.InitBinary) + } + arch := profileByKey(t, "arch") + if !strings.Contains(arch.PkgInstall, "pacman") { + t.Errorf("arch install should use pacman: %s", arch.PkgInstall) + } +} + +// The generated selector keys on the DECLARED distro id, never on which +// package-manager binary happens to exist. +func TestSelectorNoPackageManagerProbing(t *testing.T) { + t.Parallel() + sel := ShellSelector() + for _, bad := range []string{ + "command -v apt-get", "command -v dnf", "command -v yum", + "command -v microdnf", "command -v pacman", "PKG_FAMILY", + } { + if strings.Contains(sel, bad) { + t.Errorf("selector leaked package-manager probing: %q", bad) + } + } + if !strings.Contains(sel, `case "$E2B_DISTRO_ID" in`) { + t.Error("selector must switch on $E2B_DISTRO_ID (declared distro identity)") + } +} + +// Every supported id gets a case arm; an unknown id hits the failing default. +func TestSelectorCoversIDsAndRejects(t *testing.T) { + t.Parallel() + sel := ShellSelector() + for _, id := range SupportedIDs() { + if !strings.Contains(sel, id) { + t.Errorf("selector missing arm for supported id %q", id) + } + } + for _, want := range []string{"*)", "unsupported base image", "exit 1"} { + if !strings.Contains(sel, want) { + t.Errorf("selector missing fast-reject piece %q", want) + } + } + // Alpine is supported via the OpenRC track — and it must be the OpenRC + // profile, never folded into a systemd family. + alpine := profileByKey(t, "alpine") + if alpine.Init != InitOpenRC { + t.Errorf("alpine must be the OpenRC profile, got init %q", alpine.Init) + } +} + +// Every profile declares a known init system with a rendered setup body, and +// no body leaks another init system's tooling (systemctl in OpenRC or +// rc-update in systemd would fail at provisioning time). +func TestInitSystemsDeclaredAndCoherent(t *testing.T) { + t.Parallel() + for _, p := range Profiles { + setup, ok := initSetup[p.Init] + if !ok { + t.Errorf("profile %q declares init %q with no setup body", p.Key, p.Init) + + continue + } + switch p.Init { + case InitSystemd: + if strings.Contains(setup, "rc-update") { + t.Errorf("systemd init setup leaks rc-update (profile %q)", p.Key) + } + case InitOpenRC: + if strings.Contains(setup, "systemctl") { + t.Errorf("openrc init setup leaks systemctl (profile %q)", p.Key) + } + } + } + sel := ShellSelector() + if !strings.Contains(sel, "e2b_init_setup() {") { + t.Error("selector must define e2b_init_setup()") + } + // The OpenRC boot chain pieces the alpine arm must carry. + for _, want := range []string{"/etc/inittab", "rc-update add envd default", "openrc sysinit"} { + if !strings.Contains(sel, want) { + t.Errorf("selector missing OpenRC boot piece %q", want) + } + } +} + +// The cache fingerprint must cover the whole generated provisioning contract: +// stable across calls, and carrying both the selector text and the explicit +// Version (a profile change must rotate the base-layer cache key). +func TestFingerprintStableAndVersioned(t *testing.T) { + t.Parallel() + a, b := Fingerprint(), Fingerprint() + if a != b || len(a) != 64 { + t.Errorf("fingerprint must be a stable sha256 hex: %q vs %q", a, b) + } + want := sha256.Sum256([]byte(Version + "\x00" + ShellSelector())) + if a != hex.EncodeToString(want[:]) { + t.Error("fingerprint must hash Version + selector text") + } +} + +// Sanity: RHEL-family aliases (rocky/alma/oracle/amazon) all resolve to one arm. +func TestRHELFamilyAliases(t *testing.T) { + t.Parallel() + rhel := profileByKey(t, "rhel") + for _, want := range []string{"fedora", "rhel", "centos", "rocky", "almalinux", "ol", "amzn"} { + found := false + for _, id := range rhel.IDs { + if id == want { + found = true + } + } + if !found { + t.Errorf("rhel family missing alias %q", want) + } + } +} diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/init.go b/packages/orchestrator/pkg/template/build/phases/base/distro/init.go new file mode 100644 index 0000000000..0f925d7b88 --- /dev/null +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/init.go @@ -0,0 +1,132 @@ +// Init-system axis of the distro profiles: one provisioning-time shell block +// per init system, rendered into the selector as e2b_init_setup() so provision.sh +// stays init-agnostic. +package distro + +import "strings" + +// InitSystem is the guest init family a profile boots with. +type InitSystem string + +const ( + // InitSystemd — Debian/Ubuntu, RHEL/Fedora, Arch (systemd via envd.service). + InitSystemd InitSystem = "systemd" + // InitOpenRC — Alpine (busybox init → OpenRC via the baked /etc/init.d/envd). + InitOpenRC InitSystem = "openrc" +) + +// initSetup is the provisioning-time shell block per init system. Bodies may +// reference the selector's profile variables (e.g. $E2B_TIMESYNC_UNIT), defined +// in the same case arm before the function is called. +var initSetup = map[InitSystem]string{ + InitSystemd: `echo "Don't wait for ttyS0 (serial console kernel logs)" +# This is required when the Firecracker kernel args has specified console=ttyS0 +systemctl mask serial-getty@ttyS0.service + +echo "Disable network online wait" +systemctl mask systemd-networkd-wait-online.service + +echo "Disable system first boot wizard" +# This was problem with Ubuntu 24.04, that differently calculate wizard should be called +# and Linux boot was stuck in wizard until envd wait timeout +systemctl mask systemd-firstboot.service + +echo "Enable time synchronization ($E2B_TIMESYNC_UNIT)" +# Distro-correct chrony unit (chrony on Debian, chronyd on RHEL/Arch). +systemctl enable "$E2B_TIMESYNC_UNIT" + +echo "Enable envd autostart" +# Belt-and-suspenders with the baked 00-e2b.preset: on the RHEL family the +# package transaction above runs systemd's RPM scriptlet 'systemctl preset-all' +# (policy 'disable *'), which deletes the baked wants-symlink. +systemctl enable envd.service + +echo "Disable chrony-wait" +# chrony-wait blocks multi-user.target until the first clock sync (~8s); +# chrony still syncs in the background, nothing needs to wait for it. +# masking a unit that doesn't exist on this distro still succeeds (systemctl +# mask just writes the /dev/null symlink), so a failure here is real. +systemctl mask chrony-wait.service + +echo "Disable slow boot units not needed in the sandbox" +# binfmt registrations (foreign-arch exec) take ~1s of CPU early in boot and +# compete with envd start; e2scrub is for LVM-backed ext4 only. +systemctl mask systemd-binfmt.service +systemctl mask e2scrub_reap.service`, + + // OpenRC (Alpine). The image is still running the one-shot PROVISIONING + // inittab (it is what launched this script); replace it with the real + // boot sequence and wire the runlevels a container image ships without. + InitOpenRC: `echo "Installing boot inittab (busybox init -> OpenRC runlevels)" +printf '%s\n' \ + '::sysinit:/sbin/openrc sysinit' \ + '::sysinit:/sbin/openrc boot' \ + '::wait:/sbin/openrc default' \ + '::shutdown:/sbin/openrc shutdown' \ + '::ctrlaltdel:/sbin/reboot' \ + > /etc/inittab + +echo "Registering base OpenRC services" +# Container images carry no runlevel wiring at all (setup-alpine does this on +# real installs): kernel filesystems in sysinit, system prep in boot. bootmisc +# also wipes /tmp in the boot runlevel, so envd (default runlevel) can never +# race the wipe — the ordering systemd needs After= for is inherent here. +# Which scripts exist varies by image (mdev vs udev, procfs presence) — check +# and say so instead of swallowing rc-update errors. +for svc in devfs sysfs procfs dmesg mdev; do + if [ -e "/etc/init.d/$svc" ]; then + rc-update add "$svc" sysinit + else + echo "OpenRC service $svc not present on this image; skipping" + fi +done +for svc in localmount sysctl hostname bootmisc; do + if [ -e "/etc/init.d/$svc" ]; then + rc-update add "$svc" boot + else + echo "OpenRC service $svc not present on this image; skipping" + fi +done + +# The FC guest's eth0 is configured by the kernel (ip=), but OpenRC services +# declaring a "need net" dependency (chronyd) trigger the networking service, +# which errors out on a missing /etc/network/interfaces and takes chronyd +# down with it. A loopback-only interfaces file lets networking start (and +# provide "net") without touching the kernel-managed eth0. +printf 'auto lo\niface lo inet loopback\n' > /etc/network/interfaces +if [ -e /etc/init.d/networking ]; then + rc-update add networking boot +else + echo "OpenRC networking service not present on this image; services needing 'net' must not be enabled" +fi + +echo "Enable time synchronization ($E2B_TIMESYNC_UNIT)" +rc-update add "$E2B_TIMESYNC_UNIT" default + +echo "Enable envd autostart" +# The service script is baked at a neutral path (envd.openrc.tpl) so the +# Debian family's update-rc.d never sees it; install it for OpenRC here. +cp /usr/local/share/e2b/envd.openrc /etc/init.d/envd +chmod 0755 /etc/init.d/envd +rc-update add envd default + +echo "Enable sshd" +if [ -e /etc/init.d/sshd ]; then + rc-update add sshd default +else + echo "sshd service not present on this image; skipping" +fi`, +} + +// indentBlock indents every non-empty line of a shell block for embedding +// inside the generated selector function bodies. +func indentBlock(s, prefix string) string { + lines := strings.Split(s, "\n") + for i, l := range lines { + if l != "" { + lines[i] = prefix + l + } + } + + return strings.Join(lines, "\n") +} diff --git a/packages/orchestrator/pkg/template/build/phases/base/files.go b/packages/orchestrator/pkg/template/build/phases/base/files.go index afced98712..1563226a92 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/files.go +++ b/packages/orchestrator/pkg/template/build/phases/base/files.go @@ -15,6 +15,7 @@ import ( "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/config" "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs" "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/phases" + "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/phases/base/distro" artifactsregistry "github.com/e2b-dev/infra/packages/shared/pkg/artifacts-registry" "github.com/e2b-dev/infra/packages/shared/pkg/dockerhub" "github.com/e2b-dev/infra/packages/shared/pkg/featureflags" @@ -45,9 +46,10 @@ func constructLayerFilesFromOCI( featureFlags, ) provisionScript, err := getProvisionScript(ctx, ProvisionScriptParams{ - BusyBox: rootfs.SandboxBusyBoxPath, - ResultPath: provisionScriptResultPath, - Provider: buildContext.BuilderConfig.Provider, + BusyBox: rootfs.SandboxBusyBoxPath, + ResultPath: provisionScriptResultPath, + Provider: buildContext.BuilderConfig.Provider, + DistroSelector: distro.ShellSelector(), }) if err != nil { return nil, nil, containerregistry.Config{}, fmt.Errorf("error getting provision script: %w", err) diff --git a/packages/orchestrator/pkg/template/build/phases/base/hash.go b/packages/orchestrator/pkg/template/build/phases/base/hash.go index c5bc670322..39235ea5bf 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/hash.go +++ b/packages/orchestrator/pkg/template/build/phases/base/hash.go @@ -11,6 +11,7 @@ import ( "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs" "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/phases" + "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/phases/base/distro" "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/storage/cache" "github.com/e2b-dev/infra/packages/shared/pkg/featureflags" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" @@ -34,10 +35,12 @@ func (bb *BaseBuilder) Hash(ctx context.Context, _ phases.LayerResult) (string, baseSource = bb.Config.FromImage } - // For fallback/dev environments, include baked rootfs file contents in the - // provision version. In production, BuildProvisionVersion controls rollout - // invalidation explicitly. - provisionVersion := cache.HashKeys(provisionScriptFile, rootfs.FilesHash()) + // For fallback/dev environments, include baked rootfs file contents and + // the distro provisioning contract (profiles + init blocks — the rendered + // selector is part of the script but not of the raw provisionScriptFile + // hashed here) in the provision version. In production, + // BuildProvisionVersion controls rollout invalidation explicitly. + provisionVersion := cache.HashKeys(provisionScriptFile, rootfs.FilesHash(), distro.Fingerprint()) if val := bb.featureFlags.IntFlag( ctx, featureflags.BuildProvisionVersion, diff --git a/packages/orchestrator/pkg/template/build/phases/base/provision.go b/packages/orchestrator/pkg/template/build/phases/base/provision.go index 1ece2e91c2..1e94f6ce5e 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/provision.go +++ b/packages/orchestrator/pkg/template/build/phases/base/provision.go @@ -53,6 +53,9 @@ type ProvisionScriptParams struct { BusyBox string ResultPath string Provider string + // DistroSelector is the generated POSIX-sh block that selects the base + // image's distro profile by its /etc/os-release ID. + DistroSelector string } func getProvisionScript( @@ -96,6 +99,18 @@ func (bb *BaseBuilder) provisionSandbox( done.SetError(e) }() + // Rolling tail of the guest's own provisioning output (the + // prefix-marked lines only — kernel logs are unmarked). The full + // stream is logged at debug, invisible in customer build logs; on + // failure the tail is attached to the error, which IS user-visible, + // so rejections name their real reason ("unsupported base image + // distribution: …") instead of a bare exit status. + // 30 lines, because package managers emit per-package error spam that + // rotated the ROOT error ("No space left on device") out of an + // 8-line window (observed with yum on Amazon Linux 2). + const failureTailLines = 30 + var tail []string + scanner := bufio.NewScanner(exitCodeReader) for scanner.Scan() { line := scanner.Text() @@ -106,8 +121,20 @@ func (bb *BaseBuilder) provisionSandbox( return nil } + if len(tail) > 0 { + return fmt.Errorf("exit status: %s; provisioning output tail:\n%s", exitStatus, strings.Join(tail, "\n")) + } + return fmt.Errorf("exit status: %s", exitStatus) } + if after, ok := strings.CutPrefix(line, logExternalPrefix); ok { + if trimmed := strings.TrimSpace(after); trimmed != "" { + tail = append(tail, trimmed) + if len(tail) > failureTailLines { + tail = tail[1:] + } + } + } } if err := scanner.Err(); err != nil && !errors.Is(err, io.EOF) { diff --git a/packages/orchestrator/pkg/template/build/phases/base/provision.sh b/packages/orchestrator/pkg/template/build/phases/base/provision.sh index 56ecd6d176..00d81796bd 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/provision.sh +++ b/packages/orchestrator/pkg/template/build/phases/base/provision.sh @@ -13,13 +13,30 @@ echo "Starting provisioning script" echo "Making configuration immutable" $BUSYBOX chattr +i /etc/resolv.conf -# Helper function to check if a package is installed +# Identify the base image by its DECLARED /etc/os-release ID, never by probing +# for a package manager. The selector (generated from the distro profile +# registry) defines the E2B_* vars and pkg/CA shell functions, or exits 1 on an +# unsupported distribution. Images with no os-release (distroless, scratch) are +# rejected by name rather than guessed. +echo "Detecting base image distribution" +if [ -r /etc/os-release ]; then + . /etc/os-release + E2B_DISTRO_ID="${ID:-unknown}" +else + E2B_DISTRO_ID="unknown (image has no /etc/os-release)" +fi + +{{ .DistroSelector }} + +echo "Provisioning for distro '$E2B_DISTRO_ID' (init=$E2B_INIT_BIN, timesync=$E2B_TIMESYNC_UNIT, admin-group=$E2B_ADMIN_GROUP)" + +# Helper function to check if a package is installed (distro-specific query) is_package_installed() { - dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -q "install ok installed" + e2b_pkg_query "$1" } # Install required packages if not already installed -PACKAGES="systemd systemd-sysv openssh-server sudo chrony socat curl ca-certificates fuse3 iptables git nfs-common less nftables iputils-ping jq" +PACKAGES="$E2B_PACKAGES" echo "Checking presence of the following packages: $PACKAGES" MISSING="" @@ -32,18 +49,29 @@ done if [ -n "$MISSING" ]; then echo "Missing packages detected, installing:$MISSING" - apt-get -q update - DEBIAN_FRONTEND=noninteractive DEBCONF_NOWARNINGS=yes apt-get -qq -o=Dpkg::Use-Pty=0 install -y --no-install-recommends $MISSING + # shellcheck disable=SC2086 + e2b_pkg_install $MISSING else echo "All required packages are already installed." fi +# Ensure the CA trust bundle exists where envd expects it; e2b_ca_refresh +# regenerates it per family. A refresh failure fails provisioning (set -e) — a +# silently broken trust store is worse than a legible build error. +if [ ! -s "$E2B_CA_BUNDLE" ]; then + echo "CA trust bundle missing at $E2B_CA_BUNDLE — running the profile's refresh" + e2b_ca_refresh +fi + # Set /dev/fuse permissions to 666 for non-root access # Use systemd-tmpfiles to set permissions at boot mkdir -p /etc/tmpfiles.d echo 'z /dev/fuse 0666 root root -' > /etc/tmpfiles.d/fuse.conf echo "Setting up shell" +# Not every base image ships /etc/profile.d or /root; create them so the +# drop-ins below always have a home. +mkdir -p /etc/profile.d /root echo "export SHELL='/bin/bash'" >/etc/profile.d/shell.sh echo "export PS1='\w \$ '" >/etc/profile.d/prompt.sh echo "export PS1='\w \$ '" >>"/etc/profile" @@ -53,19 +81,37 @@ echo "Use .bashrc and .profile" echo "if [ -f ~/.bashrc ]; then source ~/.bashrc; fi; if [ -f ~/.profile ]; then source ~/.profile; fi" >>/etc/profile echo "Remove root password" -passwd -d root +# A minimal image may not ship /etc/passwd; only clear the root password when it +# exists rather than failing provisioning. +if [ -f /etc/passwd ]; then + passwd -d root +else + echo "No /etc/passwd on this image; skipping root password removal" +fi echo "Setting up chrony" mkdir -p /etc/chrony -cat </etc/chrony/chrony.conf -refclock PHC /dev/ptp0 poll 2 dpoll 2 -# Step (jump) the clock instead of slewing when the offset exceeds 1s, but only -# for the first 3 updates after chronyd starts. chronyd restarts on every cold -# boot/reboot, so this corrects a large boot-time offset fast (TLS needs a -# correct clock) without risking a backward jump under a running workload. -# Needed because chrony-wait is masked, so boot no longer blocks on first sync. -makestep 1.0 3 -EOF +{ + # Prefer the hypervisor's PTP clock (kvm-ptp): no network dependency and + # it tracks the host directly. It is missing where nested virtualization + # can't expose it (e.g. dev slots) — chronyd treats a missing PHC as a + # FATAL error, so only reference it when the device exists and fall back + # to NTP otherwise; a running chronyd without PHC beats a dead one. + # Device presence is probed in the provisioning VM, which runs on the same + # host/KVM as the runtime sandboxes. + if [ -e /dev/ptp0 ]; then + echo "refclock PHC /dev/ptp0 poll 2 dpoll 2" + else + echo "pool pool.ntp.org iburst maxsources 3" + fi + # Step (jump) the clock instead of slewing when the offset exceeds 1s, but + # only for the first 3 updates after chronyd starts. chronyd restarts on + # every cold boot/reboot, so this corrects a large boot-time offset fast + # (TLS needs a correct clock) without risking a backward jump under a + # running workload. Needed because chrony-wait is masked, so boot no + # longer blocks on first sync. + echo "makestep 1.0 3" +} >/etc/chrony/chrony.conf # Add a proxy config, as some environments expects it there (e.g. timemaster in Node Dockerimage) echo "include /etc/chrony/chrony.conf" >/etc/chrony.conf @@ -89,34 +135,20 @@ echo 'fs.inotify.max_user_watches=65536' | tee -a /etc/sysctl.conf echo "Disabling proactive memory compaction" echo 'vm.compaction_proactiveness=0' | tee -a /etc/sysctl.conf -echo "Don't wait for ttyS0 (serial console kernel logs)" -# This is required when the Firecracker kernel args has specified console=ttyS0 -systemctl mask serial-getty@ttyS0.service - -echo "Disable network online wait" -systemctl mask systemd-networkd-wait-online.service - -echo "Disable system first boot wizard" -# This was problem with Ubuntu 24.04, that differently calculate wizard should be called -# and Linux boot was stuck in wizard until envd wait timeout -systemctl mask systemd-firstboot.service - -echo "Disable chrony-wait" -# chrony-wait blocks multi-user.target until the first clock sync (~8s); -# chrony still syncs in the background, nothing needs to wait for it. -systemctl mask chrony-wait.service - -echo "Disable slow boot units not needed in the sandbox" -# binfmt registrations (foreign-arch exec) take ~1s of CPU early in boot and -# compete with envd start; e2scrub is for LVM-backed ext4 only. -systemctl mask systemd-binfmt.service -systemctl mask e2scrub_reap.service +# Init-system-specific boot arrangement (autostarts, boot-noise silencing, the +# OpenRC inittab swap), rendered per family by the distro selector (distro/init.go). +e2b_init_setup # Clean machine-id from Docker rm -rf /etc/machine-id -echo "Linking systemd to init" -ln -sf /lib/systemd/systemd /usr/sbin/init +echo "Linking $E2B_INIT_BIN to init" +# Not every image ships /usr/sbin; create it before linking init. +mkdir -p /usr/sbin +ln -sf "$E2B_INIT_BIN" /usr/sbin/init +# /sbin is a real directory on non-usr-merged distros (Alpine) where the line +# above doesn't reach the /sbin/init the kernel is pointed at; link it too. +[ -L /sbin ] || ln -sf "$E2B_INIT_BIN" /sbin/init echo "Unlocking immutable configuration" $BUSYBOX chattr -i /etc/resolv.conf diff --git a/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh b/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh index 76202a81a5..fd9ed077d0 100644 --- a/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh +++ b/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh @@ -10,23 +10,51 @@ TEMPLATE_ID={{ .TemplateID }} BUILD_ID={{ .BuildID }} EOF -# Create default user. -# if the /home/user directory exists, we copy the skeleton files to it because the adduser command -# will ignore the directory if it exists, but we want to include the skeleton files in the home directory -# in our case. +# Create default user. useradd is in shadow(-utils) on every supported distro +# family; -m creates the home dir, -s the shell. A creation failure is a real +# error — a template whose default user silently doesn't exist fails much more +# confusingly later. echo "Create default user 'user' (if doesn't exist yet)" -ADDUSER_OUTPUT=$(adduser -disabled-password --gecos "" user 2>&1 || true) -echo "$ADDUSER_OUTPUT" -if echo "$ADDUSER_OUTPUT" | grep -q "The home directory \`/home/user' already exists"; then - # Copy skeleton files if they don't exist in the home directory - echo "Copy skeleton files to /home/user" - cp -rn /etc/skel/. /home/user/ +if ! id -u user >/dev/null 2>&1; then + useradd -m -s /bin/bash user +fi +# useradd -m skips skeleton files when /home/user already exists, so copy them +# explicitly (no-clobber). Not every image ships /etc/skel — say so instead of +# hiding it. Walked file-by-file because `cp -n` exits non-zero when it skips an +# existing file on coreutils >= 9.2 (Fedora 40+), which is a skip, not a failure. +if [ -d /home/user ]; then + if [ -d /etc/skel ]; then + echo "Copy skeleton files to /home/user (keeping existing files)" + (cd /etc/skel && find . -type f -print | while IFS= read -r f; do + if [ ! -e "/home/user/$f" ]; then + mkdir -p "/home/user/$(dirname "$f")" + cp -a "/etc/skel/$f" "/home/user/$f" + fi + done) + else + echo "No /etc/skel on this image; skipping skeleton copy" + fi fi echo "Add sudo to 'user' with no password" -usermod -aG sudo user +# Admin group differs by distro (sudo on Debian/Ubuntu, wheel elsewhere); the +# NOPASSWD sudoers entry below is what actually grants privileges. Neither +# group existing is a real error. +if getent group sudo >/dev/null; then + usermod -aG sudo user +elif getent group wheel >/dev/null; then + usermod -aG wheel user +else + echo "ERROR: neither the sudo nor the wheel group exists on this image" >&2 + exit 1 +fi passwd -d user -echo "user ALL=(ALL:ALL) NOPASSWD: ALL" >>/etc/sudoers +# Skip the append when the entry is already present. +if grep -q '^user ALL=(ALL:ALL) NOPASSWD: ALL' /etc/sudoers; then + echo "sudoers entry already present" +else + echo "user ALL=(ALL:ALL) NOPASSWD: ALL" >>/etc/sudoers +fi echo "Give 'user' ownership to /home/user" mkdir -p /home/user From c5c9f88273a8a1e665b0d6b12cc005be3d4e954a Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Mon, 27 Jul 2026 11:59:41 +0200 Subject: [PATCH 02/17] fix(envd): degrade process-priority helpers when the image lacks them ionice/nice are util-linux/coreutils conveniences that minimal or busybox-based images may not ship; a missing helper now drops the priority adjustment instead of failing the spawn (exit 127 previously killed every process on such images). --- .../services/process/handler/handler.go | 25 +++++++++- .../services/process/handler/handler_test.go | 47 +++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 packages/envd/internal/services/process/handler/handler_test.go diff --git a/packages/envd/internal/services/process/handler/handler.go b/packages/envd/internal/services/process/handler/handler.go index e30320944d..355294b45d 100644 --- a/packages/envd/internal/services/process/handler/handler.go +++ b/packages/envd/internal/services/process/handler/handler.go @@ -89,6 +89,25 @@ func currentNice() int { return 20 - prio } +// wrapperPrefix builds the priority-tool part of the process wrapper from +// whatever the image actually ships. ionice/nice are util-linux/ +// coreutils conveniences that minimal and busybox-based images (Alpine, UBI) +// may lack or keep elsewhere than /usr/bin — a missing helper must degrade to +// running the command without that priority adjustment, never to a failed +// spawn (exit 127 killed every process on such images). lookPath is injected +// for testability; production passes exec.LookPath. +func wrapperPrefix(niceDelta int, lookPath func(string) (string, error)) string { + prefix := "" + if p, err := lookPath("ionice"); err == nil { + prefix += p + " -c 2 -n 4 " + } + if p, err := lookPath("nice"); err == nil { + prefix += fmt.Sprintf("%s -n %d ", p, niceDelta) + } + + return prefix +} + func New( ctx context.Context, user *user.User, @@ -101,9 +120,11 @@ func New( // User command string for logging (without the internal wrapper details). userCmd := strings.Join(append([]string{req.GetProcess().GetCmd()}, req.GetProcess().GetArgs()...), " ") - // Wrap in a shell that resets oom_score_adj, ioprio (ionice best-effort/4), and nice. + // Wrap in a shell that resets oom_score_adj, ioprio (ionice best-effort/4), + // and nice. The oom_score_adj write is pure /proc and always applied; the + // priority helpers are used only where the image provides them. niceDelta := defaultNice - currentNice() - oomWrapperScript := fmt.Sprintf(`echo %d > /proc/$$/oom_score_adj && exec /usr/bin/ionice -c 2 -n 4 /usr/bin/nice -n %d "${@}"`, defaultOomScore, niceDelta) + oomWrapperScript := fmt.Sprintf(`echo %d > /proc/$$/oom_score_adj && exec %s"${@}"`, defaultOomScore, wrapperPrefix(niceDelta, exec.LookPath)) wrapperArgs := append([]string{"-c", oomWrapperScript, "--", req.GetProcess().GetCmd()}, req.GetProcess().GetArgs()...) cmd := exec.CommandContext(ctx, "/bin/sh", wrapperArgs...) diff --git a/packages/envd/internal/services/process/handler/handler_test.go b/packages/envd/internal/services/process/handler/handler_test.go new file mode 100644 index 0000000000..05d670147d --- /dev/null +++ b/packages/envd/internal/services/process/handler/handler_test.go @@ -0,0 +1,47 @@ +package handler + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" +) + +// The process wrapper must degrade cleanly when the priority helpers are absent +// — the user command still runs; a present helper is applied with its resolved +// path. +func TestWrapperPrefix(t *testing.T) { + t.Parallel() + + notFound := func(string) (string, error) { return "", errors.New("not found") } + all := func(name string) (string, error) { return "/usr/bin/" + name, nil } + only := func(want string) func(string) (string, error) { + return func(name string) (string, error) { + if name == want { + return "/bin/" + name, nil + } + + return "", errors.New("not found") + } + } + + t.Run("both present", func(t *testing.T) { + t.Parallel() + assert.Equal(t, "/usr/bin/ionice -c 2 -n 4 /usr/bin/nice -n 5 ", wrapperPrefix(5, all)) + }) + + t.Run("both absent degrades to bare exec", func(t *testing.T) { + t.Parallel() + assert.Empty(t, wrapperPrefix(5, notFound)) + }) + + t.Run("only nice", func(t *testing.T) { + t.Parallel() + assert.Equal(t, "/bin/nice -n -3 ", wrapperPrefix(-3, only("nice"))) + }) + + t.Run("only ionice", func(t *testing.T) { + t.Parallel() + assert.Equal(t, "/bin/ionice -c 2 -n 4 ", wrapperPrefix(0, only("ionice"))) + }) +} From 73df3d744cc9b02998a75279355619b63f259c59 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Mon, 27 Jul 2026 12:39:09 +0200 Subject: [PATCH 03/17] fix(orch): document Arch CA bundle needs no manual link Arch ships /etc/ssl/certs/ca-certificates.crt as a symlink to the p11-kit bundle that update-ca-trust extract regenerates, so the bundle lands at $E2B_CA_BUNDLE without an explicit ln -sf (unlike the RHEL family, whose update-ca-trust writes only under /etc/pki). Restores the rationale a prior comment trim dropped. --- .../pkg/template/build/phases/base/distro/distro.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go b/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go index 4654444177..3ff92d18cf 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go @@ -97,7 +97,9 @@ var Profiles = []Profile{ TimeSyncUnit: "chronyd", AdminGroup: "wheel", CABundle: "/etc/ssl/certs/ca-certificates.crt", - CARefresh: "update-ca-trust extract", + // Arch ships /etc/ssl/certs/ca-certificates.crt as a symlink to the p11-kit + // bundle update-ca-trust extract regenerates — no manual link, unlike RHEL. + CARefresh: "update-ca-trust extract", }, { Key: "alpine", From 6e03a1de4648e0bd8ad928133280cfd1bddcfa16 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Mon, 27 Jul 2026 13:01:34 +0200 Subject: [PATCH 04/17] fix(orch): install ionice on Alpine so user processes leave realtime IO MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Alpine's busybox does not ship ionice; it comes from util-linux-misc. envd runs at realtime IO (supervise-daemon --ionice 1:4) and resets each spawned user process to best-effort via the binary, degrading to no reset when it is absent — so on Alpine user workloads inherited envd's realtime IO class. --- .../pkg/template/build/phases/base/distro/distro.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go b/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go index 3ff92d18cf..032a1036f7 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go @@ -106,10 +106,13 @@ var Profiles = []Profile{ Init: InitOpenRC, IDs: []string{"alpine"}, // shadow provides useradd/usermod (busybox's adduser takes different flags). + // util-linux-misc provides ionice, which busybox does not ship: envd runs at + // realtime IO (supervise-daemon --ionice 1:4) and resets each user process to + // best-effort with it, so without the binary they'd inherit realtime IO. Packages: []string{ "openrc", "shadow", "openssh", "sudo", "chrony", "socat", "curl", "ca-certificates", "fuse3", "iptables", "git", "nfs-utils", "less", - "nftables", "iputils", "jq", "bash", + "nftables", "iputils", "jq", "bash", "util-linux-misc", }, PkgQueryBody: `apk info -e "$1" >/dev/null 2>&1`, PkgInstall: `apk add --no-cache "$@"`, From d7408b33879fc4dca17b50d42365b3c563f6c6ff Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Mon, 27 Jul 2026 13:30:21 +0200 Subject: [PATCH 05/17] fix(orch): refresh the CA trust store per family when packing and seeding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both the finalize pack step and the boot-time seed fallback probed only for update-ca-certificates, which the RHEL family and Arch do not ship — the guard turned into a silent no-op there, so CAs added in later build layers never merged into the packed tar and a cold-start regeneration was impossible. Probe update-ca-trust as well, and on RHEL copy the bundle it writes under /etc/pki to the Debian-named path envd appends to. --- .../template/build/core/rootfs/files/seed-certs.sh.tpl | 10 +++++++++- .../pkg/template/build/phases/finalize/configure.go | 6 ++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/files/seed-certs.sh.tpl b/packages/orchestrator/pkg/template/build/core/rootfs/files/seed-certs.sh.tpl index b078a49be2..5ae34599d3 100644 --- a/packages/orchestrator/pkg/template/build/core/rootfs/files/seed-certs.sh.tpl +++ b/packages/orchestrator/pkg/template/build/core/rootfs/files/seed-certs.sh.tpl @@ -35,10 +35,18 @@ fi if [ ! -s /etc/ssl/certs/ca-certificates.crt ]; then if command -v update-ca-certificates >/dev/null 2>&1; then update-ca-certificates + elif command -v update-ca-trust >/dev/null 2>&1; then + # RHEL family and Arch refresh with update-ca-trust. Arch's extract emits + # the Debian-named bundle itself; RHEL keeps it under /etc/pki, and the + # symlink provisioning made was dereferenced into this tmpfs, so copy it. + update-ca-trust extract + if [ ! -s /etc/ssl/certs/ca-certificates.crt ] && [ -s /etc/pki/tls/certs/ca-bundle.crt ]; then + cp -L /etc/pki/tls/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt + fi else # Provisioning guarantees the bundle on every supported family; # reaching this means the image diverged after the build. - echo "e2b-seed-certs: CA bundle missing and no update-ca-certificates on this image; TLS trust will be degraded" >&2 + echo "e2b-seed-certs: CA bundle missing and no CA refresh tool on this image; TLS trust will be degraded" >&2 fi fi diff --git a/packages/orchestrator/pkg/template/build/phases/finalize/configure.go b/packages/orchestrator/pkg/template/build/phases/finalize/configure.go index 78059df3ed..f9bf9b8c7b 100644 --- a/packages/orchestrator/pkg/template/build/phases/finalize/configure.go +++ b/packages/orchestrator/pkg/template/build/phases/finalize/configure.go @@ -34,9 +34,15 @@ var ConfigureScriptTemplate = tt.Must(tt.New("provisioning-finish-script").Parse // on cold boot, trading the regen's scattered rootfs reads for one sequential // read. -h dereferences the hash-named symlinks so the real cert contents are // packed instead of links that would still fault the lazily-fetched rootfs. +// The refresh is per-family: Debian and Alpine ship update-ca-certificates, the +// RHEL family and Arch ship update-ca-trust instead, and NixOS has neither +// (its bundle comes from the image configuration). Probing both keeps the merge +// working everywhere rather than silently skipping it off Debian. const packCertBundleCmd = `set -e if command -v update-ca-certificates >/dev/null 2>&1; then update-ca-certificates +elif command -v update-ca-trust >/dev/null 2>&1; then + update-ca-trust extract fi mkdir -p /usr/local/share/e2b tar -C /etc/ssl/certs -chf /usr/local/share/e2b/ssl-certs.tar . From ae044bb6fa4be5d3222da6be15a37f690d12ebc3 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Mon, 27 Jul 2026 14:32:10 +0200 Subject: [PATCH 06/17] fix(orch): keep the finalize cert pack Debian-only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverts the update-ca-trust branch added for the RHEL family: on Fedora, extract regenerates the extracted/pem/directory-hash tree that /etc/ssl/certs points at and replaces provisioning's absolute ca-certificates.crt symlink with a relative one, which tar -h packs as a link rather than dereferencing. Verified on real KVM — the packed bundle went from a 226489-byte regular file to a 20-byte symlink, breaking the real-file contract envd's egress-CA append needs. The seed-time fallback keeps its update-ca-trust path, where cp -L dereferences. --- .../template/build/phases/finalize/configure.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/orchestrator/pkg/template/build/phases/finalize/configure.go b/packages/orchestrator/pkg/template/build/phases/finalize/configure.go index f9bf9b8c7b..2e521592e0 100644 --- a/packages/orchestrator/pkg/template/build/phases/finalize/configure.go +++ b/packages/orchestrator/pkg/template/build/phases/finalize/configure.go @@ -34,15 +34,17 @@ var ConfigureScriptTemplate = tt.Must(tt.New("provisioning-finish-script").Parse // on cold boot, trading the regen's scattered rootfs reads for one sequential // read. -h dereferences the hash-named symlinks so the real cert contents are // packed instead of links that would still fault the lazily-fetched rootfs. -// The refresh is per-family: Debian and Alpine ship update-ca-certificates, the -// RHEL family and Arch ship update-ca-trust instead, and NixOS has neither -// (its bundle comes from the image configuration). Probing both keeps the merge -// working everywhere rather than silently skipping it off Debian. +// Deliberately Debian/Alpine-only. Adding an update-ca-trust branch for the RHEL +// family regresses it: extract regenerates the extracted/pem/directory-hash tree +// that /etc/ssl/certs points at, replacing the absolute ca-certificates.crt +// symlink provisioning created with a relative one that tar -h then packs as a +// link instead of dereferencing — the packed bundle drops from ~226 KB of PEM to +// a 20-byte symlink, and envd's egress-CA append needs a real file. Provisioning +// already refreshes the store per family; this step only has to merge CAs that +// later build layers dropped in, which is a Debian/Alpine convention anyway. const packCertBundleCmd = `set -e if command -v update-ca-certificates >/dev/null 2>&1; then update-ca-certificates -elif command -v update-ca-trust >/dev/null 2>&1; then - update-ca-trust extract fi mkdir -p /usr/local/share/e2b tar -C /etc/ssl/certs -chf /usr/local/share/e2b/ssl-certs.tar . From 2f59d46ee9e4c4c46f99d3418bc2cade7997b469 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Mon, 27 Jul 2026 15:47:10 +0200 Subject: [PATCH 07/17] fix(orch): put -R before the mode in configure.sh chmods MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit chmod 777 -R relies on options being accepted after an operand, which POSIX does not require: GNU chmod only allows it when POSIXLY_CORRECT is unset, and non-GNU implementations may take -R as a filename and, under set -e, abort finalize. Busybox on Alpine happens to accept the current form, so nothing is broken today — this is the unconditionally correct spelling instead. --- .../pkg/template/build/phases/finalize/configure.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh b/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh index fd9ed077d0..7fd47e7650 100644 --- a/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh +++ b/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh @@ -61,11 +61,11 @@ mkdir -p /home/user chown -R user:user /home/user echo "Give 777 permission to /usr/local" -chmod 777 -R /usr/local +chmod -R 777 /usr/local echo "Create /code directory" mkdir -p /code echo "Give 777 permission to /code" -chmod 777 -R /code +chmod -R 777 /code echo "Finished configuration script" From 5f3899f0c658e904063f933abb06c13dfe2d8c44 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Mon, 27 Jul 2026 16:08:44 +0200 Subject: [PATCH 08/17] chore(envd): bump version to 0.6.12 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit wrapperPrefix now resolves ionice/nice through exec.LookPath and drops either helper when the image lacks it, instead of always exec'ing hardcoded /usr/bin paths — a behavioral change, which CLAUDE.md requires be reflected in pkg/version.go. --- packages/envd/pkg/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/envd/pkg/version.go b/packages/envd/pkg/version.go index abbbb74c05..21011fb23c 100644 --- a/packages/envd/pkg/version.go +++ b/packages/envd/pkg/version.go @@ -1,3 +1,3 @@ package pkg -const Version = "0.6.11" +const Version = "0.6.12" From c4ed072fb2397f74d56ce67ede49b5304a8d739a Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Mon, 27 Jul 2026 16:20:43 +0200 Subject: [PATCH 09/17] fix(orch): enable the SSH unit on the systemd families provision.sh writes the sandbox sshd_config everywhere and the OpenRC branch runs rc-update add sshd, but the systemd branch never enabled a unit. Debian's postinst and the RHEL RPM scriptlet happen to do it themselves; Arch does not, so Arch sandboxes shipped with SSH fully configured and never started (is-enabled=disabled, no sshd process). Carry the unit name per profile the way E2B_TIMESYNC_UNIT already is (ssh on Debian, sshd elsewhere) and enable it. --- .../pkg/template/build/phases/base/distro/distro.go | 6 ++++++ .../pkg/template/build/phases/base/distro/init.go | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go b/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go index 032a1036f7..52151d5078 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go @@ -36,6 +36,7 @@ type Profile struct { PkgInstall string InitBinary string TimeSyncUnit string + SSHUnit string AdminGroup string CABundle string CARefresh string @@ -55,6 +56,7 @@ var Profiles = []Profile{ PkgInstall: "apt-get -q update\n DEBIAN_FRONTEND=noninteractive DEBCONF_NOWARNINGS=yes apt-get -qq -o=Dpkg::Use-Pty=0 install -y --no-install-recommends \"$@\"", InitBinary: "/lib/systemd/systemd", TimeSyncUnit: "chrony", + SSHUnit: "ssh", AdminGroup: "sudo", CABundle: "/etc/ssl/certs/ca-certificates.crt", CARefresh: "update-ca-certificates", @@ -75,6 +77,7 @@ var Profiles = []Profile{ PkgInstall: `dnf -y --allowerasing install "$@" || microdnf -y install "$@" || yum -y install "$@"`, InitBinary: "/usr/lib/systemd/systemd", TimeSyncUnit: "chronyd", + SSHUnit: "sshd", AdminGroup: "wheel", CABundle: "/etc/ssl/certs/ca-certificates.crt", // update-ca-trust never writes ca-certificates.crt (a Debian name), so @@ -95,6 +98,7 @@ var Profiles = []Profile{ PkgInstall: `pacman -Syu --noconfirm --needed "$@"`, InitBinary: "/usr/lib/systemd/systemd", TimeSyncUnit: "chronyd", + SSHUnit: "sshd", AdminGroup: "wheel", CABundle: "/etc/ssl/certs/ca-certificates.crt", // Arch ships /etc/ssl/certs/ca-certificates.crt as a symlink to the p11-kit @@ -118,6 +122,7 @@ var Profiles = []Profile{ PkgInstall: `apk add --no-cache "$@"`, InitBinary: "/bin/busybox", TimeSyncUnit: "chronyd", + SSHUnit: "sshd", AdminGroup: "wheel", CABundle: "/etc/ssl/certs/ca-certificates.crt", CARefresh: "update-ca-certificates", @@ -148,6 +153,7 @@ func ShellSelector() string { fmt.Fprintf(&b, " e2b_pkg_install() { %s; }\n", p.PkgInstall) fmt.Fprintf(&b, " E2B_INIT_BIN=%q\n", p.InitBinary) fmt.Fprintf(&b, " E2B_TIMESYNC_UNIT=%q\n", p.TimeSyncUnit) + fmt.Fprintf(&b, " E2B_SSH_UNIT=%q\n", p.SSHUnit) fmt.Fprintf(&b, " E2B_ADMIN_GROUP=%q\n", p.AdminGroup) fmt.Fprintf(&b, " E2B_CA_BUNDLE=%q\n", p.CABundle) fmt.Fprintf(&b, " e2b_ca_refresh() { %s; }\n", p.CARefresh) diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/init.go b/packages/orchestrator/pkg/template/build/phases/base/distro/init.go index 0f925d7b88..e1bc3ca9c5 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/distro/init.go +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/init.go @@ -35,6 +35,13 @@ echo "Enable time synchronization ($E2B_TIMESYNC_UNIT)" # Distro-correct chrony unit (chrony on Debian, chronyd on RHEL/Arch). systemctl enable "$E2B_TIMESYNC_UNIT" +echo "Enable SSH ($E2B_SSH_UNIT)" +# provision.sh writes the sandbox sshd_config on every family, but nothing was +# turning the unit on: Debian's postinst and the RHEL RPM scriptlet enable it +# themselves, Arch does not, so Arch sandboxes shipped with SSH configured and +# dead. Enabling is idempotent where the packaging already did it. +systemctl enable "$E2B_SSH_UNIT.service" + echo "Enable envd autostart" # Belt-and-suspenders with the baked 00-e2b.preset: on the RHEL family the # package transaction above runs systemd's RPM scriptlet 'systemctl preset-all' From 418d92570d55d47c3844358b247a1a19742d4ec1 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Mon, 27 Jul 2026 17:08:15 +0200 Subject: [PATCH 10/17] chore(orch): make the OpenRC no-networking warning consistent The message told the reader that services needing 'net' must not be enabled, then the next line enabled chronyd, which declares 'need net'. State the actual consequence instead, and note the branch is unreachable on supported images: /etc/init.d/networking ships with openrc, which the Alpine profile installs. --- .../pkg/template/build/phases/base/distro/init.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/init.go b/packages/orchestrator/pkg/template/build/phases/base/distro/init.go index e1bc3ca9c5..99b898c601 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/distro/init.go +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/init.go @@ -104,7 +104,11 @@ printf 'auto lo\niface lo inet loopback\n' > /etc/network/interfaces if [ -e /etc/init.d/networking ]; then rc-update add networking boot else - echo "OpenRC networking service not present on this image; services needing 'net' must not be enabled" + # openrc ships this script and the profile installs openrc, so this is + # unreachable on the images we support. If it ever fires, nothing provides + # "net": chronyd is still enabled below so the failure shows up in the boot + # log rather than the sandbox silently running without time sync. + echo "OpenRC networking service not present on this image; nothing provides 'net', so time sync will fail to start" fi echo "Enable time synchronization ($E2B_TIMESYNC_UNIT)" From 0a71d3f8779ccbe2fee31b9923b5271ebbe50b49 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Tue, 28 Jul 2026 08:43:48 +0000 Subject: [PATCH 11/17] fix(orch): don't declare RHEL, Oracle Linux or Amazon Linux support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sandboxes always boot the kernel E2B supplies, never one from the base image: /lib/modules is empty, no modules can be loaded, SELinux is off, and a distro's kernel-devel resolves against a kernel that isn't running. That is most of what RHEL and Oracle Linux are chosen for — kABI, signed kernel modules, UEK — so accepting `rhel`, `ol` and `amzn` promised a fidelity the sandbox cannot deliver. RHEL's UBI repos additionally carry neither chrony nor nfs-utils, and Amazon Linux 2 exhausts the provisioning disk before finishing its transaction. Narrow the family to the rebuilds people pick for the userland — fedora, centos, rocky, almalinux — which are verified working. The removed IDs now hit the selector's explicit failure path instead of building something subtly wrong or failing for an unrelated-looking reason: `oraclelinux:9` previously spent 68s and died on "needs 23MB more space", and now stops in 12s with "unsupported base image distribution: ID='ol'" naming the supported set. Dropping the IDs changes the generated selector, so Fingerprint rotates the base-layer cache key on its own; no Version bump needed. The yum arm of the install chain stays for yum-era CentOS 7, which `centos` still covers. --- packages/orchestrator/README.md | 3 ++- .../build/phases/base/distro/distro.go | 17 ++++++++++++----- .../build/phases/base/distro/distro_test.go | 18 ++++++++++++++++-- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/packages/orchestrator/README.md b/packages/orchestrator/README.md index 161fd75505..490274e28a 100644 --- a/packages/orchestrator/README.md +++ b/packages/orchestrator/README.md @@ -303,4 +303,5 @@ Automatically set in local mode. Set before running to override: ## Limitations -- Custom template builds support base images from the declared distro families: **Debian/Ubuntu** (apt), the **RHEL family** — Fedora, RHEL, CentOS Stream, Rocky, Alma, Oracle Linux, Amazon Linux — (dnf/microdnf/yum), **Arch** (pacman), and **Alpine** (apk, OpenRC). The distro is resolved from the image's `/etc/os-release` `ID` — never by probing for package managers. Images without an os-release identity (distroless, scratch) or from an undeclared family are rejected fast, with a build-log error naming the reason and the supported families (see `pkg/template/build/phases/base/distro`). Minimal/restricted-repo images (e.g. RHEL UBI) may fail provisioning if their repos don't carry the required packages; the failure is surfaced in the build log. \ No newline at end of file +- Custom template builds support base images from the declared distro families: **Debian/Ubuntu** (apt), the **RHEL family** — Fedora, CentOS Stream, Rocky, Alma — (dnf/microdnf/yum), **Arch** (pacman), and **Alpine** (apk, OpenRC). The distro is resolved from the image's `/etc/os-release` `ID` — never by probing for package managers. Images without an os-release identity (distroless, scratch) or from an undeclared family are rejected fast, with a build-log error naming the reason and the supported families (see `pkg/template/build/phases/base/distro`). Minimal/restricted-repo images may fail provisioning if their repos don't carry the required packages; the failure is surfaced in the build log. +- Sandboxes always boot the kernel E2B supplies, never one from the base image: `/lib/modules` is empty, no kernel modules can be loaded, and SELinux is off. `kernel-devel` from a distro's repos resolves against a kernel that isn't running. **RHEL** (`rhel`, incl. UBI), **Oracle Linux** (`ol`) and **Amazon Linux** (`amzn`) are therefore not accepted even though they are RPM/dnf images — they are chosen for kABI, signed kernel modules and UEK, which this cannot honour. The community rebuilds above are supported because they are chosen for the userland. \ No newline at end of file diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go b/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go index 52151d5078..951d4bd3a9 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go @@ -62,18 +62,25 @@ var Profiles = []Profile{ CARefresh: "update-ca-certificates", }, { - Key: "rhel", // Fedora, RHEL, CentOS Stream, Rocky, Alma, Oracle Linux, Amazon Linux + Key: "rhel", // Fedora, CentOS Stream, Rocky, Alma Init: InitSystemd, - IDs: []string{"fedora", "rhel", "centos", "rocky", "almalinux", "ol", "amzn"}, - // "iptables" (not iptables-nft) and "tar" cover the yum-era members - // (CentOS 7, Amazon Linux 2) that lack the nft package or don't ship tar. + // Deliberately not rhel/ol/amzn. Sandboxes always boot E2B's kernel, so + // /lib/modules is empty and SELinux is off — which is most of what RHEL + // and Oracle Linux are chosen for (kABI, signed kmods, UEK), and + // kernel-devel would resolve against a kernel that isn't running. RHEL's + // UBI repos also carry neither chrony nor nfs-utils. Accepting those IDs + // promised a fidelity this can't deliver; they now fail fast instead. + IDs: []string{"fedora", "centos", "rocky", "almalinux"}, + // "iptables" (not iptables-nft) and "tar" cover yum-era CentOS 7, which + // lacks the nft package and doesn't ship tar. Packages: []string{ "systemd", "shadow-utils", "passwd", "openssh-server", "sudo", "chrony", "socat", "curl", "ca-certificates", "fuse3", "iptables", "git", "nfs-utils", "less", "nftables", "iputils", "jq", "bash", "tar", }, PkgQueryBody: `rpm -q "$1" >/dev/null 2>&1`, - // dnf → microdnf → yum spans the whole family; errors reach the build log. + // dnf → microdnf → yum spans the family (yum for CentOS 7); errors reach + // the build log. PkgInstall: `dnf -y --allowerasing install "$@" || microdnf -y install "$@" || yum -y install "$@"`, InitBinary: "/usr/lib/systemd/systemd", TimeSyncUnit: "chronyd", diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/distro_test.go b/packages/orchestrator/pkg/template/build/phases/base/distro/distro_test.go index 5aa9f70641..0d6a498827 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/distro/distro_test.go +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/distro_test.go @@ -158,11 +158,11 @@ func TestFingerprintStableAndVersioned(t *testing.T) { } } -// Sanity: RHEL-family aliases (rocky/alma/oracle/amazon) all resolve to one arm. +// Sanity: the RHEL-family rebuilds (centos/rocky/alma) all resolve to one arm. func TestRHELFamilyAliases(t *testing.T) { t.Parallel() rhel := profileByKey(t, "rhel") - for _, want := range []string{"fedora", "rhel", "centos", "rocky", "almalinux", "ol", "amzn"} { + for _, want := range []string{"fedora", "centos", "rocky", "almalinux"} { found := false for _, id := range rhel.IDs { if id == want { @@ -174,3 +174,17 @@ func TestRHELFamilyAliases(t *testing.T) { } } } + +// RHEL, Oracle Linux and Amazon Linux are deliberately NOT accepted: sandboxes +// boot E2B's kernel, so the kernel-level fidelity those images are chosen for +// isn't there. Rejecting them up front beats building something subtly wrong. +func TestKernelDependentIDsAreRejected(t *testing.T) { + t.Parallel() + for _, id := range []string{"rhel", "ol", "amzn"} { + for _, got := range SupportedIDs() { + if got == id { + t.Errorf("id %q must not be accepted: E2B supplies the guest kernel", id) + } + } + } +} From e91e5a493d1e9f6077fc1a9a45485df25fe1461a Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Wed, 29 Jul 2026 09:27:40 +0200 Subject: [PATCH 12/17] refactor(envd): name the priority wrapper after its tools and lift ionice args Review feedback: wrapperPrefix was ionice/nice-specific with the class and priority hardcoded in the body; ioniceNicePrefix takes them as arguments. --- .../services/process/handler/handler.go | 22 ++++++++++--------- .../services/process/handler/handler_test.go | 13 +++++++---- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/packages/envd/internal/services/process/handler/handler.go b/packages/envd/internal/services/process/handler/handler.go index 355294b45d..8a9873139e 100644 --- a/packages/envd/internal/services/process/handler/handler.go +++ b/packages/envd/internal/services/process/handler/handler.go @@ -29,6 +29,8 @@ import ( const ( defaultNice = 0 defaultOomScore = 100 + defaultIoClass = 2 // ionice best-effort + defaultIoPrio = 4 outputBufferSize = 64 systemTag = "_system" stdChunkSize = 32 << 10 // 32 KiB @@ -89,17 +91,17 @@ func currentNice() int { return 20 - prio } -// wrapperPrefix builds the priority-tool part of the process wrapper from -// whatever the image actually ships. ionice/nice are util-linux/ -// coreutils conveniences that minimal and busybox-based images (Alpine, UBI) -// may lack or keep elsewhere than /usr/bin — a missing helper must degrade to -// running the command without that priority adjustment, never to a failed -// spawn (exit 127 killed every process on such images). lookPath is injected -// for testability; production passes exec.LookPath. -func wrapperPrefix(niceDelta int, lookPath func(string) (string, error)) string { +// ioniceNicePrefix builds the ionice/nice part of the process wrapper from +// whatever the image actually ships. Both are util-linux/coreutils +// conveniences that minimal and busybox-based images (Alpine, UBI) may lack or +// keep elsewhere than /usr/bin — a missing helper must degrade to running the +// command without that priority adjustment, never to a failed spawn (exit 127 +// killed every process on such images). lookPath is injected for testability; +// production passes exec.LookPath. +func ioniceNicePrefix(ioClass, ioPrio, niceDelta int, lookPath func(string) (string, error)) string { prefix := "" if p, err := lookPath("ionice"); err == nil { - prefix += p + " -c 2 -n 4 " + prefix += fmt.Sprintf("%s -c %d -n %d ", p, ioClass, ioPrio) } if p, err := lookPath("nice"); err == nil { prefix += fmt.Sprintf("%s -n %d ", p, niceDelta) @@ -124,7 +126,7 @@ func New( // and nice. The oom_score_adj write is pure /proc and always applied; the // priority helpers are used only where the image provides them. niceDelta := defaultNice - currentNice() - oomWrapperScript := fmt.Sprintf(`echo %d > /proc/$$/oom_score_adj && exec %s"${@}"`, defaultOomScore, wrapperPrefix(niceDelta, exec.LookPath)) + oomWrapperScript := fmt.Sprintf(`echo %d > /proc/$$/oom_score_adj && exec %s"${@}"`, defaultOomScore, ioniceNicePrefix(defaultIoClass, defaultIoPrio, niceDelta, exec.LookPath)) wrapperArgs := append([]string{"-c", oomWrapperScript, "--", req.GetProcess().GetCmd()}, req.GetProcess().GetArgs()...) cmd := exec.CommandContext(ctx, "/bin/sh", wrapperArgs...) diff --git a/packages/envd/internal/services/process/handler/handler_test.go b/packages/envd/internal/services/process/handler/handler_test.go index 05d670147d..d11fc49bf1 100644 --- a/packages/envd/internal/services/process/handler/handler_test.go +++ b/packages/envd/internal/services/process/handler/handler_test.go @@ -27,21 +27,26 @@ func TestWrapperPrefix(t *testing.T) { t.Run("both present", func(t *testing.T) { t.Parallel() - assert.Equal(t, "/usr/bin/ionice -c 2 -n 4 /usr/bin/nice -n 5 ", wrapperPrefix(5, all)) + assert.Equal(t, "/usr/bin/ionice -c 2 -n 4 /usr/bin/nice -n 5 ", ioniceNicePrefix(2, 4, 5, all)) }) t.Run("both absent degrades to bare exec", func(t *testing.T) { t.Parallel() - assert.Empty(t, wrapperPrefix(5, notFound)) + assert.Empty(t, ioniceNicePrefix(2, 4, 5, notFound)) }) t.Run("only nice", func(t *testing.T) { t.Parallel() - assert.Equal(t, "/bin/nice -n -3 ", wrapperPrefix(-3, only("nice"))) + assert.Equal(t, "/bin/nice -n -3 ", ioniceNicePrefix(2, 4, -3, only("nice"))) }) t.Run("only ionice", func(t *testing.T) { t.Parallel() - assert.Equal(t, "/bin/ionice -c 2 -n 4 ", wrapperPrefix(0, only("ionice"))) + assert.Equal(t, "/bin/ionice -c 2 -n 4 ", ioniceNicePrefix(2, 4, 0, only("ionice"))) + }) + + t.Run("class and priority are caller-controlled", func(t *testing.T) { + t.Parallel() + assert.Equal(t, "/bin/ionice -c 1 -n 6 ", ioniceNicePrefix(1, 6, 0, only("ionice"))) }) } From 45ecb711981c6220684e870aff3b84649b1f0d12 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Wed, 29 Jul 2026 09:27:40 +0200 Subject: [PATCH 13/17] fix(orch): surface the fsfreeze fallback in customer build logs Review feedback: the sync-only-flush message was only visible in debug logs; prefix it with the provision log prefix like the rest of the provisioning output. --- .../template/build/core/rootfs/files/provision-runner.sh.tpl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/files/provision-runner.sh.tpl b/packages/orchestrator/pkg/template/build/core/rootfs/files/provision-runner.sh.tpl index 62827ac3ed..22135a9a63 100644 --- a/packages/orchestrator/pkg/template/build/core/rootfs/files/provision-runner.sh.tpl +++ b/packages/orchestrator/pkg/template/build/core/rootfs/files/provision-runner.sh.tpl @@ -19,8 +19,9 @@ if command -v fsfreeze >/dev/null 2>&1; then fsfreeze --freeze / else # No util-linux on this image; the double sync flushes the ext4 journal - # and the VM is paused before the snapshot is taken. - echo "fsfreeze not available on this image; using sync-only flush" + # and the VM is paused before the snapshot is taken. Prefixed so the + # fallback shows up in the customer's build logs. + echo "{{ .ProvisionLogPrefix }}fsfreeze not available on this image; using sync-only flush" $BB sync fi From 57ceb93ce2268cfc249a86efea4fd0b1f804a69b Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Wed, 29 Jul 2026 09:37:59 +0200 Subject: [PATCH 14/17] chore(orch): chain the debian install with && instead of an embedded newline Review nit: the only multi-line PkgInstall; now a one-liner like the other profiles, with identical set -e failure behavior. --- .../pkg/template/build/phases/base/distro/distro.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go b/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go index 951d4bd3a9..f19250ec02 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go @@ -53,7 +53,7 @@ var Profiles = []Profile{ "less", "nftables", "iputils-ping", "jq", }, PkgQueryBody: `dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -q "install ok installed"`, - PkgInstall: "apt-get -q update\n DEBIAN_FRONTEND=noninteractive DEBCONF_NOWARNINGS=yes apt-get -qq -o=Dpkg::Use-Pty=0 install -y --no-install-recommends \"$@\"", + PkgInstall: `apt-get -q update && DEBIAN_FRONTEND=noninteractive DEBCONF_NOWARNINGS=yes apt-get -qq -o=Dpkg::Use-Pty=0 install -y --no-install-recommends "$@"`, InitBinary: "/lib/systemd/systemd", TimeSyncUnit: "chrony", SSHUnit: "ssh", From f98cec9e6da66469936a6e05cd55a038b766a665 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Wed, 29 Jul 2026 09:42:06 +0200 Subject: [PATCH 15/17] refactor(orch): source the admin group from the distro profile in later phases Review feedback: finalize and USER steps re-probed sudo/wheel with getent, duplicating Profiles.AdminGroup. Provisioning now persists the resolved identity to /usr/local/share/e2b/distro.env and both consumers source it. --- .../pkg/template/build/commands/user.go | 18 ++++++++---------- .../template/build/phases/base/provision.sh | 9 +++++++++ .../build/phases/finalize/configure.sh | 15 +++++++-------- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/packages/orchestrator/pkg/template/build/commands/user.go b/packages/orchestrator/pkg/template/build/commands/user.go index 207c6a75e0..fafb09cac9 100644 --- a/packages/orchestrator/pkg/template/build/commands/user.go +++ b/packages/orchestrator/pkg/template/build/commands/user.go @@ -101,17 +101,15 @@ func addToSudoers( lvl, prefix, sandboxID, - // Admin group differs by distro (sudo on Debian/Ubuntu, wheel on - // RHEL/Arch/Alpine); the NOPASSWD sudoers entry below is what actually - // grants privileges. Neither group existing is a real error. - fmt.Sprintf(`if getent group sudo >/dev/null; then - usermod -aG sudo %[1]s -elif getent group wheel >/dev/null; then - usermod -aG wheel %[1]s -else - echo "neither the sudo nor the wheel group exists on this image" >&2 + // The admin group comes from the distro profile, persisted by + // provisioning; the NOPASSWD sudoers entry below is what actually + // grants privileges. + fmt.Sprintf(`. /usr/local/share/e2b/distro.env +if ! getent group "$E2B_ADMIN_GROUP" >/dev/null; then + echo "the '$E2B_ADMIN_GROUP' group from the distro profile does not exist on this image" >&2 exit 1 -fi`, userArg), +fi +usermod -aG "$E2B_ADMIN_GROUP" %s`, userArg), metadata.Context{ User: "root", EnvVars: cmdMetadata.EnvVars, diff --git a/packages/orchestrator/pkg/template/build/phases/base/provision.sh b/packages/orchestrator/pkg/template/build/phases/base/provision.sh index 00d81796bd..bdd18c8252 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/provision.sh +++ b/packages/orchestrator/pkg/template/build/phases/base/provision.sh @@ -30,6 +30,15 @@ fi echo "Provisioning for distro '$E2B_DISTRO_ID' (init=$E2B_INIT_BIN, timesync=$E2B_TIMESYNC_UNIT, admin-group=$E2B_ADMIN_GROUP)" +# Persist the resolved identity for later build phases (finalize sources it), +# so the profile's values are defined once, here. +mkdir -p /usr/local/share/e2b +{ + echo "E2B_DISTRO_ID='$E2B_DISTRO_ID'" + echo "E2B_INIT_SYSTEM='$E2B_INIT_SYSTEM'" + echo "E2B_ADMIN_GROUP='$E2B_ADMIN_GROUP'" +} > /usr/local/share/e2b/distro.env + # Helper function to check if a package is installed (distro-specific query) is_package_installed() { e2b_pkg_query "$1" diff --git a/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh b/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh index 7fd47e7650..b8f14f6b27 100644 --- a/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh +++ b/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh @@ -38,16 +38,15 @@ fi echo "Add sudo to 'user' with no password" # Admin group differs by distro (sudo on Debian/Ubuntu, wheel elsewhere); the -# NOPASSWD sudoers entry below is what actually grants privileges. Neither -# group existing is a real error. -if getent group sudo >/dev/null; then - usermod -aG sudo user -elif getent group wheel >/dev/null; then - usermod -aG wheel user -else - echo "ERROR: neither the sudo nor the wheel group exists on this image" >&2 +# NOPASSWD sudoers entry below is what actually grants privileges. The group +# comes from the distro profile, resolved and persisted by provisioning — +# defined once in distro.go, not re-probed here. +. /usr/local/share/e2b/distro.env +if ! getent group "$E2B_ADMIN_GROUP" >/dev/null; then + echo "ERROR: the '$E2B_ADMIN_GROUP' group from the distro profile does not exist on this image" >&2 exit 1 fi +usermod -aG "$E2B_ADMIN_GROUP" user passwd -d user # Skip the append when the entry is already present. if grep -q '^user ALL=(ALL:ALL) NOPASSWD: ALL' /etc/sudoers; then From 455de1e2bfaf24de2a6075e81bb531bde06c9421 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Wed, 29 Jul 2026 08:17:59 +0000 Subject: [PATCH 16/17] fix(orch): pass --allowerasing after the dnf subcommand dnf5 (Fedora 41+) rejects the flag before the subcommand with "Unknown argument" and exit 2, which sent every modern Fedora build down the microdnf/yum fallback and installed without it. dnf4 accepts either position. --- .../template/build/phases/base/distro/distro.go | 9 +++++++-- .../build/phases/base/distro/distro_test.go | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go b/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go index f19250ec02..8bac7f4480 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go @@ -80,8 +80,13 @@ var Profiles = []Profile{ }, PkgQueryBody: `rpm -q "$1" >/dev/null 2>&1`, // dnf → microdnf → yum spans the family (yum for CentOS 7); errors reach - // the build log. - PkgInstall: `dnf -y --allowerasing install "$@" || microdnf -y install "$@" || yum -y install "$@"`, + // the build log. --allowerasing goes AFTER the subcommand: dnf5 (Fedora + // 41+, where /usr/bin/dnf IS dnf5) rejects it before one with "Unknown + // argument" and exits 2, which put two errors in every modern Fedora + // build log and quietly demoted the install to the flagless + // microdnf/yum fallback, so the flag never applied. dnf4 accepts either + // position. + PkgInstall: `dnf -y install --allowerasing "$@" || microdnf -y install "$@" || yum -y install "$@"`, InitBinary: "/usr/lib/systemd/systemd", TimeSyncUnit: "chronyd", SSHUnit: "sshd", diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/distro_test.go b/packages/orchestrator/pkg/template/build/phases/base/distro/distro_test.go index 0d6a498827..7f7096cb71 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/distro/distro_test.go +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/distro_test.go @@ -68,6 +68,20 @@ func TestFamiliesDiffer(t *testing.T) { } } +// dnf5 (Fedora 41+) only accepts --allowerasing after the subcommand and exits 2 +// with "Unknown argument" when it comes first, which silently demoted every +// modern Fedora build to the flagless microdnf/yum fallback. +func TestRhelAllowErasingFollowsSubcommand(t *testing.T) { + t.Parallel() + rhel := profileByKey(t, "rhel") + if !strings.Contains(rhel.PkgInstall, "install --allowerasing") { + t.Errorf("rhel install must place --allowerasing after the subcommand: %s", rhel.PkgInstall) + } + if strings.Contains(rhel.PkgInstall, "--allowerasing install") { + t.Errorf("rhel install has --allowerasing before the subcommand, which dnf5 rejects: %s", rhel.PkgInstall) + } +} + // The generated selector keys on the DECLARED distro id, never on which // package-manager binary happens to exist. func TestSelectorNoPackageManagerProbing(t *testing.T) { From 06f2fa9e9690b5ad8fcef6c3db1b2b9ac6d97a5d Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Wed, 29 Jul 2026 10:57:13 +0200 Subject: [PATCH 17/17] fix(orch): tolerate templates provisioned before distro.env existed FROM-template builds reuse the parent rootfs without re-provisioning, so finalize and USER steps must not hard-require /usr/local/share/e2b/distro.env: fall back to probing the sudo/wheel groups when the file is absent. --- .../pkg/template/build/commands/user.go | 24 ++++++++++++++----- .../build/phases/finalize/configure.sh | 23 ++++++++++++++---- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/packages/orchestrator/pkg/template/build/commands/user.go b/packages/orchestrator/pkg/template/build/commands/user.go index fafb09cac9..3afdae39d6 100644 --- a/packages/orchestrator/pkg/template/build/commands/user.go +++ b/packages/orchestrator/pkg/template/build/commands/user.go @@ -103,13 +103,25 @@ func addToSudoers( sandboxID, // The admin group comes from the distro profile, persisted by // provisioning; the NOPASSWD sudoers entry below is what actually - // grants privileges. - fmt.Sprintf(`. /usr/local/share/e2b/distro.env -if ! getent group "$E2B_ADMIN_GROUP" >/dev/null; then - echo "the '$E2B_ADMIN_GROUP' group from the distro profile does not exist on this image" >&2 - exit 1 + // grants privileges. Templates built before distro.env existed + // (FROM-template parents keep their rootfs) fall back to probing. + fmt.Sprintf(`if [ -f /usr/local/share/e2b/distro.env ]; then + . /usr/local/share/e2b/distro.env fi -usermod -aG "$E2B_ADMIN_GROUP" %s`, userArg), +if [ -n "${E2B_ADMIN_GROUP:-}" ]; then + if ! getent group "$E2B_ADMIN_GROUP" >/dev/null; then + echo "the '$E2B_ADMIN_GROUP' group from the distro profile does not exist on this image" >&2 + exit 1 + fi + usermod -aG "$E2B_ADMIN_GROUP" %[1]s +elif getent group sudo >/dev/null; then + usermod -aG sudo %[1]s +elif getent group wheel >/dev/null; then + usermod -aG wheel %[1]s +else + echo "neither the sudo nor the wheel group exists on this image" >&2 + exit 1 +fi`, userArg), metadata.Context{ User: "root", EnvVars: cmdMetadata.EnvVars, diff --git a/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh b/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh index b8f14f6b27..185f4047f7 100644 --- a/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh +++ b/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh @@ -40,13 +40,26 @@ echo "Add sudo to 'user' with no password" # Admin group differs by distro (sudo on Debian/Ubuntu, wheel elsewhere); the # NOPASSWD sudoers entry below is what actually grants privileges. The group # comes from the distro profile, resolved and persisted by provisioning — -# defined once in distro.go, not re-probed here. -. /usr/local/share/e2b/distro.env -if ! getent group "$E2B_ADMIN_GROUP" >/dev/null; then - echo "ERROR: the '$E2B_ADMIN_GROUP' group from the distro profile does not exist on this image" >&2 +# defined once in distro.go, not re-probed here. Templates built before +# distro.env existed (FROM-template parents reuse their rootfs without +# re-provisioning) fall back to probing the two known groups. +if [ -f /usr/local/share/e2b/distro.env ]; then + . /usr/local/share/e2b/distro.env +fi +if [ -n "${E2B_ADMIN_GROUP:-}" ]; then + if ! getent group "$E2B_ADMIN_GROUP" >/dev/null; then + echo "ERROR: the '$E2B_ADMIN_GROUP' group from the distro profile does not exist on this image" >&2 + exit 1 + fi + usermod -aG "$E2B_ADMIN_GROUP" user +elif getent group sudo >/dev/null; then + usermod -aG sudo user +elif getent group wheel >/dev/null; then + usermod -aG wheel user +else + echo "ERROR: neither the sudo nor the wheel group exists on this image" >&2 exit 1 fi -usermod -aG "$E2B_ADMIN_GROUP" user passwd -d user # Skip the append when the entry is already present. if grep -q '^user ALL=(ALL:ALL) NOPASSWD: ALL' /etc/sudoers; then