feat(orch): distro-aware template base-image provisioning

packages/envd/internal/services/process/handler/handler.go

@@ -29,6 +29,8 @@ import (
 const (
 	defaultNice      = 0
 	defaultOomScore  = 100
+	defaultIoClass   = 2 // ionice best-effort
+	defaultIoPrio    = 4
 	outputBufferSize = 64
 	systemTag        = "_system"
 	stdChunkSize     = 32 << 10 // 32 KiB
@@ -161,6 +163,25 @@ func currentNice() int {
 	return 20 - prio
 }
 
+// ioniceNicePrefix builds the ionice/nice part of the process wrapper from
+// whatever the image actually ships. Both are util-linux/coreutils
+// conveniences that minimal and busybox-based images (Alpine, UBI) may lack or
+// keep elsewhere than /usr/bin — a missing helper must degrade to running the
+// command without that priority adjustment, never to a failed spawn (exit 127
+// killed every process on such images). lookPath is injected for testability;
+// production passes exec.LookPath.
+func ioniceNicePrefix(ioClass, ioPrio, niceDelta int, lookPath func(string) (string, error)) string {
+	prefix := ""
+	if p, err := lookPath("ionice"); err == nil {
+		prefix += fmt.Sprintf("%s -c %d -n %d ", p, ioClass, ioPrio)
+	}
+	if p, err := lookPath("nice"); err == nil {
+		prefix += fmt.Sprintf("%s -n %d ", p, niceDelta)
+	}
+
+	return prefix
+}
+
 func New(
 	ctx context.Context,
 	user *user.User,
@@ -173,9 +194,11 @@ func New(
 	// User command string for logging (without the internal wrapper details).
 	userCmd := strings.Join(append([]string{req.GetProcess().GetCmd()}, req.GetProcess().GetArgs()...), " ")
 
-	// Wrap in a shell that resets oom_score_adj, ioprio (ionice best-effort/4), and nice.
+	// Wrap in a shell that resets oom_score_adj, ioprio (ionice best-effort/4),
+	// and nice. The oom_score_adj write is pure /proc and always applied; the
+	// priority helpers are used only where the image provides them.
 	niceDelta := defaultNice - currentNice()
-	oomWrapperScript := fmt.Sprintf(`echo %d > /proc/$$/oom_score_adj && exec /usr/bin/ionice -c 2 -n 4 /usr/bin/nice -n %d "${@}"`, defaultOomScore, niceDelta)
+	oomWrapperScript := fmt.Sprintf(`echo %d > /proc/$$/oom_score_adj && exec %s"${@}"`, defaultOomScore, ioniceNicePrefix(defaultIoClass, defaultIoPrio, niceDelta, exec.LookPath))
 	wrapperArgs := append([]string{"-c", oomWrapperScript, "--", req.GetProcess().GetCmd()}, req.GetProcess().GetArgs()...)
 	cmd := exec.CommandContext(ctx, "/bin/sh", wrapperArgs...)
 

packages/envd/internal/services/process/handler/handler_test.go

@@ -0,0 +1,52 @@
+package handler
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// The process wrapper must degrade cleanly when the priority helpers are absent
+// — the user command still runs; a present helper is applied with its resolved
+// path.
+func TestWrapperPrefix(t *testing.T) {
+	t.Parallel()
+
+	notFound := func(string) (string, error) { return "", errors.New("not found") }
+	all := func(name string) (string, error) { return "/usr/bin/" + name, nil }
+	only := func(want string) func(string) (string, error) {
+		return func(name string) (string, error) {
+			if name == want {
+				return "/bin/" + name, nil
+			}
+
+			return "", errors.New("not found")
+		}
+	}
+
+	t.Run("both present", func(t *testing.T) {
+		t.Parallel()
+		assert.Equal(t, "/usr/bin/ionice -c 2 -n 4 /usr/bin/nice -n 5 ", ioniceNicePrefix(2, 4, 5, all))
+	})
+
+	t.Run("both absent degrades to bare exec", func(t *testing.T) {
+		t.Parallel()
+		assert.Empty(t, ioniceNicePrefix(2, 4, 5, notFound))
+	})
+
+	t.Run("only nice", func(t *testing.T) {
+		t.Parallel()
+		assert.Equal(t, "/bin/nice -n -3 ", ioniceNicePrefix(2, 4, -3, only("nice")))
+	})
+
+	t.Run("only ionice", func(t *testing.T) {
+		t.Parallel()
+		assert.Equal(t, "/bin/ionice -c 2 -n 4 ", ioniceNicePrefix(2, 4, 0, only("ionice")))
+	})
+
+	t.Run("class and priority are caller-controlled", func(t *testing.T) {
+		t.Parallel()
+		assert.Equal(t, "/bin/ionice -c 1 -n 6 ", ioniceNicePrefix(1, 6, 0, only("ionice")))
+	})
+}

packages/orchestrator/README.md

@@ -303,4 +303,5 @@ Automatically set in local mode. Set before running to override:
 
 ## Limitations
 
-- Custom template builds require Debian/Ubuntu-based base images (images that provide the `apt` package manager). Non-Debian images such as Alpine, CentOS/RHEL, or other distributions without `apt` are not supported and will fail during the template build/provisioning process. The provisioning scripts used during template build call `apt` and expect Debian-specific package names and file locations.
+- Custom template builds support base images from the declared distro families: **Debian/Ubuntu** (apt), the **RHEL family** — Fedora, CentOS Stream, Rocky, Alma — (dnf/microdnf/yum), **Arch** (pacman), and **Alpine** (apk, OpenRC). The distro is resolved from the image's `/etc/os-release` `ID` — never by probing for package managers. Images without an os-release identity (distroless, scratch) or from an undeclared family are rejected fast, with a build-log error naming the reason and the supported families (see `pkg/template/build/phases/base/distro`). Minimal/restricted-repo images may fail provisioning if their repos don't carry the required packages; the failure is surfaced in the build log.
+- Sandboxes always boot the kernel E2B supplies, never one from the base image: `/lib/modules` is empty, no kernel modules can be loaded, and SELinux is off. `kernel-devel` from a distro's repos resolves against a kernel that isn't running. **RHEL** (`rhel`, incl. UBI), **Oracle Linux** (`ol`) and **Amazon Linux** (`amzn`) are therefore not accepted even though they are RPM/dnf images — they are chosen for kABI, signed kernel modules and UEK, which this cannot honour. The community rebuilds above are supported because they are chosen for the userland.

packages/orchestrator/pkg/template/build/commands/user.go

@@ -60,7 +60,9 @@ func (u *User) Execute(
 			lvl,
 			prefix,
 			sandboxID,
-			fmt.Sprintf("adduser --disabled-password --gecos \"\" %s", userArg),
+			// useradd is in shadow(-utils) on every supported distro family; the
+			// created user has no password (locked).
+			fmt.Sprintf("useradd --create-home --shell /bin/bash %s", userArg),
 			metadata.Context{
 				User:    "root",
 				EnvVars: cmdMetadata.EnvVars,
@@ -99,7 +101,27 @@ func addToSudoers(
 		lvl,
 		prefix,
 		sandboxID,
-		fmt.Sprintf("usermod -aG sudo %s", userArg),
+		// The admin group comes from the distro profile, persisted by
+		// provisioning; the NOPASSWD sudoers entry below is what actually
+		// grants privileges. Templates built before distro.env existed
+		// (FROM-template parents keep their rootfs) fall back to probing.
+		fmt.Sprintf(`if [ -f /usr/local/share/e2b/distro.env ]; then
+    . /usr/local/share/e2b/distro.env
+fi
+if [ -n "${E2B_ADMIN_GROUP:-}" ]; then
+    if ! getent group "$E2B_ADMIN_GROUP" >/dev/null; then
+        echo "the '$E2B_ADMIN_GROUP' group from the distro profile does not exist on this image" >&2
+        exit 1
+    fi
+    usermod -aG "$E2B_ADMIN_GROUP" %[1]s
+elif getent group sudo >/dev/null; then
+    usermod -aG sudo %[1]s
+elif getent group wheel >/dev/null; then
+    usermod -aG wheel %[1]s
+else
+    echo "neither the sudo nor the wheel group exists on this image" >&2
+    exit 1
+fi`, userArg),
 		metadata.Context{
 			User:    "root",
 			EnvVars: cmdMetadata.EnvVars,

packages/orchestrator/pkg/template/build/core/rootfs/files/envd.openrc.tpl

@@ -0,0 +1,46 @@
+{{- /*gotype:github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs.templateModel*/ -}}
+{{ .WriteFile "/usr/local/share/e2b/envd.openrc" 0o755 }}
+
+#!/sbin/openrc-run
+# E2B env daemon — the OpenRC counterpart of envd.service (see envd.service.tpl
+# for the full rationale on each step; this mirrors it for the Alpine/OpenRC
+# family). Baked at a neutral path and installed to /etc/init.d/envd by the
+# OpenRC e2b_init_setup only: if it lived in /etc/init.d on every image,
+# Debian's `systemctl enable envd` would hand it to update-rc.d, which aborts
+# on a non-LSB script and fails the whole provisioning.
+#
+# /tmp-wipe ordering note: OpenRC's bootmisc (boot runlevel) wipes /tmp before
+# the default runlevel starts, so envd — in default — can never answer an
+# update-envd upload before the wipe. The race envd.service needs an explicit
+# After=systemd-tmpfiles-setup.service for cannot happen here.
+
+description="E2B env daemon"
+
+supervisor=supervise-daemon
+command=/usr/bin/envd
+# --nicelevel/--ionice/--oom-score-adj mirror envd.service's Nice=-20,
+# IOSchedulingClass=realtime + IOSchedulingPriority=4 (ionice class 1, data 4)
+# and OOMScoreAdjust=-1000 — without them envd runs at default priority and is
+# the first OOM-kill candidate on Alpine. (The unit's cgroup weights —
+# MemoryMin/CPUWeight/IOWeight — have no supervise-daemon equivalent.)
+supervise_daemon_args="--nicelevel -20 --ionice 1:4 --oom-score-adj -1000 --env GOTRACEBACK=all --env GOMEMLIMIT={{ .MemoryLimit }}MiB --stdout /var/log/envd.log --stderr /var/log/envd.log"
+# Retry forever (envd.service uses Restart=always + StartLimitIntervalSec=0).
+respawn_delay=1
+respawn_max=0
+
+depend() {
+    need localmount
+    after bootmisc
+    use net
+}
+
+start_pre() {
+    # Shared with envd.service's ExecStartPre; warns-and-continues by design.
+    /usr/local/bin/e2b-seed-certs
+    # systemd-tmpfiles applies the fuse.conf tmpfiles.d rule on the systemd
+    # family; OpenRC has no tmpfiles pass, so set the mode here.
+    if [ -e /dev/fuse ]; then
+        chmod 666 /dev/fuse
+    fi
+    return 0
+}

packages/orchestrator/pkg/template/build/core/rootfs/files/envd.service.tpl

@@ -46,7 +46,9 @@ LimitCORE=infinity
 # guaranteed present for the sandbox's routable lifetime. The only gap is guest
 # units that auto-start and egress over TLS before /init; that is accepted
 # (revisit if a template needs boot-time egress).
-ExecStartPre=/bin/sh -c 'mountpoint -q /etc/ssl/certs || { mkdir -p /run/e2b/certs && { tar -C /run/e2b/certs -xf /usr/local/share/e2b/ssl-certs.tar 2>/dev/null || cp -a /etc/ssl/certs/. /run/e2b/certs/ 2>/dev/null; }; mount --bind /run/e2b/certs /etc/ssl/certs; } && ([ -s /etc/ssl/certs/ca-certificates.crt ] || update-ca-certificates)'
+# The seeding logic is shared with the OpenRC service (Alpine) and never fails
+# the unit — a degraded trust store is recoverable, a dead envd is not.
+ExecStartPre=/usr/local/bin/e2b-seed-certs
 ExecStart=/usr/bin/envd
 Nice=-20
 IOSchedulingClass=realtime

packages/orchestrator/pkg/template/build/core/rootfs/files/inittab.tpl

@@ -1,18 +1,16 @@
 {{- /*gotype:github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs.templateModel*/ -}}
 {{ .WriteFile "/etc/inittab" 0o777 }}
 
-# Run system init
-::sysinit:/etc/init.d/rcS
-
-# Run the provision script, prefix the output with a log prefix
-::wait:/bin/sh -c '/usr/local/bin/provision.sh 2>&1 | sed "s/^/{{ .ProvisionLogPrefix }}/"'
+# Provisioning-boot inittab (busybox init). Every entry is a plain exec with
+# NO shell metacharacters: busybox init hands metachar lines to /bin/sh, and
+# minimal images (distroless) may have no /bin/sh — the pipeline logic lives in
+# e2b-provision-runner instead.
 
-# Flush filesystem changes to disk
-::wait:/usr/bin/busybox sync
-::wait:fsfreeze --freeze /
+# Run system init (mounts /proc /sys /dev /tmp /run through the baked busybox)
+::sysinit:/etc/init.d/rcS
 
-# Report the exit code of the provisioning script
-::wait:/bin/sh -c 'echo "{{ .ProvisionExitPrefix }}$(cat {{ .ProvisionResultPath }} || printf 1)"'
+# Run the provisioning pipeline and report its exit code
+::wait:/usr/bin/busybox ash /usr/local/bin/e2b-provision-runner
 
 # Wait forever to prevent the VM from exiting until the sandbox is paused and snapshot is taken
-::wait:/usr/bin/busybox sleep infinity
+::wait:/usr/bin/busybox sleep infinity

packages/orchestrator/pkg/template/build/core/rootfs/files/provision-runner.sh.tpl

@@ -0,0 +1,34 @@
+{{- /*gotype:github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs.templateModel*/ -}}
+{{ .WriteFile "usr/local/bin/e2b-provision-runner" 0o755 }}
+
+#!/usr/bin/busybox ash
+# Drives the provisioning pipeline for the busybox-init boot. This logic lives
+# in a script — NOT in /etc/inittab — because busybox init hands any inittab
+# line containing shell metacharacters to /bin/sh, and minimal images
+# (distroless) may have no /bin/sh; a plain-exec inittab line running this
+# script through the baked busybox works on every image.
+BB=/usr/bin/busybox
+
+# Run the provision script, prefix its output with the log prefix the
+# orchestrator forwards to the customer's build logs.
+$BB sh /usr/local/bin/provision.sh 2>&1 | $BB sed "s/^/{{ .ProvisionLogPrefix }}/"
+
+# Flush filesystem changes to disk before the snapshot.
+$BB sync
+if command -v fsfreeze >/dev/null 2>&1; then
+    fsfreeze --freeze /
+else
+    # No util-linux on this image; the double sync flushes the ext4 journal
+    # and the VM is paused before the snapshot is taken. Prefixed so the
+    # fallback shows up in the customer's build logs.
+    echo "{{ .ProvisionLogPrefix }}fsfreeze not available on this image; using sync-only flush"
+    $BB sync
+fi
+
+# Report the provisioning exit code: provision.sh writes "0" on success and
+# (running under set -e) leaves no file behind on failure.
+if result=$($BB cat {{ .ProvisionResultPath }} 2>/dev/null); then
+    echo "{{ .ProvisionExitPrefix }}${result}"
+else
+    echo "{{ .ProvisionExitPrefix }}1"
+fi

packages/orchestrator/pkg/template/build/core/rootfs/files/rcS.sh.tpl

@@ -2,15 +2,20 @@
 {{ .WriteFile "etc/init.d/rcS" 0o777 }}
 
 #!/usr/bin/busybox ash
+# Every command goes through the baked busybox: this runs before provisioning
+# on the raw base image, and minimal images (distroless) may have no mkdir/mount
+# on PATH at all.
+BB=/usr/bin/busybox
+
 echo "Mounting essential filesystems"
 # Ensure necessary mount points exist
-mkdir -p /proc /sys /dev /tmp /run
+$BB mkdir -p /proc /sys /dev /tmp /run
 
 # Mount essential filesystems
-mount -t proc proc /proc
-mount -t sysfs sysfs /sys
-mount -t devtmpfs devtmpfs /dev
-mount -t tmpfs tmpfs /tmp
-mount -t tmpfs tmpfs /run
+$BB mount -t proc proc /proc
+$BB mount -t sysfs sysfs /sys
+$BB mount -t devtmpfs devtmpfs /dev
+$BB mount -t tmpfs tmpfs /tmp
+$BB mount -t tmpfs tmpfs /run
 
-echo "System Init"
+echo "System Init"

packages/orchestrator/pkg/template/build/core/rootfs/files/seed-certs.sh.tpl

@@ -0,0 +1,53 @@
+{{- /*gotype:github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs.templateModel*/ -}}
+{{ .WriteFile "usr/local/bin/e2b-seed-certs" 0o755 }}
+
+#!/bin/sh
+# Seeds the tmpfs-backed /etc/ssl/certs before envd starts — shared by
+# envd.service (systemd) and /etc/init.d/envd (OpenRC). See envd.service.tpl
+# for the full rationale (why a tar, why a bind mount, the egress-CA contract).
+#
+# Every failure path here WARNS and continues deliberately: a sandbox with a
+# degraded trust store is recoverable (envd's POST /init reinstalls the egress
+# CA), a sandbox whose envd never starts is not.
+
+# The copies DEREFERENCE symlinks (-L; the build-time tar already packs with
+# -h): some images ship the /etc/ssl/certs bundle as symlinks into a read-only
+# store, and envd's egress-proxy CA install APPENDS to
+# /etc/ssl/certs/ca-certificates.crt — that only works if the tmpfs copy is a
+# real file, not a copied symlink to an immutable target.
+if ! mountpoint -q /etc/ssl/certs; then
+    mkdir -p /run/e2b/certs
+    if [ -f /usr/local/share/e2b/ssl-certs.tar ]; then
+        if ! tar -C /run/e2b/certs -xf /usr/local/share/e2b/ssl-certs.tar; then
+            echo "e2b-seed-certs: ssl-certs.tar extraction failed; seeding from the live cert dir instead" >&2
+            cp -aL /etc/ssl/certs/. /run/e2b/certs/
+        fi
+    else
+        # Only expected during the base-layer boot, before finalize packs the tar.
+        echo "e2b-seed-certs: ssl-certs.tar not packed yet; seeding from the live cert dir"
+        cp -aL /etc/ssl/certs/. /run/e2b/certs/
+    fi
+    if ! mount -o bind /run/e2b/certs /etc/ssl/certs; then
+        echo "e2b-seed-certs: bind mount failed; envd runs with the image's certs as-is" >&2
+    fi
+fi
+
+if [ ! -s /etc/ssl/certs/ca-certificates.crt ]; then
+    if command -v update-ca-certificates >/dev/null 2>&1; then
+        update-ca-certificates
+    elif command -v update-ca-trust >/dev/null 2>&1; then
+        # RHEL family and Arch refresh with update-ca-trust. Arch's extract emits
+        # the Debian-named bundle itself; RHEL keeps it under /etc/pki, and the
+        # symlink provisioning made was dereferenced into this tmpfs, so copy it.
+        update-ca-trust extract
+        if [ ! -s /etc/ssl/certs/ca-certificates.crt ] && [ -s /etc/pki/tls/certs/ca-bundle.crt ]; then
+            cp -L /etc/pki/tls/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt
+        fi
+    else
+        # Provisioning guarantees the bundle on every supported family;
+        # reaching this means the image diverged after the build.
+        echo "e2b-seed-certs: CA bundle missing and no CA refresh tool on this image; TLS trust will be degraded" >&2
+    fi
+fi
+
+exit 0