feat(orch): distro-aware template provisioning (Fedora/RHEL/Arch base images)

packages/envd/internal/services/process/handler/handler.go

@@ -89,6 +89,25 @@ func currentNice() int {
 	return 20 - prio
 }
 
+// wrapperPrefix builds the priority-tool part of the process wrapper from
+// whatever the image actually ships (FEAT-145 W3). ionice/nice are util-linux/
+// coreutils conveniences that minimal and busybox-based images (Alpine, UBI)
+// may lack or keep elsewhere than /usr/bin — a missing helper must degrade to
+// running the command without that priority adjustment, never to a failed
+// spawn (exit 127 killed every process on such images). lookPath is injected
+// for testability; production passes exec.LookPath.
+func wrapperPrefix(niceDelta int, lookPath func(string) (string, error)) string {
+	prefix := ""
+	if p, err := lookPath("ionice"); err == nil {
+		prefix += p + " -c 2 -n 4 "
+	}
+	if p, err := lookPath("nice"); err == nil {
+		prefix += fmt.Sprintf("%s -n %d ", p, niceDelta)
+	}
+
+	return prefix
+}
+
 func New(
 	ctx context.Context,
 	user *user.User,
@@ -101,9 +120,11 @@ func New(
 	// User command string for logging (without the internal wrapper details).
 	userCmd := strings.Join(append([]string{req.GetProcess().GetCmd()}, req.GetProcess().GetArgs()...), " ")
 
-	// Wrap in a shell that resets oom_score_adj, ioprio (ionice best-effort/4), and nice.
+	// Wrap in a shell that resets oom_score_adj, ioprio (ionice best-effort/4),
+	// and nice. The oom_score_adj write is pure /proc and always applied; the
+	// priority helpers are used only where the image provides them.
 	niceDelta := defaultNice - currentNice()
-	oomWrapperScript := fmt.Sprintf(`echo %d > /proc/$$/oom_score_adj && exec /usr/bin/ionice -c 2 -n 4 /usr/bin/nice -n %d "${@}"`, defaultOomScore, niceDelta)
+	oomWrapperScript := fmt.Sprintf(`echo %d > /proc/$$/oom_score_adj && exec %s"${@}"`, defaultOomScore, wrapperPrefix(niceDelta, exec.LookPath))
 	wrapperArgs := append([]string{"-c", oomWrapperScript, "--", req.GetProcess().GetCmd()}, req.GetProcess().GetArgs()...)
 	cmd := exec.CommandContext(ctx, "/bin/sh", wrapperArgs...)
 

packages/envd/internal/services/process/handler/handler_test.go

@@ -0,0 +1,47 @@
+package handler
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// TT-4 (FEAT-145 W3): the process wrapper must degrade cleanly when the
+// priority helpers are absent — the user command still runs; a present helper
+// is applied with its resolved path.
+func TestWrapperPrefix(t *testing.T) {
+	t.Parallel()
+
+	notFound := func(string) (string, error) { return "", errors.New("not found") }
+	all := func(name string) (string, error) { return "/usr/bin/" + name, nil }
+	only := func(want string) func(string) (string, error) {
+		return func(name string) (string, error) {
+			if name == want {
+				return "/bin/" + name, nil
+			}
+
+			return "", errors.New("not found")
+		}
+	}
+
+	t.Run("both present", func(t *testing.T) {
+		t.Parallel()
+		assert.Equal(t, "/usr/bin/ionice -c 2 -n 4 /usr/bin/nice -n 5 ", wrapperPrefix(5, all))
+	})
+
+	t.Run("both absent degrades to bare exec", func(t *testing.T) {
+		t.Parallel()
+		assert.Empty(t, wrapperPrefix(5, notFound))
+	})
+
+	t.Run("only nice", func(t *testing.T) {
+		t.Parallel()
+		assert.Equal(t, "/bin/nice -n -3 ", wrapperPrefix(-3, only("nice")))
+	})
+
+	t.Run("only ionice", func(t *testing.T) {
+		t.Parallel()
+		assert.Equal(t, "/bin/ionice -c 2 -n 4 ", wrapperPrefix(0, only("ionice")))
+	})
+}

packages/orchestrator/README.md

@@ -303,4 +303,4 @@ Automatically set in local mode. Set before running to override:
 
 ## Limitations
 
-- Custom template builds require Debian/Ubuntu-based base images (images that provide the `apt` package manager). Non-Debian images such as Alpine, CentOS/RHEL, or other distributions without `apt` are not supported and will fail during the template build/provisioning process. The provisioning scripts used during template build call `apt` and expect Debian-specific package names and file locations.
+- Custom template builds support base images from the declared distro families (FEAT-145): **Debian/Ubuntu** (apt), the **RHEL family** — Fedora, RHEL, CentOS Stream, Rocky, Alma, Oracle Linux, Amazon Linux — (dnf/microdnf), **Arch** (pacman), and **Alpine** (apk, OpenRC). The distro is resolved from the image's `/etc/os-release` `ID` — never by probing for package managers (ADR-010). Images without an os-release identity (pure-Nix, distroless, scratch) or from an undeclared family are rejected fast, with a build-log error naming the reason and the supported families (see `pkg/template/build/phases/base/distro`). Minimal/restricted-repo images (e.g. RHEL UBI) may fail provisioning if their repos don't carry the required packages; the failure is surfaced in the build log.

packages/orchestrator/pkg/template/build/commands/user.go

@@ -60,7 +60,11 @@ func (u *User) Execute(
 			lvl,
 			prefix,
 			sandboxID,
-			fmt.Sprintf("adduser --disabled-password --gecos \"\" %s", userArg),
+			// useradd is part of shadow(-utils) and present on every supported
+			// distro family, unlike Debian's adduser wrapper (FEAT-145). The
+			// created user has no password (locked), matching the previous
+			// adduser --disabled-password behavior.
+			fmt.Sprintf("useradd --create-home --shell /bin/bash %s", userArg),
 			metadata.Context{
 				User:    "root",
 				EnvVars: cmdMetadata.EnvVars,
@@ -99,7 +103,11 @@ func addToSudoers(
 		lvl,
 		prefix,
 		sandboxID,
-		fmt.Sprintf("usermod -aG sudo %s", userArg),
+		// Admin group differs by distro (sudo on Debian/Ubuntu, wheel on
+		// RHEL/Arch/Alpine); the NOPASSWD sudoers entry below is what
+		// actually grants privileges (FEAT-145). Neither group existing is a
+		// real error, not something to swallow.
+		fmt.Sprintf(`if getent group sudo >/dev/null; then usermod -aG sudo %[1]s; elif getent group wheel >/dev/null; then usermod -aG wheel %[1]s; else echo "neither the sudo nor the wheel group exists on this image" >&2; exit 1; fi`, userArg),
 		metadata.Context{
 			User:    "root",
 			EnvVars: cmdMetadata.EnvVars,

packages/orchestrator/pkg/template/build/core/rootfs/files/envd.openrc.tpl

@@ -0,0 +1,41 @@
+{{- /*gotype:github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs.templateModel*/ -}}
+{{ .WriteFile "/usr/local/share/e2b/envd.openrc" 0o755 }}
+
+#!/sbin/openrc-run
+# E2B env daemon — the OpenRC counterpart of envd.service (see envd.service.tpl
+# for the full rationale on each step; this mirrors it for the Alpine/OpenRC
+# family). Baked at a neutral path and installed to /etc/init.d/envd by the
+# OpenRC e2b_init_setup only: if it lived in /etc/init.d on every image,
+# Debian's `systemctl enable envd` would hand it to update-rc.d, which aborts
+# on a non-LSB script and fails the whole provisioning.
+#
+# /tmp-wipe ordering note: OpenRC's bootmisc (boot runlevel) wipes /tmp before
+# the default runlevel starts, so envd — in default — can never answer an
+# update-envd upload before the wipe. The race envd.service needs an explicit
+# After=systemd-tmpfiles-setup.service for cannot happen here.
+
+description="E2B env daemon"
+
+supervisor=supervise-daemon
+command=/usr/bin/envd
+supervise_daemon_args="--env GOTRACEBACK=all --env GOMEMLIMIT={{ .MemoryLimit }}MiB --stdout /var/log/envd.log --stderr /var/log/envd.log"
+# Retry forever (envd.service uses Restart=always + StartLimitIntervalSec=0).
+respawn_delay=1
+respawn_max=0
+
+depend() {
+    need localmount
+    after bootmisc
+    use net
+}
+
+start_pre() {
+    # Shared with envd.service's ExecStartPre; warns-and-continues by design.
+    /usr/local/bin/e2b-seed-certs
+    # systemd-tmpfiles applies the fuse.conf tmpfiles.d rule on the systemd
+    # family; OpenRC has no tmpfiles pass, so set the mode here.
+    if [ -e /dev/fuse ]; then
+        chmod 666 /dev/fuse
+    fi
+    return 0
+}

packages/orchestrator/pkg/template/build/core/rootfs/files/envd.service.tpl

@@ -46,7 +46,9 @@ LimitCORE=infinity
 # guaranteed present for the sandbox's routable lifetime. The only gap is guest
 # units that auto-start and egress over TLS before /init; that is accepted
 # (revisit if a template needs boot-time egress).
-ExecStartPre=/bin/sh -c 'mountpoint -q /etc/ssl/certs || { mkdir -p /run/e2b/certs && { tar -C /run/e2b/certs -xf /usr/local/share/e2b/ssl-certs.tar 2>/dev/null || cp -a /etc/ssl/certs/. /run/e2b/certs/ 2>/dev/null; }; mount --bind /run/e2b/certs /etc/ssl/certs; } && ([ -s /etc/ssl/certs/ca-certificates.crt ] || update-ca-certificates)'
+# The seeding logic is shared with the OpenRC service (Alpine) and never fails
+# the unit — a degraded trust store is recoverable, a dead envd is not.
+ExecStartPre=/usr/local/bin/e2b-seed-certs
 ExecStart=/usr/bin/envd
 Nice=-20
 IOSchedulingClass=realtime

packages/orchestrator/pkg/template/build/core/rootfs/files/inittab.tpl

@@ -1,18 +1,16 @@
 {{- /*gotype:github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs.templateModel*/ -}}
 {{ .WriteFile "/etc/inittab" 0o777 }}
 
-# Run system init
-::sysinit:/etc/init.d/rcS
-
-# Run the provision script, prefix the output with a log prefix
-::wait:/bin/sh -c '/usr/local/bin/provision.sh 2>&1 | sed "s/^/{{ .ProvisionLogPrefix }}/"'
+# Provisioning-boot inittab (busybox init). Every entry is a plain exec with
+# NO shell metacharacters: busybox init hands metachar lines to /bin/sh, and
+# bare images (premade NixOS, distroless) have no /bin/sh — the pipeline
+# logic lives in e2b-provision-runner instead (FEAT-145).
 
-# Flush filesystem changes to disk
-::wait:/usr/bin/busybox sync
-::wait:fsfreeze --freeze /
+# Run system init (mounts /proc /sys /dev /tmp /run through the baked busybox)
+::sysinit:/etc/init.d/rcS
 
-# Report the exit code of the provisioning script
-::wait:/bin/sh -c 'echo "{{ .ProvisionExitPrefix }}$(cat {{ .ProvisionResultPath }} || printf 1)"'
+# Run the provisioning pipeline and report its exit code
+::wait:/usr/bin/busybox ash /usr/local/bin/e2b-provision-runner
 
 # Wait forever to prevent the VM from exiting until the sandbox is paused and snapshot is taken
-::wait:/usr/bin/busybox sleep infinity
+::wait:/usr/bin/busybox sleep infinity

packages/orchestrator/pkg/template/build/core/rootfs/files/provision-runner.sh.tpl

@@ -0,0 +1,33 @@
+{{- /*gotype:github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs.templateModel*/ -}}
+{{ .WriteFile "usr/local/bin/e2b-provision-runner" 0o755 }}
+
+#!/usr/bin/busybox ash
+# Drives the provisioning pipeline for the busybox-init boot. This logic lives
+# in a script — NOT in /etc/inittab — because busybox init hands any inittab
+# line containing shell metacharacters to /bin/sh, and bare images (premade
+# NixOS, distroless) have no /bin/sh; a plain-exec inittab line running this
+# script through the baked busybox works on every image (FEAT-145).
+BB=/usr/bin/busybox
+
+# Run the provision script, prefix its output with the log prefix the
+# orchestrator forwards to the customer's build logs.
+$BB sh /usr/local/bin/provision.sh 2>&1 | $BB sed "s/^/{{ .ProvisionLogPrefix }}/"
+
+# Flush filesystem changes to disk before the snapshot.
+$BB sync
+if command -v fsfreeze >/dev/null 2>&1; then
+    fsfreeze --freeze /
+else
+    # No util-linux on this image; the double sync flushes the ext4 journal
+    # and the VM is paused before the snapshot is taken.
+    echo "fsfreeze not available on this image; using sync-only flush"
+    $BB sync
+fi
+
+# Report the provisioning exit code: provision.sh writes "0" on success and
+# (running under set -e) leaves no file behind on failure.
+if result=$($BB cat {{ .ProvisionResultPath }} 2>/dev/null); then
+    echo "{{ .ProvisionExitPrefix }}${result}"
+else
+    echo "{{ .ProvisionExitPrefix }}1"
+fi

packages/orchestrator/pkg/template/build/core/rootfs/files/rcS.sh.tpl

@@ -2,15 +2,20 @@
 {{ .WriteFile "etc/init.d/rcS" 0o777 }}
 
 #!/usr/bin/busybox ash
+# Every command goes through the baked busybox: this runs before provisioning
+# on the raw base image, and bare images (premade NixOS, distroless) have no
+# mkdir/mount on PATH at all (FEAT-145).
+BB=/usr/bin/busybox
+
 echo "Mounting essential filesystems"
 # Ensure necessary mount points exist
-mkdir -p /proc /sys /dev /tmp /run
+$BB mkdir -p /proc /sys /dev /tmp /run
 
 # Mount essential filesystems
-mount -t proc proc /proc
-mount -t sysfs sysfs /sys
-mount -t devtmpfs devtmpfs /dev
-mount -t tmpfs tmpfs /tmp
-mount -t tmpfs tmpfs /run
+$BB mount -t proc proc /proc
+$BB mount -t sysfs sysfs /sys
+$BB mount -t devtmpfs devtmpfs /dev
+$BB mount -t tmpfs tmpfs /tmp
+$BB mount -t tmpfs tmpfs /run
 
-echo "System Init"
+echo "System Init"

packages/orchestrator/pkg/template/build/core/rootfs/files/seed-certs.sh.tpl

@@ -0,0 +1,40 @@
+{{- /*gotype:github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs.templateModel*/ -}}
+{{ .WriteFile "usr/local/bin/e2b-seed-certs" 0o755 }}
+
+#!/bin/sh
+# Seeds the tmpfs-backed /etc/ssl/certs before envd starts — shared by
+# envd.service (systemd) and /etc/init.d/envd (OpenRC). See envd.service.tpl
+# for the full rationale (why a tar, why a bind mount, the egress-CA contract).
+#
+# Every failure path here WARNS and continues deliberately: a sandbox with a
+# degraded trust store is recoverable (envd's POST /init reinstalls the egress
+# CA), a sandbox whose envd never starts is not.
+
+if ! mountpoint -q /etc/ssl/certs; then
+    mkdir -p /run/e2b/certs
+    if [ -f /usr/local/share/e2b/ssl-certs.tar ]; then
+        if ! tar -C /run/e2b/certs -xf /usr/local/share/e2b/ssl-certs.tar; then
+            echo "e2b-seed-certs: ssl-certs.tar extraction failed; seeding from the live cert dir instead" >&2
+            cp -a /etc/ssl/certs/. /run/e2b/certs/
+        fi
+    else
+        # Only expected during the base-layer boot, before finalize packs the tar.
+        echo "e2b-seed-certs: ssl-certs.tar not packed yet; seeding from the live cert dir"
+        cp -a /etc/ssl/certs/. /run/e2b/certs/
+    fi
+    if ! mount -o bind /run/e2b/certs /etc/ssl/certs; then
+        echo "e2b-seed-certs: bind mount failed; envd runs with the image's certs as-is" >&2
+    fi
+fi
+
+if [ ! -s /etc/ssl/certs/ca-certificates.crt ]; then
+    if command -v update-ca-certificates >/dev/null 2>&1; then
+        update-ca-certificates
+    else
+        # Provisioning guarantees the bundle on every supported family;
+        # reaching this means the image diverged after the build.
+        echo "e2b-seed-certs: CA bundle missing and no update-ca-certificates on this image; TLS trust will be degraded" >&2
+    fi
+fi
+
+exit 0

packages/orchestrator/pkg/template/build/core/rootfs/rootfs.go

@@ -14,6 +14,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"slices"
 	"text/template"
 
 	"github.com/dustin/go-humanize"
@@ -41,16 +42,40 @@ var tracer = otel.Tracer("github.com/e2b-dev/infra/packages/orchestrator/pkg/tem
 var files embed.FS
 var fileTemplates = template.Must(template.ParseFS(files, "files/*"))
 
-// filesHash is a stable hash of the embedded rootfs file templates. It is used
-// only as part of the fallback provision version; explicit provision versions
-// remain the rollout control.
+// enableSymlinks is the content of the baked symlink layer. Package-level so
+// it feeds filesHash: a change here must rotate the fallback provision
+// version like any other baked-layer change (qa.md QA11).
+var enableSymlinks = map[string]string{
+	// Enable envd service autostart. The target MUST be absolute: the link
+	// lives in multi-user.target.wants/, so a relative target would resolve
+	// inside that directory and dangle — and provision.sh's offline
+	// `systemctl enable $E2B_TIMESYNC_UNIT` prunes dangling .wants symlinks,
+	// silently disabling envd on distros where the link dangles (FEAT-145).
+	"etc/systemd/system/multi-user.target.wants/envd.service": "/etc/systemd/system/envd.service",
+	// NOTE: chrony autostart is enabled by provision.sh via `systemctl enable
+	// $E2B_TIMESYNC_UNIT`, which picks the distro-correct unit name (chrony on
+	// Debian, chronyd on RHEL/Arch). A static chrony.service symlink here would
+	// dangle on non-Debian images where the unit is chronyd.service (FEAT-145).
+}
+
+// filesHash is a stable hash of the embedded rootfs file templates plus the
+// baked symlink layer. It is used only as part of the fallback provision
+// version; explicit provision versions remain the rollout control.
 var filesHash = func() string {
 	entries, _ := fs.ReadDir(files, "files")
 	h := sha256.New()
 	for _, e := range entries {
 		data, _ := files.ReadFile("files/" + e.Name())
 		fmt.Fprintf(h, "%s\x00%x\x00", e.Name(), data)
 	}
+	links := make([]string, 0, len(enableSymlinks))
+	for name := range enableSymlinks {
+		links = append(links, name)
+	}
+	slices.Sort(links)
+	for _, name := range links {
+		fmt.Fprintf(h, "%s\x00%s\x00", name, enableSymlinks[name])
+	}
 
 	return hex.EncodeToString(h.Sum(nil))
 }()
@@ -233,6 +258,15 @@ func additionalOCILayers(
 	filesMap := map[string]oci.File{
 		storage.GuestEnvdPath: {Bytes: envdFileData, Mode: 0o777},
 
+		// Systemd preset policy for envd. provision.sh removes /etc/machine-id,
+		// so the template's next boot is a systemd FIRST boot — and on first
+		// boot PID1 applies the distro preset policy to all units. On the
+		// RHEL family that policy ends with "disable *" (and the systemd RPM
+		// scriptlet's preset-all does the same during provisioning), which
+		// deletes envd's autostart symlink no matter how it was created.
+		// A 00- preset sorts before every distro policy file and wins.
+		"etc/systemd/system-preset/00-e2b.preset": {Bytes: []byte("enable envd.service\n"), Mode: 0o644},
+
 		// Provision script
 		"usr/local/bin/provision.sh": {Bytes: []byte(provisionScript), Mode: 0o777},
 		// Setup init system
@@ -263,14 +297,7 @@ func additionalOCILayers(
 		return nil, fmt.Errorf("error creating layer from files: %w", err)
 	}
 
-	symlinkLayer, err := oci.LayerSymlink(
-		map[string]string{
-			// Enable envd service autostart
-			"etc/systemd/system/multi-user.target.wants/envd.service": "etc/systemd/system/envd.service",
-			// Enable chrony service autostart
-			"etc/systemd/system/multi-user.target.wants/chrony.service": "etc/systemd/system/chrony.service",
-		},
-	)
+	symlinkLayer, err := oci.LayerSymlink(enableSymlinks)
 	if err != nil {
 		return nil, fmt.Errorf("error creating layer from symlinks: %w", err)
 	}