From bca4706359f549917c3c50d7efb147a0b54ea46e Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Mon, 27 Jul 2026 12:01:10 +0200 Subject: [PATCH 01/11] feat(orch): premade NixOS base-image support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a nixos distro profile (premade images — packages are declared in the image's NixOS configuration, not installed at provision time) plus the E2B NixOS base-image definition. Stacks on the base multi-distro provisioning. --- .../build/phases/base/distro/distro.go | 31 ++++++- .../template/build/phases/base/distro/init.go | 8 ++ .../base/distro/nixos-base-image/README.md | 43 +++++++++ .../base/distro/nixos-base-image/build.sh | 30 ++++++ .../distro/nixos-base-image/configuration.nix | 92 +++++++++++++++++++ 5 files changed, 202 insertions(+), 2 deletions(-) create mode 100644 packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/README.md create mode 100644 packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/build.sh create mode 100644 packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/configuration.nix diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go b/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go index 8bac7f4480..e325ab9cee 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go @@ -25,8 +25,10 @@ func Fingerprint() string { } // Profile is the declared, per-family provisioning contract. IDs are the -// /etc/os-release values that map to the family; PkgQueryBody, PkgInstall and -// CARefresh are shell fragments spliced into the generated selector. +// /etc/os-release values that map to the family; PkgQueryBody, PkgInstall, +// CARefresh and Bootstrap are shell fragments spliced into the generated +// selector (Bootstrap, if set, runs first — for premade images with no FHS +// userland yet). type Profile struct { Key string Init InitSystem @@ -40,6 +42,7 @@ type Profile struct { AdminGroup string CABundle string CARefresh string + Bootstrap string } var Profiles = []Profile{ @@ -139,6 +142,27 @@ var Profiles = []Profile{ CABundle: "/etc/ssl/certs/ca-certificates.crt", CARefresh: "update-ca-certificates", }, + { + Key: "nixos", + Init: InitNixOS, + IDs: []string{"nixos"}, + // Premade: packages and services are declared in the image's own NixOS + // configuration, so there is no package manager to drive at build time. + Packages: nil, + PkgQueryBody: "true", + PkgInstall: `echo "[provision] ERROR: NixOS images are premade — packages must be declared in the image's NixOS configuration" >&2; exit 1`, + InitBinary: "/nix/var/nix/profiles/system/init", + TimeSyncUnit: "chronyd", + AdminGroup: "wheel", + CABundle: "/etc/ssl/certs/ca-certificates.crt", + // The bundle appears at first activation; nothing to refresh pre-boot. + CARefresh: `echo "NixOS: the CA bundle is provided by the image configuration at first activation; nothing to refresh at provision time"`, + // No FHS userland pre-activation — put the baked busybox on PATH first. + Bootstrap: `E2B_BB_DIR=/run/e2b-tools + /usr/bin/busybox mkdir -p "$E2B_BB_DIR" + /usr/bin/busybox --install -s "$E2B_BB_DIR" + export PATH="$E2B_BB_DIR:$PATH"`, + }, } // SupportedIDs returns every os-release ID the selector accepts. @@ -160,6 +184,9 @@ func ShellSelector() string { b.WriteString(`case "$E2B_DISTRO_ID" in` + "\n") for _, p := range Profiles { fmt.Fprintf(&b, " %s)\n", strings.Join(p.IDs, "|")) + if p.Bootstrap != "" { + fmt.Fprintf(&b, " %s\n", p.Bootstrap) + } fmt.Fprintf(&b, " E2B_PACKAGES=%q\n", strings.Join(p.Packages, " ")) fmt.Fprintf(&b, " e2b_pkg_query() { %s; }\n", p.PkgQueryBody) fmt.Fprintf(&b, " e2b_pkg_install() { %s; }\n", p.PkgInstall) diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/init.go b/packages/orchestrator/pkg/template/build/phases/base/distro/init.go index 99b898c601..d55d811efc 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/distro/init.go +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/init.go @@ -13,6 +13,8 @@ const ( InitSystemd InitSystem = "systemd" // InitOpenRC — Alpine (busybox init → OpenRC via the baked /etc/init.d/envd). InitOpenRC InitSystem = "openrc" + // InitNixOS — premade NixOS (declarative; provisioning masks nothing). + InitNixOS InitSystem = "nixos" ) // initSetup is the provisioning-time shell block per init system. Bodies may @@ -127,6 +129,12 @@ if [ -e /etc/init.d/sshd ]; then else echo "sshd service not present on this image; skipping" fi`, + + // NixOS is declaratively configured; drop the baked systemd units so + // activation can own /etc/systemd/system as a store symlink (foreign files + // there make systemd boot with no units at all). + InitNixOS: `echo "NixOS is declaratively configured; removing the baked systemd drop-ins" +rm -rf /etc/systemd/system`, } // indentBlock indents every non-empty line of a shell block for embedding diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/README.md b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/README.md new file mode 100644 index 0000000000..af500e9981 --- /dev/null +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/README.md @@ -0,0 +1,43 @@ +# E2B premade NixOS base image + +NixOS templates work the inverse of every other family: instead of the +orchestrator provisioning the image imperatively, the image is **premade** from +`configuration.nix`, which declares everything `provision.sh` installs +elsewhere — the envd systemd unit, chrony, sshd, the default `user` (with a +matching `user` group and the exact sudoers line the build steps check for), +`/bin/bash` (build steps invoke it explicitly), and the journald watchdog +override. The orchestrator's `nixos` profile then only verifies and boots +(see `../distro.go` and the `InitNixOS` block in `../init.go`). + +## Building and publishing + +`build.sh` (run on a Linux host with docker): + +1. evaluates the NixOS system closure with `nix` inside a `nixos/nix` + container (`nixpkgs` channel pinned in the script), +2. packs the closure into a single-layer OCI rootfs tar, adding the three + pieces of glue the boot path needs: + - `/sbin/init -> /nix/var/nix/profiles/system/init` (the stage-2 init the + `nixos` profile points the kernel at), + - `/nix/var/nix/profiles/system -> `, + - a static `/etc/os-release` with `ID=nixos` so the distro selector can + identify the image *before* the first activation generates the real one, +3. `docker import`s and pushes the tar. + +**Push every rebuild under a NEW TAG.** The base-layer cache key includes the +image reference as written in the Dockerfile — republishing under the same tag +silently reuses the previously cached base layer (observed; same "default tag" +ambiguity called out in `phases/base/hash.go`). + +## Boot-path notes (all observed on real KVM) + +- Before the first activation the image has **no FHS userland** — no + `/bin/sh`, no `mkdir`. The provisioning boot runs entirely through the baked + busybox (see `core/rootfs/files/rcS.sh.tpl`, `inittab.tpl`, + `provision-runner.sh.tpl`), and the `nixos` profile's `Bootstrap` puts + busybox applets on `PATH` for the shared provisioning body. +- NixOS activation manages `/etc/systemd/system` as a symlink into the store; + the baked systemd drop-ins must be removed at provisioning (the `InitNixOS` + setup does this) or `setup-etc` refuses the symlink and systemd boots with + no units at all ("Unit default.target not found"). +- The sandbox gets the nix toolchain natively (`nix-env` on PATH for `user`). diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/build.sh b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/build.sh new file mode 100644 index 0000000000..4d5b7fe0db --- /dev/null +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/build.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -e +cd /root/nixos-e2b +# Build the toplevel closure with nix inside the nixos/nix container. +docker run --rm -v /root/nixos-e2b:/build nixos/nix:latest sh -c " +set -e +nix-build -I nixpkgs=channel:nixos-24.05 -I nixos-config=/build/configuration.nix \ + '' -A config.system.build.toplevel -o /build/result +top=\$(readlink /build/result) +echo \"TOPLEVEL=\$top\" +# Pack the full closure + the boot/identity glue into one rootfs tar. +nix-store -qR /build/result > /build/closure.txt +tar -cf /build/nixos-rootfs.tar \$(cat /build/closure.txt) +staging=/tmp/extra +mkdir -p \$staging/sbin \$staging/etc \$staging/nix/var/nix/profiles +ln -s \$top \$staging/nix/var/nix/profiles/system +ln -s /nix/var/nix/profiles/system/init \$staging/sbin/init +cat > \$staging/etc/os-release < Date: Mon, 27 Jul 2026 12:41:55 +0200 Subject: [PATCH 02/11] fix(orch): bring NixOS envd unit and chrony to provision parity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit envd: mirror the [Service] resource controls envd.service.tpl sets — Nice=-20, IOSchedulingClass=realtime/priority 4, LimitCORE=infinity, Delegate, MemoryMin/MemoryLow, CPU/IO accounting + weight, and GOMEMLIMIT (pinned to the tpl's 512 MiB ceiling since a premade image can't know the per-sandbox memory). Without these, NixOS sandboxes ran a thinner envd that could GC without a memory cap and lose CPU/IO contention. chrony: replicate provision.sh's PTP preference. A refclock line for a missing PHC is fatal to chronyd and a premade image can't probe /dev/ptp0 at build time, so a boot-time oneshot writes the source line (PHC refclock when /dev/ptp0 exists, else the NTP pool) into a runtime include. --- .../distro/nixos-base-image/configuration.nix | 49 +++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/configuration.nix b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/configuration.nix index 14b6b45045..5b555b7adc 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/configuration.nix +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/configuration.nix @@ -35,9 +35,25 @@ Restart = "always"; ExecStartPre = "/usr/local/bin/e2b-seed-certs"; ExecStart = "/usr/bin/envd"; - Environment = "GOTRACEBACK=all"; + LimitCORE = "infinity"; + # envd.service.tpl templates GOMEMLIMIT per sandbox as min(MemoryMB/2, 512)MiB; + # a premade image can't know the sandbox size, so pin the 512 MiB ceiling — + # envd must still GC under a cap, not grow unbounded. + Environment = [ "GOTRACEBACK=all" "GOMEMLIMIT=512MiB" ]; + # Priority/scheduling parity with envd.service.tpl (ionice 1:4 = realtime,4). + Nice = -20; + IOSchedulingClass = "realtime"; + IOSchedulingPriority = 4; OOMPolicy = "continue"; OOMScoreAdjust = -1000; + # Resource-control parity: reserve envd's memory and win CPU/IO contention. + Delegate = true; + MemoryMin = "50M"; + MemoryLow = "100M"; + CPUAccounting = true; + CPUWeight = 1000; + IOAccounting = true; + IOWeight = 10000; }; }; @@ -65,10 +81,37 @@ }; security.pam.services.sshd.allowNullPassword = true; + # Time-sync parity with provision.sh: prefer the hypervisor PHC refclock + # (kvm-ptp — no network, tracks the host) when /dev/ptp0 is present, else the + # NTP pool. A refclock line for a missing PHC is FATAL to chronyd, and a premade + # image can't probe the device at build time, so the source line is written at + # boot by e2b-chrony-source.service and pulled in via this include. services.chrony = { enable = true; - servers = [ "pool.ntp.org" ]; - extraConfig = "makestep 1.0 3"; + servers = [ ]; + extraConfig = '' + include /run/chrony-e2b/source.conf + makestep 1.0 3 + ''; + }; + + systemd.services.e2b-chrony-source = { + description = "E2B: select chrony time source (PHC refclock if /dev/ptp0, else NTP)"; + before = [ "chronyd.service" ]; + requiredBy = [ "chronyd.service" ]; + path = [ pkgs.coreutils ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + script = '' + mkdir -p /run/chrony-e2b + if [ -e /dev/ptp0 ]; then + echo "refclock PHC /dev/ptp0 poll 2 dpoll 2" > /run/chrony-e2b/source.conf + else + echo "pool pool.ntp.org iburst maxsources 3" > /run/chrony-e2b/source.conf + fi + ''; }; # Journald must not watchdog-reboot when the microVM is paused for From 41f17c1c1657834d6b7fcb733a13942227bd530f Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Mon, 27 Jul 2026 13:02:04 +0200 Subject: [PATCH 03/11] fix(orch): disable the NixOS firewall so envd stays reachable NixOS is the only supported family that enables a firewall by default; the other families install iptables/nftables for user workloads but never filter. Left on, it drops the orchestrator's connection to envd (TCP 49983) and every customer-exposed sandbox port, so sandboxes boot but are unreachable. --- .../phases/base/distro/nixos-base-image/configuration.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/configuration.nix b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/configuration.nix index 5b555b7adc..171a7670ce 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/configuration.nix +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/configuration.nix @@ -15,6 +15,11 @@ # eth0 is configured by the kernel command line (ip=...); nothing to manage. networking.useDHCP = false; networking.resolvconf.enable = false; + # NixOS is the only family that enables a firewall by default — provision.sh + # installs iptables/nftables for user workloads but never filters. Leaving it on + # would drop the orchestrator's connection to envd (TCP 49983) and every + # customer-exposed sandbox port; isolation is enforced by the E2B network layer. + networking.firewall.enable = false; # The E2B rootfs layer bakes an immutable /etc/resolv.conf; NixOS must not # try to regenerate it at activation. environment.etc."resolv.conf".enable = false; From 17365a26961d6e149d5d3310ac6901cd16d534d1 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Mon, 27 Jul 2026 14:34:34 +0200 Subject: [PATCH 04/11] fix(orch): put socat and iptables on the NixOS envd PATH MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit envd executes both by name — socat to forward exposed ports, iptables to pin the MMDS route — and a NixOS image has no FHS bin dir for the lookup to fall back on, so port exposure and MMDS self-heal both failed. Also declare the userland the other families get from provision.sh's package list, so a sandbox looks the same whichever base image it came from. --- .../base/distro/nixos-base-image/configuration.nix | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/configuration.nix b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/configuration.nix index 171a7670ce..5b4db9e1e2 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/configuration.nix +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/configuration.nix @@ -34,7 +34,10 @@ # tmpfs over /etc/ssl/certs seeded with DEREFERENCED copies of the trust # bundle: envd APPENDS the egress-proxy CA to ca-certificates.crt at # sandbox /init, which must not hit a symlink into the read-only store. - path = [ pkgs.coreutils pkgs.util-linux pkgs.gnutar ]; + # socat and iptables are executed by name: envd spawns socat to forward + # exposed ports and shells out to iptables to pin the MMDS route. There is no + # FHS bin dir to find them in, so they must be on the unit's PATH. + path = [ pkgs.coreutils pkgs.util-linux pkgs.gnutar pkgs.socat pkgs.iptables ]; serviceConfig = { Type = "simple"; Restart = "always"; @@ -136,5 +139,12 @@ # orchestrator invokes it explicitly, like on every FHS distro) — provide it. system.activationScripts.e2bBinBash = "mkdir -m 0755 -p /bin && ln -sfn ${pkgs.bash}/bin/bash /bin/bash"; + # Parity with the package set provision.sh installs on the other families, so + # a sandbox exposes the same userland whichever base image it was built from. + # (openssh, sudo, chrony and bash are declared as services/programs above.) + environment.systemPackages = with pkgs; [ + socat curl git jq less fuse3 iptables nftables iputils nfs-utils + ]; + system.stateVersion = "24.05"; } From 8648379dd865b116206df30003f9dd17a948619f Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Mon, 27 Jul 2026 14:47:28 +0200 Subject: [PATCH 05/11] fix(orch): keep the baked hostname and hosts on NixOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The E2B rootfs layer bakes /etc/hostname and /etc/hosts carrying e2b.local, but only resolv.conf was exempted from NixOS's etc management — activation regenerated both from networking.hostName and ran hostname, so NixOS sandboxes diverged from every other family on hostname and local hosts resolution. --- .../phases/base/distro/nixos-base-image/configuration.nix | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/configuration.nix b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/configuration.nix index 5b4db9e1e2..9783a402b6 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/configuration.nix +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/configuration.nix @@ -23,6 +23,12 @@ # The E2B rootfs layer bakes an immutable /etc/resolv.conf; NixOS must not # try to regenerate it at activation. environment.etc."resolv.conf".enable = false; + # Same for the baked /etc/hostname and /etc/hosts (both carry e2b.local, which + # every other family keeps). An empty hostName also stops activation calling + # `hostname` and overriding the running name. + networking.hostName = ""; + environment.etc."hostname".enable = false; + environment.etc."hosts".enable = false; # The env daemon: E2B bakes the static binary at /usr/bin/envd as an OCI # layer on top of this image. Mirrors envd.service.tpl (systemd family). From dd58d19c080dffc0043a5853fcae0f1e3bbaead7 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Mon, 27 Jul 2026 15:12:31 +0200 Subject: [PATCH 06/11] fix(orch): ship shadow in the NixOS image for configure.sh finalize's configure.sh creates the default user with useradd, usermod and passwd, all resolved by name. Every other family installs shadow/shadow-utils via its package list; the NixOS image has to declare it explicitly. --- .../phases/base/distro/nixos-base-image/configuration.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/configuration.nix b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/configuration.nix index 9783a402b6..743bc21ad2 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/configuration.nix +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/configuration.nix @@ -148,8 +148,11 @@ # Parity with the package set provision.sh installs on the other families, so # a sandbox exposes the same userland whichever base image it was built from. # (openssh, sudo, chrony and bash are declared as services/programs above.) + # shadow is not optional: finalize's configure.sh runs useradd, usermod and + # passwd by name, the same way it does on the families that install + # shadow/shadow-utils via Packages. environment.systemPackages = with pkgs; [ - socat curl git jq less fuse3 iptables nftables iputils nfs-utils + shadow socat curl git jq less fuse3 iptables nftables iputils nfs-utils ]; system.stateVersion = "24.05"; From 78ed8633ba23e3157641c0bbaca8c02b2cd24705 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Mon, 27 Jul 2026 15:22:35 +0200 Subject: [PATCH 07/11] fix(orch): register the nix store DB in the NixOS base image build.sh tarred the closure and the profiles/system symlink but never shipped a store registration, so nix-env and the rest of the toolchain rejected every path as invalid despite the README advertising them. Dump the DB alongside the closure and load it on first activation, guarded so a failure leaves the sandbox exactly as it is today rather than breaking boot. --- .../build/phases/base/distro/nixos-base-image/build.sh | 4 ++++ .../base/distro/nixos-base-image/configuration.nix | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/build.sh b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/build.sh index 4d5b7fe0db..f29ea53636 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/build.sh +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/build.sh @@ -14,6 +14,10 @@ tar -cf /build/nixos-rootfs.tar \$(cat /build/closure.txt) staging=/tmp/extra mkdir -p \$staging/sbin \$staging/etc \$staging/nix/var/nix/profiles ln -s \$top \$staging/nix/var/nix/profiles/system +# Tarring store paths does not make them valid to nix: the DB lives in +# /nix/var/nix/db, which the closure does not carry. Ship the registration so +# first boot can load it, otherwise nix-env and friends reject every path. +nix-store --dump-db \$(cat /build/closure.txt) > \$staging/nix/var/nix/db-registration ln -s /nix/var/nix/profiles/system/init \$staging/sbin/init cat > \$staging/etc/os-release < Date: Mon, 27 Jul 2026 16:21:22 +0200 Subject: [PATCH 08/11] chore(orch): note why the NixOS profile has no SSH unit --- .../pkg/template/build/phases/base/distro/distro.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go b/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go index e325ab9cee..cfda4874f8 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go @@ -153,8 +153,11 @@ var Profiles = []Profile{ PkgInstall: `echo "[provision] ERROR: NixOS images are premade — packages must be declared in the image's NixOS configuration" >&2; exit 1`, InitBinary: "/nix/var/nix/profiles/system/init", TimeSyncUnit: "chronyd", - AdminGroup: "wheel", - CABundle: "/etc/ssl/certs/ca-certificates.crt", + // Left empty on purpose: services.openssh is declared in the image's + // configuration, and the NixOS init setup never enables units. + SSHUnit: "", + AdminGroup: "wheel", + CABundle: "/etc/ssl/certs/ca-certificates.crt", // The bundle appears at first activation; nothing to refresh pre-boot. CARefresh: `echo "NixOS: the CA bundle is provided by the image configuration at first activation; nothing to refresh at provision time"`, // No FHS userland pre-activation — put the baked busybox on PATH first. From 9e5439f3baf7305cacd3ca9f89d5dc194bbc6a79 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Tue, 28 Jul 2026 15:16:11 +0000 Subject: [PATCH 09/11] fix(orch): make the NixOS base-image build.sh runnable It was committed non-executable and cd'd to a hardcoded /root/nixos-e2b that nothing creates, so it could not build the configuration.nix committed beside it. Run from the script's own directory and take the image tag as a required argument (the README already required a new tag per rebuild). --- .../base/distro/nixos-base-image/README.md | 15 ++++--- .../base/distro/nixos-base-image/build.sh | 39 +++++++++++++++---- 2 files changed, 41 insertions(+), 13 deletions(-) mode change 100644 => 100755 packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/build.sh diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/README.md b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/README.md index af500e9981..fc1ff9422c 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/README.md +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/README.md @@ -11,10 +11,12 @@ override. The orchestrator's `nixos` profile then only verifies and boots ## Building and publishing -`build.sh` (run on a Linux host with docker): +`./build.sh [registry]` (run on a Linux host with docker; the registry +defaults to `127.0.0.1:5000`): 1. evaluates the NixOS system closure with `nix` inside a `nixos/nix` - container (`nixpkgs` channel pinned in the script), + container (`nixpkgs` channel pinned in the script), from the + `configuration.nix` committed next to the script, 2. packs the closure into a single-layer OCI rootfs tar, adding the three pieces of glue the boot path needs: - `/sbin/init -> /nix/var/nix/profiles/system/init` (the stage-2 init the @@ -24,10 +26,11 @@ override. The orchestrator's `nixos` profile then only verifies and boots identify the image *before* the first activation generates the real one, 3. `docker import`s and pushes the tar. -**Push every rebuild under a NEW TAG.** The base-layer cache key includes the -image reference as written in the Dockerfile — republishing under the same tag -silently reuses the previously cached base layer (observed; same "default tag" -ambiguity called out in `phases/base/hash.go`). +**Push every rebuild under a NEW TAG** (hence the required `` argument). +The base-layer cache key includes the image reference as written in the +Dockerfile — republishing under the same tag silently reuses the previously +cached base layer (observed; same "default tag" ambiguity called out in +`phases/base/hash.go`). ## Boot-path notes (all observed on real KVM) diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/build.sh b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/build.sh old mode 100644 new mode 100755 index f29ea53636..71b6cbde90 --- a/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/build.sh +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/build.sh @@ -1,8 +1,32 @@ #!/bin/bash -set -e -cd /root/nixos-e2b +# Build and publish the E2B premade NixOS base image. +# +# ./build.sh [registry] +# +# Runs from its own directory so the configuration.nix committed next to it is +# the one that gets built. The tag is required: the base-layer cache key +# includes the image reference as written in the Dockerfile, so republishing +# under a tag that was already built silently reuses the cached base layer. +set -euo pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +tag=${1:-} +registry=${2:-127.0.0.1:5000} +if [ -z "$tag" ]; then + echo "usage: ${BASH_SOURCE[0]} [registry] # push every rebuild under a NEW tag" >&2 + exit 1 +fi +image="$registry/e2b-nixos:$tag" + +# Staged outside the repo: the closure tar is ~700 MB, and the repo checkout can +# be a slow network mount on a dev box. +work=${E2B_NIXOS_WORKDIR:-/var/tmp/e2b-nixos-base} +mkdir -p "$work" +cp "$here/configuration.nix" "$work/configuration.nix" +rm -f "$work/result" + # Build the toplevel closure with nix inside the nixos/nix container. -docker run --rm -v /root/nixos-e2b:/build nixos/nix:latest sh -c " +docker run --rm -v "$work:/build" nixos/nix:latest sh -c " set -e nix-build -I nixpkgs=channel:nixos-24.05 -I nixos-config=/build/configuration.nix \ '' -A config.system.build.toplevel -o /build/result @@ -12,6 +36,7 @@ echo \"TOPLEVEL=\$top\" nix-store -qR /build/result > /build/closure.txt tar -cf /build/nixos-rootfs.tar \$(cat /build/closure.txt) staging=/tmp/extra +rm -rf \$staging mkdir -p \$staging/sbin \$staging/etc \$staging/nix/var/nix/profiles ln -s \$top \$staging/nix/var/nix/profiles/system # Tarring store paths does not make them valid to nix: the DB lives in @@ -28,7 +53,7 @@ OSR tar -rf /build/nixos-rootfs.tar -C \$staging sbin etc nix echo PACKED " -ls -lh /root/nixos-e2b/nixos-rootfs.tar -docker import /root/nixos-e2b/nixos-rootfs.tar 127.0.0.1:5000/e2b-nixos:latest -docker push 127.0.0.1:5000/e2b-nixos:latest -echo IMAGE_PUSHED +ls -lh "$work/nixos-rootfs.tar" +docker import "$work/nixos-rootfs.tar" "$image" +docker push "$image" +echo "IMAGE_PUSHED $image" From 7d6c80620817885d9ec7cb6160fefe81bf5fe621 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Tue, 28 Jul 2026 16:09:06 +0000 Subject: [PATCH 10/11] fix(orch): put the system profile on the NixOS envd unit PATH --- .../distro/nixos-base-image/configuration.nix | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/configuration.nix b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/configuration.nix index 8808fe5cd5..6b9e049705 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/configuration.nix +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/configuration.nix @@ -43,7 +43,21 @@ # socat and iptables are executed by name: envd spawns socat to forward # exposed ports and shells out to iptables to pin the MMDS route. There is no # FHS bin dir to find them in, so they must be on the unit's PATH. - path = [ pkgs.coreutils pkgs.util-linux pkgs.gnutar pkgs.socat pkgs.iptables ]; + # + # envd hands its OWN environment's PATH to every process it spawns, so this + # list is also the PATH of anything started through the exec API that does + # not go through a login shell (`sh -c` — what the orchestrator's pre-pause + # reclaim/sync use — or a bare argv[0]). Without the system profile that PATH + # has no `sh`, `curl`, `git`, `jq`: on the FHS families the same unit inherits + # systemd's default /usr/bin:/bin, i.e. the whole userland, so append NixOS's + # equivalent. /run/wrappers first, like the login PATH, so setuid wrappers win + # over the plain store copies; the explicit packages stay ahead of the profile + # so envd's own helpers always resolve inside this closure. + path = [ + "/run/wrappers" + pkgs.coreutils pkgs.util-linux pkgs.gnutar pkgs.socat pkgs.iptables + "/run/current-system/sw" + ]; serviceConfig = { Type = "simple"; Restart = "always"; From f873b3fdf6bac5ff9a9c9f70b7d5c8681a319f8e Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Wed, 29 Jul 2026 16:13:58 +0200 Subject: [PATCH 11/11] Fix version on Nix:2.35.1 --- .../template/build/phases/base/distro/nixos-base-image/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/build.sh b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/build.sh index 71b6cbde90..457c1db885 100755 --- a/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/build.sh +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/build.sh @@ -26,7 +26,7 @@ cp "$here/configuration.nix" "$work/configuration.nix" rm -f "$work/result" # Build the toplevel closure with nix inside the nixos/nix container. -docker run --rm -v "$work:/build" nixos/nix:latest sh -c " +docker run --rm -v "$work:/build" nixos/nix:2.35.1 sh -c " set -e nix-build -I nixpkgs=channel:nixos-24.05 -I nixos-config=/build/configuration.nix \ '' -A config.system.build.toplevel -o /build/result