diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go b/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go index 8bac7f4480..cfda4874f8 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/distro.go @@ -25,8 +25,10 @@ func Fingerprint() string { } // Profile is the declared, per-family provisioning contract. IDs are the -// /etc/os-release values that map to the family; PkgQueryBody, PkgInstall and -// CARefresh are shell fragments spliced into the generated selector. +// /etc/os-release values that map to the family; PkgQueryBody, PkgInstall, +// CARefresh and Bootstrap are shell fragments spliced into the generated +// selector (Bootstrap, if set, runs first — for premade images with no FHS +// userland yet). type Profile struct { Key string Init InitSystem @@ -40,6 +42,7 @@ type Profile struct { AdminGroup string CABundle string CARefresh string + Bootstrap string } var Profiles = []Profile{ @@ -139,6 +142,30 @@ var Profiles = []Profile{ CABundle: "/etc/ssl/certs/ca-certificates.crt", CARefresh: "update-ca-certificates", }, + { + Key: "nixos", + Init: InitNixOS, + IDs: []string{"nixos"}, + // Premade: packages and services are declared in the image's own NixOS + // configuration, so there is no package manager to drive at build time. + Packages: nil, + PkgQueryBody: "true", + PkgInstall: `echo "[provision] ERROR: NixOS images are premade — packages must be declared in the image's NixOS configuration" >&2; exit 1`, + InitBinary: "/nix/var/nix/profiles/system/init", + TimeSyncUnit: "chronyd", + // Left empty on purpose: services.openssh is declared in the image's + // configuration, and the NixOS init setup never enables units. + SSHUnit: "", + AdminGroup: "wheel", + CABundle: "/etc/ssl/certs/ca-certificates.crt", + // The bundle appears at first activation; nothing to refresh pre-boot. + CARefresh: `echo "NixOS: the CA bundle is provided by the image configuration at first activation; nothing to refresh at provision time"`, + // No FHS userland pre-activation — put the baked busybox on PATH first. + Bootstrap: `E2B_BB_DIR=/run/e2b-tools + /usr/bin/busybox mkdir -p "$E2B_BB_DIR" + /usr/bin/busybox --install -s "$E2B_BB_DIR" + export PATH="$E2B_BB_DIR:$PATH"`, + }, } // SupportedIDs returns every os-release ID the selector accepts. @@ -160,6 +187,9 @@ func ShellSelector() string { b.WriteString(`case "$E2B_DISTRO_ID" in` + "\n") for _, p := range Profiles { fmt.Fprintf(&b, " %s)\n", strings.Join(p.IDs, "|")) + if p.Bootstrap != "" { + fmt.Fprintf(&b, " %s\n", p.Bootstrap) + } fmt.Fprintf(&b, " E2B_PACKAGES=%q\n", strings.Join(p.Packages, " ")) fmt.Fprintf(&b, " e2b_pkg_query() { %s; }\n", p.PkgQueryBody) fmt.Fprintf(&b, " e2b_pkg_install() { %s; }\n", p.PkgInstall) diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/init.go b/packages/orchestrator/pkg/template/build/phases/base/distro/init.go index 99b898c601..d55d811efc 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/distro/init.go +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/init.go @@ -13,6 +13,8 @@ const ( InitSystemd InitSystem = "systemd" // InitOpenRC — Alpine (busybox init → OpenRC via the baked /etc/init.d/envd). InitOpenRC InitSystem = "openrc" + // InitNixOS — premade NixOS (declarative; provisioning masks nothing). + InitNixOS InitSystem = "nixos" ) // initSetup is the provisioning-time shell block per init system. Bodies may @@ -127,6 +129,12 @@ if [ -e /etc/init.d/sshd ]; then else echo "sshd service not present on this image; skipping" fi`, + + // NixOS is declaratively configured; drop the baked systemd units so + // activation can own /etc/systemd/system as a store symlink (foreign files + // there make systemd boot with no units at all). + InitNixOS: `echo "NixOS is declaratively configured; removing the baked systemd drop-ins" +rm -rf /etc/systemd/system`, } // indentBlock indents every non-empty line of a shell block for embedding diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/README.md b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/README.md new file mode 100644 index 0000000000..fc1ff9422c --- /dev/null +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/README.md @@ -0,0 +1,46 @@ +# E2B premade NixOS base image + +NixOS templates work the inverse of every other family: instead of the +orchestrator provisioning the image imperatively, the image is **premade** from +`configuration.nix`, which declares everything `provision.sh` installs +elsewhere — the envd systemd unit, chrony, sshd, the default `user` (with a +matching `user` group and the exact sudoers line the build steps check for), +`/bin/bash` (build steps invoke it explicitly), and the journald watchdog +override. The orchestrator's `nixos` profile then only verifies and boots +(see `../distro.go` and the `InitNixOS` block in `../init.go`). + +## Building and publishing + +`./build.sh [registry]` (run on a Linux host with docker; the registry +defaults to `127.0.0.1:5000`): + +1. evaluates the NixOS system closure with `nix` inside a `nixos/nix` + container (`nixpkgs` channel pinned in the script), from the + `configuration.nix` committed next to the script, +2. packs the closure into a single-layer OCI rootfs tar, adding the three + pieces of glue the boot path needs: + - `/sbin/init -> /nix/var/nix/profiles/system/init` (the stage-2 init the + `nixos` profile points the kernel at), + - `/nix/var/nix/profiles/system -> `, + - a static `/etc/os-release` with `ID=nixos` so the distro selector can + identify the image *before* the first activation generates the real one, +3. `docker import`s and pushes the tar. + +**Push every rebuild under a NEW TAG** (hence the required `` argument). +The base-layer cache key includes the image reference as written in the +Dockerfile — republishing under the same tag silently reuses the previously +cached base layer (observed; same "default tag" ambiguity called out in +`phases/base/hash.go`). + +## Boot-path notes (all observed on real KVM) + +- Before the first activation the image has **no FHS userland** — no + `/bin/sh`, no `mkdir`. The provisioning boot runs entirely through the baked + busybox (see `core/rootfs/files/rcS.sh.tpl`, `inittab.tpl`, + `provision-runner.sh.tpl`), and the `nixos` profile's `Bootstrap` puts + busybox applets on `PATH` for the shared provisioning body. +- NixOS activation manages `/etc/systemd/system` as a symlink into the store; + the baked systemd drop-ins must be removed at provisioning (the `InitNixOS` + setup does this) or `setup-etc` refuses the symlink and systemd boots with + no units at all ("Unit default.target not found"). +- The sandbox gets the nix toolchain natively (`nix-env` on PATH for `user`). diff --git a/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/build.sh b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/build.sh new file mode 100755 index 0000000000..457c1db885 --- /dev/null +++ b/packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/build.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Build and publish the E2B premade NixOS base image. +# +# ./build.sh [registry] +# +# Runs from its own directory so the configuration.nix committed next to it is +# the one that gets built. The tag is required: the base-layer cache key +# includes the image reference as written in the Dockerfile, so republishing +# under a tag that was already built silently reuses the cached base layer. +set -euo pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +tag=${1:-} +registry=${2:-127.0.0.1:5000} +if [ -z "$tag" ]; then + echo "usage: ${BASH_SOURCE[0]} [registry] # push every rebuild under a NEW tag" >&2 + exit 1 +fi +image="$registry/e2b-nixos:$tag" + +# Staged outside the repo: the closure tar is ~700 MB, and the repo checkout can +# be a slow network mount on a dev box. +work=${E2B_NIXOS_WORKDIR:-/var/tmp/e2b-nixos-base} +mkdir -p "$work" +cp "$here/configuration.nix" "$work/configuration.nix" +rm -f "$work/result" + +# Build the toplevel closure with nix inside the nixos/nix container. +docker run --rm -v "$work:/build" nixos/nix:2.35.1 sh -c " +set -e +nix-build -I nixpkgs=channel:nixos-24.05 -I nixos-config=/build/configuration.nix \ + '' -A config.system.build.toplevel -o /build/result +top=\$(readlink /build/result) +echo \"TOPLEVEL=\$top\" +# Pack the full closure + the boot/identity glue into one rootfs tar. +nix-store -qR /build/result > /build/closure.txt +tar -cf /build/nixos-rootfs.tar \$(cat /build/closure.txt) +staging=/tmp/extra +rm -rf \$staging +mkdir -p \$staging/sbin \$staging/etc \$staging/nix/var/nix/profiles +ln -s \$top \$staging/nix/var/nix/profiles/system +# Tarring store paths does not make them valid to nix: the DB lives in +# /nix/var/nix/db, which the closure does not carry. Ship the registration so +# first boot can load it, otherwise nix-env and friends reject every path. +nix-store --dump-db \$(cat /build/closure.txt) > \$staging/nix/var/nix/db-registration +ln -s /nix/var/nix/profiles/system/init \$staging/sbin/init +cat > \$staging/etc/os-release < /run/chrony-e2b/source.conf + else + echo "pool pool.ntp.org iburst maxsources 3" > /run/chrony-e2b/source.conf + fi + ''; + }; + + # Journald must not watchdog-reboot when the microVM is paused for + # snapshots (mirrors the systemd-family override baked into other images). + systemd.services.systemd-journald.serviceConfig.WatchdogSec = 0; + + # No serial getty fighting the console; keep the closure lean. + systemd.services."serial-getty@ttyS0".enable = false; + documentation.enable = false; + + # Kernel tunables provision.sh writes to /etc/sysctl.conf on other + # families (NixOS reads sysctl.d from its own config instead). + boot.kernel.sysctl."fs.inotify.max_user_watches" = 65536; + boot.kernel.sysctl."vm.compaction_proactiveness" = 0; + + # E2B build steps and customer commands are executed via /bin/bash (the + # orchestrator invokes it explicitly, like on every FHS distro) — provide it. + system.activationScripts.e2bBinBash = "mkdir -m 0755 -p /bin && ln -sfn ${pkgs.bash}/bin/bash /bin/bash"; + + # Load the store registration build.sh packed, once, so the nix tooling sees + # the closure as valid. Never fail activation over it: without the DB the nix + # commands are broken, but the sandbox itself is fine — which is the status quo + # this repairs, not a regression it could introduce. + system.activationScripts.e2bNixDb = '' + if [ -f /nix/var/nix/db-registration ] && [ ! -e /nix/var/nix/db/db.sqlite ]; then + ${pkgs.nix}/bin/nix-store --load-db < /nix/var/nix/db-registration || true + fi + ''; + + # Parity with the package set provision.sh installs on the other families, so + # a sandbox exposes the same userland whichever base image it was built from. + # (openssh, sudo, chrony and bash are declared as services/programs above.) + # shadow is not optional: finalize's configure.sh runs useradd, usermod and + # passwd by name, the same way it does on the families that install + # shadow/shadow-utils via Packages. + environment.systemPackages = with pkgs; [ + shadow socat curl git jq less fuse3 iptables nftables iputils nfs-utils + ]; + + system.stateVersion = "24.05"; +}