diff --git a/packages/envd/internal/services/process/handler/handler.go b/packages/envd/internal/services/process/handler/handler.go index e30320944d..468d6aaa96 100644 --- a/packages/envd/internal/services/process/handler/handler.go +++ b/packages/envd/internal/services/process/handler/handler.go @@ -89,6 +89,17 @@ func currentNice() int { return 20 - prio } +// findBinary returns the first existing path from candidates, falling back to +// the bare name (relying on PATH lookup at exec time) if none exist on disk. +func findBinary(name string, candidates ...string) string { + for _, p := range candidates { + if _, err := os.Stat(p); err == nil { + return p + } + } + return name +} + func New( ctx context.Context, user *user.User, @@ -102,8 +113,12 @@ func New( userCmd := strings.Join(append([]string{req.GetProcess().GetCmd()}, req.GetProcess().GetArgs()...), " ") // Wrap in a shell that resets oom_score_adj, ioprio (ionice best-effort/4), and nice. + // Use PATH-based lookup for ionice/nice to support Alpine (busybox applets at /bin/) + // in addition to standard distros (binaries at /usr/bin/). niceDelta := defaultNice - currentNice() - oomWrapperScript := fmt.Sprintf(`echo %d > /proc/$$/oom_score_adj && exec /usr/bin/ionice -c 2 -n 4 /usr/bin/nice -n %d "${@}"`, defaultOomScore, niceDelta) + ionicePath := findBinary("ionice", "/usr/bin/ionice", "/bin/ionice") + nicePath := findBinary("nice", "/usr/bin/nice", "/bin/nice") + oomWrapperScript := fmt.Sprintf(`echo %d > /proc/$$/oom_score_adj && exec %s -c 2 -n 4 %s -n %d "${@}"`, defaultOomScore, ionicePath, nicePath, niceDelta) wrapperArgs := append([]string{"-c", oomWrapperScript, "--", req.GetProcess().GetCmd()}, req.GetProcess().GetArgs()...) cmd := exec.CommandContext(ctx, "/bin/sh", wrapperArgs...) diff --git a/packages/orchestrator/README.md b/packages/orchestrator/README.md index 47bd5b72d6..2e84f83034 100644 --- a/packages/orchestrator/README.md +++ b/packages/orchestrator/README.md @@ -303,4 +303,4 @@ Automatically set in local mode. Set before running to override: ## Limitations -- Custom template builds require Debian/Ubuntu-based base images (images that provide the `apt` package manager). Non-Debian images such as Alpine, CentOS/RHEL, or other distributions without `apt` are not supported and will fail during the template build/provisioning process. The provisioning scripts used during template build call `apt` and expect Debian-specific package names and file locations. \ No newline at end of file +- Custom template builds now support mainstream Linux distributions, including Debian/Ubuntu, Alpine, CentOS/RHEL. The provisioning logic automatically adapts to the native package manager (apt, apk, dnf/yum) and distro-specific package naming rules, file paths for each target OS during template building and provisioning. \ No newline at end of file diff --git a/packages/orchestrator/pkg/sandbox/fc/process.go b/packages/orchestrator/pkg/sandbox/fc/process.go index cf28fe7ed9..5e36359b02 100644 --- a/packages/orchestrator/pkg/sandbox/fc/process.go +++ b/packages/orchestrator/pkg/sandbox/fc/process.go @@ -405,6 +405,9 @@ func (p *Process) Create( delete(args, "quiet") args["console"] = "ttyS0" args["loglevel"] = "5" // KERN_NOTICE + // Add systemd debug logging for troubleshooting + args["systemd.log_level"] = "debug" + args["systemd.log_target"] = "console" } kernelArgs := args.String() diff --git a/packages/orchestrator/pkg/template/build/commands/user.go b/packages/orchestrator/pkg/template/build/commands/user.go index 0b1f0cb94e..ba550e8bb6 100644 --- a/packages/orchestrator/pkg/template/build/commands/user.go +++ b/packages/orchestrator/pkg/template/build/commands/user.go @@ -53,6 +53,8 @@ func (u *User) Execute( // Only create user if it doesn't exist if !userExists { + // Try Debian-style adduser first, fall back to useradd for RHEL/CentOS/Alpine + createUserCmd := buildCreateUserCmd(userArg) err = sandboxtools.RunCommandWithLogger( ctx, proxy, @@ -60,7 +62,7 @@ func (u *User) Execute( lvl, prefix, sandboxID, - fmt.Sprintf("adduser --disabled-password --gecos \"\" %s", userArg), + createUserCmd, metadata.Context{ User: "root", EnvVars: cmdMetadata.EnvVars, @@ -91,7 +93,7 @@ func addToSudoers( cmdMetadata metadata.Context, userArg string, ) (metadata.Context, error) { - // Add user to sudo group + // Add user to sudo/wheel group (sudo for Debian/Ubuntu, wheel for RHEL/CentOS/Alpine) err := sandboxtools.RunCommandWithLogger( ctx, proxy, @@ -99,7 +101,7 @@ func addToSudoers( lvl, prefix, sandboxID, - fmt.Sprintf("usermod -aG sudo %s", userArg), + buildAddToGroupCmd(userArg), metadata.Context{ User: "root", EnvVars: cmdMetadata.EnvVars, @@ -109,7 +111,7 @@ func addToSudoers( return metadata.Context{}, fmt.Errorf("failed to add user to sudo group: %w", err) } - // Remove password + // Remove password (passwd may not exist on minimal images) err = sandboxtools.RunCommandWithLogger( ctx, proxy, @@ -117,7 +119,7 @@ func addToSudoers( lvl, prefix, sandboxID, - fmt.Sprintf("passwd -d %s", userArg), + buildRemovePasswordCmd(userArg), metadata.Context{ User: "root", EnvVars: cmdMetadata.EnvVars, @@ -135,7 +137,7 @@ func addToSudoers( lvl, prefix, sandboxID, - fmt.Sprintf("grep -q '^%s ALL=(ALL:ALL) NOPASSWD: ALL' /etc/sudoers || echo '%s ALL=(ALL:ALL) NOPASSWD: ALL' >>/etc/sudoers", userArg, userArg), + buildSudoersCmd(userArg), metadata.Context{ User: "root", EnvVars: cmdMetadata.EnvVars, @@ -172,3 +174,32 @@ func saveUserMeta( return cmdMetadata, err } + +// buildCreateUserCmd returns the shell command that creates a user, trying +// Debian-style adduser first, then useradd (RHEL/CentOS), then Alpine adduser. +func buildCreateUserCmd(username string) string { + return fmt.Sprintf( + "adduser --disabled-password --gecos \"\" %s 2>/dev/null || useradd -m %s 2>/dev/null || adduser -D %s", + username, username, username, + ) +} + +// buildAddToGroupCmd returns the shell command that adds a user to the sudo or +// wheel group, depending on the distro. +func buildAddToGroupCmd(username string) string { + return fmt.Sprintf("usermod -aG sudo %s 2>/dev/null || usermod -aG wheel %s 2>/dev/null || true", username, username) +} + +// buildRemovePasswordCmd returns the shell command that removes a user's password. +func buildRemovePasswordCmd(username string) string { + return fmt.Sprintf("passwd -d %s 2>/dev/null || true", username) +} + +// buildSudoersCmd returns the shell command that appends a NOPASSWD sudoers +// entry for the given user if it is not already present. +func buildSudoersCmd(username string) string { + return fmt.Sprintf( + "touch /etc/sudoers && (grep -q '^%s ALL=(ALL:ALL) NOPASSWD: ALL' /etc/sudoers || echo '%s ALL=(ALL:ALL) NOPASSWD: ALL' >>/etc/sudoers)", + username, username, + ) +} diff --git a/packages/orchestrator/pkg/template/build/commands/user_test.go b/packages/orchestrator/pkg/template/build/commands/user_test.go new file mode 100644 index 0000000000..b1821d6a82 --- /dev/null +++ b/packages/orchestrator/pkg/template/build/commands/user_test.go @@ -0,0 +1,154 @@ +//go:build linux + +package commands + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +// ─── buildCreateUserCmd ─────────────────────────────────────────────────────── + +func TestBuildCreateUserCmd_ContainsUsername(t *testing.T) { + t.Parallel() + cmd := buildCreateUserCmd("alice") + assert.Contains(t, cmd, "alice", "command should reference the username") +} + +func TestBuildCreateUserCmd_DebianStyleFirst(t *testing.T) { + t.Parallel() + cmd := buildCreateUserCmd("alice") + // adduser (Debian) must appear before useradd (RHEL) in the fallback chain. + debianIdx := strings.Index(cmd, "adduser --disabled-password") + rhelIdx := strings.Index(cmd, "useradd -m") + assert.Greater(t, debianIdx, -1, "Debian-style adduser should be present") + assert.Greater(t, rhelIdx, -1, "useradd should be present as fallback") + assert.Less(t, debianIdx, rhelIdx, "Debian-style adduser should come before useradd") +} + +func TestBuildCreateUserCmd_AlpineFallback(t *testing.T) { + t.Parallel() + cmd := buildCreateUserCmd("alice") + // Alpine's adduser -D must be the last fallback. + assert.Contains(t, cmd, "adduser -D alice", "Alpine adduser -D should be present") + alpineIdx := strings.Index(cmd, "adduser -D") + rhelIdx := strings.Index(cmd, "useradd -m") + assert.Greater(t, alpineIdx, rhelIdx, "Alpine adduser -D should come after useradd") +} + +func TestBuildCreateUserCmd_SuppressesStderr(t *testing.T) { + t.Parallel() + cmd := buildCreateUserCmd("bob") + // Each alternative should redirect stderr to /dev/null so that the + // fallback chain works correctly (non-zero exit triggers the next ||). + assert.Contains(t, cmd, "2>/dev/null", "stderr should be suppressed") +} + +func TestBuildCreateUserCmd_SpecialUsername(t *testing.T) { + t.Parallel() + cmd := buildCreateUserCmd("www-data") + assert.Contains(t, cmd, "www-data") +} + +// ─── buildAddToGroupCmd ─────────────────────────────────────────────────────── + +func TestBuildAddToGroupCmd_ContainsSudoGroup(t *testing.T) { + t.Parallel() + cmd := buildAddToGroupCmd("alice") + assert.Contains(t, cmd, "sudo", "should add to sudo group (Debian/Ubuntu)") +} + +func TestBuildAddToGroupCmd_ContainsWheelGroup(t *testing.T) { + t.Parallel() + cmd := buildAddToGroupCmd("alice") + assert.Contains(t, cmd, "wheel", "should add to wheel group (RHEL/CentOS/Alpine)") +} + +func TestBuildAddToGroupCmd_SudoBeforeWheel(t *testing.T) { + t.Parallel() + cmd := buildAddToGroupCmd("alice") + sudoIdx := strings.Index(cmd, "sudo") + wheelIdx := strings.Index(cmd, "wheel") + assert.Less(t, sudoIdx, wheelIdx, "sudo group should be tried before wheel") +} + +func TestBuildAddToGroupCmd_EndsWithTrue(t *testing.T) { + t.Parallel() + cmd := buildAddToGroupCmd("alice") + // The command must end with `|| true` so it never fails even on distros + // that have neither sudo nor wheel groups. + assert.True(t, strings.HasSuffix(strings.TrimSpace(cmd), "|| true"), + "command should end with '|| true' to be non-fatal") +} + +func TestBuildAddToGroupCmd_ContainsUsername(t *testing.T) { + t.Parallel() + cmd := buildAddToGroupCmd("charlie") + assert.Contains(t, cmd, "charlie") +} + +// ─── buildRemovePasswordCmd ─────────────────────────────────────────────────── + +func TestBuildRemovePasswordCmd_ContainsUsername(t *testing.T) { + t.Parallel() + cmd := buildRemovePasswordCmd("alice") + assert.Contains(t, cmd, "alice") +} + +func TestBuildRemovePasswordCmd_UsesPasswdD(t *testing.T) { + t.Parallel() + cmd := buildRemovePasswordCmd("alice") + assert.Contains(t, cmd, "passwd -d", "should use passwd -d to remove password") +} + +func TestBuildRemovePasswordCmd_NonFatal(t *testing.T) { + t.Parallel() + cmd := buildRemovePasswordCmd("alice") + // Must be non-fatal because passwd may not exist on minimal images. + assert.True(t, strings.HasSuffix(strings.TrimSpace(cmd), "|| true"), + "command should end with '|| true' to be non-fatal") +} + +// ─── buildSudoersCmd ───────────────────────────────────────────────────────── + +func TestBuildSudoersCmd_ContainsUsername(t *testing.T) { + t.Parallel() + cmd := buildSudoersCmd("alice") + assert.Contains(t, cmd, "alice") +} + +func TestBuildSudoersCmd_ContainsNopasswd(t *testing.T) { + t.Parallel() + cmd := buildSudoersCmd("alice") + assert.Contains(t, cmd, "NOPASSWD: ALL", "sudoers entry should grant passwordless sudo") +} + +func TestBuildSudoersCmd_IdempotentViaGrep(t *testing.T) { + t.Parallel() + cmd := buildSudoersCmd("alice") + // The command must check for an existing entry before appending. + assert.Contains(t, cmd, "grep -q", "should check for existing entry before appending") +} + +func TestBuildSudoersCmd_TouchesSudoers(t *testing.T) { + t.Parallel() + cmd := buildSudoersCmd("alice") + // Must ensure /etc/sudoers exists even on minimal images. + assert.Contains(t, cmd, "touch /etc/sudoers") +} + +func TestBuildSudoersCmd_AppendsEntry(t *testing.T) { + t.Parallel() + cmd := buildSudoersCmd("alice") + assert.Contains(t, cmd, ">>", "should append to sudoers file") + assert.Contains(t, cmd, "/etc/sudoers") +} + +func TestBuildSudoersCmd_FullEntryFormat(t *testing.T) { + t.Parallel() + cmd := buildSudoersCmd("alice") + // Verify the exact sudoers rule format. + assert.Contains(t, cmd, "alice ALL=(ALL:ALL) NOPASSWD: ALL") +} diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/files/80-envd.preset.tpl b/packages/orchestrator/pkg/template/build/core/rootfs/files/80-envd.preset.tpl new file mode 100644 index 0000000000..a4be418d3c --- /dev/null +++ b/packages/orchestrator/pkg/template/build/core/rootfs/files/80-envd.preset.tpl @@ -0,0 +1,3 @@ +{{- /*gotype:github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs.templateModel*/ -}} +{{ .WriteFile "/etc/systemd/system-preset/80-envd.preset" 0o644 }} +enable envd.service diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/files/envd-openrc.tpl b/packages/orchestrator/pkg/template/build/core/rootfs/files/envd-openrc.tpl new file mode 100644 index 0000000000..6a5c2b9bc1 --- /dev/null +++ b/packages/orchestrator/pkg/template/build/core/rootfs/files/envd-openrc.tpl @@ -0,0 +1,29 @@ +{{- /*gotype:github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs.templateModel*/ -}} +{{ .WriteFile "/etc/init.d/envd" 0o755 }} +#!/sbin/openrc-run + +description="Env Daemon Service" + +command="/usr/bin/envd" +command_background=true +pidfile="/run/envd.pid" + +# Start as early as possible +start_stop_daemon_args="--nicelevel -20" + +depend() { + need localmount + after bootmisc +} + +start_pre() { + # Seed the tmpfs from the tar packed as the build's last guest step + if ! mountpoint -q /etc/ssl/certs 2>/dev/null; then + mkdir -p /run/e2b/certs + if tar -C /run/e2b/certs -xf /usr/local/share/e2b/ssl-certs.tar 2>/dev/null || \ + cp -a /etc/ssl/certs/. /run/e2b/certs/ 2>/dev/null; then + mount --bind /run/e2b/certs /etc/ssl/certs + fi + [ -s /etc/ssl/certs/ca-certificates.crt ] || update-ca-certificates 2>/dev/null || true + fi +} diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.service.tpl b/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.service.tpl index 7f59a239ad..2159f69f6b 100644 --- a/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.service.tpl +++ b/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.service.tpl @@ -41,7 +41,7 @@ LimitCORE=infinity # guaranteed present for the sandbox's routable lifetime. The only gap is guest # units that auto-start and egress over TLS before /init; that is accepted # (revisit if a template needs boot-time egress). -ExecStartPre=/bin/sh -c 'mountpoint -q /etc/ssl/certs || { mkdir -p /run/e2b/certs && { tar -C /run/e2b/certs -xf /usr/local/share/e2b/ssl-certs.tar 2>/dev/null || cp -a /etc/ssl/certs/. /run/e2b/certs/ 2>/dev/null; }; mount --bind /run/e2b/certs /etc/ssl/certs; } && ([ -s /etc/ssl/certs/ca-certificates.crt ] || update-ca-certificates)' +ExecStartPre=-/bin/sh -c 'mountpoint -q /etc/ssl/certs || { mkdir -p /run/e2b/certs && { tar -C /run/e2b/certs -xf /usr/local/share/e2b/ssl-certs.tar 2>/dev/null || cp -a /etc/ssl/certs/. /run/e2b/certs/ 2>/dev/null; }; mount --bind /run/e2b/certs /etc/ssl/certs; } && ([ -s /etc/ssl/certs/ca-certificates.crt ] || update-ca-certificates 2>/dev/null || update-ca-trust extract 2>/dev/null || true)' ExecStart=/usr/bin/envd Nice=-20 IOSchedulingClass=realtime diff --git a/packages/orchestrator/pkg/template/build/phases/base/builder.go b/packages/orchestrator/pkg/template/build/phases/base/builder.go index 24dd56726d..7b1c56af6f 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/builder.go +++ b/packages/orchestrator/pkg/template/build/phases/base/builder.go @@ -255,6 +255,16 @@ func (bb *BaseBuilder) buildLayerFromOCI( return metadata.Template{}, fmt.Errorf("error enlarging disk after provisioning: %w", err) } + // Fix envd preset files after provisioning + // systemd 219 (CentOS 7) re-creates .wants/ symlinks from preset files on first boot + // when /etc is unpopulated. We need to ensure the preset file exists in /etc/systemd/system-preset/ + // which has higher priority than /usr/lib/systemd/system-preset/ + err = bb.fixEnvdPresetFiles(ctx, rootfsPath) + if err != nil { + userLogger.Warn(ctx, "Warning: failed to fix envd preset files", zap.Error(err)) + // Don't fail the build, just warn + } + // Create sandbox for building template userLogger.Debug(ctx, "Creating base sandbox template layer") @@ -395,3 +405,96 @@ func (bb *BaseBuilder) Layer( }, nil } } + +// fixEnvdPresetFiles ensures the envd preset file exists in /etc/systemd/system-preset/ +// after provisioning. systemd 219 (CentOS 7) re-creates .wants/ symlinks from preset files +// on first boot when /etc is unpopulated, so we need to ensure the preset file is present. +func (bb *BaseBuilder) fixEnvdPresetFiles(ctx context.Context, rootfsPath string) error { + // Mount the rootfs to access it + // Use a build-specific directory name to avoid conflicts when multiple builds run concurrently. + mountPoint := filepath.Join(bb.BuilderConfig.TemplatesDir, "preset-fix-mount-"+bb.Template.BuildID) + err := os.MkdirAll(mountPoint, 0o755) + if err != nil { + return fmt.Errorf("error creating mount point: %w", err) + } + defer os.RemoveAll(mountPoint) + + // Mount the rootfs + err = filesystem.Mount(ctx, rootfsPath, mountPoint) + if err != nil { + return fmt.Errorf("error mounting rootfs: %w", err) + } + defer filesystem.Unmount(ctx, mountPoint) + + return applyEnvdPresetFiles(mountPoint) +} + +// applyEnvdPresetFiles writes systemd preset files and re-creates service +// symlinks inside an already-mounted rootfs at mountPoint. +// It is a pure filesystem operation with no mount/unmount side-effects, which +// makes it straightforward to unit-test against a temporary directory. +func applyEnvdPresetFiles(mountPoint string) error { + // Create /etc/systemd/system-preset directory if it doesn't exist + presetDir := filepath.Join(mountPoint, "etc/systemd/system-preset") + if err := os.MkdirAll(presetDir, 0o755); err != nil { + return fmt.Errorf("error creating preset directory: %w", err) + } + + // Write the preset file + presetContent := "enable envd.service\n" + presetFile := filepath.Join(presetDir, "80-envd.preset") + if err := os.WriteFile(presetFile, []byte(presetContent), 0o644); err != nil { + return fmt.Errorf("error writing preset file: %w", err) + } + + // Also ensure /usr/lib/systemd/system-preset/80-envd.preset exists + usrPresetDir := filepath.Join(mountPoint, "usr/lib/systemd/system-preset") + if err := os.MkdirAll(usrPresetDir, 0o755); err != nil { + return fmt.Errorf("error creating usr preset directory: %w", err) + } + + usrPresetFile := filepath.Join(usrPresetDir, "80-envd.preset") + if err := os.WriteFile(usrPresetFile, []byte(presetContent), 0o644); err != nil { + return fmt.Errorf("error writing usr preset file: %w", err) + } + + // Directly re-create the envd.service symlink in multi-user.target.wants + // This is critical for CentOS 7 (systemd 219) where provisioning may have + // removed or overwritten the symlink, and systemd's preset-based re-creation + // on first boot is unreliable. + wantsDir := filepath.Join(mountPoint, "etc/systemd/system/multi-user.target.wants") + if err := os.MkdirAll(wantsDir, 0o755); err != nil { + return fmt.Errorf("error creating wants directory: %w", err) + } + + envdSymlink := filepath.Join(wantsDir, "envd.service") + // Remove existing (possibly broken) symlink + os.Remove(envdSymlink) + // Create symlink pointing to the envd.service unit file + // Try /etc/systemd/system/envd.service first (where rootfs layer puts it) + envdServicePath := "/etc/systemd/system/envd.service" + if _, statErr := os.Lstat(filepath.Join(mountPoint, "etc/systemd/system/envd.service")); statErr != nil { + // Fallback: maybe it's in /usr/lib/systemd/system/ + if _, statErr2 := os.Lstat(filepath.Join(mountPoint, "usr/lib/systemd/system/envd.service")); statErr2 == nil { + envdServicePath = "/usr/lib/systemd/system/envd.service" + } + } + if err := os.Symlink(envdServicePath, envdSymlink); err != nil { + return fmt.Errorf("error creating envd.service symlink: %w", err) + } + + // Also re-create chrony.service symlink + chronySymlink := filepath.Join(wantsDir, "chrony.service") + os.Remove(chronySymlink) + chronyServicePath := "/etc/systemd/system/chrony.service" + if _, statErr := os.Lstat(filepath.Join(mountPoint, "etc/systemd/system/chrony.service")); statErr != nil { + if _, statErr2 := os.Lstat(filepath.Join(mountPoint, "usr/lib/systemd/system/chronyd.service")); statErr2 == nil { + chronyServicePath = "/usr/lib/systemd/system/chronyd.service" + } else if _, statErr3 := os.Lstat(filepath.Join(mountPoint, "usr/lib/systemd/system/chrony.service")); statErr3 == nil { + chronyServicePath = "/usr/lib/systemd/system/chrony.service" + } + } + os.Symlink(chronyServicePath, chronySymlink) + + return nil +} diff --git a/packages/orchestrator/pkg/template/build/phases/base/builder_test.go b/packages/orchestrator/pkg/template/build/phases/base/builder_test.go new file mode 100644 index 0000000000..9d87913dd6 --- /dev/null +++ b/packages/orchestrator/pkg/template/build/phases/base/builder_test.go @@ -0,0 +1,202 @@ +package base +//go:build linux + +package base + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// setupFakeRootfs creates a minimal fake rootfs directory tree that simulates +// a mounted ext4 image. It returns the path to the fake mount point. +func setupFakeRootfs(t *testing.T) string { + t.Helper() + root := t.TempDir() + + // Create the directory structure that a real rootfs would have. + dirs := []string{ + "etc/systemd/system", + "etc/systemd/system-preset", + "usr/lib/systemd/system", + "usr/lib/systemd/system-preset", + } + for _, d := range dirs { + require.NoError(t, os.MkdirAll(filepath.Join(root, d), 0o755)) + } + + return root +} + +// createFakeServiceFile creates a dummy unit file at the given path inside root. +func createFakeServiceFile(t *testing.T, root, relPath string) { + t.Helper() + full := filepath.Join(root, relPath) + require.NoError(t, os.MkdirAll(filepath.Dir(full), 0o755)) + require.NoError(t, os.WriteFile(full, []byte("[Unit]\nDescription=fake\n"), 0o644)) +} + +// readSymlink reads the target of a symlink and returns it. +func readSymlink(t *testing.T, path string) string { + t.Helper() + target, err := os.Readlink(path) + require.NoError(t, err, "expected a symlink at %s", path) + return target +} + +// ─── preset file content ────────────────────────────────────────────────────── + +func TestApplyEnvdPresetFiles_EtcPresetFileContent(t *testing.T) { + t.Parallel() + root := setupFakeRootfs(t) + + require.NoError(t, applyEnvdPresetFiles(root)) + + content, err := os.ReadFile(filepath.Join(root, "etc/systemd/system-preset/80-envd.preset")) + require.NoError(t, err) + assert.Equal(t, "enable envd.service\n", string(content)) +} + +func TestApplyEnvdPresetFiles_UsrLibPresetFileContent(t *testing.T) { + t.Parallel() + root := setupFakeRootfs(t) + + require.NoError(t, applyEnvdPresetFiles(root)) + + content, err := os.ReadFile(filepath.Join(root, "usr/lib/systemd/system-preset/80-envd.preset")) + require.NoError(t, err) + assert.Equal(t, "enable envd.service\n", string(content)) +} + +// ─── envd.service symlink ───────────────────────────────────────────────────── + +func TestApplyEnvdPresetFiles_EnvdSymlinkDefaultsToEtcSystemd(t *testing.T) { + t.Parallel() + root := setupFakeRootfs(t) + // Place envd.service in /etc/systemd/system/ (Debian/Ubuntu layout). + createFakeServiceFile(t, root, "etc/systemd/system/envd.service") + + require.NoError(t, applyEnvdPresetFiles(root)) + + symlink := filepath.Join(root, "etc/systemd/system/multi-user.target.wants/envd.service") + target := readSymlink(t, symlink) + assert.Equal(t, "/etc/systemd/system/envd.service", target) +} + +func TestApplyEnvdPresetFiles_EnvdSymlinkFallsBackToUsrLib(t *testing.T) { + t.Parallel() + root := setupFakeRootfs(t) + // Only place envd.service in /usr/lib/systemd/system/ (RHEL/CentOS layout). + createFakeServiceFile(t, root, "usr/lib/systemd/system/envd.service") + + require.NoError(t, applyEnvdPresetFiles(root)) + + symlink := filepath.Join(root, "etc/systemd/system/multi-user.target.wants/envd.service") + target := readSymlink(t, symlink) + assert.Equal(t, "/usr/lib/systemd/system/envd.service", target) +} + +func TestApplyEnvdPresetFiles_EnvdSymlinkReplacesExistingBrokenSymlink(t *testing.T) { + t.Parallel() + root := setupFakeRootfs(t) + createFakeServiceFile(t, root, "etc/systemd/system/envd.service") + + // Pre-create a broken symlink. + wantsDir := filepath.Join(root, "etc/systemd/system/multi-user.target.wants") + require.NoError(t, os.MkdirAll(wantsDir, 0o755)) + brokenSymlink := filepath.Join(wantsDir, "envd.service") + require.NoError(t, os.Symlink("/nonexistent/path", brokenSymlink)) + + require.NoError(t, applyEnvdPresetFiles(root)) + + target := readSymlink(t, brokenSymlink) + assert.Equal(t, "/etc/systemd/system/envd.service", target, + "broken symlink should be replaced with the correct target") +} + +func TestApplyEnvdPresetFiles_WantsDirCreatedIfMissing(t *testing.T) { + t.Parallel() + root := t.TempDir() // intentionally empty — no pre-created dirs + + require.NoError(t, applyEnvdPresetFiles(root)) + + wantsDir := filepath.Join(root, "etc/systemd/system/multi-user.target.wants") + assert.DirExists(t, wantsDir) +} + +// ─── chrony.service symlink ─────────────────────────────────────────────────── + +func TestApplyEnvdPresetFiles_ChronySymlinkDefaultsToEtcSystemd(t *testing.T) { + t.Parallel() + root := setupFakeRootfs(t) + createFakeServiceFile(t, root, "etc/systemd/system/chrony.service") + + require.NoError(t, applyEnvdPresetFiles(root)) + + symlink := filepath.Join(root, "etc/systemd/system/multi-user.target.wants/chrony.service") + target := readSymlink(t, symlink) + assert.Equal(t, "/etc/systemd/system/chrony.service", target) +} + +func TestApplyEnvdPresetFiles_ChronySymlinkFallsBackToChronyd(t *testing.T) { + t.Parallel() + root := setupFakeRootfs(t) + // RHEL/CentOS uses chronyd.service, not chrony.service. + createFakeServiceFile(t, root, "usr/lib/systemd/system/chronyd.service") + + require.NoError(t, applyEnvdPresetFiles(root)) + + symlink := filepath.Join(root, "etc/systemd/system/multi-user.target.wants/chrony.service") + target := readSymlink(t, symlink) + assert.Equal(t, "/usr/lib/systemd/system/chronyd.service", target) +} + +func TestApplyEnvdPresetFiles_ChronySymlinkFallsBackToUsrLibChrony(t *testing.T) { + t.Parallel() + root := setupFakeRootfs(t) + // Some distros ship chrony.service under /usr/lib/systemd/system/. + createFakeServiceFile(t, root, "usr/lib/systemd/system/chrony.service") + + require.NoError(t, applyEnvdPresetFiles(root)) + + symlink := filepath.Join(root, "etc/systemd/system/multi-user.target.wants/chrony.service") + target := readSymlink(t, symlink) + assert.Equal(t, "/usr/lib/systemd/system/chrony.service", target) +} + +func TestApplyEnvdPresetFiles_ChronySymlinkChronydBeforeUsrLibChrony(t *testing.T) { + t.Parallel() + root := setupFakeRootfs(t) + // Both chronyd.service and chrony.service exist; chronyd should win. + createFakeServiceFile(t, root, "usr/lib/systemd/system/chronyd.service") + createFakeServiceFile(t, root, "usr/lib/systemd/system/chrony.service") + + require.NoError(t, applyEnvdPresetFiles(root)) + + symlink := filepath.Join(root, "etc/systemd/system/multi-user.target.wants/chrony.service") + target := readSymlink(t, symlink) + assert.Equal(t, "/usr/lib/systemd/system/chronyd.service", target, + "chronyd.service should take priority over chrony.service in usr/lib") +} + +// ─── idempotency ───────────────────────────────────────────────────────────── + +func TestApplyEnvdPresetFiles_Idempotent(t *testing.T) { + t.Parallel() + root := setupFakeRootfs(t) + createFakeServiceFile(t, root, "etc/systemd/system/envd.service") + createFakeServiceFile(t, root, "etc/systemd/system/chrony.service") + + // Run twice; second call should succeed without error. + require.NoError(t, applyEnvdPresetFiles(root)) + require.NoError(t, applyEnvdPresetFiles(root)) + + // Files should still have the correct content. + content, err := os.ReadFile(filepath.Join(root, "etc/systemd/system-preset/80-envd.preset")) + require.NoError(t, err) + assert.Equal(t, "enable envd.service\n", string(content)) +} diff --git a/packages/orchestrator/pkg/template/build/phases/base/provision.go b/packages/orchestrator/pkg/template/build/phases/base/provision.go index 0923203332..917cd17a45 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/provision.go +++ b/packages/orchestrator/pkg/template/build/phases/base/provision.go @@ -36,7 +36,7 @@ import ( ) const ( - provisionTimeout = 5 * time.Minute + provisionTimeout = 15 * time.Minute ) //go:embed provision.sh diff --git a/packages/orchestrator/pkg/template/build/phases/base/provision.sh b/packages/orchestrator/pkg/template/build/phases/base/provision.sh index 56ecd6d176..670bbe07b3 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/provision.sh +++ b/packages/orchestrator/pkg/template/build/phases/base/provision.sh @@ -10,54 +10,202 @@ echo "Starting provisioning script" # GCP Specific logic {{ end }} +# Detect the distro family +detect_distro() { + if [ -f /etc/os-release ]; then + . /etc/os-release + case "$ID" in + alpine) echo "alpine" ;; + centos|rhel|rocky|alma) echo "rhel" ;; + fedora) echo "rhel" ;; + debian|ubuntu|linuxmint) echo "debian" ;; + *) echo "unknown" ;; + esac + elif [ -f /etc/centos-release ] || [ -f /etc/redhat-release ]; then + echo "rhel" + elif [ -f /etc/alpine-release ]; then + echo "alpine" + else + echo "unknown" + fi +} + +DISTRO=$(detect_distro) +echo "Detected distro family: $DISTRO" + echo "Making configuration immutable" $BUSYBOX chattr +i /etc/resolv.conf -# Helper function to check if a package is installed -is_package_installed() { - dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -q "install ok installed" +# --- Package installation per distro --- +install_packages_debian() { + is_pkg_installed() { + dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -q "install ok installed" + } + PACKAGES="systemd systemd-sysv openssh-server sudo chrony socat curl ca-certificates iptables git nfs-common less nftables iputils-ping jq" + echo "Checking presence of the following packages: $PACKAGES" + MISSING="" + for pkg in $PACKAGES; do + if ! is_pkg_installed "$pkg"; then + echo "Package $pkg is missing, will install it." + MISSING="$MISSING $pkg" + fi + done + + # Handle EOL releases by switching to archive mirrors + if [ -f /etc/os-release ]; then + . /etc/os-release + APT_OUTPUT=$(apt-get -q update 2>&1) && APT_RC=0 || APT_RC=$? + echo "$APT_OUTPUT" + # Some old apt versions return 0 even when repos are 404, so also check output + if [ "$APT_RC" -ne 0 ] || echo "$APT_OUTPUT" | grep -qE "does not have a Release file|Failed to fetch"; then + echo "apt-get update failed or has repo errors (exit code $APT_RC), attempting EOL mirror fallback for $ID $VERSION_CODENAME" + if [ "$ID" = "ubuntu" ]; then + sed -i -e 's|archive.ubuntu.com|old-releases.ubuntu.com|g' \ + -e 's|security.ubuntu.com|old-releases.ubuntu.com|g' \ + /etc/apt/sources.list 2>/dev/null || true + elif [ "$ID" = "debian" ]; then + # Debian EOL releases (e.g. buster, stretch) are moved to archive.debian.org + sed -i -e 's|deb.debian.org|archive.debian.org|g' \ + -e 's|security.debian.org|archive.debian.org|g' \ + /etc/apt/sources.list 2>/dev/null || true + # Remove lines with -updates or -backports as they don't exist in archive + sed -i '/-updates/d; /-backports/d' /etc/apt/sources.list 2>/dev/null || true + # Also handle /etc/apt/sources.list.d/ if present + if [ -d /etc/apt/sources.list.d ]; then + sed -i -e 's|deb.debian.org|archive.debian.org|g' \ + -e 's|security.debian.org|archive.debian.org|g' \ + /etc/apt/sources.list.d/*.list 2>/dev/null || true + sed -i '/-updates/d; /-backports/d' /etc/apt/sources.list.d/*.list 2>/dev/null || true + fi + fi + apt-get -q update + fi + else + apt-get -q update + fi + + # fuse3 is not available on older distros (e.g. Ubuntu 18.04, Debian 9), fall back to fuse + # Use apt-cache policy to check if fuse3 has an installation candidate (not just metadata) + if apt-cache policy fuse3 2>/dev/null | grep -q "Candidate:" && ! apt-cache policy fuse3 2>/dev/null | grep -q "Candidate: (none)"; then + if ! is_pkg_installed "fuse3"; then + echo "Package fuse3 is missing, will install it." + MISSING="$MISSING fuse3" + fi + else + echo "fuse3 not available, falling back to fuse" + if ! is_pkg_installed "fuse"; then + echo "Package fuse is missing, will install it." + MISSING="$MISSING fuse" + fi + fi + + if [ -n "$MISSING" ]; then + echo "Missing packages detected, installing:$MISSING" + DEBIAN_FRONTEND=noninteractive DEBCONF_NOWARNINGS=yes apt-get -qq -o=Dpkg::Use-Pty=0 install -y --no-install-recommends $MISSING + else + echo "All required packages are already installed." + fi } -# Install required packages if not already installed -PACKAGES="systemd systemd-sysv openssh-server sudo chrony socat curl ca-certificates fuse3 iptables git nfs-common less nftables iputils-ping jq" -echo "Checking presence of the following packages: $PACKAGES" +install_packages_rhel() { + PACKAGES="systemd openssh-server sudo chrony socat curl ca-certificates iptables git nfs-utils less nftables iputils jq passwd" -MISSING="" -for pkg in $PACKAGES; do - if ! is_package_installed "$pkg"; then - echo "Package $pkg is missing, will install it." - MISSING="$MISSING $pkg" + # Handle EOL CentOS releases by switching mirrorlist to vault.centos.org + # CentOS 7 EOL: 2024-06-30, CentOS 8 EOL: 2021-12-31 + if [ -f /etc/os-release ]; then + . /etc/os-release + if [ "$ID" = "centos" ]; then + case "$VERSION_ID" in + 7*|8*) + echo "CentOS $VERSION_ID is EOL, switching repos to vault.centos.org" + sed -i 's/^mirrorlist=/#mirrorlist=/g' /etc/yum.repos.d/CentOS-*.repo 2>/dev/null || true + sed -i 's|^#\s*baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*.repo 2>/dev/null || true + ;; + esac + fi fi -done -if [ -n "$MISSING" ]; then - echo "Missing packages detected, installing:$MISSING" - apt-get -q update - DEBIAN_FRONTEND=noninteractive DEBCONF_NOWARNINGS=yes apt-get -qq -o=Dpkg::Use-Pty=0 install -y --no-install-recommends $MISSING -else - echo "All required packages are already installed." -fi + # fuse3 is not available on older RHEL/CentOS (e.g. CentOS 7), fall back to fuse + if command -v dnf >/dev/null 2>&1; then + if dnf info fuse3 >/dev/null 2>&1; then + PACKAGES="$PACKAGES fuse3" + else + PACKAGES="$PACKAGES fuse" + fi + elif command -v yum >/dev/null 2>&1; then + if yum info fuse3 >/dev/null 2>&1; then + PACKAGES="$PACKAGES fuse3" + else + PACKAGES="$PACKAGES fuse" + fi + fi + echo "Installing packages (yum/dnf): $PACKAGES" + if command -v dnf >/dev/null 2>&1; then + dnf install -y --allowerasing $PACKAGES + elif command -v yum >/dev/null 2>&1; then + yum install -y $PACKAGES + fi +} + +install_packages_alpine() { + PACKAGES="openrc openssh-server sudo chrony socat curl ca-certificates iptables git nfs-utils less nftables iputils jq bash shadow" + # fuse3 may not be available on older Alpine versions + if apk info -e fuse3 >/dev/null 2>&1 || apk search -x fuse3 2>/dev/null | grep -q fuse3; then + PACKAGES="$PACKAGES fuse3" + else + PACKAGES="$PACKAGES fuse" + fi + echo "Installing packages (apk): $PACKAGES" + apk update + apk add --no-cache $PACKAGES +} + +case "$DISTRO" in + debian) install_packages_debian ;; + rhel) install_packages_rhel ;; + alpine) install_packages_alpine ;; + *) + echo "WARNING: Unknown distro, skipping package installation" + ;; +esac # Set /dev/fuse permissions to 666 for non-root access -# Use systemd-tmpfiles to set permissions at boot -mkdir -p /etc/tmpfiles.d -echo 'z /dev/fuse 0666 root root -' > /etc/tmpfiles.d/fuse.conf +# Use systemd-tmpfiles on systemd distros (Alpine uses local.d, configured below) +if command -v systemctl >/dev/null 2>&1; then + mkdir -p /etc/tmpfiles.d + echo 'z /dev/fuse 0666 root root -' > /etc/tmpfiles.d/fuse.conf +fi echo "Setting up shell" -echo "export SHELL='/bin/bash'" >/etc/profile.d/shell.sh +if [ -x /bin/bash ]; then + DEFAULT_SHELL="/bin/bash" +else + DEFAULT_SHELL="/bin/sh" +fi +echo "export SHELL='$DEFAULT_SHELL'" >/etc/profile.d/shell.sh echo "export PS1='\w \$ '" >/etc/profile.d/prompt.sh echo "export PS1='\w \$ '" >>"/etc/profile" +mkdir -p /root +touch /root/.bashrc echo "export PS1='\w \$ '" >>"/root/.bashrc" echo "Use .bashrc and .profile" -echo "if [ -f ~/.bashrc ]; then source ~/.bashrc; fi; if [ -f ~/.profile ]; then source ~/.profile; fi" >>/etc/profile +echo "if [ -f ~/.bashrc ]; then . ~/.bashrc; fi; if [ -f ~/.profile ]; then . ~/.profile; fi" >>/etc/profile echo "Remove root password" -passwd -d root +passwd -d root 2>/dev/null || true echo "Setting up chrony" -mkdir -p /etc/chrony -cat </etc/chrony/chrony.conf +# Determine chrony config path based on distro +if [ "$DISTRO" = "rhel" ]; then + CHRONY_CONF_DIR="/etc" + CHRONY_CONF="$CHRONY_CONF_DIR/chrony.conf" +else + CHRONY_CONF_DIR="/etc/chrony" + CHRONY_CONF="$CHRONY_CONF_DIR/chrony.conf" +fi +mkdir -p "$CHRONY_CONF_DIR" +cat <"$CHRONY_CONF" refclock PHC /dev/ptp0 poll 2 dpoll 2 # Step (jump) the clock instead of slewing when the offset exceeds 1s, but only # for the first 3 updates after chronyd starts. chronyd restarts on every cold @@ -68,7 +216,9 @@ makestep 1.0 3 EOF # Add a proxy config, as some environments expects it there (e.g. timemaster in Node Dockerimage) -echo "include /etc/chrony/chrony.conf" >/etc/chrony.conf +if [ "$DISTRO" != "rhel" ]; then + echo "include /etc/chrony/chrony.conf" >/etc/chrony.conf +fi echo "Setting up SSH" mkdir -p /etc/ssh @@ -91,32 +241,111 @@ echo 'vm.compaction_proactiveness=0' | tee -a /etc/sysctl.conf echo "Don't wait for ttyS0 (serial console kernel logs)" # This is required when the Firecracker kernel args has specified console=ttyS0 -systemctl mask serial-getty@ttyS0.service +if command -v systemctl >/dev/null 2>&1; then + systemctl mask serial-getty@ttyS0.service -echo "Disable network online wait" -systemctl mask systemd-networkd-wait-online.service + echo "Disable network online wait" + systemctl mask systemd-networkd-wait-online.service 2>/dev/null || true + # RHEL/CentOS may use NetworkManager-wait-online instead + systemctl mask NetworkManager-wait-online.service 2>/dev/null || true -echo "Disable system first boot wizard" -# This was problem with Ubuntu 24.04, that differently calculate wizard should be called -# and Linux boot was stuck in wizard until envd wait timeout -systemctl mask systemd-firstboot.service + echo "Disable system first boot wizard" + # This was problem with Ubuntu 24.04, that differently calculate wizard should be called + # and Linux boot was stuck in wizard until envd wait timeout + systemctl mask systemd-firstboot.service 2>/dev/null || true -echo "Disable chrony-wait" -# chrony-wait blocks multi-user.target until the first clock sync (~8s); -# chrony still syncs in the background, nothing needs to wait for it. -systemctl mask chrony-wait.service + echo "Disable chrony-wait" + # chrony-wait blocks multi-user.target until the first clock sync (~8s); + # chrony still syncs in the background, nothing needs to wait for it. + systemctl mask chrony-wait.service 2>/dev/null || true -echo "Disable slow boot units not needed in the sandbox" -# binfmt registrations (foreign-arch exec) take ~1s of CPU early in boot and -# compete with envd start; e2scrub is for LVM-backed ext4 only. -systemctl mask systemd-binfmt.service -systemctl mask e2scrub_reap.service + echo "Disable slow boot units not needed in the sandbox" + # binfmt registrations (foreign-arch exec) take ~1s of CPU early in boot and + # compete with envd start; e2scrub is for LVM-backed ext4 only. + systemctl mask systemd-binfmt.service 2>/dev/null || true + systemctl mask e2scrub_reap.service 2>/dev/null || true + + echo "Enable envd service" + systemctl enable envd.service 2>/dev/null || true +else + echo "systemctl not found, configuring OpenRC (Alpine)" + # Create /etc/inittab for busybox init to boot OpenRC + cat > /etc/inittab <<'INITTAB' +::sysinit:/sbin/openrc sysinit +::sysinit:/sbin/openrc boot +::wait:/sbin/openrc default +ttyS0::respawn:/sbin/getty 38400 ttyS0 +::ctrlaltdel:/sbin/reboot +::shutdown:/sbin/openrc shutdown +INITTAB + + # Create /etc/network/interfaces so the networking service doesn't fail. + # Kernel boot args already configure eth0 via ip=..., so we just need + # a valid (minimal) interfaces file for ifupdown-ng to parse. + mkdir -p /etc/network + cat > /etc/network/interfaces <<'NETIF' +auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet manual +NETIF + + # Remove chronyd's hard dependency on networking — it only uses PHC + # /dev/ptp0 and doesn't need a network stack. Replace "need net" with + # "use net" (soft dependency) so chronyd starts even if networking fails. + if [ -f /etc/init.d/chronyd ]; then + sed -i 's/need net/use net/' /etc/init.d/chronyd + fi + + # Set /dev/fuse permissions via a local.d script (Alpine has no systemd-tmpfiles) + mkdir -p /etc/local.d + cat > /etc/local.d/fuse-permissions.start <<'FUSE' +#!/bin/sh +[ -e /dev/fuse ] && chmod 0666 /dev/fuse +FUSE + chmod +x /etc/local.d/fuse-permissions.start + rc-update add local default 2>/dev/null || true + + # Enable envd service via OpenRC + if [ -f /etc/init.d/envd ]; then + rc-update add envd default 2>/dev/null || true + fi + # Enable chrony and sshd via OpenRC + rc-update add chronyd default 2>/dev/null || true + rc-update add sshd default 2>/dev/null || true + # Ensure OpenRC default runlevel boots properly + rc-update add devfs sysinit 2>/dev/null || true + rc-update add dmesg sysinit 2>/dev/null || true + rc-update add mdev sysinit 2>/dev/null || true + # hwdrivers needs 'dev' service which mdev provides; only add if mdev is present + if [ -f /etc/init.d/mdev ]; then + rc-update add hwdrivers sysinit 2>/dev/null || true + fi + rc-update add hostname boot 2>/dev/null || true + rc-update add bootmisc boot 2>/dev/null || true + rc-update add sysctl boot 2>/dev/null || true + rc-update add localmount boot 2>/dev/null || true + rc-update add networking boot 2>/dev/null || true + rc-update add syslog boot 2>/dev/null || true +fi # Clean machine-id from Docker rm -rf /etc/machine-id echo "Linking systemd to init" -ln -sf /lib/systemd/systemd /usr/sbin/init +if [ -f /lib/systemd/systemd ]; then + ln -sf /lib/systemd/systemd /usr/sbin/init +elif [ -f /usr/lib/systemd/systemd ]; then + # RHEL/CentOS use /usr/lib/systemd/systemd + ln -sf /usr/lib/systemd/systemd /usr/sbin/init +else + echo "systemd not found, skipping init link (Alpine/OpenRC)" + # envd's process handler uses hardcoded /usr/bin/ionice and /usr/bin/nice paths. + # On Alpine these are busybox applets at /bin/, so create symlinks. + [ ! -e /usr/bin/ionice ] && [ -e /bin/ionice ] && ln -sf /bin/ionice /usr/bin/ionice + [ ! -e /usr/bin/nice ] && [ -e /bin/nice ] && ln -sf /bin/nice /usr/bin/nice +fi echo "Unlocking immutable configuration" $BUSYBOX chattr -i /etc/resolv.conf diff --git a/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh b/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh index 76202a81a5..7a32ee8dc7 100644 --- a/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh +++ b/packages/orchestrator/pkg/template/build/phases/finalize/configure.sh @@ -1,6 +1,5 @@ -#!/bin/bash -export BASH_XTRACEFD=1 -set -euo pipefail +#!/bin/sh +set -eu echo "Starting configuration script" @@ -10,22 +9,88 @@ TEMPLATE_ID={{ .TemplateID }} BUILD_ID={{ .BuildID }} EOF +# Detect the distro family +detect_distro() { + if [ -f /etc/os-release ]; then + . /etc/os-release + case "$ID" in + alpine) echo "alpine" ;; + centos|rhel|rocky|alma) echo "rhel" ;; + fedora) echo "rhel" ;; + debian|ubuntu|linuxmint) echo "debian" ;; + *) echo "unknown" ;; + esac + elif [ -f /etc/centos-release ] || [ -f /etc/redhat-release ]; then + echo "rhel" + elif [ -f /etc/alpine-release ]; then + echo "alpine" + else + echo "unknown" + fi +} + +DISTRO=$(detect_distro) +echo "Detected distro family: $DISTRO" + # Create default user. # if the /home/user directory exists, we copy the skeleton files to it because the adduser command # will ignore the directory if it exists, but we want to include the skeleton files in the home directory # in our case. echo "Create default user 'user' (if doesn't exist yet)" -ADDUSER_OUTPUT=$(adduser -disabled-password --gecos "" user 2>&1 || true) -echo "$ADDUSER_OUTPUT" -if echo "$ADDUSER_OUTPUT" | grep -q "The home directory \`/home/user' already exists"; then - # Copy skeleton files if they don't exist in the home directory - echo "Copy skeleton files to /home/user" - cp -rn /etc/skel/. /home/user/ -fi - -echo "Add sudo to 'user' with no password" -usermod -aG sudo user -passwd -d user + +create_user_debian() { + ADDUSER_OUTPUT=$(adduser --disabled-password --gecos "" user 2>&1 || true) + echo "$ADDUSER_OUTPUT" + if echo "$ADDUSER_OUTPUT" | grep -q "The home directory.*already exists"; then + echo "Copy skeleton files to /home/user" + cp -rn /etc/skel/. /home/user/ + fi + echo "Add sudo to 'user' with no password" + usermod -aG sudo user +} + +create_user_rhel() { + if ! id user >/dev/null 2>&1; then + useradd -m user 2>&1 || true + else + echo "User 'user' already exists" + # Copy skeleton files if home exists + if [ -d /home/user ]; then + cp -rn /etc/skel/. /home/user/ 2>/dev/null || true + fi + fi + echo "Add sudo to 'user' with no password (wheel group)" + usermod -aG wheel user 2>/dev/null || true + # Also try sudo group in case it exists + usermod -aG sudo user 2>/dev/null || true +} + +create_user_alpine() { + if ! id user >/dev/null 2>&1; then + adduser -D -s /bin/sh user 2>&1 || true + else + echo "User 'user' already exists" + fi + echo "Add sudo to 'user' with no password (wheel group)" + addgroup user wheel 2>/dev/null || true +} + +case "$DISTRO" in + debian) create_user_debian ;; + rhel) create_user_rhel ;; + alpine) create_user_alpine ;; + *) + # Fallback: try useradd first, then adduser + if command -v useradd >/dev/null 2>&1; then + useradd -m user 2>/dev/null || true + usermod -aG wheel user 2>/dev/null || usermod -aG sudo user 2>/dev/null || true + elif command -v adduser >/dev/null 2>&1; then + adduser -D user 2>/dev/null || adduser --disabled-password --gecos "" user 2>/dev/null || true + fi + ;; +esac + +passwd -d user 2>/dev/null || true echo "user ALL=(ALL:ALL) NOPASSWD: ALL" >>/etc/sudoers echo "Give 'user' ownership to /home/user" diff --git a/packages/orchestrator/pkg/template/build/sandboxtools/command.go b/packages/orchestrator/pkg/template/build/sandboxtools/command.go index 558761ab64..628fabcd17 100644 --- a/packages/orchestrator/pkg/template/build/sandboxtools/command.go +++ b/packages/orchestrator/pkg/template/build/sandboxtools/command.go @@ -19,7 +19,6 @@ import ( "github.com/e2b-dev/infra/packages/orchestrator/pkg/proxy" "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox" - "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs" "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/metadata" "github.com/e2b-dev/infra/packages/shared/pkg/grpc" "github.com/e2b-dev/infra/packages/shared/pkg/grpc/envd/process" @@ -246,7 +245,7 @@ func SyncChangesToDisk( ctx, proxy, sandboxID, - rootfs.SandboxBusyBoxPath+" sync", + "sync", metadata.Context{ User: "root", }, diff --git a/packages/orchestrator/pkg/template/cache/build_cache.go b/packages/orchestrator/pkg/template/cache/build_cache.go index eba0744146..de6d410453 100644 --- a/packages/orchestrator/pkg/template/cache/build_cache.go +++ b/packages/orchestrator/pkg/template/cache/build_cache.go @@ -17,7 +17,7 @@ import ( ) const ( - buildInfoExpiration = time.Minute * 10 // 10 minutes + buildInfoExpiration = time.Minute * 70 // Must exceed the API-side buildTimeout (1 hour) ) type BuildInfoResult struct { @@ -57,6 +57,9 @@ func (b *BuildInfo) GetResult() *BuildInfoResult { } func (b *BuildInfo) SetSuccess(metadata *template_manager.TemplateBuildMetadata) { + logger.L().Info(context.Background(), "build marked as SUCCESS in cache", + zap.String("team.id", b.TeamID), + ) _ = b.Result.SetValue(BuildInfoResult{ Status: template_manager.TemplateBuildState_Completed, Metadata: metadata, @@ -65,6 +68,14 @@ func (b *BuildInfo) SetSuccess(metadata *template_manager.TemplateBuildMetadata) } func (b *BuildInfo) SetFail(reason *template_manager.TemplateBuildStatusReason) { + var msg string + if reason != nil { + msg = reason.GetMessage() + } + logger.L().Warn(context.Background(), "build marked as FAILED in cache", + zap.String("team.id", b.TeamID), + zap.String("reason", msg), + ) _ = b.Result.SetValue(BuildInfoResult{ Status: template_manager.TemplateBuildState_Failed, Reason: reason, @@ -83,7 +94,27 @@ type BuildCache struct { func NewBuildCache(ctx context.Context, meterProvider metric.MeterProvider) *BuildCache { meter := meterProvider.Meter("github.com/e2b-dev/infra/packages/orchestrator/pkg/template/cache") + logger.L().Info(ctx, "creating build cache", zap.Duration("ttl", buildInfoExpiration)) + cache := ttlcache.New(ttlcache.WithTTL[string, *BuildInfo](buildInfoExpiration)) + + // Log when cache entries are evicted so we can diagnose premature expiry. + cache.OnEviction(func(_ context.Context, reason ttlcache.EvictionReason, item *ttlcache.Item[string, *BuildInfo]) { + buildID := item.Key() + info := item.Value() + var status string + if info != nil { + status = info.GetStatus().String() + } else { + status = "nil" + } + logger.L().Warn(ctx, "build cache entry evicted", + zap.String("build.id", buildID), + zap.Int("eviction_reason", int(reason)), + zap.String("build_status", status), + zap.Duration("ttl", buildInfoExpiration), + ) + }) _, err := telemetry.GetObservableUpDownCounter(meter, telemetry.BuildCounterMeterName, func(_ context.Context, observer metric.Int64Observer) error { items := utils.MapValues(cache.Items()) @@ -133,12 +164,25 @@ func NewBuildCache(ctx context.Context, meterProvider metric.MeterProvider) *Bui func (c *BuildCache) Get(buildID string) (*BuildInfo, error) { item := c.cache.Get(buildID) if item == nil { - return nil, fmt.Errorf("build %s not found in cache", buildID) + // Log all current cache keys to help diagnose missing entries. + var keys []string + for k := range c.cache.Items() { + keys = append(keys, k) + } + logger.L().Warn(context.Background(), "build cache miss", + zap.String("build.id", buildID), + zap.Int("cache_size", len(keys)), + zap.Strings("cached_builds", keys), + ) + return nil, fmt.Errorf("build %s not found in cache (cache has %d entries)", buildID, len(keys)) } value := item.Value() if value == nil { - return nil, fmt.Errorf("build %s not found in cache", buildID) + logger.L().Warn(context.Background(), "build cache hit but nil value", + zap.String("build.id", buildID), + ) + return nil, fmt.Errorf("build %s not found in cache (nil value)", buildID) } return value, nil @@ -154,15 +198,24 @@ func (c *BuildCache) Create(teamID string, buildID string, logs *buildlogger.Log _, found := c.cache.GetOrSet(buildID, info, ttlcache.WithTTL[string, *BuildInfo](buildInfoExpiration), - ttlcache.WithDisableTouchOnHit[string, *BuildInfo](), ) if found { return nil, fmt.Errorf("build %s already exists in cache", buildID) } + logger.L().Info(context.Background(), "build cache entry created", + zap.String("build.id", buildID), + zap.String("team.id", teamID), + zap.Duration("ttl", buildInfoExpiration), + zap.Int("cache_size", c.cache.Len()), + ) + return info, nil } func (c *BuildCache) Delete(buildID string) { + logger.L().Info(context.Background(), "build cache entry explicitly deleted", + zap.String("build.id", buildID), + ) c.cache.Delete(buildID) } diff --git a/packages/orchestrator/pkg/template/cache/build_cache_test.go b/packages/orchestrator/pkg/template/cache/build_cache_test.go new file mode 100644 index 0000000000..81d931810b --- /dev/null +++ b/packages/orchestrator/pkg/template/cache/build_cache_test.go @@ -0,0 +1,226 @@ +package cache + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/metric/noop" + + "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/buildlogger" + template_manager "github.com/e2b-dev/infra/packages/shared/pkg/grpc/template-manager" + "github.com/e2b-dev/infra/packages/shared/pkg/utils" +) + +// newSetOnce is a convenience wrapper for the generic SetOnce constructor. +func newSetOnce() *utils.SetOnce[BuildInfoResult] { + return utils.NewSetOnce[BuildInfoResult]() +} + +// newTestBuildCache creates a BuildCache with a noop meter provider for testing. +func newTestBuildCache(t *testing.T) *BuildCache { + t.Helper() + ctx := context.Background() + bc := NewBuildCache(ctx, noop.NewMeterProvider()) + t.Cleanup(func() { bc.cache.Stop() }) + return bc +} + +// newTestLogs creates a LogEntryLogger for testing. +func newTestLogs() *buildlogger.LogEntryLogger { + return buildlogger.NewLogEntryLogger() +} + +// ─── BuildInfo state machine ────────────────────────────────────────────────── + +func TestBuildInfo_InitialStateIsBuilding(t *testing.T) { + t.Parallel() + info := &BuildInfo{ + TeamID: "team-1", + logs: newTestLogs(), + Result: newSetOnce(), + } + + assert.Equal(t, template_manager.TemplateBuildState_Building, info.GetStatus()) + assert.True(t, info.IsRunning()) + assert.Nil(t, info.GetResult()) +} + +func TestBuildInfo_SetSuccess(t *testing.T) { + t.Parallel() + info := &BuildInfo{ + TeamID: "team-1", + logs: newTestLogs(), + Result: newSetOnce(), + } + + meta := &template_manager.TemplateBuildMetadata{EnvdVersionKey: "1.0.0"} + info.SetSuccess(meta) + + assert.Equal(t, template_manager.TemplateBuildState_Completed, info.GetStatus()) + assert.False(t, info.IsRunning()) + + result := info.GetResult() + require.NotNil(t, result) + assert.Equal(t, template_manager.TemplateBuildState_Completed, result.Status) + assert.Equal(t, meta, result.Metadata) + assert.Nil(t, result.Reason) +} + +func TestBuildInfo_SetFail(t *testing.T) { + t.Parallel() + info := &BuildInfo{ + TeamID: "team-1", + logs: newTestLogs(), + Result: newSetOnce(), + } + + reason := &template_manager.TemplateBuildStatusReason{Message: "out of disk"} + info.SetFail(reason) + + assert.Equal(t, template_manager.TemplateBuildState_Failed, info.GetStatus()) + assert.False(t, info.IsRunning()) + + result := info.GetResult() + require.NotNil(t, result) + assert.Equal(t, template_manager.TemplateBuildState_Failed, result.Status) + assert.Equal(t, reason, result.Reason) + assert.Nil(t, result.Metadata) +} + +func TestBuildInfo_SetFailWithNilReason(t *testing.T) { + t.Parallel() + info := &BuildInfo{ + TeamID: "team-1", + logs: newTestLogs(), + Result: newSetOnce(), + } + + // Should not panic when reason is nil. + info.SetFail(nil) + + assert.Equal(t, template_manager.TemplateBuildState_Failed, info.GetStatus()) +} + +func TestBuildInfo_SetSuccessIdempotent(t *testing.T) { + t.Parallel() + info := &BuildInfo{ + TeamID: "team-1", + logs: newTestLogs(), + Result: newSetOnce(), + } + + meta1 := &template_manager.TemplateBuildMetadata{EnvdVersionKey: "1.0.0"} + meta2 := &template_manager.TemplateBuildMetadata{EnvdVersionKey: "2.0.0"} + + info.SetSuccess(meta1) + info.SetSuccess(meta2) // second call should be silently ignored + + result := info.GetResult() + require.NotNil(t, result) + // First value wins. + assert.Equal(t, meta1, result.Metadata) +} + +func TestBuildInfo_GetLogs(t *testing.T) { + t.Parallel() + info := &BuildInfo{ + TeamID: "team-1", + logs: newTestLogs(), + Result: newSetOnce(), + } + + logs := info.GetLogs() + assert.NotNil(t, logs) + assert.Empty(t, logs) +} + +// ─── BuildCache CRUD ────────────────────────────────────────────────────────── + +func TestBuildCache_CreateAndGet(t *testing.T) { + t.Parallel() + bc := newTestBuildCache(t) + + info, err := bc.Create("team-1", "build-abc", newTestLogs()) + require.NoError(t, err) + require.NotNil(t, info) + + got, err := bc.Get("build-abc") + require.NoError(t, err) + assert.Equal(t, info, got) +} + +func TestBuildCache_GetMissingReturnsError(t *testing.T) { + t.Parallel() + bc := newTestBuildCache(t) + + _, err := bc.Get("nonexistent-build") + require.Error(t, err) + assert.Contains(t, err.Error(), "nonexistent-build") +} + +func TestBuildCache_CreateDuplicateReturnsError(t *testing.T) { + t.Parallel() + bc := newTestBuildCache(t) + + _, err := bc.Create("team-1", "build-dup", newTestLogs()) + require.NoError(t, err) + + _, err = bc.Create("team-1", "build-dup", newTestLogs()) + require.Error(t, err) + assert.Contains(t, err.Error(), "build-dup") +} + +func TestBuildCache_DeleteRemovesEntry(t *testing.T) { + t.Parallel() + bc := newTestBuildCache(t) + + _, err := bc.Create("team-1", "build-del", newTestLogs()) + require.NoError(t, err) + + bc.Delete("build-del") + + _, err = bc.Get("build-del") + require.Error(t, err) +} + +func TestBuildCache_DeleteNonExistentIsNoOp(t *testing.T) { + t.Parallel() + bc := newTestBuildCache(t) + + // Should not panic. + bc.Delete("does-not-exist") +} + +func TestBuildCache_MultipleBuildsIsolated(t *testing.T) { + t.Parallel() + bc := newTestBuildCache(t) + + info1, err := bc.Create("team-1", "build-1", newTestLogs()) + require.NoError(t, err) + + info2, err := bc.Create("team-2", "build-2", newTestLogs()) + require.NoError(t, err) + + info1.SetSuccess(&template_manager.TemplateBuildMetadata{EnvdVersionKey: "v1"}) + + // build-2 should still be running. + got2, err := bc.Get("build-2") + require.NoError(t, err) + assert.True(t, got2.IsRunning()) + assert.Equal(t, info2, got2) +} + +func TestBuildCache_GetAfterDeleteReturnsError(t *testing.T) { + t.Parallel() + bc := newTestBuildCache(t) + + _, err := bc.Create("team-1", "build-x", newTestLogs()) + require.NoError(t, err) + + bc.Delete("build-x") + + _, err = bc.Get("build-x") + require.Error(t, err) +} diff --git a/packages/orchestrator/pkg/template/server/create_template.go b/packages/orchestrator/pkg/template/server/create_template.go index 7aeb1aeb90..6ceb84badf 100644 --- a/packages/orchestrator/pkg/template/server/create_template.go +++ b/packages/orchestrator/pkg/template/server/create_template.go @@ -174,8 +174,19 @@ func (s *ServerStore) TemplateCreate(ctx context.Context, templateRequest *templ telemetry.ReportError(ctx, "error while building template", err, attrs...) } + s.buildLogger.Detach(ctx).Warn("build goroutine setting FAIL", + zap.String("template.id", cfg.GetTemplateID()), + zap.String("build.id", cfg.GetBuildID()), + zap.Error(err), + zap.String("user_error", userError.GetMessage()), + ) buildInfo.SetFail(userError) } else { + s.buildLogger.Detach(ctx).Info("build goroutine setting SUCCESS", + zap.String("template.id", cfg.GetTemplateID()), + zap.String("build.id", cfg.GetBuildID()), + zap.Int64("rootfs_size_mb", int64(res.RootfsSizeMB)), + ) buildInfo.SetSuccess(&templatemanager.TemplateBuildMetadata{ RootfsSizeKey: int32(res.RootfsSizeMB), EnvdVersionKey: res.EnvdVersion, diff --git a/packages/orchestrator/pkg/template/server/template_status.go b/packages/orchestrator/pkg/template/server/template_status.go index b162ef29fa..ff0a9f926c 100644 --- a/packages/orchestrator/pkg/template/server/template_status.go +++ b/packages/orchestrator/pkg/template/server/template_status.go @@ -8,7 +8,10 @@ import ( "slices" "time" + "go.uber.org/zap" + template_manager "github.com/e2b-dev/infra/packages/shared/pkg/grpc/template-manager" + "github.com/e2b-dev/infra/packages/shared/pkg/logger" ) const ( @@ -24,6 +27,11 @@ func (s *ServerStore) TemplateBuildStatus(ctx context.Context, in *template_mana buildInfo, err := s.buildCache.Get(in.GetBuildID()) if err != nil { + logger.L().Warn(ctx, "TemplateBuildStatus: cache lookup failed", + zap.String("build.id", in.GetBuildID()), + zap.String("template.id", in.GetTemplateID()), + zap.Error(err), + ) return nil, fmt.Errorf("error while getting build info, maybe already expired: %w", err) } diff --git a/tests/integration/internal/tests/api/templates/build_template_multidistro_test.go b/tests/integration/internal/tests/api/templates/build_template_multidistro_test.go new file mode 100644 index 0000000000..6683431755 --- /dev/null +++ b/tests/integration/internal/tests/api/templates/build_template_multidistro_test.go @@ -0,0 +1,154 @@ +package api_templates + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/e2b-dev/infra/tests/integration/internal/api" +) + +// TestTemplateBuildAlpine verifies template builds from Alpine base images. +func TestTemplateBuildAlpine(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + templateName string + fromImage string + }{ + {"alpine:latest", "test-alpine-latest", "alpine:latest"}, + {"alpine:3.22", "test-alpine-3-22", "alpine:3.22"}, + {"alpine:3.21", "test-alpine-3-21", "alpine:3.21"}, + {"alpine:3.20", "test-alpine-3-20", "alpine:3.20"}, + {"alpine:3.19", "test-alpine-3-19", "alpine:3.19"}, + {"alpine:3.18", "test-alpine-3-18", "alpine:3.18"}, + {"alpine:3.17", "test-alpine-3-17", "alpine:3.17"}, + {"alpine:edge", "test-alpine-edge", "alpine:edge"}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + assert.True(t, buildTemplate(t, tc.templateName, api.TemplateBuildStartV2{ + Force: new(ForceBaseBuild), + FromImage: new(tc.fromImage), + Steps: new([]api.TemplateStep{ + { + Type: "RUN", + Force: new(true), + Args: new([]string{"echo 'Hello from Alpine'"}), + }, + }), + }, defaultBuildLogHandler(t))) + }) + } +} + +// TestTemplateBuildCentOS verifies template builds from CentOS base images. +func TestTemplateBuildCentOS(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + templateName string + fromImage string + }{ + {"centos:7", "test-centos-7", "centos:7"}, + {"centos:8", "test-centos-8", "centos:8"}, + {"centos:stream8", "test-centos-stream8", "quay.io/centos/centos:stream8"}, + {"centos:stream9", "test-centos-stream9", "quay.io/centos/centos:stream9"}, + {"centos:stream10", "test-centos-stream10", "quay.io/centos/centos:stream10"}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + assert.True(t, buildTemplate(t, tc.templateName, api.TemplateBuildStartV2{ + Force: new(ForceBaseBuild), + FromImage: new(tc.fromImage), + Steps: new([]api.TemplateStep{ + { + Type: "RUN", + Force: new(true), + Args: new([]string{"echo 'Hello from CentOS'"}), + }, + }), + }, defaultBuildLogHandler(t))) + }) + } +} + +// TestTemplateBuildRHELCompat verifies template builds from RHEL-compatible distros. +func TestTemplateBuildRHELCompat(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + templateName string + fromImage string + }{ + {"rockylinux:9", "test-rocky-9", "rockylinux/rockylinux:9"}, + {"rockylinux:8", "test-rocky-8", "rockylinux/rockylinux:8"}, + {"almalinux:9", "test-alma-9", "almalinux/almalinux:9"}, + {"almalinux:8", "test-alma-8", "almalinux/almalinux:8"}, + {"oraclelinux:9", "test-oracle-9", "oraclelinux:9"}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + assert.True(t, buildTemplate(t, tc.templateName, api.TemplateBuildStartV2{ + Force: new(ForceBaseBuild), + FromImage: new(tc.fromImage), + Steps: new([]api.TemplateStep{ + { + Type: "RUN", + Force: new(true), + Args: new([]string{"echo 'Hello from RHEL-compat'"}), + }, + }), + }, defaultBuildLogHandler(t))) + }) + } +} + +// TestTemplateBuildMultiDistroUSER verifies the USER step works across distro families. +func TestTemplateBuildMultiDistroUSER(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + templateName string + fromImage string + }{ + {"debian-user", "test-debian-user-step", "ubuntu:22.04"}, + {"alpine-user", "test-alpine-user-step", "alpine:3.21"}, + {"rhel-user", "test-rhel-user-step", "rockylinux/rockylinux:9"}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + assert.True(t, buildTemplate(t, tc.templateName, api.TemplateBuildStartV2{ + Force: new(ForceBaseBuild), + FromImage: new(tc.fromImage), + Steps: new([]api.TemplateStep{ + { + Type: "USER", + Force: new(true), + Args: new([]string{"testuser", "true"}), + }, + { + Type: "RUN", + Args: new([]string{"whoami && id testuser"}), + }, + }), + }, defaultBuildLogHandler(t))) + }) + } +}