diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index a8158dc3f2..e8b2bfae9b 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -174,9 +174,18 @@ The agent inside every VM (started by systemd very early in boot), port 49983, c stdout/stderr, stdin, signals, PTYs — this is what SDKs use to "run code". - **Filesystem service** (`spec/filesystem/filesystem.proto`): stat/list/make/move/remove/watch. - **REST**: `/health`, `/metrics`, `/files` upload/download, `/init` (orchestrator pushes env - vars, access token, metadata after boot/resume), freeze/thaw hooks used during pause. + vars, access token, metadata after boot/resume), `/upgrade` (live self-upgrade, below), + freeze/thaw hooks used during pause. - **Auth**: `X-Access-Token` header checked against a token delivered via Firecracker MMDS; signed URLs for file endpoints. +- **Live upgrade** (`internal/services/process/upgrade.go`): an authenticated `POST /upgrade` lets + the orchestrator swap envd inside a *running* sandbox at resume. It streams the new binary in the + request body and envd `syscall.Exec`s into it **with the same PID**, carrying the workload's + stdio/PTY fds, process table, recently-retained exit codes and filesystem watchers forward via a + tmpfs handover blob. The workload cgroups stay frozen until the post-upgrade `/init` restores the + access token (so no re-adopted process runs unauthenticated), and the handover outcome + (procs/watchers re-adopted, plus any failures) rides back on that `/init`'s `X-Envd-Handover` + header for fleet visibility. - Scans guest ports and forwards them so any port a user process opens becomes reachable through sandbox URLs. **`pkg/version.go` must be bumped on every behavioral change** — the API and the orchestrator gate features on the envd version recorded in each template build. @@ -290,6 +299,12 @@ sequenceDiagram - **Resume**: same path as creation, but placement prefers the **origin node** — if the snapshot is still in its local cache, resume avoids any object-storage reads. `Checkpoint` is a pause+resume in place used to persist state while keeping the sandbox running. +- **Envd live-upgrade on resume**: the orchestrator can upgrade the sandbox's envd to a newer + node-local build during resume (gated by the `envd-upgrade-target` flag in + `packages/shared/pkg/featureflags`), via envd's `POST /upgrade` (see the envd section). It is + best-effort — a delivery failure before the `exec` leaves the old envd serving — except an + unrecoverable post-`exec` failure (the new envd never re-initializes), which fails the resume + rather than return a permanently unusable sandbox. - Auto-pause/auto-resume make sandboxes effectively serverless: idle sandboxes pause, traffic resumes them (see traffic flow above). diff --git a/packages/envd/internal/api/auth.go b/packages/envd/internal/api/auth.go index 5323169d88..b47bb76508 100644 --- a/packages/envd/internal/api/auth.go +++ b/packages/envd/internal/api/auth.go @@ -31,13 +31,27 @@ var authExcludedPaths = []string{ "POST/init", } +// handoverPreInitAllowedPaths is the MINIMAL set reachable on a live-upgraded +// envd before its post-upgrade /init has restored the access token: only /init +// (which restores auth and lifts this gate, self-authenticated via MMDS) and +// the health check. It deliberately omits /files — unlike authExcludedPaths — +// so a re-adopted (possibly hostile) guest process can't reach the +// root-privileged file API unauthenticated in that window. The orchestrator +// delivers the upgrade over /upgrade's body, not /files, so nothing legitimate +// needs /files before /init. +var handoverPreInitAllowedPaths = []string{ + "GET/health", + "POST/init", +} + func (a *API) WithAuthorization(handler http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - if a.accessToken.IsSet() { - authHeader := req.Header.Get(accessTokenHeader) + // check if this path is allowed without authentication (e.g., health check, endpoints supporting signing) + allowedPath := slices.Contains(authExcludedPaths, req.Method+req.URL.Path) - // check if this path is allowed without authentication (e.g., health check, endpoints supporting signing) - allowedPath := slices.Contains(authExcludedPaths, req.Method+req.URL.Path) + switch { + case a.accessToken.IsSet(): + authHeader := req.Header.Get(accessTokenHeader) if !a.accessToken.Equals(authHeader) && !allowedPath { a.logger.Error().Msg("Trying to access secured envd without correct access token") @@ -47,6 +61,24 @@ func (a *API) WithAuthorization(handler http.Handler) http.Handler { return } + + case a.handover != nil && !a.initialized.Load() && !slices.Contains(handoverPreInitAllowedPaths, req.Method+req.URL.Path): + // A live-upgraded envd serves before its post-upgrade /init has + // restored the access token — and the fallback thaw may already be + // running the re-adopted workload. The sandbox HAD a token (it is a + // resume), so treat the unset token as "not yet restored" and fail + // CLOSED here rather than falling through to the open path below, + // which would let a re-adopted (and possibly hostile) guest process + // reach control endpoints unauthenticated in this window. Only the + // minimal handoverPreInitAllowedPaths (/init, /health) get through — + // notably NOT /files, whose root file API is otherwise unauthenticated + // (it is in authExcludedPaths). /init self-authenticates via MMDS and + // lifts this gate. + a.logger.Warn().Msg("blocking pre-init request on live-upgraded envd (auth not yet restored)") + + jsonError(w, http.StatusUnauthorized, errors.New("envd not initialized")) + + return } handler.ServeHTTP(w, req) diff --git a/packages/envd/internal/api/compose_test.go b/packages/envd/internal/api/compose_test.go index 483cdde5ee..b2cc4e5d7b 100644 --- a/packages/envd/internal/api/compose_test.go +++ b/packages/envd/internal/api/compose_test.go @@ -33,7 +33,7 @@ func newComposeTestAPI(t *testing.T) (*API, *user.User) { User: currentUser.Username, } - return New(&logger, defaults, nil, false, cgroups.NewNoopManager()), currentUser + return New(&logger, defaults, nil, false, cgroups.NewWorkloadFreezer(cgroups.NewNoopManager())), currentUser } func writeSourceFile(t *testing.T, dir string, name string, data []byte) string { diff --git a/packages/envd/internal/api/download_test.go b/packages/envd/internal/api/download_test.go index 22f9ce672d..42cf31282b 100644 --- a/packages/envd/internal/api/download_test.go +++ b/packages/envd/internal/api/download_test.go @@ -96,7 +96,7 @@ func TestGetFilesContentDisposition(t *testing.T) { EnvVars: utils.NewEnvVars(), User: currentUser.Username, } - api := New(&logger, defaults, nil, false, cgroups.NewNoopManager()) + api := New(&logger, defaults, nil, false, cgroups.NewWorkloadFreezer(cgroups.NewNoopManager())) // Create request and response recorder req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/files?path="+url.QueryEscape(tempFile), nil) @@ -145,7 +145,7 @@ func TestGetFilesContentDispositionWithNestedPath(t *testing.T) { EnvVars: utils.NewEnvVars(), User: currentUser.Username, } - api := New(&logger, defaults, nil, false, cgroups.NewNoopManager()) + api := New(&logger, defaults, nil, false, cgroups.NewWorkloadFreezer(cgroups.NewNoopManager())) // Create request and response recorder req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/files?path="+url.QueryEscape(tempFile), nil) @@ -188,7 +188,7 @@ func TestGetFiles_GzipEncoding_ExplicitIdentityOffWithRange(t *testing.T) { EnvVars: utils.NewEnvVars(), User: currentUser.Username, } - api := New(&logger, defaults, nil, false, cgroups.NewNoopManager()) + api := New(&logger, defaults, nil, false, cgroups.NewWorkloadFreezer(cgroups.NewNoopManager())) // Create request and response recorder req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/files?path="+url.QueryEscape(tempFile), nil) @@ -229,7 +229,7 @@ func TestGetFiles_GzipDownload(t *testing.T) { EnvVars: utils.NewEnvVars(), User: currentUser.Username, } - api := New(&logger, defaults, nil, false, cgroups.NewNoopManager()) + api := New(&logger, defaults, nil, false, cgroups.NewWorkloadFreezer(cgroups.NewNoopManager())) req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/files?path="+url.QueryEscape(tempFile), nil) req.Header.Set("Accept-Encoding", "gzip") @@ -294,7 +294,7 @@ func TestPostFiles_GzipUpload(t *testing.T) { EnvVars: utils.NewEnvVars(), User: currentUser.Username, } - api := New(&logger, defaults, nil, false, cgroups.NewNoopManager()) + api := New(&logger, defaults, nil, false, cgroups.NewWorkloadFreezer(cgroups.NewNoopManager())) req := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/files?path="+url.QueryEscape(destPath), &gzBuf) req.Header.Set("Content-Type", mpWriter.FormDataContentType()) @@ -334,7 +334,7 @@ func TestPostFiles_RawBodyUpload(t *testing.T) { EnvVars: utils.NewEnvVars(), User: currentUser.Username, } - api := New(&logger, defaults, nil, false, cgroups.NewNoopManager()) + api := New(&logger, defaults, nil, false, cgroups.NewWorkloadFreezer(cgroups.NewNoopManager())) req := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/files?path="+url.QueryEscape(destPath), bytes.NewReader(originalContent)) req.Header.Set("Content-Type", "application/octet-stream") @@ -372,7 +372,7 @@ func TestPostFiles_RawBodyUploadCreatesDirectories(t *testing.T) { EnvVars: utils.NewEnvVars(), User: currentUser.Username, } - api := New(&logger, defaults, nil, false, cgroups.NewNoopManager()) + api := New(&logger, defaults, nil, false, cgroups.NewWorkloadFreezer(cgroups.NewNoopManager())) req := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/files?path="+url.QueryEscape(destPath), bytes.NewReader(originalContent)) req.Header.Set("Content-Type", "application/octet-stream") @@ -405,7 +405,7 @@ func TestPostFiles_RawBodyUploadRequiresPath(t *testing.T) { EnvVars: utils.NewEnvVars(), User: currentUser.Username, } - api := New(&logger, defaults, nil, false, cgroups.NewNoopManager()) + api := New(&logger, defaults, nil, false, cgroups.NewWorkloadFreezer(cgroups.NewNoopManager())) req := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/files", bytes.NewReader([]byte("some content"))) req.Header.Set("Content-Type", "application/octet-stream") @@ -440,7 +440,7 @@ func TestPostFiles_RawBodyUploadOverwritesExisting(t *testing.T) { EnvVars: utils.NewEnvVars(), User: currentUser.Username, } - api := New(&logger, defaults, nil, false, cgroups.NewNoopManager()) + api := New(&logger, defaults, nil, false, cgroups.NewWorkloadFreezer(cgroups.NewNoopManager())) req := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/files?path="+url.QueryEscape(destPath), bytes.NewReader(newContent)) req.Header.Set("Content-Type", "application/octet-stream") @@ -486,7 +486,7 @@ func TestPostFiles_RawBodyGzipUpload(t *testing.T) { EnvVars: utils.NewEnvVars(), User: currentUser.Username, } - api := New(&logger, defaults, nil, false, cgroups.NewNoopManager()) + api := New(&logger, defaults, nil, false, cgroups.NewWorkloadFreezer(cgroups.NewNoopManager())) req := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/files?path="+url.QueryEscape(destPath), &gzBuf) req.Header.Set("Content-Type", "application/octet-stream") @@ -520,7 +520,7 @@ func TestPostFiles_UnsupportedContentType(t *testing.T) { EnvVars: utils.NewEnvVars(), User: currentUser.Username, } - api := New(&logger, defaults, nil, false, cgroups.NewNoopManager()) + api := New(&logger, defaults, nil, false, cgroups.NewWorkloadFreezer(cgroups.NewNoopManager())) tempDir := t.TempDir() destPath := filepath.Join(tempDir, "test.txt") @@ -566,7 +566,7 @@ func TestPostFiles_MultipartStillWorksWithoutContentType(t *testing.T) { EnvVars: utils.NewEnvVars(), User: currentUser.Username, } - api := New(&logger, defaults, nil, false, cgroups.NewNoopManager()) + api := New(&logger, defaults, nil, false, cgroups.NewWorkloadFreezer(cgroups.NewNoopManager())) req := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/files?path="+url.QueryEscape(destPath), &multipartBuf) req.Header.Set("Content-Type", mpWriter.FormDataContentType()) @@ -624,7 +624,7 @@ func TestGzipUploadThenGzipDownload(t *testing.T) { EnvVars: utils.NewEnvVars(), User: currentUser.Username, } - api := New(&logger, defaults, nil, false, cgroups.NewNoopManager()) + api := New(&logger, defaults, nil, false, cgroups.NewWorkloadFreezer(cgroups.NewNoopManager())) uploadReq := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/files?path="+url.QueryEscape(destPath), &gzBuf) uploadReq.Header.Set("Content-Type", mpWriter.FormDataContentType()) diff --git a/packages/envd/internal/api/init.go b/packages/envd/internal/api/init.go index 620204b0a4..85eb09bb8d 100644 --- a/packages/envd/internal/api/init.go +++ b/packages/envd/internal/api/init.go @@ -22,7 +22,7 @@ import ( "github.com/e2b-dev/infra/packages/envd/internal/host" "github.com/e2b-dev/infra/packages/envd/internal/logs" "github.com/e2b-dev/infra/packages/envd/internal/logs/ratelimit" - "github.com/e2b-dev/infra/packages/envd/internal/services/cgroups" + "github.com/e2b-dev/infra/packages/envd/pkg" "github.com/e2b-dev/infra/packages/shared/pkg/keys" ) @@ -115,6 +115,22 @@ func (a *API) checkMMDSHash(ctx context.Context, requestToken *SecureToken) (boo func (a *API) PostInit(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() + // Report the running envd version on every /init response (even error/stale + // ones) so the orchestrator can read the live version off the resume-path + // call it already makes — no extra round-trip. Set before any WriteHeader. + w.Header().Set("X-Envd-Version", pkg.Version) + + // If this envd booted from a live-upgrade handover, advertise its outcome on + // /init so the orchestrator can record it — the envd-side result (re-adopted + // procs, restored retained exits, watcher re-arm success/failures) is + // otherwise only logged in-guest and invisible fleet-wide. Set before any + // WriteHeader. + if a.handover != nil { + if b, err := json.Marshal(a.handover); err == nil { + w.Header().Set("X-Envd-Handover", string(b)) + } + } + ctx := r.Context() operationID := logs.AssignOperationID() @@ -167,7 +183,14 @@ func (a *API) PostInit(w http.ResponseWriter, r *http.Request) { // requests still thaw cgroups after pre-pause freeze. defer a.unfreezeUserCgroups(ctx, logger) - // Update data only if the request is newer or if there's no timestamp at all + // Restore the access token (and env) BEFORE marking the envd initialized. + // On a live-upgraded envd, WithAuthorization only fails CLOSED while + // !initialized; flipping initialized first, with the token not yet + // restored, falls through to the fail-OPEN path and lets a re-adopted + // (and possibly hostile) guest process reach control endpoints — including + // /upgrade — unauthenticated in that window. Update only if the request is + // newer (or carries no timestamp); a stale/replayed /init keeps the token + // already set by the first one. if initRequest.Timestamp == nil || a.lastSetTime.SetToGreater(initRequest.Timestamp.UnixNano()) { if err := a.SetData(ctx, logger, initRequest); err != nil { writeInitError(w, logger, err) @@ -175,6 +198,12 @@ func (a *API) PostInit(w http.ResponseWriter, r *http.Request) { return } } + + // Auth passed and token restored: mark the envd initialized so the + // live-upgrade /upgrade endpoint and the post-upgrade fallback thaw open + // up — a guest process that can't pass auth can't flip this and drive an + // unauthenticated upgrade. + a.initialized.Store(true) } go func() { //nolint:contextcheck // TODO: fix this later @@ -245,11 +274,6 @@ func (a *API) SetData(ctx context.Context, logger zerolog.Logger, data PostInitJ } // userCgroupsToFreeze is the cgroup set frozen pre-pause and thawed on /init. -var userCgroupsToFreeze = []cgroups.ProcessType{ - cgroups.ProcessTypeUser, - cgroups.ProcessTypePTY, -} - // PostFreeze freezes user/pty cgroups directly (no Process.Start / shell). // Orchestrator calls this just before pause; the frozen state persists into the // snapshot and /init thaws on resume. Best-effort: tries every cgroup even if @@ -259,22 +283,16 @@ func (a *API) PostFreeze(w http.ResponseWriter, r *http.Request) { logger := a.logger.With().Str(string(logs.OperationIDKey), logs.AssignOperationID()).Logger() - if err := a.freezeLock.Acquire(r.Context(), 1); err != nil { - w.WriteHeader(http.StatusServiceUnavailable) - - return - } - defer a.freezeLock.Release(1) + if err := a.workloadFreezer.Freeze(r.Context()); err != nil { + // A failed lock acquire means the request ctx was cancelled; a failed + // sweep leaves ctx intact. + if r.Context().Err() != nil { + w.WriteHeader(http.StatusServiceUnavailable) - var errs []error - for _, pt := range userCgroupsToFreeze { - if err := a.cgroupManager.Freeze(pt); err != nil { - logger.Error().Err(err).Msgf("freeze %s cgroup", pt) - errs = append(errs, fmt.Errorf("freeze %s cgroup: %w", pt, err)) + return } - } - if len(errs) > 0 { - jsonError(w, http.StatusInternalServerError, errors.Join(errs...)) + logger.Error().Err(err).Msg("freeze workload cgroups") + jsonError(w, http.StatusInternalServerError, err) return } @@ -290,25 +308,11 @@ func (a *API) PostFreeze(w http.ResponseWriter, r *http.Request) { func (a *API) PostUnfreeze(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() - ctx := r.Context() logger := a.logger.With().Str(string(logs.OperationIDKey), logs.AssignOperationID()).Logger() - if err := a.freezeLock.Acquire(context.WithoutCancel(ctx), 1); err != nil { - w.WriteHeader(http.StatusServiceUnavailable) - - return - } - defer a.freezeLock.Release(1) - - var errs []error - for _, pt := range userCgroupsToFreeze { - if err := a.cgroupManager.Unfreeze(pt); err != nil { - logger.Error().Err(err).Msgf("unfreeze %s cgroup", pt) - errs = append(errs, fmt.Errorf("unfreeze %s cgroup: %w", pt, err)) - } - } - if len(errs) > 0 { - jsonError(w, http.StatusInternalServerError, errors.Join(errs...)) + if err := a.workloadFreezer.Unfreeze(r.Context()); err != nil { + logger.Error().Err(err).Msg("unfreeze workload cgroups") + jsonError(w, http.StatusInternalServerError, err) return } @@ -318,15 +322,11 @@ func (a *API) PostUnfreeze(w http.ResponseWriter, r *http.Request) { } // unfreezeUserCgroups unfreezes user/pty cgroups (idempotent if not frozen). -// Wraps the context with WithoutCancel so the unfreeze always completes. +// The freezer detaches the wait from ctx cancellation so the unfreeze always +// completes. func (a *API) unfreezeUserCgroups(ctx context.Context, logger zerolog.Logger) { - _ = a.freezeLock.Acquire(context.WithoutCancel(ctx), 1) - defer a.freezeLock.Release(1) - - for _, pt := range userCgroupsToFreeze { - if err := a.cgroupManager.Unfreeze(pt); err != nil { - logger.Warn().Err(err).Msgf("unfreeze %s cgroup", pt) - } + if err := a.workloadFreezer.Unfreeze(ctx); err != nil { + logger.Warn().Err(err).Msg("unfreeze workload cgroups") } } diff --git a/packages/envd/internal/api/init_test.go b/packages/envd/internal/api/init_test.go index af24b718e5..0cc3a9393c 100644 --- a/packages/envd/internal/api/init_test.go +++ b/packages/envd/internal/api/init_test.go @@ -148,7 +148,7 @@ func newTestAPI(accessToken *SecureToken, mmdsClient MMDSClient) *API { defaults := &execcontext.Defaults{ EnvVars: utils.NewEnvVars(), } - api := New(&logger, defaults, nil, false, cgroups.NewNoopManager()) + api := New(&logger, defaults, nil, false, cgroups.NewWorkloadFreezer(cgroups.NewNoopManager())) if accessToken != nil { api.accessToken.TakeFrom(accessToken) } @@ -637,7 +637,7 @@ func (f *fakeCgroupManager) Close() error { return nil } func newAPIWithCgroupManager(mgr cgroups.Manager) *API { logger := zerolog.Nop() - return New(&logger, &execcontext.Defaults{EnvVars: utils.NewEnvVars()}, nil, false, mgr) + return New(&logger, &execcontext.Defaults{EnvVars: utils.NewEnvVars()}, nil, false, cgroups.NewWorkloadFreezer(mgr)) } func TestPostFreeze(t *testing.T) { @@ -654,7 +654,7 @@ func TestPostFreeze(t *testing.T) { api.PostFreeze(rec, req) require.Equal(t, http.StatusNoContent, rec.Code) - assert.Equal(t, userCgroupsToFreeze, mgr.frozen) + assert.Equal(t, cgroups.WorkloadProcessTypes, mgr.frozen) }) t.Run("returns 500 on freeze error", func(t *testing.T) { @@ -686,7 +686,7 @@ func TestPostUnfreeze(t *testing.T) { api.PostUnfreeze(rec, req) require.Equal(t, http.StatusNoContent, rec.Code) - assert.Equal(t, userCgroupsToFreeze, mgr.unfrozen) + assert.Equal(t, cgroups.WorkloadProcessTypes, mgr.unfrozen) }) t.Run("returns 500 but attempts every cgroup on unfreeze error", func(t *testing.T) { @@ -701,7 +701,7 @@ func TestPostUnfreeze(t *testing.T) { assert.Equal(t, http.StatusInternalServerError, rec.Code) assert.Empty(t, mgr.unfrozen) - assert.Equal(t, userCgroupsToFreeze, mgr.unfreezeAttempts) + assert.Equal(t, cgroups.WorkloadProcessTypes, mgr.unfreezeAttempts) }) } @@ -732,7 +732,7 @@ func TestPostInit_UnfreezeOnStaleTimestamp(t *testing.T) { require.Equal(t, http.StatusNoContent, rec.Code) _, ok := api.defaults.EnvVars.Load("SHOULD_NOT_BE_SET") assert.False(t, ok, "stale /init should not apply EnvVars") - assert.Equal(t, userCgroupsToFreeze, mgr.unfrozen, "stale /init must still unfreeze") + assert.Equal(t, cgroups.WorkloadProcessTypes, mgr.unfrozen, "stale /init must still unfreeze") } // Unauthorized /init must NOT thaw cgroups. diff --git a/packages/envd/internal/api/mounts_handover.go b/packages/envd/internal/api/mounts_handover.go new file mode 100644 index 0000000000..df3126d09a --- /dev/null +++ b/packages/envd/internal/api/mounts_handover.go @@ -0,0 +1,34 @@ +package api + +import ( + "github.com/e2b-dev/infra/packages/envd/internal/services/spec/upgrade" +) + +// ExportMounts snapshots the NFS mount ledger (path -> lifecycle id) as typed +// MountEntry messages, carried across an envd live-upgrade. The kernel mounts +// survive the same-PID execve, but the ledger lives only in envd's heap; without +// it the new envd's post-upgrade /init would see an empty ledger, decide every +// volume needs (re)mounting, and force-unmount + remount a still-live mount — +// risking ESTALE for the workload and a failed resume. Carrying it lets /init +// recognize a matching-lifecycle mount and leave it in place. +func (a *API) ExportMounts() []*upgrade.MountEntry { + out := make([]*upgrade.MountEntry, 0) + a.mountedPaths.Range(func(k, v any) bool { + path, _ := k.(string) + lifecycle, _ := v.(string) + out = append(out, &upgrade.MountEntry{Path: path, LifecycleId: lifecycle}) + + return true + }) + + return out +} + +// ImportMounts restores the NFS mount ledger from a live-upgrade handover so the +// new envd knows which paths are already mounted (and for which lifecycle) +// before its first post-upgrade /init runs setupNFS. +func (a *API) ImportMounts(mounts []*upgrade.MountEntry) { + for _, m := range mounts { + a.mountedPaths.Store(m.GetPath(), m.GetLifecycleId()) + } +} diff --git a/packages/envd/internal/api/mounts_handover_test.go b/packages/envd/internal/api/mounts_handover_test.go new file mode 100644 index 0000000000..cd6bfa3dfc --- /dev/null +++ b/packages/envd/internal/api/mounts_handover_test.go @@ -0,0 +1,50 @@ +package api + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/e2b-dev/infra/packages/envd/internal/services/spec/upgrade" +) + +// TestMountLedger_HandoverRoundTrip verifies the NFS mount ledger survives an +// Export -> Import cycle, so the new envd knows which paths were mounted (and for +// which lifecycle) after a live-upgrade. +func TestMountLedger_HandoverRoundTrip(t *testing.T) { + t.Parallel() + + a := &API{} + a.ImportMounts([]*upgrade.MountEntry{ + {Path: "/mnt/a", LifecycleId: "lc-1"}, + {Path: "/mnt/b", LifecycleId: "lc-2"}, + }) + + got := map[string]string{} + for _, m := range a.ExportMounts() { + got[m.GetPath()] = m.GetLifecycleId() + } + assert.Equal(t, map[string]string{"/mnt/a": "lc-1", "/mnt/b": "lc-2"}, got) +} + +// TestImportedMountLedger_SkipsRemountForSameLifecycle is the point of carrying +// the ledger: setupNFS consults it (mountedPaths.Load + shouldRemountNFS), and a +// path carried across the upgrade with an unchanged lifecycle must NOT be +// remounted — leaving the still-live kernel mount in place (no ESTALE). A changed +// lifecycle still triggers a remount. +func TestImportedMountLedger_SkipsRemountForSameLifecycle(t *testing.T) { + t.Parallel() + + a := &API{} + a.ImportMounts([]*upgrade.MountEntry{{Path: "/mnt/a", LifecycleId: "lc-1"}}) + + v, ok := a.mountedPaths.Load("/mnt/a") + require.True(t, ok, "carried mount must be present in the ledger") + mountedLifecycle, _ := v.(string) + + assert.False(t, shouldRemountNFS(true, mountedLifecycle, "lc-1"), + "a live mount carried across the upgrade must not be remounted for the same lifecycle") + assert.True(t, shouldRemountNFS(true, mountedLifecycle, "lc-2"), + "a lifecycle change must still trigger a remount") +} diff --git a/packages/envd/internal/api/store.go b/packages/envd/internal/api/store.go index c1d7b01d40..395b6c5a51 100644 --- a/packages/envd/internal/api/store.go +++ b/packages/envd/internal/api/store.go @@ -43,22 +43,73 @@ type API struct { initLock *semaphore.Weighted caCertInstaller *host.CACertInstaller - cgroupManager cgroups.Manager - // freezeLock serializes the per-cgroup sweep across /freeze, /unfreeze - // and the /init deferred unfreeze. PostFreeze acquires with the request - // ctx; unfreeze paths acquire with Background so they always land - // regardless of HTTP-client cancellation. - freezeLock *semaphore.Weighted - isMountingNFS atomic.Bool - mountedPaths sync.Map // map[path]lifecycleID - tracks which lifecycle each path was mounted for + // workloadFreezer freezes/thaws the user+pty cgroups. Shared with the process + // service (the live-upgrade handover) so every freeze/unfreeze caller — this + // API's /freeze, /unfreeze and /init deferred thaw, plus the upgrade — is + // serialized through one lock. + workloadFreezer *cgroups.WorkloadFreezer + isMountingNFS atomic.Bool + mountedPaths sync.Map // map[path]lifecycleID - tracks which lifecycle each path was mounted for // fsFreezer freezes/thaws the guest rootfs for filesystem-only pauses; // fsFreezeLock serializes /fsfreeze and /fsthaw. fsFreezer fsfreeze.Freezer fsFreezeLock *semaphore.Weighted + + // handover, when non-nil, is the outcome of the live-upgrade handover this + // envd booted from; PostInit advertises it to the orchestrator via the + // X-Envd-Handover header. Set once at startup, before serving. + handover *handoverResult + + // initialized flips true on the first authenticated /init. It gates the + // live-upgrade /upgrade endpoint and the handover fallback thaw so a + // re-adopted (possibly hostile) guest process can neither drive an upgrade + // nor be handed a running workload before /init has re-established auth. + initialized atomic.Bool +} + +// Initialized reports whether the first authenticated /init has completed. +func (a *API) Initialized() bool { + return a.initialized.Load() +} + +// handoverResult is the outcome of a live-upgrade handover, reported to the +// orchestrator on the next /init so the envd-side result (otherwise only logged +// in-guest) is observable fleet-wide. +type handoverResult struct { + // Failed is true when ResumeFromHandover itself errored/panicked post-execve: + // the workload was NOT re-adopted (it is orphaned and won't be reaped), so + // the version flipping to the target does NOT mean a healthy sandbox. Distinct + // from the per-item *Failed counts below, which are partial degradations of an + // otherwise-successful handover. + Failed bool `json:"failed"` + // Every item is total-carried + failed-subset (ok = total - failed). + Procs int `json:"procs"` + ProcsFailed int `json:"procs_failed"` + Retained int `json:"retained"` + RetainedFailed int `json:"retained_failed"` + Watchers int `json:"watchers"` + WatchersFailed int `json:"watchers_failed"` +} + +// SetHandoverResult records the live-upgrade handover outcome so PostInit can +// advertise it. Called once at startup (before serving) when this envd booted +// via --resume-handover. failed reports whether the handover itself failed (the +// workload was not re-adopted), so the orchestrator can tear the sandbox down +// rather than mistake the version flip for success. +func (a *API) SetHandoverResult(failed bool, procs, procsFailed, retained, retainedFailed, watchers, watchersFailed int) { + a.handover = &handoverResult{ + Failed: failed, + Procs: procs, + ProcsFailed: procsFailed, + Retained: retained, + RetainedFailed: retainedFailed, + Watchers: watchers, + WatchersFailed: watchersFailed, + } } -func New(l *zerolog.Logger, defaults *execcontext.Defaults, mmdsChan chan *host.MMDSOpts, isNotFC bool, cgroupManager cgroups.Manager) *API { +func New(l *zerolog.Logger, defaults *execcontext.Defaults, mmdsChan chan *host.MMDSOpts, isNotFC bool, workloadFreezer *cgroups.WorkloadFreezer) *API { return &API{ logger: l, defaults: defaults, @@ -68,9 +119,8 @@ func New(l *zerolog.Logger, defaults *execcontext.Defaults, mmdsChan chan *host. lastSetTime: utils.NewAtomicMax(), accessToken: &SecureToken{}, caCertInstaller: host.NewCACertInstaller(l), - cgroupManager: cgroupManager, + workloadFreezer: workloadFreezer, initLock: semaphore.NewWeighted(1), - freezeLock: semaphore.NewWeighted(1), fsFreezer: fsfreeze.New(), fsFreezeLock: semaphore.NewWeighted(1), } diff --git a/packages/envd/internal/api/version_test.go b/packages/envd/internal/api/version_test.go new file mode 100644 index 0000000000..d8cbed25c1 --- /dev/null +++ b/packages/envd/internal/api/version_test.go @@ -0,0 +1,71 @@ +package api + +import ( + "bytes" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/e2b-dev/infra/packages/envd/internal/services/cgroups" + "github.com/e2b-dev/infra/packages/envd/pkg" +) + +// TestPostInit_ReportsEnvdVersion verifies /init always advertises the running +// envd version via the X-Envd-Version header. The orchestrator reads it off the +// resume-path /init it already makes, to decide, label, and confirm live +// upgrades against the actual running version. The header is set before any +// parsing/auth, so it is present regardless of the request outcome. +func TestPostInit_ReportsEnvdVersion(t *testing.T) { + t.Parallel() + + api := newAPIWithCgroupManager(cgroups.NewNoopManager()) + + // A malformed body makes PostInit return early — the header must still be set. + req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, "/init", bytes.NewReader([]byte("{not json"))) + require.NoError(t, err) + rec := httptest.NewRecorder() + + api.PostInit(rec, req) + + assert.Equal(t, pkg.Version, rec.Header().Get("X-Envd-Version")) +} + +// TestPostInit_ReportsHandover verifies /init advertises the live-upgrade +// handover outcome via X-Envd-Handover once SetHandoverResult has been called +// (and omits it otherwise), so the orchestrator can record what the new envd +// re-adopted — otherwise only logged in-guest. +func TestPostInit_ReportsHandover(t *testing.T) { + t.Parallel() + + postInit := func(a *API) *httptest.ResponseRecorder { + req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, "/init", bytes.NewReader([]byte("{not json"))) + require.NoError(t, err) + rec := httptest.NewRecorder() + a.PostInit(rec, req) + + return rec + } + + // No handover happened: the header is absent. + plain := newAPIWithCgroupManager(cgroups.NewNoopManager()) + assert.Empty(t, postInit(plain).Header().Get("X-Envd-Handover")) + + // After a successful handover: the outcome is advertised as JSON, failed=false. + upgraded := newAPIWithCgroupManager(cgroups.NewNoopManager()) + upgraded.SetHandoverResult(false, 3, 1, 2, 1, 6, 1) + assert.JSONEq(t, + `{"failed":false,"procs":3,"procs_failed":1,"retained":2,"retained_failed":1,"watchers":6,"watchers_failed":1}`, + postInit(upgraded).Header().Get("X-Envd-Handover")) + + // A FAILED handover (workload not re-adopted) advertises failed=true so the + // orchestrator can tear the sandbox down instead of mistaking the version + // flip for success. + failed := newAPIWithCgroupManager(cgroups.NewNoopManager()) + failed.SetHandoverResult(true, 0, 0, 0, 0, 0, 0) + assert.JSONEq(t, + `{"failed":true,"procs":0,"procs_failed":0,"retained":0,"retained_failed":0,"watchers":0,"watchers_failed":0}`, + postInit(failed).Header().Get("X-Envd-Handover")) +} diff --git a/packages/envd/internal/port/forward.go b/packages/envd/internal/port/forward.go index 9953c1dbf9..3ed6cbeaa3 100644 --- a/packages/envd/internal/port/forward.go +++ b/packages/envd/internal/port/forward.go @@ -11,11 +11,13 @@ import ( "fmt" "net" "os/exec" + "sync" "syscall" "github.com/rs/zerolog" "github.com/e2b-dev/infra/packages/envd/internal/services/cgroups" + "github.com/e2b-dev/infra/packages/envd/internal/services/spec/upgrade" ) type PortState string @@ -29,6 +31,10 @@ var defaultGatewayIP = net.IPv4(169, 254, 0, 21) type PortToForward struct { socat *exec.Cmd + // socatPid is the pid of a socat re-adopted across a live-upgrade, for which + // there is no *exec.Cmd (the old envd's runtime — and its Wait goroutine — + // were replaced by execve). 0 for socats this envd spawned itself. + socatPid int // Process ID of the process that's listening on port. pid int32 // family version of the ip. @@ -37,9 +43,24 @@ type PortToForward struct { port uint32 } +// socatPID returns the pid of the forwarding socat regardless of whether it was +// spawned by this envd (*exec.Cmd) or re-adopted across a live-upgrade (pid +// only). Returns 0 when there is no socat. +func (p *PortToForward) socatPID() int { + if p.socat != nil && p.socat.Process != nil { + return p.socat.Process.Pid + } + + return p.socatPid +} + type Forwarder struct { logger *zerolog.Logger cgroupManager cgroups.Manager + // mu guards the ports map. The scan loop is single-goroutine, but the + // live-upgrade export (ExportForwards) reads the map from the upgrade + // goroutine concurrently, so map access is serialized. + mu sync.Mutex // Map of ports that are being currently forwarded. ports map[string]*PortToForward scannerSubscriber *ScannerSubscriber @@ -87,6 +108,11 @@ func (f *Forwarder) StartForwarding(ctx context.Context) { return } + // Serialize the whole refresh against a concurrent ExportForwards + // (live-upgrade). stop/startPortForwarding below are called with the + // lock held and must not take it themselves. + f.mu.Lock() + // Now we are going to refresh all ports that are being forwarded in the `ports` map. Maybe add new ones // and maybe remove some. @@ -134,6 +160,8 @@ func (f *Forwarder) StartForwarding(ctx context.Context) { f.stopPortForwarding(v) } } + + f.mu.Unlock() } } } @@ -187,14 +215,15 @@ func (f *Forwarder) startPortForwarding(ctx context.Context, p *PortToForward) { } func (f *Forwarder) stopPortForwarding(p *PortToForward) { - if p.socat == nil { + pid := p.socatPID() + if pid <= 0 { return } - defer func() { p.socat = nil }() + defer func() { p.socat = nil; p.socatPid = 0 }() logger := f.logger.With(). - Str("socatCmd", p.socat.String()). + Int("socatPid", pid). Int32("pid", p.pid). Uint32("family", p.family). IPAddr("sourceIP", f.sourceIP.To4()). @@ -203,7 +232,10 @@ func (f *Forwarder) stopPortForwarding(p *PortToForward) { logger.Debug().Msg("Stopping port forwarding") - if err := syscall.Kill(-p.socat.Process.Pid, syscall.SIGKILL); err != nil { + // Kill the socat's process group. A re-adopted socat has no *exec.Cmd Wait + // goroutine to reap it, so ImportForwards started a wait4 reaper for it; a + // self-spawned socat is reaped by its startPortForwarding Wait goroutine. + if err := syscall.Kill(-pid, syscall.SIGKILL); err != nil { logger.Error().Err(err).Msg("Failed to kill process group") return @@ -212,6 +244,88 @@ func (f *Forwarder) stopPortForwarding(p *PortToForward) { logger.Debug().Msg("Stopped port forwarding") } +// ExportForwards snapshots the active port-forwards as typed ForwardedPort +// messages, carried across an envd live-upgrade. The socat children survive the +// same-PID execve; carrying their pids lets the new forwarder re-adopt them +// (ImportForwards) instead of spawning duplicate socats that would contend on +// the same bind address and leak the originals as un-reaped zombies. +func (f *Forwarder) ExportForwards() []*upgrade.ForwardedPort { + f.mu.Lock() + defer f.mu.Unlock() + + return f.exportLocked() +} + +// ExportForwardsHold is ExportForwards but keeps the forwarder mutex held, +// returning a release func the caller invokes to resume scanning. The +// live-upgrade path holds it from the snapshot through the execve so the scan +// loop cannot spawn a new socat in that window — a socat started after the +// snapshot would be orphaned by the swap (never carried, never re-adopted) or +// duplicate a port the new envd re-adopts, contending on the same bind address. +// A successful execve replaces this process and the held lock vanishes with it; +// on failure the caller's release restores scanning under the old envd. +func (f *Forwarder) ExportForwardsHold() ([]*upgrade.ForwardedPort, func()) { + f.mu.Lock() + + return f.exportLocked(), f.mu.Unlock +} + +// exportLocked snapshots the active port-forwards. The caller must hold f.mu. +func (f *Forwarder) exportLocked() []*upgrade.ForwardedPort { + out := make([]*upgrade.ForwardedPort, 0, len(f.ports)) + for key, p := range f.ports { + pid := p.socatPID() + if pid <= 0 { + continue + } + out = append(out, &upgrade.ForwardedPort{ + Key: key, + Port: p.port, + ListenerPid: p.pid, + Family: p.family, + SocatPid: int32(pid), + }) + } + + return out +} + +// ImportForwards re-adopts the socats carried across a live-upgrade: it seeds the +// ports map so the next scan recognizes each already-forwarded port (and does +// not spawn a duplicate socat), and starts a reaper for each re-adopted socat +// (there is no surviving *exec.Cmd Wait goroutine for it). It MUST run before +// StartForwarding so the seeding is race-free. A socat that did not survive the +// handover is skipped, so the next scan respawns a fresh one for the still-open +// port. +func (f *Forwarder) ImportForwards(forwards []*upgrade.ForwardedPort) (readopted int) { + f.mu.Lock() + defer f.mu.Unlock() + + for _, fp := range forwards { + pid := int(fp.GetSocatPid()) + if pid <= 0 { + continue + } + // Only re-adopt a socat that is still alive (kill -0). One that exited + // during the handover window is left out so the next scan respawns rather + // than recording a dead socat as "forwarded". + if syscall.Kill(pid, 0) != nil { + continue + } + f.ports[fp.GetKey()] = &PortToForward{ + pid: fp.GetListenerPid(), + port: fp.GetPort(), + family: fp.GetFamily(), + socatPid: pid, + state: PortStateForward, + } + go reapAdoptedSocat(pid) + readopted++ + } + + return readopted +} + func familyToIPVersion(family uint32) uint32 { switch family { case syscall.AF_INET: diff --git a/packages/envd/internal/port/forward_handover_test.go b/packages/envd/internal/port/forward_handover_test.go new file mode 100644 index 0000000000..05d0fcf12a --- /dev/null +++ b/packages/envd/internal/port/forward_handover_test.go @@ -0,0 +1,83 @@ +package port + +import ( + "os/exec" + "syscall" + "testing" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/e2b-dev/infra/packages/envd/internal/services/spec/upgrade" +) + +func newHandoverTestForwarder() *Forwarder { + l := zerolog.Nop() + + return &Forwarder{ + logger: &l, + ports: make(map[string]*PortToForward), + sourceIP: defaultGatewayIP, + } +} + +// TestForwarder_ExportForwards exports only ports that actually have a socat, so +// a half-set-up entry is not carried as if it were forwarded. +func TestForwarder_ExportForwards(t *testing.T) { + t.Parallel() + + f := newHandoverTestForwarder() + f.ports["100-8080"] = &PortToForward{pid: 100, port: 8080, family: 4, socatPid: 555, state: PortStateForward} + f.ports["101-9090"] = &PortToForward{pid: 101, port: 9090, family: 4, state: PortStateForward} // no socat + + out := f.ExportForwards() + require.Len(t, out, 1) + assert.Equal(t, "100-8080", out[0].GetKey()) + assert.Equal(t, uint32(8080), out[0].GetPort()) + assert.Equal(t, int32(100), out[0].GetListenerPid()) + assert.Equal(t, uint32(4), out[0].GetFamily()) + assert.Equal(t, int32(555), out[0].GetSocatPid()) +} + +// TestForwarder_ImportForwards_SkipsDeadSocat leaves out a socat that did not +// survive the handover, so the next scan respawns a fresh one for the still-open +// port instead of recording a dead pid as "forwarded". +func TestForwarder_ImportForwards_SkipsDeadSocat(t *testing.T) { + t.Parallel() + + f := newHandoverTestForwarder() + // A very high pid that is overwhelmingly unlikely to exist: kill -0 fails. + n := f.ImportForwards([]*upgrade.ForwardedPort{ + {Key: "1-1", Port: 1, ListenerPid: 1, Family: 4, SocatPid: 2147483646}, + }) + assert.Zero(t, n) + assert.Empty(t, f.ports) +} + +// TestForwarder_ImportForwards_ReadoptsLiveSocat re-adopts a still-running socat +// into the ports map (so the next scan won't spawn a duplicate) and records it as +// pid-only (no *exec.Cmd survives the execve). A real child process stands in for +// the socat. +func TestForwarder_ImportForwards_ReadoptsLiveSocat(t *testing.T) { + t.Parallel() + + cmd := exec.CommandContext(t.Context(), "sleep", "30") + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + require.NoError(t, cmd.Start()) + pid := cmd.Process.Pid + // The re-adopt path starts a wait4 reaper for the pid; killing the group lets + // it reap so the child does not linger. + t.Cleanup(func() { _ = syscall.Kill(-pid, syscall.SIGKILL) }) + + f := newHandoverTestForwarder() + n := f.ImportForwards([]*upgrade.ForwardedPort{ + {Key: "100-8080", Port: 8080, ListenerPid: 100, Family: 4, SocatPid: int32(pid)}, + }) + require.Equal(t, 1, n) + + p, ok := f.ports["100-8080"] + require.True(t, ok, "a live socat must be re-adopted into the ports map") + assert.Equal(t, pid, p.socatPID()) + assert.Nil(t, p.socat, "a re-adopted socat has no *exec.Cmd") +} diff --git a/packages/envd/internal/port/forward_reap_linux.go b/packages/envd/internal/port/forward_reap_linux.go new file mode 100644 index 0000000000..6fae85e6bc --- /dev/null +++ b/packages/envd/internal/port/forward_reap_linux.go @@ -0,0 +1,21 @@ +//go:build linux + +package port + +import "syscall" + +// reapAdoptedSocat blocks until the re-adopted socat pid exits, then reaps it so +// it does not linger as a zombie under the same-PID envd. A socat spawned by +// this envd is reaped by its os/exec Wait goroutine, but a socat re-adopted +// across a live-upgrade has no such goroutine (the old runtime was replaced by +// execve), so we wait4 its pid directly. The pid is a child of this process (the +// PID is unchanged across the upgrade), so wait4 succeeds; it targets one +// specific pid, so it never steals os/exec's other children. +func reapAdoptedSocat(pid int) { + var ws syscall.WaitStatus + for { + if _, err := syscall.Wait4(pid, &ws, 0, nil); err != syscall.EINTR { + return + } + } +} diff --git a/packages/envd/internal/port/forward_reap_other.go b/packages/envd/internal/port/forward_reap_other.go new file mode 100644 index 0000000000..2ac9f4c8ba --- /dev/null +++ b/packages/envd/internal/port/forward_reap_other.go @@ -0,0 +1,7 @@ +//go:build !linux + +package port + +// reapAdoptedSocat is a no-op on non-Linux platforms; the port forwarder and its +// socat children are Linux-only. +func reapAdoptedSocat(_ int) {} diff --git a/packages/envd/internal/services/cgroups/freeze.go b/packages/envd/internal/services/cgroups/freeze.go new file mode 100644 index 0000000000..9f7bb84d96 --- /dev/null +++ b/packages/envd/internal/services/cgroups/freeze.go @@ -0,0 +1,129 @@ +package cgroups + +import ( + "context" + "errors" + "fmt" + "sync" + + "golang.org/x/sync/semaphore" +) + +// WorkloadProcessTypes are the cgroups holding the customer workload: the +// processes/shells envd spawns (user) and PTY sessions (ptys). These are frozen +// before a pause and thawed on resume; envd's own system processes are excluded. +var WorkloadProcessTypes = []ProcessType{ProcessTypeUser, ProcessTypePTY} + +// WorkloadFreezer serializes freeze/unfreeze of the workload cgroups across +// every caller — the pre-pause /freeze, the pause-rollback /unfreeze, the /init +// deferred resume-thaw, and the live-upgrade handover — through a single lock, +// so their per-cgroup sweeps can never interleave and strand the workload +// frozen. Freeze and Unfreeze are best-effort: each attempts every cgroup even +// if one fails and returns the joined error. +// +// A single WorkloadFreezer instance must be shared by all of those callers for +// the serialization to hold; construct one and pass it to each. +type WorkloadFreezer struct { + mgr Manager + lock *semaphore.Weighted + + // thawMu guards thawedCh, the channel closed on the next Unfreeze. It lets + // callers block until the workload is next thawed (see Thawed). + thawMu sync.Mutex + thawedCh chan struct{} +} + +// NewWorkloadFreezer wraps a cgroup manager with the shared freeze lock. +func NewWorkloadFreezer(mgr Manager) *WorkloadFreezer { + return &WorkloadFreezer{mgr: mgr, lock: semaphore.NewWeighted(1)} +} + +// Thawed returns a channel that is closed the next time the workload is +// unfrozen (a fresh one is installed after each Unfreeze). A re-adopted +// process's carried kill-timer selects on it so the timeout only starts once the +// workload actually runs — the new envd runs while the re-adopted workload is +// still frozen (until the post-upgrade /init), and the timeout must measure +// running time, not frozen time. Call it while the workload is frozen (before +// the thaw you want to observe) so the close isn't missed. +func (f *WorkloadFreezer) Thawed() <-chan struct{} { + f.thawMu.Lock() + defer f.thawMu.Unlock() + if f.thawedCh == nil { + f.thawedCh = make(chan struct{}) + } + + return f.thawedCh +} + +// signalThawed closes the pending Thawed channel (if any) to wake waiters. +func (f *WorkloadFreezer) signalThawed() { + f.thawMu.Lock() + defer f.thawMu.Unlock() + if f.thawedCh != nil { + close(f.thawedCh) + f.thawedCh = nil + } +} + +// Manager returns the underlying cgroup manager, for callers that also need it +// for non-freeze work such as process placement. +func (f *WorkloadFreezer) Manager() Manager { return f.mgr } + +// Freeze freezes the workload cgroups, serialized against all other callers. The +// ctx bounds only the wait for the lock. +func (f *WorkloadFreezer) Freeze(ctx context.Context) error { + release, err := f.FreezeHold(ctx) + release() + + return err +} + +// FreezeHold freezes the workload cgroups and KEEPS the lock held, returning a +// release func. Unlike Freeze (which releases as soon as the sweep is done), this +// lets a caller keep the freeze uninterruptible across a critical section — the +// live-upgrade handover — so a concurrent Unfreeze (e.g. /init's deferred +// resume-thaw or /unfreeze) blocks on the lock until release is called and cannot +// thaw the workload mid-handover. The frozen cgroup state persists after release; +// release only drops the lock and is idempotent. On a lock-acquire failure it +// returns a no-op release and the error. +func (f *WorkloadFreezer) FreezeHold(ctx context.Context) (release func(), err error) { + if err := f.lock.Acquire(ctx, 1); err != nil { + return func() {}, err + } + + var once sync.Once + release = func() { once.Do(func() { f.lock.Release(1) }) } + + var errs []error + for _, pt := range WorkloadProcessTypes { + if e := f.mgr.Freeze(pt); e != nil { + errs = append(errs, fmt.Errorf("freeze %s cgroup: %w", pt, e)) + } + } + + return release, errors.Join(errs...) +} + +// Unfreeze thaws the workload cgroups, serialized against all other callers. It +// detaches the lock wait from ctx cancellation so the thaw always lands even if +// the caller's request context is cancelled — a dropped unfreeze would strand +// the workload frozen. Thawing a non-frozen cgroup is a no-op, so it is safe to +// call unconditionally on every upgrade/resume outcome. +func (f *WorkloadFreezer) Unfreeze(ctx context.Context) error { + if err := f.lock.Acquire(context.WithoutCancel(ctx), 1); err != nil { + return err + } + defer f.lock.Release(1) + + var errs []error + for _, pt := range WorkloadProcessTypes { + if err := f.mgr.Unfreeze(pt); err != nil { + errs = append(errs, fmt.Errorf("unfreeze %s cgroup: %w", pt, err)) + } + } + + // Wake anyone waiting for the workload to run again (carried kill-timers). + f.signalThawed() + + return errors.Join(errs...) +} diff --git a/packages/envd/internal/services/cgroups/freeze_test.go b/packages/envd/internal/services/cgroups/freeze_test.go new file mode 100644 index 0000000000..403167c72c --- /dev/null +++ b/packages/envd/internal/services/cgroups/freeze_test.go @@ -0,0 +1,46 @@ +package cgroups + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestWorkloadFreezer_FreezeHoldBlocksUnfreeze verifies FreezeHold keeps the +// shared lock held so a concurrent Unfreeze cannot thaw the workload until +// release is called — the serialization the live-upgrade handover relies on. +func TestWorkloadFreezer_FreezeHoldBlocksUnfreeze(t *testing.T) { + t.Parallel() + + f := NewWorkloadFreezer(NewNoopManager()) + + release, err := f.FreezeHold(context.Background()) + require.NoError(t, err) + + unfrozen := make(chan struct{}) + go func() { + _ = f.Unfreeze(context.Background()) + close(unfrozen) + }() + + select { + case <-unfrozen: + t.Fatal("Unfreeze thawed the workload while the freeze hold was active") + case <-time.After(100 * time.Millisecond): + // expected: blocked on the held lock + } + + release() + + select { + case <-unfrozen: + // expected: proceeds once the hold is released + case <-time.After(2 * time.Second): + t.Fatal("Unfreeze did not proceed after the hold was released") + } + + assert.NotPanics(t, release, "release must be idempotent") +} diff --git a/packages/envd/internal/services/filesystem/service.go b/packages/envd/internal/services/filesystem/service.go index 631a002f89..6365f830bc 100644 --- a/packages/envd/internal/services/filesystem/service.go +++ b/packages/envd/internal/services/filesystem/service.go @@ -1,6 +1,8 @@ package filesystem import ( + "sync" + "connectrpc.com/connect" "github.com/go-chi/chi/v5" "github.com/rs/zerolog" @@ -16,13 +18,22 @@ type Service struct { logger *zerolog.Logger watchers *utils.Map[string, *FileWatcher] defaults *execcontext.Defaults + // watchersMu serializes watcher membership changes (CreateWatcher/ + // RemoveWatcher) and the event drain (GetWatcherEvents) against the + // live-upgrade handover snapshot (ExportWatchers/ExportWatchersHold/ + // ImportWatchers), so the exported set is a consistent view, a watcher + // created/removed concurrently isn't dropped or resurrected across the swap, + // and no event is drained after the snapshot carried it (double-delivery). + // A pointer so it stays shared across the value-type Service's copies. + watchersMu *sync.Mutex } -func Handle(server *chi.Mux, l *zerolog.Logger, defaults *execcontext.Defaults) { +func Handle(server *chi.Mux, l *zerolog.Logger, defaults *execcontext.Defaults) Service { service := Service{ - logger: l, - watchers: utils.NewMap[string, *FileWatcher](), - defaults: defaults, + logger: l, + watchers: utils.NewMap[string, *FileWatcher](), + defaults: defaults, + watchersMu: &sync.Mutex{}, } interceptors := connect.WithInterceptors( @@ -33,4 +44,8 @@ func Handle(server *chi.Mux, l *zerolog.Logger, defaults *execcontext.Defaults) path, handler := spec.NewFilesystemHandler(service, interceptors) server.Mount(path, handler) + + // Returned so the live-upgrade handover (main.go) can export/import watcher + // state. Service is a value type but its watchers map is a shared pointer. + return service } diff --git a/packages/envd/internal/services/filesystem/service_test.go b/packages/envd/internal/services/filesystem/service_test.go index 9a6d421f94..ce531d61b0 100644 --- a/packages/envd/internal/services/filesystem/service_test.go +++ b/packages/envd/internal/services/filesystem/service_test.go @@ -1,6 +1,8 @@ package filesystem import ( + "sync" + "github.com/rs/zerolog" "github.com/e2b-dev/infra/packages/envd/internal/execcontext" @@ -11,8 +13,9 @@ func mockService() Service { logger := zerolog.Nop() return Service{ - logger: &logger, - watchers: utils.NewMap[string, *FileWatcher](), + logger: &logger, + watchers: utils.NewMap[string, *FileWatcher](), + watchersMu: &sync.Mutex{}, defaults: &execcontext.Defaults{ EnvVars: utils.NewEnvVars(), }, diff --git a/packages/envd/internal/services/filesystem/watch_handover.go b/packages/envd/internal/services/filesystem/watch_handover.go new file mode 100644 index 0000000000..3e163ae65c --- /dev/null +++ b/packages/envd/internal/services/filesystem/watch_handover.go @@ -0,0 +1,119 @@ +package filesystem + +import ( + "context" + + rpc "github.com/e2b-dev/infra/packages/envd/internal/services/spec/filesystem" + "github.com/e2b-dev/infra/packages/envd/internal/services/spec/upgrade" +) + +// ExportWatchers snapshots the active watchers (id, config, and any buffered +// events not yet fetched) as typed upgrade.HandoverWatcher messages, carried in +// the process-service handover across an envd live-upgrade so the new envd can +// re-arm each watcher with the same id. Pending FilesystemEvents ride natively +// (no protojson round-trip). This package owns the watcher schema; the process +// service only carries the messages. +func (s Service) ExportWatchers() []*upgrade.HandoverWatcher { + // Snapshot the watcher set under watchersMu so a concurrent CreateWatcher/ + // RemoveWatcher can't make the export an inconsistent view. + s.watchersMu.Lock() + defer s.watchersMu.Unlock() + + return s.exportWatchersLocked() +} + +// ExportWatchersHold is ExportWatchers but keeps watchersMu held, returning a +// release func the caller invokes to resume serving. The live-upgrade path +// holds it from the snapshot through the execve so a concurrent GetWatcherEvents +// cannot drain a watcher's buffered events in that window — draining there would +// consume events the snapshot already carried, so the re-armed watcher would +// re-deliver them after the swap (double-delivery). GetWatcherEvents takes the +// same lock. A successful execve replaces this process and the held lock +// vanishes with it; on failure the caller's release restores serving. +func (s Service) ExportWatchersHold() ([]*upgrade.HandoverWatcher, func()) { + s.watchersMu.Lock() + + return s.exportWatchersLocked(), s.watchersMu.Unlock +} + +// exportWatchersLocked snapshots the active watchers. The caller must hold +// watchersMu. +func (s Service) exportWatchersLocked() []*upgrade.HandoverWatcher { + out := make([]*upgrade.HandoverWatcher, 0) + + s.watchers.Range(func(id string, fw *FileWatcher) bool { + fw.Lock.Lock() + // Copy the event pointers into an independent slice under fw.Lock so a + // later append to fw.Events (or the eventual proto.Marshal, which runs + // outside this lock) sees a stable snapshot. + pending := make([]*rpc.FilesystemEvent, len(fw.Events)) + copy(pending, fw.Events) + fw.Lock.Unlock() + + out = append(out, &upgrade.HandoverWatcher{ + Id: id, + Path: fw.WatchPath, + Recursive: fw.Recursive, + IncludeEntryInfo: fw.IncludeEntryInfo, + PendingEvents: pending, + }) + + return true + }) + + return out +} + +// ImportWatchers re-arms watchers carried across a live-upgrade: it re-creates a +// fresh fsnotify watch for each carried watcher, preserving the original watcher +// id (so GetWatcherEvents keeps working without a client change) and re-queuing +// any pending events. The resume freeze makes this lossless — nothing mutates +// the filesystem during the handover window. +func (s Service) ImportWatchers(watchers []*upgrade.HandoverWatcher) (rearmed, failed int) { + if len(watchers) == 0 { + return 0, 0 + } + + // Re-arm under watchersMu so the rebuilt membership is consistent (symmetric + // with ExportWatchers). No contention in practice — the new envd re-adopts + // before it serves client RPCs. + s.watchersMu.Lock() + defer s.watchersMu.Unlock() + + for _, wh := range watchers { + fw, err := CreateFileWatcher(context.Background(), s.logger, wh.GetPath(), wh.GetRecursive(), wh.GetIncludeEntryInfo()) + if err != nil { + failed++ + // Don't log wh.Path — it can be a customer code/data location, and + // these events go to centralized logs; the opaque watcher_id suffices. + s.logger.Warn().Err(err).Str("watcher_id", wh.GetId()).Msg("handover: re-arm watcher failed") + + continue + } + rearmed++ + + pending := wh.GetPendingEvents() + if len(pending) > 0 { + fw.Lock.Lock() + fw.Events = append(pending, fw.Events...) + fw.Lock.Unlock() + } + + s.watchers.Store(wh.GetId(), fw) + + s.logger.Info(). + Str("event_type", "watcher_readopted"). + Str("watcher_id", wh.GetId()). + Int("pending_events", len(pending)). + Msg("re-armed watcher after envd self-upgrade") + } + + // Loki-queryable summary (rollout observability). + s.logger.Info(). + Str("event_type", "watchers_rearmed"). + Int("rearmed", rearmed). + Int("failed", failed). + Msg("re-armed filesystem watchers after envd self-upgrade") + + return rearmed, failed +} diff --git a/packages/envd/internal/services/filesystem/watch_handover_test.go b/packages/envd/internal/services/filesystem/watch_handover_test.go new file mode 100644 index 0000000000..b9d80ad326 --- /dev/null +++ b/packages/envd/internal/services/filesystem/watch_handover_test.go @@ -0,0 +1,81 @@ +package filesystem + +import ( + "os" + "os/user" + "path/filepath" + "testing" + + "connectrpc.com/authn" + "connectrpc.com/connect" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/e2b-dev/infra/packages/envd/internal/services/spec/filesystem" +) + +// TestWatcherHandoverReArm verifies the live-upgrade watcher re-arm: a watcher +// exported from one service and imported into a fresh one keeps its id, its +// config, and continues to deliver new filesystem events. +func TestWatcherHandoverReArm(t *testing.T) { + t.Parallel() + + u, err := user.Current() + require.NoError(t, err) + + root := t.TempDir() + ctx := authn.SetInfo(t.Context(), u) + + src := mockService() + created, err := src.CreateWatcher(ctx, connect.NewRequest(&filesystem.CreateWatcherRequest{ + Path: root, + Recursive: false, + })) + require.NoError(t, err) + wid := created.Msg.GetWatcherId() + + // Export from the outgoing service, import into the incoming one. + blob := src.ExportWatchers() + require.NotEmpty(t, blob) + + dst := mockService() + dst.ImportWatchers(blob) + + // The re-armed watcher keeps the same id... + _, ok := dst.watchers.Load(wid) + assert.True(t, ok, "re-armed watcher should preserve its id") + + // ...and delivers events for changes made after the handover. + require.NoError(t, os.WriteFile(filepath.Join(root, "after.txt"), []byte("x"), 0o644)) + events := collectEvents(t, ctx, dst, wid) + require.NotEmpty(t, events, "re-armed watcher should deliver post-handover events under the preserved id") +} + +// TestWatcherHandoverPreservesPending verifies buffered events not yet fetched +// survive the export/import round-trip. +func TestWatcherHandoverPreservesPending(t *testing.T) { + t.Parallel() + + root := t.TempDir() + src := mockService() + + fw, err := CreateFileWatcher(t.Context(), src.logger, root, false, false) + require.NoError(t, err) + fw.Lock.Lock() + fw.Events = append(fw.Events, &filesystem.FilesystemEvent{ + Name: "pending.txt", + Type: filesystem.EventType_EVENT_TYPE_CREATE, + }) + fw.Lock.Unlock() + src.watchers.Store("wpending", fw) + + dst := mockService() + dst.ImportWatchers(src.ExportWatchers()) + + got, ok := dst.watchers.Load("wpending") + require.True(t, ok) + got.Lock.Lock() + defer got.Lock.Unlock() + require.GreaterOrEqual(t, len(got.Events), 1) + assert.Equal(t, "pending.txt", got.Events[0].GetName()) +} diff --git a/packages/envd/internal/services/filesystem/watch_sync.go b/packages/envd/internal/services/filesystem/watch_sync.go index 8935597a86..0d3a37f0a0 100644 --- a/packages/envd/internal/services/filesystem/watch_sync.go +++ b/packages/envd/internal/services/filesystem/watch_sync.go @@ -24,6 +24,13 @@ type FileWatcher struct { cancel func() Error error + // Config captured so the watcher can be re-armed after an envd + // live-upgrade. WatchPath is the already-resolved + // absolute path. + WatchPath string + Recursive bool + IncludeEntryInfo bool + Lock sync.Mutex } @@ -44,10 +51,13 @@ func CreateFileWatcher(ctx context.Context, logger *zerolog.Logger, watchPath st return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("error adding path %s to watcher: %w", watchPath, err)) } fw := &FileWatcher{ - watcher: w, - cancel: cancel, - Events: []*rpc.FilesystemEvent{}, - Error: nil, + watcher: w, + cancel: cancel, + Events: []*rpc.FilesystemEvent{}, + Error: nil, + WatchPath: watchPath, + Recursive: recursive, + IncludeEntryInfo: includeEntryInfo, } go func() { @@ -179,7 +189,9 @@ func (s Service) CreateWatcher(ctx context.Context, req *connect.Request[rpc.Cre return nil, err } + s.watchersMu.Lock() s.watchers.Store(watcherId, w) + s.watchersMu.Unlock() return connect.NewResponse(&rpc.CreateWatcherResponse{ WatcherId: watcherId, @@ -194,6 +206,14 @@ func (s Service) GetWatcherEvents(_ context.Context, req *connect.Request[rpc.Ge return nil, connect.NewError(connect.CodeNotFound, fmt.Errorf("watcher with id %s not found", watcherId)) } + // Serialize the drain against a live-upgrade handover snapshot + // (ExportWatchersHold holds watchersMu across the execve). Draining a + // watcher's events in that window would consume events the snapshot already + // carried, so the re-armed watcher would re-deliver them after the swap. + // watchersMu is taken before w.Lock, matching Export/CreateWatcher ordering. + s.watchersMu.Lock() + defer s.watchersMu.Unlock() + w.Lock.Lock() defer w.Lock.Unlock() @@ -218,7 +238,9 @@ func (s Service) RemoveWatcher(_ context.Context, req *connect.Request[rpc.Remov } w.Close() + s.watchersMu.Lock() s.watchers.Delete(watcherId) + s.watchersMu.Unlock() return connect.NewResponse(&rpc.RemoveWatcherResponse{}), nil } diff --git a/packages/envd/internal/services/process/connect.go b/packages/envd/internal/services/process/connect.go index 321ada5541..b837349041 100644 --- a/packages/envd/internal/services/process/connect.go +++ b/packages/envd/internal/services/process/connect.go @@ -22,6 +22,14 @@ func (s *Service) handleConnect(ctx context.Context, req *connect.Request[rpc.Co proc, err := s.getProcess(req.Msg.GetProcess()) if err != nil { + // The process is gone from the live map. It may have exited during a + // window when no client was subscribed (e.g. a live-upgrade handover + // gap). Serve the retained terminal event if we still have it, so the + // caller still learns the exit code. + if ret, ok := s.lookupTerminated(req.Msg.GetProcess()); ok { + return s.serveTerminated(stream, ret) + } + return err } @@ -99,6 +107,20 @@ func (s *Service) handleConnect(ctx context.Context, req *connect.Request[rpc.Co return case event, ok := <-end: if !ok { + // The EndEvent was already emitted and closed before we + // subscribed (the process exited during the no-subscriber + // window). Serve the retained terminal event for THIS process if + // we still have it. Checked here — not before subscribing — so a + // reused PID is never served a previous process's exit. + if ret, rok := s.terminated.Load(proc.Pid()); rok { + if serr := stream.Send(&rpc.ConnectResponse{ + Event: &rpc.ProcessEvent{Event: &rpc.ProcessEvent_End{End: ret.end}}, + }); serr != nil { + cancel(connect.NewError(connect.CodeUnknown, fmt.Errorf("error sending retained end event: %w", serr))) + } + + return + } cancel(connect.NewError(connect.CodeUnknown, errors.New("end event channel closed before sending end event"))) return @@ -122,3 +144,58 @@ func (s *Service) handleConnect(ctx context.Context, req *connect.Request[rpc.Co return ctx.Err() } + +// lookupTerminated finds a retained terminal event by the same selector shape +// Connect accepts (pid or tag). It backs the work-item-#8 late-Connect path. +func (s *Service) lookupTerminated(selector *rpc.ProcessSelector) (*retainedExit, bool) { + switch selector.GetSelector().(type) { + case *rpc.ProcessSelector_Pid: + return s.terminated.Load(selector.GetPid()) + case *rpc.ProcessSelector_Tag: + tag := selector.GetTag() + + var found *retainedExit + s.terminated.Range(func(_ uint32, v *retainedExit) bool { + if v.tag != nil && *v.tag == tag { + found = v + + return false + } + + return true + }) + + return found, found != nil + default: + return nil, false + } +} + +// serveTerminated streams the retained Start+End pair for an already-exited +// process and returns, so a client that (re)connects after the exit still +// observes the terminal event and exit code. +func (s *Service) serveTerminated(stream *connect.ServerStream[rpc.ConnectResponse], ret *retainedExit) error { + if err := stream.Send(&rpc.ConnectResponse{ + Event: &rpc.ProcessEvent{ + Event: &rpc.ProcessEvent_Start{ + Start: &rpc.ProcessEvent_StartEvent{ + Pid: ret.pid, + }, + }, + }, + }); err != nil { + return connect.NewError(connect.CodeUnknown, fmt.Errorf("error sending start event: %w", err)) + } + + if err := stream.Send(&rpc.ConnectResponse{ + Event: &rpc.ProcessEvent{ + Event: &rpc.ProcessEvent_End{ + End: ret.end, + }, + }, + }); err != nil { + return connect.NewError(connect.CodeUnknown, fmt.Errorf("error sending retained end event: %w", err)) + } + + return nil +} diff --git a/packages/envd/internal/services/process/connect_test.go b/packages/envd/internal/services/process/connect_test.go new file mode 100644 index 0000000000..c82b856cf0 --- /dev/null +++ b/packages/envd/internal/services/process/connect_test.go @@ -0,0 +1,189 @@ +package process + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "connectrpc.com/connect" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/e2b-dev/infra/packages/envd/internal/execcontext" + "github.com/e2b-dev/infra/packages/envd/internal/services/cgroups" + "github.com/e2b-dev/infra/packages/envd/internal/services/process/handler" + rpc "github.com/e2b-dev/infra/packages/envd/internal/services/spec/process" + spec "github.com/e2b-dev/infra/packages/envd/internal/services/spec/process/processconnect" + "github.com/e2b-dev/infra/packages/envd/internal/utils" +) + +// newRetentionTestService builds a process Service and its Connect client +// without spawning any child processes, so — unlike newTestService — it does +// not require root. It is used to exercise the terminal-event retention cache +// in isolation. +func newRetentionTestService(t *testing.T) (spec.ProcessClient, *Service, func()) { + t.Helper() + + cwd := t.TempDir() + logger := zerolog.Nop() + + svc := newService(&logger, &execcontext.Defaults{ + EnvVars: utils.NewEnvVars(), + Workdir: &cwd, + }, cgroups.NewWorkloadFreezer(cgroups.NewNoopManager())) + + mux := http.NewServeMux() + path, handler := spec.NewProcessHandler(svc) + mux.Handle(path, handler) + + srv := httptest.NewServer(mux) + client := spec.NewProcessClient(srv.Client(), srv.URL) + + return client, svc, srv.Close +} + +func drainConnect(t *testing.T, stream *connect.ServerStreamForClient[rpc.ConnectResponse]) []*rpc.ProcessEvent { + t.Helper() + + var events []*rpc.ProcessEvent + for stream.Receive() { + events = append(events, stream.Msg().GetEvent()) + } + require.NoError(t, stream.Err()) + require.NoError(t, stream.Close()) + + return events +} + +// TestConnect_ServesRetainedExitByPid verifies that a Connect issued after the +// process has exited — the process is no longer in the live map, mirroring a +// gap-exit during a live-upgrade handover — still returns the Start+End pair +// with the retained exit code when selected by pid. +func TestConnect_ServesRetainedExitByPid(t *testing.T) { + t.Parallel() + + client, svc, cleanup := newRetentionTestService(t) + defer cleanup() + + const pid = uint32(4242) + svc.terminated.Store(pid, &retainedExit{ + pid: pid, + end: &rpc.ProcessEvent_EndEvent{Exited: true, ExitCode: 7, Status: "exited"}, + }) + + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + + stream, err := client.Connect(ctx, connect.NewRequest(&rpc.ConnectRequest{ + Process: &rpc.ProcessSelector{Selector: &rpc.ProcessSelector_Pid{Pid: pid}}, + })) + require.NoError(t, err) + + events := drainConnect(t, stream) + + require.Len(t, events, 2, "expected Start + retained End") + assert.NotNil(t, events[0].GetStart(), "first event should be Start") + require.NotNil(t, events[1].GetEnd(), "second event should be End") + assert.True(t, events[1].GetEnd().GetExited()) + assert.Equal(t, int32(7), events[1].GetEnd().GetExitCode()) +} + +// TestConnect_SubscriberServedRetainedOnEndClose covers the reaper→Connect +// contract that the Wait fix restores for fresh processes: a Connect that finds +// the process still live and subscribes, but whose terminal event was already +// fanned out before it subscribed, must be served the retained exit the instant +// EndEvent closes — not block forever. (Wait now closes EndEvent, mirroring the +// re-adopt reaper; here the close is driven directly so the test needs no root.) +func TestConnect_SubscriberServedRetainedOnEndClose(t *testing.T) { + t.Parallel() + + client, svc, cleanup := newRetentionTestService(t) + defer cleanup() + + const pid = uint32(7373) + logger := zerolog.Nop() + h := handler.Readopt(handler.ReadoptArgs{Pid: pid}, &logger) + svc.processes.Store(pid, h) // live when Connect subscribes + svc.terminated.Store(pid, &retainedExit{ + pid: pid, + end: &rpc.ProcessEvent_EndEvent{Exited: true, ExitCode: 9, Status: "exited"}, + }) + + // The event was fanned out before this Connect subscribed; the close (what + // Wait now does) is what makes the retention fallback fire instead of hanging. + go func() { + time.Sleep(150 * time.Millisecond) + close(h.EndEvent.Source) + }() + + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + + stream, err := client.Connect(ctx, connect.NewRequest(&rpc.ConnectRequest{ + Process: &rpc.ProcessSelector{Selector: &rpc.ProcessSelector_Pid{Pid: pid}}, + })) + require.NoError(t, err) + + events := drainConnect(t, stream) + require.NotEmpty(t, events) + last := events[len(events)-1] + require.NotNil(t, last.GetEnd(), "must serve the retained End on channel close, not hang") + assert.Equal(t, int32(9), last.GetEnd().GetExitCode()) +} + +// TestConnect_ServesRetainedExitByTag verifies tag-selected lookup of a +// retained terminal event (the code-interpreter kernel is addressed by tag). +func TestConnect_ServesRetainedExitByTag(t *testing.T) { + t.Parallel() + + client, svc, cleanup := newRetentionTestService(t) + defer cleanup() + + tag := "kernel" + svc.terminated.Store(99, &retainedExit{ + pid: 99, + tag: &tag, + end: &rpc.ProcessEvent_EndEvent{Exited: true, ExitCode: 0, Status: "exited"}, + }) + + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + + stream, err := client.Connect(ctx, connect.NewRequest(&rpc.ConnectRequest{ + Process: &rpc.ProcessSelector{Selector: &rpc.ProcessSelector_Tag{Tag: tag}}, + })) + require.NoError(t, err) + + events := drainConnect(t, stream) + + require.Len(t, events, 2) + assert.Equal(t, uint32(99), events[0].GetStart().GetPid()) + require.NotNil(t, events[1].GetEnd()) + assert.True(t, events[1].GetEnd().GetExited()) +} + +// TestConnect_UnknownProcessNotFound verifies the regression guard: a Connect +// for a process that is neither live nor retained still returns NotFound. +func TestConnect_UnknownProcessNotFound(t *testing.T) { + t.Parallel() + + client, _, cleanup := newRetentionTestService(t) + defer cleanup() + + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + + stream, err := client.Connect(ctx, connect.NewRequest(&rpc.ConnectRequest{ + Process: &rpc.ProcessSelector{Selector: &rpc.ProcessSelector_Pid{Pid: 1}}, + })) + require.NoError(t, err) + + for stream.Receive() { + } + require.Error(t, stream.Err()) + assert.Equal(t, connect.CodeNotFound, connect.CodeOf(stream.Err())) + _ = stream.Close() +} diff --git a/packages/envd/internal/services/process/dup3_linux.go b/packages/envd/internal/services/process/dup3_linux.go new file mode 100644 index 0000000000..702ace800e --- /dev/null +++ b/packages/envd/internal/services/process/dup3_linux.go @@ -0,0 +1,12 @@ +//go:build linux + +package process + +import "golang.org/x/sys/unix" + +// dup3 duplicates oldfd onto newfd via dup3(2) with the given flags. Linux-only; +// the live upgrade uses it (with flags 0) to relocate carried fds with CLOEXEC +// cleared so they survive execve. +func dup3(oldfd, newfd, flags int) error { + return unix.Dup3(oldfd, newfd, flags) +} diff --git a/packages/envd/internal/services/process/dup3_other.go b/packages/envd/internal/services/process/dup3_other.go new file mode 100644 index 0000000000..49b98e51ca --- /dev/null +++ b/packages/envd/internal/services/process/dup3_other.go @@ -0,0 +1,11 @@ +//go:build !linux + +package process + +import "errors" + +// dup3 is unsupported off Linux. The live upgrade only runs in the (Linux) +// guest; this stub keeps the package compiling on non-Linux dev machines. +func dup3(oldfd, newfd, flags int) error { + return errors.New("dup3 is only supported on linux") +} diff --git a/packages/envd/internal/services/process/handler/handler.go b/packages/envd/internal/services/process/handler/handler.go index e30320944d..d7122f99ab 100644 --- a/packages/envd/internal/services/process/handler/handler.go +++ b/packages/envd/internal/services/process/handler/handler.go @@ -65,11 +65,83 @@ type Handler struct { DataEvent *MultiplexedChannel[rpc.ProcessEvent_Data] EndEvent *MultiplexedChannel[rpc.ProcessEvent_End] + + // --- live-upgrade handover --- + // pid is stored at Start so it survives an envd self-upgrade where cmd is + // gone (a re-adopted handler has cmd == nil). cgType records the cgroup the + // child runs in. stdoutF/stderrF/stdinF are the raw pipe fds captured at + // New() so they can be carried across execve; tty is the PTY master. + pid uint32 + cgType cgroups.ProcessType + readopted bool + stdoutF *os.File + stderrF *os.File + stdinF *os.File + // deadline is the process's timeout deadline (zero = no timeout). Captured + // so it can be carried across a live-upgrade and re-armed on the new envd. + // deadlineMu guards it: the readopt reaper writes it asynchronously once the + // workload thaws, while Deadline() may be read concurrently by a handover. + deadlineMu sync.Mutex + deadline time.Time + // readoptTimeout is the remaining timeout carried across a live-upgrade, + // re-armed when BeginReaping is called. + readoptTimeout time.Duration + // thawed, if non-nil (re-adopted handlers only), is closed when the workload + // is unfrozen after the upgrade; the carried kill-timer waits on it so the + // timeout is not burned down while the process is still frozen. + thawed <-chan struct{} + // OnExit, if set, is invoked by the re-adopt reaper with the terminal event + // immediately before EndEvent is closed, so the service can retain the exit + // synchronously. A Connect that forks after the close then always finds the + // retained exit in the cache rather than racing an asynchronous retain. + OnExit func(*rpc.ProcessEvent_EndEvent) } // This method must be called only after the process has been started func (p *Handler) Pid() uint32 { - return uint32(p.cmd.Process.Pid) + if p.cmd != nil && p.cmd.Process != nil { + return uint32(p.cmd.Process.Pid) + } + + return p.pid +} + +// CgType returns the cgroup type the child was placed in (for handover). +func (p *Handler) CgType() cgroups.ProcessType { return p.cgType } + +// Deadline returns the process's timeout deadline and whether one is set, so a +// live-upgrade handover can carry the remaining timeout. +func (p *Handler) Deadline() (time.Time, bool) { + p.deadlineMu.Lock() + d := p.deadline + p.deadlineMu.Unlock() + if d.IsZero() { + return time.Time{}, false + } + + return d, true +} + +// setDeadline records the process's timeout deadline under deadlineMu. +func (p *Handler) setDeadline(t time.Time) { + p.deadlineMu.Lock() + p.deadline = t + p.deadlineMu.Unlock() +} + +// HandoverFds returns the raw fds to carry across an envd self-upgrade: +// stdout/stderr read ends, stdin write end, and the PTY master. Absent fds +// are -1. The fds remain owned by the Handler. +func (p *Handler) HandoverFds() (stdout, stderr, stdin, tty int) { + fd := func(f *os.File) int { + if f == nil { + return -1 + } + + return int(f.Fd()) + } + + return fd(p.stdoutF), fd(p.stderrF), fd(p.stdinF), fd(p.tty) } // userCommand returns a human-readable representation of the user's original command, @@ -188,6 +260,13 @@ func New( EndEvent: NewMultiplexedChannel[rpc.ProcessEvent_End](0), logger: logger, } + h.cgType = getProcType(req) + + // Capture the process timeout deadline (if any) so it can be carried across + // a live-upgrade and re-armed on the new envd. + if d, ok := ctx.Deadline(); ok { + h.setDeadline(d) + } if req.GetPty() != nil { // The pty should ideally start only in the Start method, but the package does not support that and we would have to code it manually. @@ -240,6 +319,9 @@ func New( if err != nil { return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("error creating stdout pipe for command '%s': %w", userCmd, err)) } + if f, ok := stdout.(*os.File); ok { + h.stdoutF = f // captured for live-upgrade handover + } outWg.Go(func() { readBuf := make([]byte, stdChunkSize) @@ -279,6 +361,9 @@ func New( if err != nil { return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("error creating stderr pipe for command '%s': %w", userCmd, err)) } + if f, ok := stderr.(*os.File); ok { + h.stderrF = f // captured for live-upgrade handover + } outWg.Go(func() { readBuf := make([]byte, stdChunkSize) @@ -323,6 +408,9 @@ func New( } h.stdin = stdin + if f, ok := stdin.(*os.File); ok { + h.stdinF = f // captured for live-upgrade handover + } } } @@ -350,14 +438,23 @@ func getProcType(req *rpc.StartRequest) cgroups.ProcessType { } func (p *Handler) SendSignal(signal syscall.Signal) error { - if p.cmd.Process == nil { - return errors.New("process not started") - } - if signal == syscall.SIGKILL || signal == syscall.SIGTERM { p.outCancel() } + // Re-adopted handler (post live-upgrade): no cmd, signal by stored pid. + if p.cmd == nil { + if p.pid == 0 { + return errors.New("process not started") + } + + return syscall.Kill(int(p.pid), signal) + } + + if p.cmd.Process == nil { + return errors.New("process not started") + } + return p.cmd.Process.Signal(signal) } @@ -383,7 +480,7 @@ func (p *Handler) WriteStdin(data []byte) error { _, err := p.stdin.Write(data) if err != nil { - return fmt.Errorf("error writing to stdin of process '%d': %w", p.cmd.Process.Pid, err) + return fmt.Errorf("error writing to stdin of process '%d': %w", p.Pid(), err) } return nil @@ -418,7 +515,7 @@ func (p *Handler) WriteTty(data []byte) error { _, err := p.tty.Write(data) if err != nil { - return fmt.Errorf("error writing to tty of process '%d': %w", p.cmd.Process.Pid, err) + return fmt.Errorf("error writing to tty of process '%d': %w", p.Pid(), err) } return nil @@ -433,6 +530,8 @@ func (p *Handler) Start(requestTimeout time.Duration) (uint32, error) { } } + p.pid = uint32(p.cmd.Process.Pid) + p.logger. Info(). Str("event_type", "process_start"). @@ -471,6 +570,18 @@ func (p *Handler) Wait() { } p.EndEvent.Source <- event + // Retain the terminal event synchronously — BEFORE closing the source — so a + // Connect that forks after the close and falls back to the retention cache is + // guaranteed to find this exit rather than race an asynchronous retain. This + // mirrors the re-adopt reaper's ordering (readopt.go). + if p.OnExit != nil { + p.OnExit(endEvent) + } + // Close the source after the terminal event, mirroring the re-adopt reaper. + // A late Connect that subscribes after the event was fanned out (the process + // exited during the no-subscriber window) sees the closed channel and falls + // back to the (now-populated) retention cache instead of blocking forever. + close(p.EndEvent.Source) p.logger. Info(). diff --git a/packages/envd/internal/services/process/handler/pidfd_linux.go b/packages/envd/internal/services/process/handler/pidfd_linux.go new file mode 100644 index 0000000000..b51edbc768 --- /dev/null +++ b/packages/envd/internal/services/process/handler/pidfd_linux.go @@ -0,0 +1,11 @@ +//go:build linux + +package handler + +import "golang.org/x/sys/unix" + +// pidfdOpen returns a pidfd referring to pid via pidfd_open(2). Linux-only; the +// live-upgrade reaper uses it to wait on a re-adopted process it did not fork. +func pidfdOpen(pid int) (int, error) { + return unix.PidfdOpen(pid, 0) +} diff --git a/packages/envd/internal/services/process/handler/pidfd_other.go b/packages/envd/internal/services/process/handler/pidfd_other.go new file mode 100644 index 0000000000..53e5265126 --- /dev/null +++ b/packages/envd/internal/services/process/handler/pidfd_other.go @@ -0,0 +1,12 @@ +//go:build !linux + +package handler + +import "errors" + +// pidfdOpen is unsupported off Linux. envd only runs the live-upgrade reaper in +// the (Linux) guest; this stub keeps the package compiling on non-Linux dev +// machines (the reaper falls back to Wait4 when it errors). +func pidfdOpen(int) (int, error) { + return -1, errors.New("pidfd_open is only supported on linux") +} diff --git a/packages/envd/internal/services/process/handler/readopt.go b/packages/envd/internal/services/process/handler/readopt.go new file mode 100644 index 0000000000..30adb30ce9 --- /dev/null +++ b/packages/envd/internal/services/process/handler/readopt.go @@ -0,0 +1,248 @@ +package handler + +import ( + "context" + "errors" + "io" + "os" + "slices" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/rs/zerolog" + "golang.org/x/sys/unix" + + "github.com/e2b-dev/infra/packages/envd/internal/services/cgroups" + rpc "github.com/e2b-dev/infra/packages/envd/internal/services/spec/process" +) + +// ReadoptArgs carries the per-process state from a live-upgrade handover blob +// plus the inherited fds (already present in this process's fd table — the +// outgoing envd dup'd them across execve with CLOEXEC cleared). +type ReadoptArgs struct { + Pid uint32 + Tag *string + Config *rpc.ProcessConfig + CgType cgroups.ProcessType + Stdout *os.File // nil if pty + Stderr *os.File // nil if pty + Stdin *os.File // nil if disabled / pty + Tty *os.File // nil if non-pty + // Timeout is the process's remaining timeout carried across the upgrade + // (0 = none). Re-armed as a kill-timer so a timed-out process is still + // killed after the swap. + Timeout time.Duration + // Thawed, if non-nil, is closed when the workload is next unfrozen. The + // carried kill-timer waits on it before starting, so the remaining timeout + // measures the process's running time and is not burned down while it is + // still frozen (until the post-upgrade /init thaws it). + Thawed <-chan struct{} +} + +// Readopt reconstructs a Handler for a process that survived an envd +// self-upgrade. cmd is nil: the process keeps running with the +// same PID and the inherited pipe/PTY fds, so its I/O is uninterrupted. It is +// reaped via its pidfd (we are still the parent — execve preserves the PID), +// using a pid-specific wait4 so os/exec's reaping of post-upgrade processes is +// never disturbed. +func Readopt(args ReadoptArgs, logger *zerolog.Logger) *Handler { + outMultiplex := NewMultiplexedChannel[rpc.ProcessEvent_Data](outputBufferSize) + outCtx, outCancel := context.WithCancel(context.Background()) + _, cancel := context.WithCancel(context.Background()) + + h := &Handler{ + Config: args.Config, + Tag: args.Tag, + logger: logger, + cancel: cancel, + outCtx: outCtx, + outCancel: outCancel, + DataEvent: outMultiplex, + EndEvent: NewMultiplexedChannel[rpc.ProcessEvent_End](0), + pid: args.Pid, + cgType: args.CgType, + readopted: true, + tty: args.Tty, + stdoutF: args.Stdout, + stderrF: args.Stderr, + stdinF: args.Stdin, + } + if args.Stdin != nil { + h.stdin = args.Stdin + } + + var outWg sync.WaitGroup + if args.Tty != nil { + outWg.Go(func() { + h.pump(args.Tty, ptyChunkSize, &h.ptyBytes, func(b []byte) *rpc.ProcessEvent_DataEvent { + return &rpc.ProcessEvent_DataEvent{Output: &rpc.ProcessEvent_DataEvent_Pty{Pty: b}} + }) + }) + } else { + if args.Stdout != nil { + outWg.Go(func() { + h.pump(args.Stdout, stdChunkSize, &h.stdoutBytes, func(b []byte) *rpc.ProcessEvent_DataEvent { + return &rpc.ProcessEvent_DataEvent{Output: &rpc.ProcessEvent_DataEvent_Stdout{Stdout: b}} + }) + }) + } + if args.Stderr != nil { + outWg.Go(func() { + h.pump(args.Stderr, stdChunkSize, &h.stderrBytes, func(b []byte) *rpc.ProcessEvent_DataEvent { + return &rpc.ProcessEvent_DataEvent{Output: &rpc.ProcessEvent_DataEvent_Stderr{Stderr: b}} + }) + }) + } + } + + go func() { + outWg.Wait() + close(outMultiplex.Source) + outCancel() + }() + + h.readoptTimeout = args.Timeout + h.thawed = args.Thawed + + // NB: the reaper is NOT started here. It emits the terminal EndEvent, and a + // process whose sleep timer expired during the freeze can exit the instant + // it is unfrozen — before a caller (the service's trackTermination) has + // subscribed to EndEvent. Starting the reaper here would race that + // subscription and drop the exit code. The caller must subscribe first and + // then call BeginReaping. + return h +} + +// BeginReaping starts the pidfd reaper (and re-arms the carried timeout). It +// must be called AFTER the caller has subscribed to the handler's EndEvent, so +// a fast-exiting re-adopted process cannot emit its terminal event before there +// is a subscriber to retain it. +func (p *Handler) BeginReaping() { + go p.reapByPidfd() + + // Re-arm the carried process timeout: kill the process once the remaining + // timeout elapses, unless it exits first (outCtx is cancelled on exit). This + // restores the deadline that the pre-upgrade exec.CommandContext enforced. + if p.readoptTimeout > 0 { + go func() { + // The new envd runs while the re-adopted workload is still frozen + // (until the post-upgrade /init, or the fallback, thaws it). Don't + // start the kill countdown until then, so the carried timeout measures + // the process's running time and a short remaining timeout isn't burned + // down — killing the process — before it can run again. + if p.thawed != nil { + select { + case <-p.thawed: + case <-p.outCtx.Done(): + return + } + } + + // Record the absolute deadline so Deadline() reports it: Upgrade + // carries a process's remaining timeout forward across a (further, + // chained) upgrade exclusively via Deadline(), so without this a + // re-adopted timed process would run unbounded after a second handover. + p.setDeadline(time.Now().Add(p.readoptTimeout)) + + t := time.NewTimer(p.readoptTimeout) + defer t.Stop() + + select { + case <-t.C: + _ = p.SendSignal(syscall.SIGKILL) + case <-p.outCtx.Done(): + } + }() + } +} + +// pump mirrors the New() reader loops: read a chunk, account it, and fan it out +// to subscribers (dropped if none — same semantics as the live path). +func (p *Handler) pump(r io.Reader, chunk int, counter *atomic.Int64, mk func([]byte) *rpc.ProcessEvent_DataEvent) { + buf := make([]byte, chunk) + for { + n, readErr := r.Read(buf) + if n > 0 { + counter.Add(int64(n)) + if p.DataEvent.HasSubscribers() { + p.DataEvent.Source <- rpc.ProcessEvent_Data{Data: mk(slices.Clone(buf[:n]))} + } + } + if errors.Is(readErr, io.EOF) || errors.Is(readErr, syscall.EIO) { + return + } + if readErr != nil { + p.logger.Error().Err(readErr).Msg("readopt: error reading process output") + + return + } + } +} + +// reapByPidfd blocks on the child's pidfd until it exits, then harvests the +// status with a pid-specific wait4 and emits the EndEvent (never wait4(-1), +// which would steal os/exec's post-upgrade children). +func (p *Handler) reapByPidfd() { + // Prefer a pidfd to wait for exit, but if pidfd_open fails (the process + // already exited, or fd exhaustion) fall back to a blocking pid-specific + // wait4 below. Either way a terminal event is still emitted — never leave the + // process orphaned in the live map with clients blocked on an EndEvent that + // would otherwise never fire. + if pidfd, err := pidfdOpen(int(p.pid)); err != nil { + p.logger.Warn().Err(err).Uint32("pid", p.pid).Msg("readopt: pidfd_open failed; falling back to blocking wait4") + } else { + pfd := []unix.PollFd{{Fd: int32(pidfd), Events: unix.POLLIN}} + for { + if _, perr := unix.Poll(pfd, -1); perr != nil { + if errors.Is(perr, unix.EINTR) { + continue + } + } + + break + } + unix.Close(pidfd) + } + + var ws syscall.WaitStatus + _, werr := syscall.Wait4(int(p.pid), &ws, 0, nil) + + end := &rpc.ProcessEvent_EndEvent{} + switch { + case werr != nil: + msg := werr.Error() + end.Error = &msg + end.Status = "wait4 error" + case ws.Exited(): + end.Exited = true + end.ExitCode = int32(ws.ExitStatus()) + end.Status = "exited" + case ws.Signaled(): + end.ExitCode = int32(128 + int(ws.Signal())) + end.Status = ws.Signal().String() + } + + p.EndEvent.Source <- rpc.ProcessEvent_End{End: end} + // Retain the terminal event synchronously — before closing the source — so a + // Connect that forks after the close and falls back to the retention cache is + // guaranteed to find this exit rather than race an asynchronous retain. + if p.OnExit != nil { + p.OnExit(end) + } + // Close the source after the terminal event has fanned out to whatever + // subscribers existed at send time. A Connect that forks afterwards then + // gets a closed channel (rather than a live subscriber that would block + // forever, since no further end event is ever produced) and falls back to + // the service's retained-terminal-event cache. + close(p.EndEvent.Source) + p.outCancel() + p.cancel() + + p.logger.Info(). + Str("event_type", "process_end_readopted"). + Uint32("pid", p.pid). + Interface("process_result", end). + Msg("re-adopted process ended") +} diff --git a/packages/envd/internal/services/process/handler/readopt_test.go b/packages/envd/internal/services/process/handler/readopt_test.go new file mode 100644 index 0000000000..982bf31e21 --- /dev/null +++ b/packages/envd/internal/services/process/handler/readopt_test.go @@ -0,0 +1,66 @@ +package handler + +import ( + "testing" + "time" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestReapByPidfd_EmitsEndWhenPidfdOpenFails guards against orphaning a +// re-adopted process: if pidfd_open fails, the reaper must still emit a terminal +// event (via the blocking-wait4 fallback) so the process is finalized rather +// than left in the live map with clients blocked on an EndEvent that never fires. +func TestReapByPidfd_EmitsEndWhenPidfdOpenFails(t *testing.T) { + t.Parallel() + + logger := zerolog.Nop() + // A PID above pid_max can never exist, so pidfd_open fails deterministically. + h := Readopt(ReadoptArgs{Pid: 0x7FFFFFFF}, &logger) + + endCh, cancel := h.EndEvent.Fork() + defer cancel() + + h.BeginReaping() + + select { + case ev, ok := <-endCh: + require.True(t, ok, "EndEvent channel closed without a terminal event") + assert.NotNil(t, ev.End, "reaper must emit a terminal event on pidfd_open failure") + case <-time.After(5 * time.Second): + t.Fatal("re-adopted process orphaned: no EndEvent after pidfd_open failure") + } +} + +// TestBeginReaping_ArmsDeadlineForChainedUpgrade guards the timeout carried +// across chained live-upgrades: a re-adopted handler with a remaining timeout +// must report it via Deadline(), because Upgrade re-carries the timeout forward +// exclusively through Deadline(). Without it, a timed process that survives a +// second handover would run unbounded. +func TestBeginReaping_ArmsDeadlineForChainedUpgrade(t *testing.T) { + t.Parallel() + + logger := zerolog.Nop() + // A PID above pid_max can never exist, so the reaper's pidfd_open fails and + // its wait4 fallback returns immediately — no real child needed. + h := Readopt(ReadoptArgs{Pid: 0x7FFFFFFF, Timeout: time.Hour}, &logger) + + if _, ok := h.Deadline(); ok { + t.Fatal("deadline should be armed by BeginReaping, not Readopt") + } + + h.BeginReaping() + + // The deadline is armed asynchronously by the reaper goroutine once the + // workload thaws; with no Thawed channel it arms immediately, so wait for it. + require.Eventually(t, func() bool { + _, ok := h.Deadline() + + return ok + }, time.Second, 5*time.Millisecond, + "a re-adopted handler with a timeout must report a deadline once reaping begins") + d, _ := h.Deadline() + assert.WithinDuration(t, time.Now().Add(time.Hour), d, time.Minute) +} diff --git a/packages/envd/internal/services/process/pid_reuse_test.go b/packages/envd/internal/services/process/pid_reuse_test.go new file mode 100644 index 0000000000..5e267232e2 --- /dev/null +++ b/packages/envd/internal/services/process/pid_reuse_test.go @@ -0,0 +1,258 @@ +package process + +import ( + "context" + "os/exec" + "testing" + "time" + + "connectrpc.com/connect" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/e2b-dev/infra/packages/envd/internal/services/process/handler" + rpc "github.com/e2b-dev/infra/packages/envd/internal/services/spec/process" + "github.com/e2b-dev/infra/packages/envd/internal/services/spec/upgrade" +) + +// TestConnect_LivePidNotServedStaleExit guards against PID reuse: a live process +// that happens to share a PID with a still-cached terminal event of an earlier +// process must have its OWN exit served, not the stale cached one. +func TestConnect_LivePidNotServedStaleExit(t *testing.T) { + t.Parallel() + + client, svc, cleanup := newRetentionTestService(t) + defer cleanup() + + const pid = uint32(6161) + logger := zerolog.Nop() + live := handler.Readopt(handler.ReadoptArgs{Pid: pid}, &logger) + svc.processes.Store(pid, live) + // A stale exit left by a PRIOR process that used this PID. + svc.terminated.Store(pid, &retainedExit{ + pid: pid, + end: &rpc.ProcessEvent_EndEvent{Exited: true, ExitCode: 99, Status: "stale"}, + }) + + // Once Connect has subscribed to the live process, end it with code 7. + go func() { + for !live.EndEvent.HasSubscribers() { + time.Sleep(5 * time.Millisecond) + } + live.EndEvent.Source <- rpc.ProcessEvent_End{ + End: &rpc.ProcessEvent_EndEvent{Exited: true, ExitCode: 7, Status: "exited"}, + } + close(live.EndEvent.Source) + }() + + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + stream, err := client.Connect(ctx, connect.NewRequest(&rpc.ConnectRequest{ + Process: &rpc.ProcessSelector{Selector: &rpc.ProcessSelector_Pid{Pid: pid}}, + })) + require.NoError(t, err) + + events := drainConnect(t, stream) + require.NotEmpty(t, events) + last := events[len(events)-1].GetEnd() + require.NotNil(t, last, "last event should be End") + assert.Equal(t, int32(7), last.GetExitCode(), "must serve the live process's exit, not the stale cached 99") +} + +// TestTrackTermination_PidReuseKeepsSuccessor verifies the retention/eviction is +// identity-guarded: when a PID is reused before the previous process's exit is +// processed, the late exit must neither evict the successor from the live map nor +// cache a stale terminal event under the reused PID. +func TestTrackTermination_PidReuseKeepsSuccessor(t *testing.T) { + t.Parallel() + + _, svc, cleanup := newRetentionTestService(t) + defer cleanup() + + const pid = uint32(7272) + logger := zerolog.Nop() + a := handler.Readopt(handler.ReadoptArgs{Pid: pid}, &logger) + b := handler.Readopt(handler.ReadoptArgs{Pid: pid}, &logger) + + svc.processes.Store(pid, a) + svc.trackTermination(pid, a) // subscribes to A's EndEvent + + // The PID is reused by a new process before A's exit is processed. + svc.processes.Store(pid, b) + + // A exits (late). + for !a.EndEvent.HasSubscribers() { + time.Sleep(5 * time.Millisecond) + } + a.EndEvent.Source <- rpc.ProcessEvent_End{ + End: &rpc.ProcessEvent_EndEvent{Exited: true, ExitCode: 1, Status: "exited"}, + } + + assert.Eventually(t, func() bool { + cur, ok := svc.processes.Load(pid) + _, retained := svc.terminated.Load(pid) + + return ok && cur == b && !retained + }, 2*time.Second, 10*time.Millisecond, "reused PID must keep successor B and cache no stale exit") +} + +// TestRestoreTerminated_SkipsLivePid verifies the handover retention restore +// does not cache a terminal event for a PID that was re-adopted as live: a PID +// carried in both the process table and the retention cache (the retain-before- +// delete race caught in the freeze window) must resolve to the live process, not +// a stale exit that Connect could later serve. +func TestRestoreTerminated_SkipsLivePid(t *testing.T) { + t.Parallel() + + _, svc, cleanup := newRetentionTestService(t) + defer cleanup() + + logger := zerolog.Nop() + // pid 100 is re-adopted as live (Readopt does not start the reaper, so it + // simply sits in the live map); pid 200 is genuinely terminated. + svc.processes.Store(uint32(100), handler.Readopt(handler.ReadoptArgs{Pid: 100}, &logger)) + + svc.restoreTerminated([]*upgrade.HandoverExit{ + {Pid: 100, RemainingMs: 10_000}, // live -> must be skipped + {Pid: 200, RemainingMs: 10_000}, // gone -> restored + }) + + _, live := svc.terminated.Load(100) + assert.False(t, live, "a live re-adopted PID must not get a stale retained exit") + _, gone := svc.terminated.Load(200) + assert.True(t, gone, "a genuinely terminated PID should be restored") +} + +// TestReapByPidfd_RetainsExitBeforeClose guards the retain-before-close ordering +// on the re-adopt path: the reaper must retain the terminal event (via the +// Handler.OnExit hook) BEFORE it closes EndEvent, so a Connect that forks after +// the close always recovers the exit from the retention cache instead of racing +// an asynchronous retain and getting an error. It drives the real reaper with a +// controlled child, so moving the close ahead of the hook fails the test. +func TestReapByPidfd_RetainsExitBeforeClose(t *testing.T) { + t.Parallel() + + _, svc, cleanup := newRetentionTestService(t) + defer cleanup() + + // A real short-lived child so the reaper's wait4 harvests a genuine exit. + cmd := exec.CommandContext(t.Context(), "sh", "-c", "exit 5") + require.NoError(t, cmd.Start()) + pid := uint32(cmd.Process.Pid) //nolint:gosec // pid fits uint32 + // Deliberately not cmd.Wait(): the reaper's wait4 reaps the child. + + logger := zerolog.Nop() + h := handler.Readopt(handler.ReadoptArgs{Pid: pid}, &logger) + svc.processes.Store(pid, h) + h.OnExit = func(end *rpc.ProcessEvent_EndEvent) { + svc.finalizeTermination(pid, h, end) + } + + h.BeginReaping() + + // Drain a subscription until EndEvent closes — the instant the close is + // observable, the retain must already have happened. + drained := make(chan struct{}) + go func() { + ch, cancelFork := h.EndEvent.Fork() + defer cancelFork() + for { + if _, open := <-ch; !open { + break + } + } + close(drained) + }() + + select { + case <-drained: + case <-time.After(10 * time.Second): + t.Fatal("reaper did not close EndEvent") + } + + got, ok := svc.terminated.Load(pid) + require.True(t, ok, "exit must be retained by the time the EndEvent close is observable") + assert.Equal(t, int32(5), got.end.GetExitCode()) +} + +// TestRetain_StaleTimerDoesNotEvictNewer verifies the eviction timer is +// identity-guarded: replacing a retained exit for a PID with a newer one must +// not let the earlier entry's still-armed timer evict the newer entry early. +func TestRetain_StaleTimerDoesNotEvictNewer(t *testing.T) { + t.Parallel() + + _, svc, cleanup := newRetentionTestService(t) + defer cleanup() + + const pid = uint32(8383) + old := &retainedExit{ + pid: pid, + end: &rpc.ProcessEvent_EndEvent{Exited: true, ExitCode: 1}, + expiry: time.Now().Add(50 * time.Millisecond), + } + svc.retain(pid, old) + + // Replace with a newer entry (long TTL) before the old timer fires. + newer := &retainedExit{ + pid: pid, + end: &rpc.ProcessEvent_EndEvent{Exited: true, ExitCode: 2}, + expiry: time.Now().Add(10 * time.Second), + } + svc.retain(pid, newer) + + // Past when the OLD timer fires, the newer entry must still be present. + time.Sleep(120 * time.Millisecond) + + got, ok := svc.terminated.Load(pid) + require.True(t, ok, "newer retained entry must survive the stale timer") + assert.Equal(t, int32(2), got.end.GetExitCode()) +} + +// TestClearTerminatedForTag_DropsPredecessorExit verifies a tagged restart drops +// the predecessor's retained exit under that tag (Start clears the reused pid; +// this covers the tag), while a different tag's entry is left alone. +func TestClearTerminatedForTag_DropsPredecessorExit(t *testing.T) { + t.Parallel() + + _, svc, cleanup := newRetentionTestService(t) + defer cleanup() + + worker, other := "worker", "other" + future := time.Now().Add(time.Minute) + svc.retain(100, &retainedExit{pid: 100, tag: &worker, end: &rpc.ProcessEvent_EndEvent{Exited: true, ExitCode: 1}, expiry: future}) + svc.retain(200, &retainedExit{pid: 200, tag: &other, end: &rpc.ProcessEvent_EndEvent{Exited: true, ExitCode: 9}, expiry: future}) + + svc.clearTerminatedForTag(worker) + + _, ok := svc.terminated.Load(100) + assert.False(t, ok, "predecessor exit under the reused tag must be cleared") + _, ok = svc.terminated.Load(200) + assert.True(t, ok, "a different tag's retained exit must be left alone") +} + +// TestLookupByTag_AfterTaggedRestart_ServesSuccessorExit is the end-to-end guard +// for the finding: after a tagged process restarts within the retention TTL, a +// Connect-by-tag must be served the successor's exit code, never the +// predecessor's stale one (lookupTerminated returns the first tag match, so the +// predecessor entry has to be cleared when the tag is reused). +func TestLookupByTag_AfterTaggedRestart_ServesSuccessorExit(t *testing.T) { + t.Parallel() + + _, svc, cleanup := newRetentionTestService(t) + defer cleanup() + + tag := "worker" + future := time.Now().Add(time.Minute) + // Predecessor P1 (pid 100) exited, retained under tag with code 1. + svc.retain(100, &retainedExit{pid: 100, tag: &tag, end: &rpc.ProcessEvent_EndEvent{Exited: true, ExitCode: 1}, expiry: future}) + // A successor claims the same tag (as handleStart would), then exits at pid + // 200 with code 2. + svc.clearTerminatedForTag(tag) + svc.retain(200, &retainedExit{pid: 200, tag: &tag, end: &rpc.ProcessEvent_EndEvent{Exited: true, ExitCode: 2}, expiry: future}) + + got, ok := svc.lookupTerminated(&rpc.ProcessSelector{Selector: &rpc.ProcessSelector_Tag{Tag: tag}}) + require.True(t, ok) + assert.Equal(t, int32(2), got.end.GetExitCode(), + "a tag-Connect after a restart must get the successor's exit, never the predecessor's stale one") +} diff --git a/packages/envd/internal/services/process/service.go b/packages/envd/internal/services/process/service.go index 01b839ba9c..7efde5c697 100644 --- a/packages/envd/internal/services/process/service.go +++ b/packages/envd/internal/services/process/service.go @@ -2,6 +2,8 @@ package process import ( "fmt" + "sync" + "time" "connectrpc.com/connect" "github.com/go-chi/chi/v5" @@ -16,24 +18,141 @@ import ( "github.com/e2b-dev/infra/packages/envd/internal/utils" ) +// terminatedRetentionTTL is how long a process's terminal event is retained +// after it exits, so a late Connect can still recover the exit code. This +// covers the live-upgrade handover gap: if a process +// exits while no client is subscribed, its EndEvent is dropped by the +// multiplex, and a reconnecting client would otherwise never learn the exit +// code. The cache keeps only the terminal event (the exit code), which is +// enough for a reconnecting client to learn how the process ended; buffering +// the full missed output stream for replay is a possible later enhancement, not +// done here. +const terminatedRetentionTTL = 30 * time.Second + +// retainedExit is a process's terminal event, kept in the retention cache for +// terminatedRetentionTTL after the process exits. +type retainedExit struct { + pid uint32 + tag *string + end *rpc.ProcessEvent_EndEvent + expiry time.Time // when this entry is evicted; carried across a live-upgrade +} + type Service struct { - processes *utils.Map[uint32, *handler.Handler] - logger *zerolog.Logger - defaults *execcontext.Defaults - cgroupManager cgroups.Manager + processes *utils.Map[uint32, *handler.Handler] + terminated *utils.Map[uint32, *retainedExit] + logger *zerolog.Logger + defaults *execcontext.Defaults + // snapshotMu serializes a live-upgrade's process-table snapshot against + // concurrent Start registration. handleStart holds it RLocked across + // fork+Store so a child is never spawned-but-unregistered when Upgrade takes + // the write lock to snapshot; without it that child would survive the execve + // with no carried handler and be left unconnectable/unreaped. RWMutex so + // concurrent Starts don't serialize against each other, only against Upgrade. + snapshotMu sync.RWMutex + // cgroupManager places spawned processes into their cgroup; workloadFreezer + // freezes/thaws the workload during a live-upgrade handover. The freezer is + // shared with the HTTP API so both serialize on one lock. + cgroupManager cgroups.Manager + workloadFreezer *cgroups.WorkloadFreezer } -func newService(l *zerolog.Logger, defaults *execcontext.Defaults, cgroupManager cgroups.Manager) *Service { +func newService(l *zerolog.Logger, defaults *execcontext.Defaults, workloadFreezer *cgroups.WorkloadFreezer) *Service { return &Service{ - logger: l, - processes: utils.NewMap[uint32, *handler.Handler](), - defaults: defaults, - cgroupManager: cgroupManager, + logger: l, + processes: utils.NewMap[uint32, *handler.Handler](), + terminated: utils.NewMap[uint32, *retainedExit](), + defaults: defaults, + cgroupManager: workloadFreezer.Manager(), + workloadFreezer: workloadFreezer, + } +} + +// trackTermination subscribes to a handler's terminal event so the exit code +// survives even if no client is attached when the process exits. On exit it +// caches the EndEvent (retained for terminatedRetentionTTL) and removes the +// process from the live map. This is what lets a Connect issued after a +// live-upgrade handover gap still return the exit code. +// +// It must be called once, right after the handler is registered in +// s.processes (both the fresh-Start and the readopt paths). +func (s *Service) trackTermination(pid uint32, proc *handler.Handler) { + endCh, cancel := proc.EndEvent.Fork() + + go func() { + defer cancel() + + ev, ok := <-endCh + var end *rpc.ProcessEvent_EndEvent + if ok { + end = ev.End + } + s.finalizeTermination(pid, proc, end) + }() +} + +// finalizeTermination caches the terminal event for the retention window and +// then removes the process from the live map, guarded against PID reuse. It is +// shared by the asynchronous live-exit watcher (trackTermination) and the +// synchronous re-adopt reaper hook (Handler.OnExit). The exit is cached before +// the process is removed so a racing Connect that still finds it live can fall +// back to the cache instead of blocking on an EndEvent that will never re-fire. +func (s *Service) finalizeTermination(pid uint32, proc *handler.Handler, end *rpc.ProcessEvent_EndEvent) { + // If the PID was already reused by a newer process, this exit is not ours to + // cache or evict — leave the successor untouched. + if cur, live := s.processes.Load(pid); !live || cur != proc { + return + } + if end != nil { + s.retain(pid, &retainedExit{ + pid: pid, + tag: proc.Tag, + end: end, + expiry: time.Now().Add(terminatedRetentionTTL), + }) } + + s.processes.CompareAndDelete(pid, proc) +} + +// retain stores a terminal event in the retention cache and schedules its +// eviction at r.expiry. Shared by the live exit path (trackTermination) and the +// live-upgrade handover restore, so a process that exited shortly before the +// upgrade still has its exit code available on the new envd. +func (s *Service) retain(pid uint32, r *retainedExit) { + d := time.Until(r.expiry) + if d <= 0 { + return + } + + s.terminated.Store(pid, r) + time.AfterFunc(d, func() { + // Only evict this entry: a later retain for the same PID replaces it and + // arms its own timer, so this stale timer must not delete the newer one. + s.terminated.CompareAndDelete(pid, r) + }) +} + +// clearTerminatedForTag drops every retained terminal event carrying tag. Start +// already evicts the retained exit for a reused *pid*; this covers the *tag* +// dimension: a Connect can resolve by tag, and lookupTerminated returns the +// first tag match, so when a new process claims a tag (a tagged restart) any +// predecessor's retained exit under that tag must go too — otherwise a late +// Connect-by-tag could be served the predecessor's stale exit code. sync.Map +// permits Delete during Range; CompareAndDelete avoids clobbering a newer entry +// stored under the same pid between the read and the delete. +func (s *Service) clearTerminatedForTag(tag string) { + s.terminated.Range(func(pid uint32, r *retainedExit) bool { + if r.tag != nil && *r.tag == tag { + s.terminated.CompareAndDelete(pid, r) + } + + return true + }) } -func Handle(server *chi.Mux, l *zerolog.Logger, defaults *execcontext.Defaults, cgroupManager cgroups.Manager) *Service { - service := newService(l, defaults, cgroupManager) +func Handle(server *chi.Mux, l *zerolog.Logger, defaults *execcontext.Defaults, workloadFreezer *cgroups.WorkloadFreezer) *Service { + service := newService(l, defaults, workloadFreezer) interceptors := connect.WithInterceptors(logs.NewUnaryLogInterceptor(l)) diff --git a/packages/envd/internal/services/process/start.go b/packages/envd/internal/services/process/start.go index e9e9f70e19..bbb5e62a57 100644 --- a/packages/envd/internal/services/process/start.go +++ b/packages/envd/internal/services/process/start.go @@ -43,6 +43,15 @@ func (s *Service) handleStart(ctx context.Context, req *connect.Request[rpc.Star procCtx, cancelProc = context.WithTimeout(procCtx, requestTimeout) } + // Hold snapshotMu.RLock across the whole fork+register span so a live-upgrade + // snapshot (Upgrade takes the write lock) can never observe a child spawned + // but not yet in s.processes — which would leave it surviving the execve with + // no carried handler. It must be taken BEFORE handler.New: for a PTY, + // handler.New's pty.StartWithSize already forks the child, so acquiring it + // only before proc.Start would miss that fork. RLock so concurrent Starts + // don't serialize with each other, only against the (rare) upgrade. + s.snapshotMu.RLock() + proc, err := handler.New( //nolint:contextcheck // TODO: fix this later procCtx, u, @@ -53,6 +62,7 @@ func (s *Service) handleStart(ctx context.Context, req *connect.Request[rpc.Star cancelProc, ) if err != nil { + s.snapshotMu.RUnlock() // Ensure the process cancel is called to cleanup resources. cancelProc() @@ -162,10 +172,31 @@ func (s *Service) handleStart(ctx context.Context, req *connect.Request[rpc.Star pid, err := proc.Start(requestTimeout) if err != nil { + s.snapshotMu.RUnlock() + return connect.NewError(connect.CodeInvalidArgument, err) } + // Drop any retained exit left over from a previous process that used this + // PID, so a Connect to the new process can't be served the old exit code. + s.terminated.Delete(pid) + // A Connect can also resolve by tag (lookupTerminated returns the first tag + // match), so if this process reuses a tag, drop any predecessor's retained + // exit under that tag too — else a late Connect-by-tag could get a stale code. + if proc.Tag != nil { + s.clearTerminatedForTag(*proc.Tag) + } s.processes.Store(pid, proc) + s.snapshotMu.RUnlock() + + // Retain the terminal event synchronously when the process exits — Wait + // invokes this hook before it closes EndEvent, so a late Connect that falls + // back to the retention cache is guaranteed to find the exit (no race with an + // async retain). Set before the reaper goroutine below can run. Same + // mechanism the re-adopt path uses. + proc.OnExit = func(end *rpc.ProcessEvent_EndEvent) { + s.finalizeTermination(pid, proc, end) + } start <- rpc.ProcessEvent_Start{ Start: &rpc.ProcessEvent_StartEvent{ @@ -174,8 +205,11 @@ func (s *Service) handleStart(ctx context.Context, req *connect.Request[rpc.Star } go func() { - defer s.processes.Delete(pid) - + // Reap the process. Removal from s.processes is owned solely by + // finalizeTermination (invoked via proc.OnExit above), which retains the + // exit code BEFORE deleting and is identity-guarded against PID reuse. + // Deleting here too would race that retention away and lose the exit for a + // late Connect or the pre-upgrade handover. proc.Wait() }() diff --git a/packages/envd/internal/services/process/start_test.go b/packages/envd/internal/services/process/start_test.go index 29e98d522c..3df5fb0b26 100644 --- a/packages/envd/internal/services/process/start_test.go +++ b/packages/envd/internal/services/process/start_test.go @@ -40,7 +40,7 @@ func newTestService(t *testing.T, middleware ...func(http.Handler) http.Handler) EnvVars: utils.NewEnvVars(), User: u.Username, Workdir: &cwd, - }, cgroups.NewNoopManager()) + }, cgroups.NewWorkloadFreezer(cgroups.NewNoopManager())) mux := http.NewServeMux() path, handler := spec.NewProcessHandler(svc) diff --git a/packages/envd/internal/services/process/upgrade.go b/packages/envd/internal/services/process/upgrade.go new file mode 100644 index 0000000000..af798cee51 --- /dev/null +++ b/packages/envd/internal/services/process/upgrade.go @@ -0,0 +1,494 @@ +package process + +import ( + "context" + "fmt" + "os" + "syscall" + "time" + + "golang.org/x/sys/unix" + "google.golang.org/protobuf/proto" + + "github.com/e2b-dev/infra/packages/envd/internal/services/cgroups" + "github.com/e2b-dev/infra/packages/envd/internal/services/process/handler" + rpc "github.com/e2b-dev/infra/packages/envd/internal/services/spec/process" + "github.com/e2b-dev/infra/packages/envd/internal/services/spec/upgrade" + "github.com/e2b-dev/infra/packages/envd/pkg" +) + +// HandoverPath is the tmpfs blob the outgoing envd writes and the incoming one +// reads across a live self-upgrade. The format is a protobuf-encoded +// upgrade.HandoverState (spec/upgrade/handover.proto) — an additive schema, so +// an outgoing and incoming envd built at different versions stay compatible. +// /run is tmpfs, so it never touches the rootfs diff. A var (not a const) so +// tests can point it at a temp file. +var HandoverPath = "/run/e2b/envd-handover.pb" + +// handoverSchema is the version of the HandoverState layout this envd writes and +// the maximum it will read. A reader refuses a blob whose schema exceeds this +// (design §6.4: decode every schema <= N, abort on schema > N) rather than +// mis-read a newer-than-known layout — the outgoing envd then keeps running the +// old binary and the resume proceeds unupgraded. +const handoverSchema = 1 + +// fdBase is where carried fds are dup3'd to; high enough that the fresh runtime +// in the new image won't have grabbed these numbers during early startup. +const fdBase = 200 + +// DefaultUpgradeBinPath is the only filesystem path a live upgrade will write a +// delivered binary to and re-exec into. Constraining the target (rather than +// trusting a request header or marker file) keeps a malformed or forged upgrade +// request from writing to / executing an arbitrary path. The /upgrade endpoint +// is authenticated, but this is defense-in-depth around a same-PID exec. +const DefaultUpgradeBinPath = "/usr/bin/envd.next" + +// dupKeep dup3's oldfd onto target with CLOEXEC cleared so it survives execve. +// Returns (target, nil) on success and (-1, nil) when oldfd is absent. It errors +// if target is already open: dup3 silently closes the occupant, so a target +// collision must abort the upgrade rather than corrupt a live fd. The caller +// holds syscall.ForkLock, so no concurrent Go fd allocation can claim target +// between the F_GETFD check and the dup3. +func dupKeep(oldfd, target int) (int, error) { + if oldfd < 0 { + return -1, nil + } + if _, err := unix.FcntlInt(uintptr(target), unix.F_GETFD, 0); err == nil { + return -1, fmt.Errorf("handover fd target %d already in use", target) + } + if err := dup3(oldfd, target, 0); err != nil { + return -1, fmt.Errorf("dup3 %d->%d: %w", oldfd, target, err) + } + + return target, nil +} + +// Upgrade is the outgoing side of a live self-upgrade. It must be +// called with the workload frozen and envd's own spawners quiesced (caller's +// responsibility — see main's /upgrade handler). It serializes the process +// table, carries the I/O fds across execve, and re-execs newBin with the same +// PID. It does not return on success. +func (s *Service) Upgrade(newBin, fromVer string, watchers []*upgrade.HandoverWatcher, mounts []*upgrade.MountEntry, forwards []*upgrade.ForwardedPort) error { + // Only re-exec self (empty) or the fixed delivered-binary path — never an + // arbitrary caller-supplied path. Checked first, before any side effects. + if newBin != "" && newBin != DefaultUpgradeBinPath { + return fmt.Errorf("refusing upgrade to unexpected binary %q", newBin) + } + + st := &upgrade.HandoverState{ + Schema: handoverSchema, + FromVer: fromVer, + Watchers: watchers, + Mounts: mounts, + Forwards: forwards, + } + + // Serialize the process table and relocate each carried fd to its fixed + // target, holding syscall.ForkLock so no concurrent Go fd allocation can + // claim a target between dupKeep's free-check and its dup3. A collision or + // dup3 failure aborts the upgrade with the workload intact (handled below). + var ( + dupErr error + dupped []int + ) + i := 0 + // snapshotMu (write) blocks concurrent Start registration for the whole + // snapshot→execve window, so no child is spawned-but-unregistered (and thus + // left behind, unconnectable, across the swap). Taken before ForkLock and + // released on every exit path alongside it — consistent order, no deadlock + // (handleStart takes RLock then ForkLock via fork; Upgrade takes them in the + // same order). + s.snapshotMu.Lock() + syscall.ForkLock.Lock() + s.processes.Range(func(_ uint32, h *handler.Handler) bool { + stdout, stderr, stdin, tty := h.HandoverFds() + slot := fdBase + i*5 + + hp := &upgrade.HandoverProc{ + Pid: h.Pid(), + CgType: string(h.CgType()), + // Native nested message — no protojson round-trip. The live + // *rpc.ProcessConfig is carried directly and proto.Marshal encodes it. + Config: h.Config, + } + for _, m := range []struct { + old int + tgt int + dst *int32 + }{ + {stdout, slot + 0, &hp.StdoutFd}, + {stderr, slot + 1, &hp.StderrFd}, + {stdin, slot + 2, &hp.StdinFd}, + {tty, slot + 3, &hp.TtyFd}, + } { + fd, err := dupKeep(m.old, m.tgt) + if err != nil { + dupErr = err + + return false + } + *m.dst = int32(fd) + if fd >= 0 { + dupped = append(dupped, fd) + } + } + // Carry the remaining timeout so it is re-armed on the new envd. A + // deadline already in the past is clamped to 1ms (kill ASAP). + if d, ok := h.Deadline(); ok { + if rem := time.Until(d).Milliseconds(); rem > 0 { + hp.TimeoutMs = rem + } else { + hp.TimeoutMs = 1 + } + } + if h.Tag != nil { + hp.Tag = *h.Tag + hp.HasTag = true + } + st.Processes = append(st.Processes, hp) + i++ + + return true + }) + if dupErr != nil { + // Close the fds already relocated so a retry sees a clean target window, + // then keep running the old binary. + for _, fd := range dupped { + _ = unix.Close(fd) + } + syscall.ForkLock.Unlock() + s.snapshotMu.Unlock() + + return fmt.Errorf("relocate handover fds: %w", dupErr) + } + + // Keep ForkLock held from the CLOEXEC-clearing relocation above all the way + // through the execve below. Dropping it here would leave the carried fds + // (now CLOEXEC-cleared) exposed to a concurrent os/exec fork — the port + // scanner's socat, an in-flight Start — which would inherit them. The + // intervening marshal / write / os.Executable never fork, so holding it is + // safe. On any error return before the execve (which never returns on + // success — it replaces the image), close the relocated dups so they don't + // leak into the still-running old envd and its future children, and release + // the lock. Mirrors the dupErr cleanup above. + defer func() { + for _, fd := range dupped { + _ = unix.Close(fd) + } + syscall.ForkLock.Unlock() + s.snapshotMu.Unlock() + }() + + // Carry the retention cache so a process that exited shortly before the + // upgrade keeps its exit code retrievable on the new envd. + s.terminated.Range(func(pid uint32, r *retainedExit) bool { + rem := time.Until(r.expiry).Milliseconds() + if rem <= 0 { + return true + } + + he := &upgrade.HandoverExit{Pid: pid, End: r.end, RemainingMs: rem} + if r.tag != nil { + he.Tag = *r.tag + he.HasTag = true + } + st.Terminated = append(st.Terminated, he) + + return true + }) + + blob, err := proto.Marshal(st) + if err != nil { + return fmt.Errorf("marshal handover: %w", err) + } + if err := os.WriteFile(HandoverPath, blob, 0o600); err != nil { + return fmt.Errorf("write handover: %w", err) + } + + exe, err := os.Executable() + if err != nil { + return fmt.Errorf("resolve executable: %w", err) + } + if newBin == "" { + newBin = exe + } + + // argv: keep original flags, append --resume-handover. + argv := append([]string{newBin}, os.Args[1:]...) + argv = append(argv, "--resume-handover") + + // Carry the dup'd fds across execve. ForkLock is still held from the + // relocation above (released by the deferred cleanup), so no concurrent fork + // can inherit the CLOEXEC-cleared fds. Exec only returns on failure + // (corrupt/missing staged binary); the deferred cleanup then closes the + // relocated dups and releases the lock while the old envd keeps running. + err = syscall.Exec(newBin, argv, os.Environ()) + + return fmt.Errorf("execve %s: %w", newBin, err) +} + +// ResumeFromHandover is the incoming side: read the blob, re-adopt +// each process from its inherited fds, register it, then thaw the workload. +// No-op if no handover file is present. +// It returns the opaque filesystem-watcher blob (if any) so the caller can hand +// it to the filesystem service to re-arm watches. +// HandoverResult summarizes a completed incoming handover. It is surfaced to the +// orchestrator via the /init X-Envd-Handover header so the envd-side outcome +// (which envd otherwise only logs) is observable fleet-wide. +type HandoverResult struct { + // Every item is total-carried + failed-subset (ok = total - failed). + Procs int + ProcsFailed int + Retained int + RetainedFailed int + Watchers int + WatchersFailed int + // Mounts and Forwards are decoded from the blob and handed back to their + // owners (the API service's mount ledger and the port forwarder), which are + // constructed after ResumeFromHandover runs — so unlike watchers they are + // returned rather than applied via a callback. + Mounts []*upgrade.MountEntry + Forwards []*upgrade.ForwardedPort +} + +func (s *Service) ResumeFromHandover(reArmWatchers func([]*upgrade.HandoverWatcher) (rearmed, failed int)) (HandoverResult, error) { + // Thaw the workload on every FAILURE path — a bad blob, partial re-adopt, or + // panic must never leave the sandbox frozen (a degraded-but-running workload + // beats a hung one). On SUCCESS the workload is deliberately left frozen: the + // orchestrator's post-upgrade /init thaws it (deferred unfreeze in PostInit) + // only after it has re-established the access token. This closes the window + // in which a re-adopted — and possibly hostile — guest process could run + // before /init restores auth and reach the unauthenticated /upgrade endpoint + // (which execs request-body bytes as root). + keepFrozen := false + defer func() { + if !keepFrozen { + s.UnfreezeWorkload() + } + }() + + blob, err := os.ReadFile(HandoverPath) + if os.IsNotExist(err) { + return HandoverResult{}, nil + } + if err != nil { + return HandoverResult{}, fmt.Errorf("read handover: %w", err) + } + + st := &upgrade.HandoverState{} + if err := proto.Unmarshal(blob, st); err != nil { + return HandoverResult{}, fmt.Errorf("unmarshal handover: %w", err) + } + + // Schema gate (design §6.4): refuse a blob written by a newer envd whose + // layout this binary does not understand, rather than mis-read it. The + // outgoing envd is already gone (this runs post-execve), so we can't fall + // back to it — but the orchestrator's post-upgrade /init readiness wait will + // fail and the resume is reported unupgraded rather than corrupt. + if st.GetSchema() > handoverSchema { + return HandoverResult{}, fmt.Errorf("handover schema %d exceeds max supported %d", st.GetSchema(), handoverSchema) + } + + // journald-visible proof of the running image after the swap (from_ver is + // the outgoing version; pkg.Version is what this new image is). + fmt.Fprintf(os.Stderr, "envd: resumed as v%s after handover (from v%s, %d procs)\n", + pkg.Version, st.GetFromVer(), len(st.GetProcesses())) + + // fileOrNil wraps a carried fd (inherited at its fixed fdBase slot with + // CLOEXEC cleared, from the outgoing envd's execve) as an *os.File, but first + // RELOCATES it off the fdBase range onto a fresh fd with CLOEXEC set. This + // frees the slot for the NEXT upgrade's relocation: a chained upgrade dup3's + // the carried fds back onto the same fdBase slots, and dupKeep aborts if a + // target is already in use — so leaving them parked at fdBase would break the + // second swap. Setting CLOEXEC also stops the fd from leaking into processes + // this envd spawns before the next upgrade (the next dupKeep re-clears it). + fileOrNil := func(fd int, name string) *os.File { + if fd < 0 { + return nil + } + + fresh, err := unix.Dup(fd) + if err != nil { + // Fall back to the slot as-is: the current re-adoption still works; + // only a later chained upgrade might collide. + s.logger.Warn().Err(err).Int("fd", fd).Msg("handover: relocate carried fd off fdBase failed") + + return os.NewFile(uintptr(fd), name) + } + unix.CloseOnExec(fresh) + _ = unix.Close(fd) + + return os.NewFile(uintptr(fresh), name) + } + + // Captured before re-adoption (while the workload is still frozen): closed + // when the post-upgrade /init — or the fallback — next thaws the workload, so + // each re-adopted process's carried kill-timer only starts counting once the + // process can actually run again. + thawed := s.workloadFreezer.Thawed() + + procsFailed := 0 + for _, hp := range st.GetProcesses() { + // Native nested message — carried directly by proto, no protojson + // round-trip. A malformed config would have failed the top-level + // proto.Unmarshal above (schema-gated), so here it is either the real + // config or absent; default an absent one so Readopt always gets non-nil. + cfg := hp.GetConfig() + if cfg == nil { + cfg = &rpc.ProcessConfig{} + } + + var tag *string + if hp.GetHasTag() { + t := hp.GetTag() + tag = &t + } + + var timeout time.Duration + if hp.GetTimeoutMs() > 0 { + timeout = time.Duration(hp.GetTimeoutMs()) * time.Millisecond + } + + h := handler.Readopt(handler.ReadoptArgs{ + Pid: hp.GetPid(), + Tag: tag, + Config: cfg, + CgType: cgroups.ProcessType(hp.GetCgType()), + Stdout: fileOrNil(int(hp.GetStdoutFd()), fmt.Sprintf("p%d-stdout", hp.GetPid())), + Stderr: fileOrNil(int(hp.GetStderrFd()), fmt.Sprintf("p%d-stderr", hp.GetPid())), + Stdin: fileOrNil(int(hp.GetStdinFd()), fmt.Sprintf("p%d-stdin", hp.GetPid())), + Tty: fileOrNil(int(hp.GetTtyFd()), fmt.Sprintf("p%d-tty", hp.GetPid())), + Timeout: timeout, + Thawed: thawed, + }, s.logger) + + pid := hp.GetPid() + s.processes.Store(pid, h) + // Retain the terminal event synchronously on exit, before the reaper + // closes EndEvent, so a Connect arriving in the handover gap always + // recovers the exit code even if it forks after the close. Set before + // BeginReaping: a process whose timeout expired during the freeze can + // exit the instant it is unfrozen, so the reaper — which invokes the + // hook — must never start before the hook is in place. + h.OnExit = func(end *rpc.ProcessEvent_EndEvent) { + s.finalizeTermination(pid, h, end) + } + h.BeginReaping() + + s.logger.Info(). + Str("event_type", "process_readopted"). + Uint32("pid", hp.GetPid()). + Msg("re-adopted process after envd self-upgrade") + } + + // Restore the retention cache: terminal events of processes that exited + // shortly before the upgrade, so a Connect can still recover their exit code. + retainedFailed := s.restoreTerminated(st.GetTerminated()) + + _ = os.Remove(HandoverPath) + + // Re-arm filesystem watchers while the workload is STILL frozen (it stays + // frozen past this return until the post-upgrade /init thaws it), so no + // filesystem event is lost in the gap between the thaw and the re-arm. + watchersRearmed, watchersFailed := 0, 0 + if reArmWatchers != nil { + watchersRearmed, watchersFailed = reArmWatchers(st.GetWatchers()) + } + + // Loki-queryable summary of what the handover carried + how it fared + // (rollout observability; the counts also ride to the orchestrator via the + // /init X-Envd-Handover header — see HandoverResult). + s.logger.Info(). + Str("event_type", "handover_resumed"). + Str("from_ver", st.GetFromVer()). + Int("procs", len(st.GetProcesses())). + Int("procs_failed", procsFailed). + Int("retained", len(st.GetTerminated())). + Int("retained_failed", retainedFailed). + Int("watchers", watchersRearmed+watchersFailed). + Int("watchers_failed", watchersFailed). + Msg("re-adopted workload after envd self-upgrade") + + // Handover succeeded: keep the workload frozen (see the deferred thaw above); + // the orchestrator's post-upgrade /init thaws it once auth is restored. + keepFrozen = true + + return HandoverResult{ + Procs: len(st.GetProcesses()), + ProcsFailed: procsFailed, + Retained: len(st.GetTerminated()), + RetainedFailed: retainedFailed, + Watchers: watchersRearmed + watchersFailed, + WatchersFailed: watchersFailed, + // Handed back to the API service (mount ledger) and port forwarder, which + // are constructed after this returns. + Mounts: st.GetMounts(), + Forwards: st.GetForwards(), + }, nil +} + +// restoreTerminated re-populates the terminal-event retention cache from the +// handover. It skips any PID that was already re-adopted as a *live* process +// above: a PID can't be both live and terminated, and caching a stale exit +// under a live PID could let a Connect (or a later reuse of that PID) be served +// the wrong exit code. Mirrors handleStart's clear-on-register. +func (s *Service) restoreTerminated(entries []*upgrade.HandoverExit) (failed int) { + for _, he := range entries { + if _, live := s.processes.Load(he.GetPid()); live { + continue + } + + // Native nested message — carried directly by proto (schema-gated at + // decode). Default an absent event so retain always gets a non-nil End. + end := he.GetEnd() + if end == nil { + end = &rpc.ProcessEvent_EndEvent{} + } + + var tag *string + if he.GetHasTag() { + t := he.GetTag() + tag = &t + } + + s.retain(he.GetPid(), &retainedExit{ + pid: he.GetPid(), + tag: tag, + end: end, + expiry: time.Now().Add(time.Duration(he.GetRemainingMs()) * time.Millisecond), + }) + } + + return failed +} + +// FreezeWorkload freezes the user/pty cgroups ahead of an Upgrade, serialized +// (via the shared freezer) against the HTTP API's freeze/unfreeze paths. +func (s *Service) FreezeWorkload() { + if err := s.workloadFreezer.Freeze(context.Background()); err != nil { + s.logger.Warn().Err(err).Msg("handover: freeze failed") + } +} + +// FreezeWorkloadHold freezes the workload and keeps the shared freeze lock held, +// returning a release func, so the freeze stays uninterruptible across the +// handover: a concurrent /init or /unfreeze thaw blocks until release. The caller +// MUST release on any path that does not execve; a successful execve drops the +// lock with the process image. +func (s *Service) FreezeWorkloadHold() (release func(), err error) { + release, err = s.workloadFreezer.FreezeHold(context.Background()) + if err != nil { + s.logger.Warn().Err(err).Msg("handover: freeze failed") + } + + return release, err +} + +// UnfreezeWorkload thaws the user/pty cgroups. Idempotent (thawing a non-frozen +// cgroup is a no-op), so it is safe to call on every upgrade outcome — success, +// failure, or panic — guaranteeing a failed swap never leaves the workload frozen. +func (s *Service) UnfreezeWorkload() { + if err := s.workloadFreezer.Unfreeze(context.Background()); err != nil { + s.logger.Warn().Err(err).Msg("handover: unfreeze failed") + } +} diff --git a/packages/envd/internal/services/process/upgrade_test.go b/packages/envd/internal/services/process/upgrade_test.go new file mode 100644 index 0000000000..33c8d79ed1 --- /dev/null +++ b/packages/envd/internal/services/process/upgrade_test.go @@ -0,0 +1,216 @@ +package process + +import ( + "os" + "path/filepath" + "sync/atomic" + "testing" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + "github.com/e2b-dev/infra/packages/envd/internal/execcontext" + "github.com/e2b-dev/infra/packages/envd/internal/services/cgroups" + fs "github.com/e2b-dev/infra/packages/envd/internal/services/spec/filesystem" + rpc "github.com/e2b-dev/infra/packages/envd/internal/services/spec/process" + "github.com/e2b-dev/infra/packages/envd/internal/services/spec/upgrade" + "github.com/e2b-dev/infra/packages/envd/internal/utils" +) + +// mustMarshalHandover proto-encodes a HandoverState for the on-disk blob tests. +func mustMarshalHandover(t *testing.T, st *upgrade.HandoverState) []byte { + t.Helper() + b, err := proto.Marshal(st) + require.NoError(t, err) + + return b +} + +// spyCgroupManager records Unfreeze calls so we can assert the workload is +// always thawed. Everything else is a no-op. +type spyCgroupManager struct { + unfreezes atomic.Int64 +} + +func (m *spyCgroupManager) GetFileDescriptor(cgroups.ProcessType) (int, bool) { return -1, false } +func (m *spyCgroupManager) Freeze(cgroups.ProcessType) error { return nil } + +func (m *spyCgroupManager) Unfreeze(cgroups.ProcessType) error { + m.unfreezes.Add(1) + + return nil +} +func (m *spyCgroupManager) Close() error { return nil } + +func newHandoverTestService(t *testing.T, spy *spyCgroupManager) *Service { + t.Helper() + logger := zerolog.Nop() + cwd := t.TempDir() + + return newService(&logger, &execcontext.Defaults{ + EnvVars: utils.NewEnvVars(), + Workdir: &cwd, + }, cgroups.NewWorkloadFreezer(spy)) +} + +// TestUpgrade_RejectsUnexpectedBinary verifies the exec target is constrained: +// a caller-supplied path other than the fixed DefaultUpgradeBinPath (or empty +// self-exec) is refused before any side effects, so a malformed/forged upgrade +// request can't turn the same-PID exec into arbitrary code execution. +func TestUpgrade_RejectsUnexpectedBinary(t *testing.T) { + t.Parallel() + + s := newHandoverTestService(t, &spyCgroupManager{}) + + err := s.Upgrade("/tmp/attacker-controlled", "0.6.11", nil, nil, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "refusing upgrade") +} + +// TestResumeFromHandoverAlwaysUnfreezes is the safety guarantee: however the +// handover resume ends — a malformed blob (error) or no blob at all — the +// workload is thawed. A failed upgrade must never leave the sandbox frozen. +// +//nolint:paralleltest // mutates the package-global HandoverPath; must run serially +func TestResumeFromHandoverAlwaysUnfreezes(t *testing.T) { + orig := HandoverPath + t.Cleanup(func() { HandoverPath = orig }) + + // Malformed blob -> ResumeFromHandover errors, but must still thaw. A lone + // varint tag with no payload is invalid protobuf wire format. + spy := &spyCgroupManager{} + s := newHandoverTestService(t, spy) + HandoverPath = filepath.Join(t.TempDir(), "handover.pb") + require.NoError(t, os.WriteFile(HandoverPath, []byte{0x08}, 0o600)) + + _, err := s.ResumeFromHandover(nil) + require.Error(t, err, "a malformed blob should surface an error") + assert.Positive(t, spy.unfreezes.Load(), "workload must be thawed even on a bad blob") + + // No blob -> no-op return, but the deferred thaw must still run. + spy2 := &spyCgroupManager{} + s2 := newHandoverTestService(t, spy2) + HandoverPath = filepath.Join(t.TempDir(), "absent.pb") + + _, err = s2.ResumeFromHandover(nil) + require.NoError(t, err) + assert.Positive(t, spy2.unfreezes.Load(), "the deferred thaw must run on the no-blob path too") +} + +// TestResumeFromHandover_ReArmsWatchersBeforeThaw verifies the incoming handover +// re-arms filesystem watchers (via the callback) while the workload is STILL +// frozen — before the deferred thaw — so no filesystem event can be missed in +// the gap between the thaw and the re-arm. +// +//nolint:paralleltest // mutates the package-global HandoverPath; must run serially +func TestResumeFromHandover_ReArmsWatchersBeforeThaw(t *testing.T) { + orig := HandoverPath + t.Cleanup(func() { HandoverPath = orig }) + + spy := &spyCgroupManager{} + s := newHandoverTestService(t, spy) + HandoverPath = filepath.Join(t.TempDir(), "handover.pb") + // A valid proto blob with no processes: drives the success path through to + // the watcher-rearm callback. + require.NoError(t, os.WriteFile(HandoverPath, + mustMarshalHandover(t, &upgrade.HandoverState{Schema: handoverSchema, FromVer: "0.6.12"}), 0o600)) + + unfreezesAtCallback := int64(-1) + _, err := s.ResumeFromHandover(func([]*upgrade.HandoverWatcher) (int, int) { + unfreezesAtCallback = spy.unfreezes.Load() + + return 0, 0 + }) + require.NoError(t, err) + + assert.Equal(t, int64(0), unfreezesAtCallback, "watchers must be re-armed before any thaw") + assert.Equal(t, int64(0), spy.unfreezes.Load(), "on success the workload stays frozen for the post-upgrade /init to thaw (so no re-adopted process runs before /init restores auth)") +} + +// TestHandoverState_ProtoRoundTrip is the core wire-format contract: a +// HandoverState carrying native nested messages (a process config, a retained +// terminal event, and a watcher with buffered filesystem events) survives a +// proto.Marshal -> proto.Unmarshal round-trip unchanged. This is what replaced +// the old JSON+protojson encoding. +func TestHandoverState_ProtoRoundTrip(t *testing.T) { + t.Parallel() + + orig := &upgrade.HandoverState{ + Schema: handoverSchema, + FromVer: "0.6.12", + Processes: []*upgrade.HandoverProc{{ + Pid: 1214, + Tag: "worker", + HasTag: true, + CgType: "user", + Config: &rpc.ProcessConfig{Cmd: "/bin/sh", Cwd: new("/root"), Args: []string{"-c", "echo hi"}}, + StdoutFd: 200, + StderrFd: 201, + StdinFd: 202, + TtyFd: -1, + TimeoutMs: 5000, + }}, + Terminated: []*upgrade.HandoverExit{{ + Pid: 99, + End: &rpc.ProcessEvent_EndEvent{ExitCode: 42, Status: "exited"}, + RemainingMs: 1000, + }}, + Watchers: []*upgrade.HandoverWatcher{{ + Id: "w1", + Path: "/data", + Recursive: true, + IncludeEntryInfo: true, + PendingEvents: []*fs.FilesystemEvent{{Name: "a.txt", Type: fs.EventType_EVENT_TYPE_CREATE}}, + }}, + Mounts: []*upgrade.MountEntry{{Path: "/mnt/vol", LifecycleId: "lc-1"}}, + Forwards: []*upgrade.ForwardedPort{{ + Key: "100-8080", Port: 8080, ListenerPid: 100, Family: 4, SocatPid: 555, + }}, + } + + blob, err := proto.Marshal(orig) + require.NoError(t, err) + + got := &upgrade.HandoverState{} + require.NoError(t, proto.Unmarshal(blob, got)) + + assert.True(t, proto.Equal(orig, got), "round-tripped HandoverState must equal the original") + // Spot-check the nested messages explicitly so a failure points at the field. + require.Len(t, got.GetProcesses(), 1) + assert.Equal(t, "/bin/sh", got.GetProcesses()[0].GetConfig().GetCmd()) + assert.Equal(t, int32(-1), got.GetProcesses()[0].GetTtyFd()) + require.Len(t, got.GetTerminated(), 1) + assert.Equal(t, int32(42), got.GetTerminated()[0].GetEnd().GetExitCode()) + require.Len(t, got.GetWatchers(), 1) + require.Len(t, got.GetWatchers()[0].GetPendingEvents(), 1) + assert.Equal(t, "a.txt", got.GetWatchers()[0].GetPendingEvents()[0].GetName()) + require.Len(t, got.GetMounts(), 1) + assert.Equal(t, "/mnt/vol", got.GetMounts()[0].GetPath()) + assert.Equal(t, "lc-1", got.GetMounts()[0].GetLifecycleId()) + require.Len(t, got.GetForwards(), 1) + assert.Equal(t, "100-8080", got.GetForwards()[0].GetKey()) + assert.Equal(t, int32(555), got.GetForwards()[0].GetSocatPid()) +} + +// TestResumeFromHandover_RejectsNewerSchema is the §6.4 version gate: a blob +// written by a hypothetical newer envd (schema > this binary's max) is refused +// rather than mis-read, and the workload is still thawed. +// +//nolint:paralleltest // mutates the package-global HandoverPath; must run serially +func TestResumeFromHandover_RejectsNewerSchema(t *testing.T) { + orig := HandoverPath + t.Cleanup(func() { HandoverPath = orig }) + + spy := &spyCgroupManager{} + s := newHandoverTestService(t, spy) + HandoverPath = filepath.Join(t.TempDir(), "handover.pb") + require.NoError(t, os.WriteFile(HandoverPath, + mustMarshalHandover(t, &upgrade.HandoverState{Schema: handoverSchema + 1, FromVer: "9.9.9"}), 0o600)) + + _, err := s.ResumeFromHandover(nil) + require.Error(t, err, "a newer-than-known schema must be refused") + assert.Contains(t, err.Error(), "schema") + assert.Positive(t, spy.unfreezes.Load(), "workload must be thawed even when the schema is rejected") +} diff --git a/packages/envd/internal/services/spec/upgrade/handover.pb.go b/packages/envd/internal/services/spec/upgrade/handover.pb.go new file mode 100644 index 0000000000..15ddb02381 --- /dev/null +++ b/packages/envd/internal/services/spec/upgrade/handover.pb.go @@ -0,0 +1,798 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc (unknown) +// source: upgrade/handover.proto + +package upgrade + +import ( + filesystem "github.com/e2b-dev/infra/packages/envd/internal/services/spec/filesystem" + process "github.com/e2b-dev/infra/packages/envd/internal/services/spec/process" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// HandoverState is the entire cross-version compatibility contract for an envd +// live-upgrade (design §5.2, §6.4): the outgoing envd serializes it to a tmpfs +// file, the incoming (new-binary) envd decodes it after execve to reconstruct +// its world. It is an additive protobuf — an older envd omits fields a newer one +// added, and the reader treats absent fields as defaults. +// +// Schema is the versioning discipline §6.4 requires: it is bumped on any change, +// and a reader MUST refuse a blob whose schema exceeds the maximum it understands +// (decode every schema <= N, abort on schema > N) rather than mis-read a +// newer-than-known layout — the outgoing envd keeps running the old binary. +type HandoverState struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Schema uint32 `protobuf:"varint,1,opt,name=schema,proto3" json:"schema,omitempty"` + FromVer string `protobuf:"bytes,2,opt,name=from_ver,json=fromVer,proto3" json:"from_ver,omitempty"` + Processes []*HandoverProc `protobuf:"bytes,3,rep,name=processes,proto3" json:"processes,omitempty"` + // terminated carries the retention cache (recently-exited terminal events not + // yet drained). + Terminated []*HandoverExit `protobuf:"bytes,4,rep,name=terminated,proto3" json:"terminated,omitempty"` + // watchers is the filesystem service's active-watcher set. The filesystem + // service owns building/consuming these; the process handover only carries + // them. + Watchers []*HandoverWatcher `protobuf:"bytes,5,rep,name=watchers,proto3" json:"watchers,omitempty"` + // mounts is the API service's NFS mount ledger (path -> lifecycle), carried so + // the new envd's post-upgrade /init skips re-mounting a still-live mount. + Mounts []*MountEntry `protobuf:"bytes,6,rep,name=mounts,proto3" json:"mounts,omitempty"` + // forwards is the port forwarder's active socat set, carried so the new envd + // re-adopts the running socat children instead of spawning duplicates. + Forwards []*ForwardedPort `protobuf:"bytes,7,rep,name=forwards,proto3" json:"forwards,omitempty"` +} + +func (x *HandoverState) Reset() { + *x = HandoverState{} + if protoimpl.UnsafeEnabled { + mi := &file_upgrade_handover_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *HandoverState) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HandoverState) ProtoMessage() {} + +func (x *HandoverState) ProtoReflect() protoreflect.Message { + mi := &file_upgrade_handover_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HandoverState.ProtoReflect.Descriptor instead. +func (*HandoverState) Descriptor() ([]byte, []int) { + return file_upgrade_handover_proto_rawDescGZIP(), []int{0} +} + +func (x *HandoverState) GetSchema() uint32 { + if x != nil { + return x.Schema + } + return 0 +} + +func (x *HandoverState) GetFromVer() string { + if x != nil { + return x.FromVer + } + return "" +} + +func (x *HandoverState) GetProcesses() []*HandoverProc { + if x != nil { + return x.Processes + } + return nil +} + +func (x *HandoverState) GetTerminated() []*HandoverExit { + if x != nil { + return x.Terminated + } + return nil +} + +func (x *HandoverState) GetWatchers() []*HandoverWatcher { + if x != nil { + return x.Watchers + } + return nil +} + +func (x *HandoverState) GetMounts() []*MountEntry { + if x != nil { + return x.Mounts + } + return nil +} + +func (x *HandoverState) GetForwards() []*ForwardedPort { + if x != nil { + return x.Forwards + } + return nil +} + +// MountEntry is one NFS volume mount the outgoing envd had set up, keyed by the +// lifecycle it was mounted for. The kernel mount survives execve; carrying the +// ledger lets the new envd recognize a matching-lifecycle mount and leave it +// in place rather than force-unmounting and remounting it (ESTALE risk). +type MountEntry struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` + LifecycleId string `protobuf:"bytes,2,opt,name=lifecycle_id,json=lifecycleId,proto3" json:"lifecycle_id,omitempty"` +} + +func (x *MountEntry) Reset() { + *x = MountEntry{} + if protoimpl.UnsafeEnabled { + mi := &file_upgrade_handover_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MountEntry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MountEntry) ProtoMessage() {} + +func (x *MountEntry) ProtoReflect() protoreflect.Message { + mi := &file_upgrade_handover_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MountEntry.ProtoReflect.Descriptor instead. +func (*MountEntry) Descriptor() ([]byte, []int) { + return file_upgrade_handover_proto_rawDescGZIP(), []int{1} +} + +func (x *MountEntry) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *MountEntry) GetLifecycleId() string { + if x != nil { + return x.LifecycleId + } + return "" +} + +// ForwardedPort is one active port-forward. The socat child survives execve; +// carrying its pid lets the new forwarder re-adopt it (suppressing a duplicate +// socat and reaping it when the port closes) instead of respawning. +type ForwardedPort struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` // the forwarder map key: "-" + Port uint32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"` + ListenerPid int32 `protobuf:"varint,3,opt,name=listener_pid,json=listenerPid,proto3" json:"listener_pid,omitempty"` // pid of the guest process listening on the port + Family uint32 `protobuf:"varint,4,opt,name=family,proto3" json:"family,omitempty"` // IP version (4 or 6) + SocatPid int32 `protobuf:"varint,5,opt,name=socat_pid,json=socatPid,proto3" json:"socat_pid,omitempty"` // pid of the running socat child to re-adopt +} + +func (x *ForwardedPort) Reset() { + *x = ForwardedPort{} + if protoimpl.UnsafeEnabled { + mi := &file_upgrade_handover_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ForwardedPort) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ForwardedPort) ProtoMessage() {} + +func (x *ForwardedPort) ProtoReflect() protoreflect.Message { + mi := &file_upgrade_handover_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ForwardedPort.ProtoReflect.Descriptor instead. +func (*ForwardedPort) Descriptor() ([]byte, []int) { + return file_upgrade_handover_proto_rawDescGZIP(), []int{2} +} + +func (x *ForwardedPort) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *ForwardedPort) GetPort() uint32 { + if x != nil { + return x.Port + } + return 0 +} + +func (x *ForwardedPort) GetListenerPid() int32 { + if x != nil { + return x.ListenerPid + } + return 0 +} + +func (x *ForwardedPort) GetFamily() uint32 { + if x != nil { + return x.Family + } + return 0 +} + +func (x *ForwardedPort) GetSocatPid() int32 { + if x != nil { + return x.SocatPid + } + return 0 +} + +// HandoverProc is a live child re-adopted across the same-PID execve. The fd +// fields carry the *numbers* (kept valid across execve, CLOEXEC cleared); the +// kernel objects ride the fd table. +type HandoverProc struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Pid uint32 `protobuf:"varint,1,opt,name=pid,proto3" json:"pid,omitempty"` // K-anchor: live child, survives same-PID execve + Tag string `protobuf:"bytes,2,opt,name=tag,proto3" json:"tag,omitempty"` // K-metadata: not in /proc, unreconstructable if lost + HasTag bool `protobuf:"varint,3,opt,name=has_tag,json=hasTag,proto3" json:"has_tag,omitempty"` // distinguishes an empty tag from no tag + CgType string `protobuf:"bytes,4,opt,name=cg_type,json=cgType,proto3" json:"cg_type,omitempty"` // user | pty + Config *process.ProcessConfig `protobuf:"bytes,5,opt,name=config,proto3" json:"config,omitempty"` + StdoutFd int32 `protobuf:"varint,6,opt,name=stdout_fd,json=stdoutFd,proto3" json:"stdout_fd,omitempty"` + StderrFd int32 `protobuf:"varint,7,opt,name=stderr_fd,json=stderrFd,proto3" json:"stderr_fd,omitempty"` + StdinFd int32 `protobuf:"varint,8,opt,name=stdin_fd,json=stdinFd,proto3" json:"stdin_fd,omitempty"` + TtyFd int32 `protobuf:"varint,9,opt,name=tty_fd,json=ttyFd,proto3" json:"tty_fd,omitempty"` + TimeoutMs int64 `protobuf:"varint,10,opt,name=timeout_ms,json=timeoutMs,proto3" json:"timeout_ms,omitempty"` // per-process kill deadline remaining, re-armed on readopt +} + +func (x *HandoverProc) Reset() { + *x = HandoverProc{} + if protoimpl.UnsafeEnabled { + mi := &file_upgrade_handover_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *HandoverProc) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HandoverProc) ProtoMessage() {} + +func (x *HandoverProc) ProtoReflect() protoreflect.Message { + mi := &file_upgrade_handover_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HandoverProc.ProtoReflect.Descriptor instead. +func (*HandoverProc) Descriptor() ([]byte, []int) { + return file_upgrade_handover_proto_rawDescGZIP(), []int{3} +} + +func (x *HandoverProc) GetPid() uint32 { + if x != nil { + return x.Pid + } + return 0 +} + +func (x *HandoverProc) GetTag() string { + if x != nil { + return x.Tag + } + return "" +} + +func (x *HandoverProc) GetHasTag() bool { + if x != nil { + return x.HasTag + } + return false +} + +func (x *HandoverProc) GetCgType() string { + if x != nil { + return x.CgType + } + return "" +} + +func (x *HandoverProc) GetConfig() *process.ProcessConfig { + if x != nil { + return x.Config + } + return nil +} + +func (x *HandoverProc) GetStdoutFd() int32 { + if x != nil { + return x.StdoutFd + } + return 0 +} + +func (x *HandoverProc) GetStderrFd() int32 { + if x != nil { + return x.StderrFd + } + return 0 +} + +func (x *HandoverProc) GetStdinFd() int32 { + if x != nil { + return x.StdinFd + } + return 0 +} + +func (x *HandoverProc) GetTtyFd() int32 { + if x != nil { + return x.TtyFd + } + return 0 +} + +func (x *HandoverProc) GetTimeoutMs() int64 { + if x != nil { + return x.TimeoutMs + } + return 0 +} + +// HandoverExit is a recently-terminated process whose terminal event has not yet +// been drained by a client — carried so an exit code is not lost across the swap. +type HandoverExit struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Pid uint32 `protobuf:"varint,1,opt,name=pid,proto3" json:"pid,omitempty"` + Tag string `protobuf:"bytes,2,opt,name=tag,proto3" json:"tag,omitempty"` + HasTag bool `protobuf:"varint,3,opt,name=has_tag,json=hasTag,proto3" json:"has_tag,omitempty"` + End *process.ProcessEvent_EndEvent `protobuf:"bytes,4,opt,name=end,proto3" json:"end,omitempty"` + RemainingMs int64 `protobuf:"varint,5,opt,name=remaining_ms,json=remainingMs,proto3" json:"remaining_ms,omitempty"` // retention TTL remaining +} + +func (x *HandoverExit) Reset() { + *x = HandoverExit{} + if protoimpl.UnsafeEnabled { + mi := &file_upgrade_handover_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *HandoverExit) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HandoverExit) ProtoMessage() {} + +func (x *HandoverExit) ProtoReflect() protoreflect.Message { + mi := &file_upgrade_handover_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HandoverExit.ProtoReflect.Descriptor instead. +func (*HandoverExit) Descriptor() ([]byte, []int) { + return file_upgrade_handover_proto_rawDescGZIP(), []int{4} +} + +func (x *HandoverExit) GetPid() uint32 { + if x != nil { + return x.Pid + } + return 0 +} + +func (x *HandoverExit) GetTag() string { + if x != nil { + return x.Tag + } + return "" +} + +func (x *HandoverExit) GetHasTag() bool { + if x != nil { + return x.HasTag + } + return false +} + +func (x *HandoverExit) GetEnd() *process.ProcessEvent_EndEvent { + if x != nil { + return x.End + } + return nil +} + +func (x *HandoverExit) GetRemainingMs() int64 { + if x != nil { + return x.RemainingMs + } + return 0 +} + +// HandoverWatcher re-arms a persistent CreateWatcher on the incoming envd under +// the preserved id (design §6.9, Option C: re-arm from metadata, do NOT carry an +// inotify fd). +type HandoverWatcher struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Path string `protobuf:"bytes,2,opt,name=path,proto3" json:"path,omitempty"` + Recursive bool `protobuf:"varint,3,opt,name=recursive,proto3" json:"recursive,omitempty"` + IncludeEntryInfo bool `protobuf:"varint,4,opt,name=include_entry_info,json=includeEntryInfo,proto3" json:"include_entry_info,omitempty"` + PendingEvents []*filesystem.FilesystemEvent `protobuf:"bytes,5,rep,name=pending_events,json=pendingEvents,proto3" json:"pending_events,omitempty"` // un-polled GetWatcherEvents buffer +} + +func (x *HandoverWatcher) Reset() { + *x = HandoverWatcher{} + if protoimpl.UnsafeEnabled { + mi := &file_upgrade_handover_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *HandoverWatcher) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HandoverWatcher) ProtoMessage() {} + +func (x *HandoverWatcher) ProtoReflect() protoreflect.Message { + mi := &file_upgrade_handover_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HandoverWatcher.ProtoReflect.Descriptor instead. +func (*HandoverWatcher) Descriptor() ([]byte, []int) { + return file_upgrade_handover_proto_rawDescGZIP(), []int{5} +} + +func (x *HandoverWatcher) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *HandoverWatcher) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *HandoverWatcher) GetRecursive() bool { + if x != nil { + return x.Recursive + } + return false +} + +func (x *HandoverWatcher) GetIncludeEntryInfo() bool { + if x != nil { + return x.IncludeEntryInfo + } + return false +} + +func (x *HandoverWatcher) GetPendingEvents() []*filesystem.FilesystemEvent { + if x != nil { + return x.PendingEvents + } + return nil +} + +var File_upgrade_handover_proto protoreflect.FileDescriptor + +var file_upgrade_handover_proto_rawDesc = []byte{ + 0x0a, 0x16, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x2f, 0x68, 0x61, 0x6e, 0x64, 0x6f, 0x76, + 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x07, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, + 0x65, 0x1a, 0x15, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x63, 0x65, + 0x73, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1b, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x79, + 0x73, 0x74, 0x65, 0x6d, 0x2f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xc5, 0x02, 0x0a, 0x0d, 0x48, 0x61, 0x6e, 0x64, 0x6f, 0x76, + 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, + 0x19, 0x0a, 0x08, 0x66, 0x72, 0x6f, 0x6d, 0x5f, 0x76, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x66, 0x72, 0x6f, 0x6d, 0x56, 0x65, 0x72, 0x12, 0x33, 0x0a, 0x09, 0x70, 0x72, + 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, + 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x2e, 0x48, 0x61, 0x6e, 0x64, 0x6f, 0x76, 0x65, 0x72, + 0x50, 0x72, 0x6f, 0x63, 0x52, 0x09, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, 0x73, 0x12, + 0x35, 0x0a, 0x0a, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x64, 0x18, 0x04, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x2e, 0x48, 0x61, + 0x6e, 0x64, 0x6f, 0x76, 0x65, 0x72, 0x45, 0x78, 0x69, 0x74, 0x52, 0x0a, 0x74, 0x65, 0x72, 0x6d, + 0x69, 0x6e, 0x61, 0x74, 0x65, 0x64, 0x12, 0x34, 0x0a, 0x08, 0x77, 0x61, 0x74, 0x63, 0x68, 0x65, + 0x72, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x75, 0x70, 0x67, 0x72, 0x61, + 0x64, 0x65, 0x2e, 0x48, 0x61, 0x6e, 0x64, 0x6f, 0x76, 0x65, 0x72, 0x57, 0x61, 0x74, 0x63, 0x68, + 0x65, 0x72, 0x52, 0x08, 0x77, 0x61, 0x74, 0x63, 0x68, 0x65, 0x72, 0x73, 0x12, 0x2b, 0x0a, 0x06, + 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x75, + 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x2e, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x06, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x12, 0x32, 0x0a, 0x08, 0x66, 0x6f, 0x72, + 0x77, 0x61, 0x72, 0x64, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x75, 0x70, + 0x67, 0x72, 0x61, 0x64, 0x65, 0x2e, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x65, 0x64, 0x50, + 0x6f, 0x72, 0x74, 0x52, 0x08, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x73, 0x22, 0x43, 0x0a, + 0x0a, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x70, + 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, + 0x21, 0x0a, 0x0c, 0x6c, 0x69, 0x66, 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6c, 0x69, 0x66, 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, + 0x49, 0x64, 0x22, 0x8d, 0x01, 0x0a, 0x0d, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x65, 0x64, + 0x50, 0x6f, 0x72, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x6c, 0x69, + 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x5f, 0x70, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x0b, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x50, 0x69, 0x64, 0x12, 0x16, 0x0a, + 0x06, 0x66, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x66, + 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x6f, 0x63, 0x61, 0x74, 0x5f, 0x70, + 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x73, 0x6f, 0x63, 0x61, 0x74, 0x50, + 0x69, 0x64, 0x22, 0x9f, 0x02, 0x0a, 0x0c, 0x48, 0x61, 0x6e, 0x64, 0x6f, 0x76, 0x65, 0x72, 0x50, + 0x72, 0x6f, 0x63, 0x12, 0x10, 0x0a, 0x03, 0x70, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x03, 0x70, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x61, 0x67, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x74, 0x61, 0x67, 0x12, 0x17, 0x0a, 0x07, 0x68, 0x61, 0x73, 0x5f, 0x74, + 0x61, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x68, 0x61, 0x73, 0x54, 0x61, 0x67, + 0x12, 0x17, 0x0a, 0x07, 0x63, 0x67, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x63, 0x67, 0x54, 0x79, 0x70, 0x65, 0x12, 0x2e, 0x0a, 0x06, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x63, + 0x65, 0x73, 0x73, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x74, 0x64, + 0x6f, 0x75, 0x74, 0x5f, 0x66, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x73, 0x74, + 0x64, 0x6f, 0x75, 0x74, 0x46, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x74, 0x64, 0x65, 0x72, 0x72, + 0x5f, 0x66, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x73, 0x74, 0x64, 0x65, 0x72, + 0x72, 0x46, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x74, 0x64, 0x69, 0x6e, 0x5f, 0x66, 0x64, 0x18, + 0x08, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x73, 0x74, 0x64, 0x69, 0x6e, 0x46, 0x64, 0x12, 0x15, + 0x0a, 0x06, 0x74, 0x74, 0x79, 0x5f, 0x66, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, + 0x74, 0x74, 0x79, 0x46, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, + 0x5f, 0x6d, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x6f, + 0x75, 0x74, 0x4d, 0x73, 0x22, 0xa0, 0x01, 0x0a, 0x0c, 0x48, 0x61, 0x6e, 0x64, 0x6f, 0x76, 0x65, + 0x72, 0x45, 0x78, 0x69, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x70, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x03, 0x70, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x61, 0x67, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x74, 0x61, 0x67, 0x12, 0x17, 0x0a, 0x07, 0x68, 0x61, 0x73, + 0x5f, 0x74, 0x61, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x68, 0x61, 0x73, 0x54, + 0x61, 0x67, 0x12, 0x30, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, + 0x73, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, + 0x03, 0x65, 0x6e, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x72, 0x65, 0x6d, 0x61, 0x69, 0x6e, 0x69, 0x6e, + 0x67, 0x5f, 0x6d, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x72, 0x65, 0x6d, 0x61, + 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x4d, 0x73, 0x22, 0xc5, 0x01, 0x0a, 0x0f, 0x48, 0x61, 0x6e, 0x64, + 0x6f, 0x76, 0x65, 0x72, 0x57, 0x61, 0x74, 0x63, 0x68, 0x65, 0x72, 0x12, 0x0e, 0x0a, 0x02, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x70, + 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, + 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x09, 0x72, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x12, 0x2c, 0x0a, + 0x12, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x5f, 0x69, + 0x6e, 0x66, 0x6f, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x69, 0x6e, 0x63, 0x6c, 0x75, + 0x64, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x42, 0x0a, 0x0e, 0x70, + 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x05, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, + 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, + 0x52, 0x0d, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x42, + 0x9f, 0x01, 0x0a, 0x0b, 0x63, 0x6f, 0x6d, 0x2e, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x42, + 0x0d, 0x48, 0x61, 0x6e, 0x64, 0x6f, 0x76, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, + 0x5a, 0x45, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x32, 0x62, + 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x69, 0x6e, 0x66, 0x72, 0x61, 0x2f, 0x70, 0x61, 0x63, 0x6b, 0x61, + 0x67, 0x65, 0x73, 0x2f, 0x65, 0x6e, 0x76, 0x64, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x73, 0x70, 0x65, 0x63, 0x2f, + 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0xa2, 0x02, 0x03, 0x55, 0x58, 0x58, 0xaa, 0x02, 0x07, + 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0xca, 0x02, 0x07, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, + 0x65, 0xe2, 0x02, 0x13, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x5c, 0x47, 0x50, 0x42, 0x4d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x07, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, + 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_upgrade_handover_proto_rawDescOnce sync.Once + file_upgrade_handover_proto_rawDescData = file_upgrade_handover_proto_rawDesc +) + +func file_upgrade_handover_proto_rawDescGZIP() []byte { + file_upgrade_handover_proto_rawDescOnce.Do(func() { + file_upgrade_handover_proto_rawDescData = protoimpl.X.CompressGZIP(file_upgrade_handover_proto_rawDescData) + }) + return file_upgrade_handover_proto_rawDescData +} + +var file_upgrade_handover_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_upgrade_handover_proto_goTypes = []interface{}{ + (*HandoverState)(nil), // 0: upgrade.HandoverState + (*MountEntry)(nil), // 1: upgrade.MountEntry + (*ForwardedPort)(nil), // 2: upgrade.ForwardedPort + (*HandoverProc)(nil), // 3: upgrade.HandoverProc + (*HandoverExit)(nil), // 4: upgrade.HandoverExit + (*HandoverWatcher)(nil), // 5: upgrade.HandoverWatcher + (*process.ProcessConfig)(nil), // 6: process.ProcessConfig + (*process.ProcessEvent_EndEvent)(nil), // 7: process.ProcessEvent.EndEvent + (*filesystem.FilesystemEvent)(nil), // 8: filesystem.FilesystemEvent +} +var file_upgrade_handover_proto_depIdxs = []int32{ + 3, // 0: upgrade.HandoverState.processes:type_name -> upgrade.HandoverProc + 4, // 1: upgrade.HandoverState.terminated:type_name -> upgrade.HandoverExit + 5, // 2: upgrade.HandoverState.watchers:type_name -> upgrade.HandoverWatcher + 1, // 3: upgrade.HandoverState.mounts:type_name -> upgrade.MountEntry + 2, // 4: upgrade.HandoverState.forwards:type_name -> upgrade.ForwardedPort + 6, // 5: upgrade.HandoverProc.config:type_name -> process.ProcessConfig + 7, // 6: upgrade.HandoverExit.end:type_name -> process.ProcessEvent.EndEvent + 8, // 7: upgrade.HandoverWatcher.pending_events:type_name -> filesystem.FilesystemEvent + 8, // [8:8] is the sub-list for method output_type + 8, // [8:8] is the sub-list for method input_type + 8, // [8:8] is the sub-list for extension type_name + 8, // [8:8] is the sub-list for extension extendee + 0, // [0:8] is the sub-list for field type_name +} + +func init() { file_upgrade_handover_proto_init() } +func file_upgrade_handover_proto_init() { + if File_upgrade_handover_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_upgrade_handover_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HandoverState); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_upgrade_handover_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MountEntry); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_upgrade_handover_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ForwardedPort); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_upgrade_handover_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HandoverProc); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_upgrade_handover_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HandoverExit); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_upgrade_handover_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HandoverWatcher); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_upgrade_handover_proto_rawDesc, + NumEnums: 0, + NumMessages: 6, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_upgrade_handover_proto_goTypes, + DependencyIndexes: file_upgrade_handover_proto_depIdxs, + MessageInfos: file_upgrade_handover_proto_msgTypes, + }.Build() + File_upgrade_handover_proto = out.File + file_upgrade_handover_proto_rawDesc = nil + file_upgrade_handover_proto_goTypes = nil + file_upgrade_handover_proto_depIdxs = nil +} diff --git a/packages/envd/internal/utils/map.go b/packages/envd/internal/utils/map.go index 256b44bf4a..57fdf2b7d1 100644 --- a/packages/envd/internal/utils/map.go +++ b/packages/envd/internal/utils/map.go @@ -49,3 +49,10 @@ func (m *Map[K, V]) Range(f func(key K, value V) bool) { func (m *Map[K, V]) Store(key K, value V) { m.m.Store(key, value) } + +// CompareAndDelete deletes the entry for key only if its value is still old, +// returning whether it deleted. Lets a caller evict its own entry without +// clobbering a newer value stored under the same key (e.g. a reused PID). +func (m *Map[K, V]) CompareAndDelete(key K, old V) (deleted bool) { + return m.m.CompareAndDelete(key, old) +} diff --git a/packages/envd/main.go b/packages/envd/main.go index a9f462110a..f61e0b9e8e 100644 --- a/packages/envd/main.go +++ b/packages/envd/main.go @@ -4,6 +4,7 @@ import ( "context" "flag" "fmt" + "io" "log" "net/http" "os" @@ -38,6 +39,12 @@ const ( portScannerInterval = 1000 * time.Millisecond + // handoverFallbackThawTimeout bounds how long a live-upgraded envd keeps its + // re-adopted workload frozen waiting for the orchestrator's post-upgrade + // /init to thaw it. Comfortably past the orchestrator's upgrade readiness + // budget, so it only fires when that /init genuinely never arrives. + handoverFallbackThawTimeout = 60 * time.Second + // This is the default user used in the container if not specified otherwise. // It should be always overridden by the user in /init when building the template. defaultUser = "root" @@ -57,6 +64,11 @@ var ( cgroupRoot string noCgroups bool verbose bool + + // resumeHandover: set by the outgoing envd when it re-execs itself during a + // live self-upgrade. On boot the new image re-adopts the + // processes described in the tmpfs handover blob. + resumeHandover bool ) func parseFlags() { @@ -109,6 +121,13 @@ func parseFlags() { "write envd logs to stdout", ) + flag.BoolVar( + &resumeHandover, + "resume-handover", + false, + "internal: re-adopt processes from the live-upgrade handover blob (set by the outgoing envd)", + ) + flag.Parse() } @@ -186,7 +205,7 @@ func run() error { envLogger := l.With().Str("logger", "envd").Logger() fsLogger := l.With().Str("logger", "filesystem").Logger() - filesystemRpc.Handle(m, &fsLogger, defaults) + filesystemService := filesystemRpc.Handle(m, &fsLogger, defaults) cgroupManager := createCgroupManager() defer func() { @@ -196,10 +215,78 @@ func run() error { } }() + // One freezer shared by the process service (live-upgrade handover) and the + // HTTP API (/freeze, /unfreeze, /init thaw) so every freeze/unfreeze caller + // serializes on a single lock. + workloadFreezer := cgroups.NewWorkloadFreezer(cgroupManager) + processLogger := l.With().Str("logger", "process").Logger() - processRpc.Handle(m, &processLogger, defaults, cgroupManager) + processService := processRpc.Handle(m, &processLogger, defaults, workloadFreezer) + + // Live-upgrade incoming side: if this envd was re-exec'd by an outgoing + // envd, re-adopt the handed-over processes before serving. + var handover processRpc.HandoverResult + // handoverFailed is true if ResumeFromHandover errored or panicked: the + // workload was not re-adopted, so /init must advertise the failure (below) — + // otherwise the orchestrator, which confirms the upgrade by version alone, + // would record a broken sandbox (orphaned, unreaped workload) as a success. + var handoverFailed bool + if resumeHandover { + func() { + // A panic here must not crash the new envd: systemd would restart a + // fresh (old-binary) envd with a new PID that can neither re-adopt nor + // reap the frozen workload. Recover and keep serving; the workload is + // thawed by ResumeFromHandover's deferred unfreeze regardless. + defer func() { + if r := recover(); r != nil { + fmt.Fprintf(os.Stderr, "resume-from-handover panic (recovered): %v\n", r) + handoverFailed = true + processService.UnfreezeWorkload() + } + }() + + // Pass ImportWatchers as a callback so watchers are re-armed while the + // workload is still frozen inside ResumeFromHandover (on success it + // stays frozen until the post-upgrade /init thaws it), leaving no gap + // in which a filesystem event could be missed between the thaw and the + // re-arm. + res, err := processService.ResumeFromHandover(filesystemService.ImportWatchers) + if err != nil { + fmt.Fprintf(os.Stderr, "resume-from-handover failed: %v\n", err) + handoverFailed = true + } + handover = res + }() + } - service := api.New(&envLogger, defaults, mmdsChan, isNotFC, cgroupManager) + service := api.New(&envLogger, defaults, mmdsChan, isNotFC, workloadFreezer) + if resumeHandover { + // Restore the NFS mount ledger carried across the upgrade before the + // post-upgrade /init runs setupNFS, so it recognizes a still-live mount + // (matching lifecycle) instead of force-unmounting and remounting it. + service.ImportMounts(handover.Mounts) + + // Surface the handover outcome on the next /init so the orchestrator can + // record it — the envd-side result (re-adopted procs, restored retained + // exits, watcher re-arm success/failures) is otherwise only logged + // in-guest and invisible fleet-wide. + service.SetHandoverResult(handoverFailed, handover.Procs, handover.ProcsFailed, handover.Retained, handover.RetainedFailed, handover.Watchers, handover.WatchersFailed) + + // Safety net for the freeze-until-/init handover: ResumeFromHandover left + // the workload frozen for the orchestrator's post-upgrade /init to thaw. + // If that /init never lands (e.g. WaitForEnvd fails / deadlines), thaw + // anyway after a grace window so the sandbox degrades rather than hangs + // for the rest of its life. Gated on Initialized() so it never races a + // legitimate later /freeze, and /upgrade stays gated until /init even on + // this path, so the fallback can't reopen the unauthenticated-upgrade + // window. + time.AfterFunc(handoverFallbackThawTimeout, func() { + if !service.Initialized() { + fmt.Fprintf(os.Stderr, "envd: post-upgrade /init did not arrive within %s; thawing workload as fallback\n", handoverFallbackThawTimeout) + processService.UnfreezeWorkload() + } + }) + } handler := api.HandlerFromMux(service, m) middleware := authn.NewMiddleware(permissions.AuthenticateUsername) @@ -222,10 +309,131 @@ func run() error { portLogger := l.With().Str("logger", "port-forwarder").Logger() portForwarder := publicport.NewForwarder(&portLogger, portScanner, cgroupManager) + if resumeHandover { + // Re-adopt the socats carried across the upgrade before the forwarder's + // first scan, so it recognizes already-forwarded ports instead of spawning + // duplicate socats (and leaking the originals as un-reaped zombies). + if readopted := portForwarder.ImportForwards(handover.Forwards); readopted > 0 { + fmt.Fprintf(os.Stderr, "envd: re-adopted %d forwarded port(s) after handover\n", readopted) + } + } go portForwarder.StartForwarding(ctx) go portScanner.ScanAndBroadcast() + // Live-upgrade outgoing trigger. doUpgrade performs the outgoing + // choreography: freeze the workload, then re-exec into newBin (empty + // = re-exec self). It does not return on success. The port scanner is left + // running: the fd relocation runs under syscall.ForkLock (see Upgrade), which + // already serializes it against the scanner's fd/socat creation, so there is + // no CLOEXEC race to quiesce — and leaving it up means a failed upgrade never + // disturbs port forwarding. + doUpgrade := func(newBin string) error { + fmt.Fprintf(os.Stderr, "envd: self-upgrade (from v%s, newBin=%q)\n", pkg.Version, newBin) + // Hold the freeze lock across the WHOLE handover (freeze -> serialize -> + // execve), not just the freeze sweep, so a concurrent /init or /unfreeze + // can't acquire the lock and thaw the workload while Upgrade is + // snapshotting the process table. + releaseFreeze, freezeErr := processService.FreezeWorkloadHold() //nolint:contextcheck // (un)freeze uses a non-cancellable context internally so the thaw always lands + // Release the freeze hold and thaw on ANY non-exec exit — an error OR a + // panic in ExportWatchers/Upgrade (net/http recovers the handler, so envd + // keeps serving; a leaked hold would then deadlock every later Unfreeze, + // including /init, and strand the workload frozen). Order matters: drop + // the lock BEFORE thawing, since Unfreeze acquires it. A successful execve + // replaces this process, so this defer never runs on success. + defer func() { + releaseFreeze() + processService.UnfreezeWorkload() //nolint:contextcheck // (un)freeze uses a non-cancellable context internally so the thaw always lands + }() + if freezeErr != nil { + // The workload isn't fully frozen, so snapshotting the process table + // would be racy/stale and the pre-init security assumptions (nothing + // runs until /init restores auth) wouldn't hold. Abort the swap; the + // deferred release+thaw keeps the old envd serving. + return fmt.Errorf("handover aborted: freeze workload: %w", freezeErr) + } + + // Export the state owned by the other services so the new envd can restore + // it after the swap: filesystem watches, the NFS mount ledger (so /init + // doesn't remount a live mount), and the active port-forwards (so socats + // are re-adopted, not duplicated). + // + // Watches and forwards are exported with the owning lock HELD across the + // execve (the *Hold variants). The freeze quiesces the workload, but the + // filesystem watcher-drain (GetWatcherEvents) and the port scanner are + // envd-internal and keep running; holding their locks through the swap + // stops a post-snapshot event-drain or socat-spawn that the snapshot could + // not capture. On a successful execve this process is replaced and the + // held locks vanish with it; the defers below only run on the failure + // path, restoring both services under the old envd. + watchers, releaseWatchers := filesystemService.ExportWatchersHold() + defer releaseWatchers() + mounts := service.ExportMounts() + forwards, releaseForwards := portForwarder.ExportForwardsHold() + defer releaseForwards() + + // Upgrade only returns on failure — a successful execve replaces this + // process (and drops the held freeze/watcher/forward locks with it). On + // failure the OLD envd is still running; the deferred releases + thaw keep + // the old version serving rather than leaving the sandbox hung. + return processService.Upgrade(newBin, pkg.Version, watchers, mounts, forwards) + } + + // Orchestrator-driven trigger: authenticated POST /upgrade with the target + // binary path in the X-Envd-Upgrade-Bin header (empty = re-exec self). This + // is the production trigger the orchestrator calls at resume after delivering + // the new binary into the guest. On success it never responds (envd execs); + // the caller treats a dropped connection as success. + m.Post("/upgrade", func(w http.ResponseWriter, r *http.Request) { + // Refuse upgrades until the first authenticated /init. A re-exec'd envd + // serves before /init with its access token cleared, so without this a + // guest process could drive an unauthenticated upgrade in that window + // (and after the fallback thaw). The orchestrator only triggers /upgrade + // on an already-initialized envd, so this never blocks the real caller. + if !service.Initialized() { + http.Error(w, "envd not initialized", http.StatusConflict) + + return + } + // The delivered binary is always written to and exec'd from a fixed path; + // reject a caller asking for anything else rather than writing/exec'ing an + // arbitrary path (defense-in-depth — the endpoint is authenticated). + if hdr := r.Header.Get("X-Envd-Upgrade-Bin"); hdr != "" && hdr != processRpc.DefaultUpgradeBinPath { + http.Error(w, "unsupported upgrade target", http.StatusBadRequest) + + return + } + newBin := "" + // If the new binary is streamed in the request body (the orchestrator's + // authenticated host-side delivery), write it to the fixed path before + // swapping. This avoids the unauthenticated /files path that fails on a + // runtime sandbox. + if r.ContentLength != 0 { + newBin = processRpc.DefaultUpgradeBinPath + // On a chained upgrade this envd is itself running from + // DefaultUpgradeBinPath, so opening it O_TRUNC would fail with ETXTBSY. + // Unlink first: the create then lands on a fresh inode while the + // running process keeps executing its now-unlinked image. + _ = os.Remove(processRpc.DefaultUpgradeBinPath) + f, err := os.OpenFile(processRpc.DefaultUpgradeBinPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o755) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + + return + } + if _, err := io.Copy(f, r.Body); err != nil { + f.Close() + http.Error(w, err.Error(), http.StatusInternalServerError) + + return + } + f.Close() + } + if err := doUpgrade(newBin); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + }) + err := s.ListenAndServe() // Signal goroutines to stop before deferred cleanup closes their resources. // TODO: shutdown synchronization needs to be revisited. diff --git a/packages/envd/pkg/version.go b/packages/envd/pkg/version.go index abbbb74c05..21011fb23c 100644 --- a/packages/envd/pkg/version.go +++ b/packages/envd/pkg/version.go @@ -1,3 +1,3 @@ package pkg -const Version = "0.6.11" +const Version = "0.6.12" diff --git a/packages/envd/spec/upgrade/handover.proto b/packages/envd/spec/upgrade/handover.proto new file mode 100644 index 0000000000..08ae543ac8 --- /dev/null +++ b/packages/envd/spec/upgrade/handover.proto @@ -0,0 +1,92 @@ +syntax = "proto3"; + +package upgrade; + +import "process/process.proto"; +import "filesystem/filesystem.proto"; + +// HandoverState is the entire cross-version compatibility contract for an envd +// live-upgrade (design §5.2, §6.4): the outgoing envd serializes it to a tmpfs +// file, the incoming (new-binary) envd decodes it after execve to reconstruct +// its world. It is an additive protobuf — an older envd omits fields a newer one +// added, and the reader treats absent fields as defaults. +// +// Schema is the versioning discipline §6.4 requires: it is bumped on any change, +// and a reader MUST refuse a blob whose schema exceeds the maximum it understands +// (decode every schema <= N, abort on schema > N) rather than mis-read a +// newer-than-known layout — the outgoing envd keeps running the old binary. +message HandoverState { + uint32 schema = 1; + string from_ver = 2; + repeated HandoverProc processes = 3; + // terminated carries the retention cache (recently-exited terminal events not + // yet drained). + repeated HandoverExit terminated = 4; + // watchers is the filesystem service's active-watcher set. The filesystem + // service owns building/consuming these; the process handover only carries + // them. + repeated HandoverWatcher watchers = 5; + // mounts is the API service's NFS mount ledger (path -> lifecycle), carried so + // the new envd's post-upgrade /init skips re-mounting a still-live mount. + repeated MountEntry mounts = 6; + // forwards is the port forwarder's active socat set, carried so the new envd + // re-adopts the running socat children instead of spawning duplicates. + repeated ForwardedPort forwards = 7; +} + +// MountEntry is one NFS volume mount the outgoing envd had set up, keyed by the +// lifecycle it was mounted for. The kernel mount survives execve; carrying the +// ledger lets the new envd recognize a matching-lifecycle mount and leave it +// in place rather than force-unmounting and remounting it (ESTALE risk). +message MountEntry { + string path = 1; + string lifecycle_id = 2; +} + +// ForwardedPort is one active port-forward. The socat child survives execve; +// carrying its pid lets the new forwarder re-adopt it (suppressing a duplicate +// socat and reaping it when the port closes) instead of respawning. +message ForwardedPort { + string key = 1; // the forwarder map key: "-" + uint32 port = 2; + int32 listener_pid = 3; // pid of the guest process listening on the port + uint32 family = 4; // IP version (4 or 6) + int32 socat_pid = 5; // pid of the running socat child to re-adopt +} + +// HandoverProc is a live child re-adopted across the same-PID execve. The fd +// fields carry the *numbers* (kept valid across execve, CLOEXEC cleared); the +// kernel objects ride the fd table. +message HandoverProc { + uint32 pid = 1; // K-anchor: live child, survives same-PID execve + string tag = 2; // K-metadata: not in /proc, unreconstructable if lost + bool has_tag = 3; // distinguishes an empty tag from no tag + string cg_type = 4; // user | pty + process.ProcessConfig config = 5; + int32 stdout_fd = 6; + int32 stderr_fd = 7; + int32 stdin_fd = 8; + int32 tty_fd = 9; + int64 timeout_ms = 10; // per-process kill deadline remaining, re-armed on readopt +} + +// HandoverExit is a recently-terminated process whose terminal event has not yet +// been drained by a client — carried so an exit code is not lost across the swap. +message HandoverExit { + uint32 pid = 1; + string tag = 2; + bool has_tag = 3; + process.ProcessEvent.EndEvent end = 4; + int64 remaining_ms = 5; // retention TTL remaining +} + +// HandoverWatcher re-arms a persistent CreateWatcher on the incoming envd under +// the preserved id (design §6.9, Option C: re-arm from metadata, do NOT carry an +// inotify fd). +message HandoverWatcher { + string id = 1; + string path = 2; + bool recursive = 3; + bool include_entry_info = 4; + repeated filesystem.FilesystemEvent pending_events = 5; // un-polled GetWatcherEvents buffer +} diff --git a/packages/orchestrator/cmd/resume-build/main.go b/packages/orchestrator/cmd/resume-build/main.go index 699e9e88d0..a635b9deec 100644 --- a/packages/orchestrator/cmd/resume-build/main.go +++ b/packages/orchestrator/cmd/resume-build/main.go @@ -406,7 +406,7 @@ func (r *runner) startSandbox(ctx context.Context, runtime sandbox.RuntimeMetada }) } - return r.factory.RebootSandbox(ctx, r.tmpl, r.sbxConfig, runtime, end, nil, procOpts...) + return r.factory.RebootSandbox(ctx, r.tmpl, r.sbxConfig, runtime, end, nil, false, procOpts...) } return r.factory.ResumeSandbox(ctx, r.tmpl, r.sbxConfig, runtime, start, end, nil) diff --git a/packages/orchestrator/pkg/sandbox/envd.go b/packages/orchestrator/pkg/sandbox/envd.go index c83de86235..18e6b5324f 100644 --- a/packages/orchestrator/pkg/sandbox/envd.go +++ b/packages/orchestrator/pkg/sandbox/envd.go @@ -9,7 +9,10 @@ import ( "errors" "fmt" "io" + "net" "net/http" + "os" + "syscall" "time" "go.opentelemetry.io/otel/attribute" @@ -208,6 +211,157 @@ func (s *Sandbox) postEnvd(ctx context.Context, timeout time.Duration, path stri return nil } +// CallEnvdUpgrade triggers envd's POST /upgrade — the orchestrator-driven +// live-upgrade trigger. It streams the new envd binary +// from localSrcPath as the (authenticated) request body; envd writes it to +// guestBinPath inside the guest and then same-PID re-execs into it. Delivering +// over the token-authenticated /upgrade endpoint avoids the unauthenticated +// /files path that a runtime (post-/init) sandbox rejects. +// +// envd reads the whole body, then execs and never responds, so the connection +// drops without a reply: a transport error after the body was sent is the +// expected success path. The caller must follow with WaitForEnvd. +// +// execConfirmed reports whether the same-PID exec is CONFIRMED to have happened +// (a connection reset/EOF after the body was sent). A deadline OR a cancelled +// ctx returns (false, nil): ambiguous — envd may still be mid-handover on the +// old binary — so the caller must NOT treat a follow-up not-ready as an +// unrecoverable brick. The request is bounded by the timeout ctx deadline, not +// sandboxHttpClient's shorter client-level Timeout. +func (s *Sandbox) CallEnvdUpgrade(ctx context.Context, localSrcPath, guestBinPath string, timeout time.Duration) (execConfirmed bool, err error) { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + f, err := os.Open(localSrcPath) + if err != nil { + return false, fmt.Errorf("open envd source %s: %w", localSrcPath, err) + } + defer f.Close() + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.envdServerURL()+"/upgrade", f) + if err != nil { + return false, fmt.Errorf("build upgrade request: %w", err) + } + if fi, statErr := f.Stat(); statErr == nil { + req.ContentLength = fi.Size() + } + if s.Config.Envd.AccessToken != nil { + req.Header.Set("X-Access-Token", *s.Config.Envd.AccessToken) + } + req.Header.Set("X-Envd-Upgrade-Bin", guestBinPath) + + // Reuse the shared transport but drop sandboxHttpClient's short client-level + // Timeout (10s) — it would preempt the deliverTimeout ctx above and cut the + // upgrade off before envd finishes reading the body and exec'ing. The ctx + // deadline is the sole delivery budget. + resp, err := (&http.Client{Transport: sandboxHttpClient.Transport}).Do(req) + if err != nil { + // envd reads the whole body, then execs without responding, so a + // transport error AFTER the request reached it (connection reset/EOF) is + // the expected success path. But a failure to even reach a running envd + // (connection refused, or a dial-phase failure) means the upgrade was + // never delivered — surface it so the caller doesn't record a false + // success. A deadline is deliberately NOT treated as delivery failure + // (ambiguous — see isUpgradeDeliveryFailure — left to version confirm). + if isUpgradeDeliveryFailure(err) { + return false, fmt.Errorf("deliver upgrade to envd: %w", err) + } + // A context error — the deliverTimeout deadline OR a cancelled parent ctx + // (e.g. the resume budget ran out) — means we gave up before observing the + // exec, so it is NOT confirmed: envd may still be mid-handover on the old + // binary. Report success-but-unconfirmed so the caller keeps a follow-up + // not-ready recoverable rather than a fatal brick. + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { + return false, nil + } + // Anything else here reached envd (it passed isUpgradeDeliveryFailure) and + // isn't a context error, so it's the expected post-send connection + // reset/EOF = the same-PID exec fired. + return true, nil + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + body, _ := io.ReadAll(resp.Body) + + return false, fmt.Errorf("upgrade returned %d: %s", resp.StatusCode, utils.Truncate(string(body), 100)) + } + + // envd answered instead of exec'ing — no swap happened, exec not confirmed. + return false, nil +} + +// isUpgradeDeliveryFailure reports whether an error from the /upgrade request +// means the binary never reached a running envd — a genuine failure — as opposed +// to the expected post-send connection drop when envd execs mid-response. +// +// A deadline is deliberately NOT treated as failure: it's ambiguous (envd may +// have exec'd and simply never answered), so it's left to the post-upgrade +// version confirmation to decide the true outcome. +func isUpgradeDeliveryFailure(err error) bool { + // Covered (=> true, "the binary never reached a running envd, so no swap"): + // - syscall.ECONNREFUSED: nothing is listening on the envd port. + // - net.OpError with Op == "dial": the connection could not be established + // (DNS / dial-phase failure) before any byte was sent. + // Deliberately NOT covered (=> false, i.e. treated as the expected + // post-send drop): connection reset / EOF / unexpected EOF after the body + // was sent (envd exec'd mid-response), and context deadline / timeout — the + // latter is ambiguous (envd may have exec'd and simply never answered), so + // it is left to the post-upgrade version confirmation to decide the outcome. + if errors.Is(err, syscall.ECONNREFUSED) { + return true + } + var opErr *net.OpError + if errors.As(err, &opErr) && opErr.Op == "dial" { + return true + } + + return false +} + +// setLiveEnvdVersion records the version the running envd last reported. +func (s *Sandbox) setLiveEnvdVersion(v string) { + s.liveEnvdVersion.Store(&v) +} + +// LiveEnvdVersion returns the version the running envd last reported on /init, +// or "" if none has been captured yet. +func (s *Sandbox) LiveEnvdVersion() string { + if p := s.liveEnvdVersion.Load(); p != nil { + return *p + } + + return "" +} + +// EnvdHandoverResult is the live-upgrade handover outcome the running envd +// reported on /init (X-Envd-Handover). Its JSON tags match envd's +// api.handoverResult so the header unmarshals directly. +type EnvdHandoverResult struct { + // Failed is true when the in-guest handover itself failed post-execve (the + // workload was not re-adopted), so a version flip to the target is NOT a + // healthy upgrade — the trigger fails the resume on it. + Failed bool `json:"failed"` + // Every item is total-carried + failed-subset (ok = total - failed). + Procs int `json:"procs"` + ProcsFailed int `json:"procs_failed"` + Retained int `json:"retained"` + RetainedFailed int `json:"retained_failed"` + Watchers int `json:"watchers"` + WatchersFailed int `json:"watchers_failed"` +} + +// setHandoverResult records the handover outcome the running envd last reported. +func (s *Sandbox) setHandoverResult(h *EnvdHandoverResult) { + s.handoverResult.Store(h) +} + +// HandoverResult returns the last handover outcome the running envd reported on +// /init, or nil if it never booted from a live-upgrade handover. +func (s *Sandbox) HandoverResult() *EnvdHandoverResult { + return s.handoverResult.Load() +} + // envdServerURL returns the base URL (scheme://host:port) of the sandbox's envd // HTTP server. A non-empty internalConfig.envdServerURLOverride redirects it // (test-only; production always uses the slot IP and the default envd port). @@ -256,7 +410,7 @@ func (s *Sandbox) convertMounts(mounts []VolumeMountConfig) []envd.VolumeMount { return results } -func (s *Sandbox) initEnvd(ctx context.Context, startType StartType) (e error) { +func (s *Sandbox) initEnvd(ctx context.Context, startType StartType, recordMetrics bool) (e error) { ctx, span := tracer.Start(ctx, "envd-init", trace.WithAttributes(telemetry.WithEnvdVersion(s.Config.Envd.Version))) defer func() { if e != nil { @@ -293,20 +447,42 @@ func (s *Sandbox) initEnvd(ctx context.Context, startType StartType) (e error) { ) exit := classifyEnvdInitExit(err) - envdInitCalls.Add(ctx, count, metric.WithAttributes(callAttributes(exit)...)) + // Count only on the first WaitForEnvd (the real start); a later re-check + // on the same handler (post-upgrade readiness, template-build swap) must + // not double-count the resume KPI. + if recordMetrics { + envdInitCalls.Add(ctx, count, metric.WithAttributes(callAttributes(exit)...)) + } return fmt.Errorf("failed to init envd: %w", err) } - if count > 1 { + if recordMetrics && count > 1 { // Retried attempts were transient per-request failures that preceded the success. envdInitCalls.Add(ctx, count-1, metric.WithAttributes(callAttributes(envdInitExitTransient)...)) } - // Track successful envd init - envdInitCalls.Add(ctx, 1, metric.WithAttributes(callAttributes(envdInitExitSuccess)...)) + // Track successful envd init (first WaitForEnvd only — see recordMetrics). + if recordMetrics { + envdInitCalls.Add(ctx, 1, metric.WithAttributes(callAttributes(envdInitExitSuccess)...)) + } defer response.Body.Close() + // Capture the version the running envd reports (X-Envd-Version). This rides + // on the /init call the resume path already makes — before and after an + // upgrade — so the upgrade trigger can decide/label/confirm against the live + // version with no extra round-trip. + if v := response.Header.Get("X-Envd-Version"); v != "" { + s.setLiveEnvdVersion(v) + } + // Alongside the version, capture the handover outcome the new envd advertises + // after a live upgrade (X-Envd-Handover) so the trigger can record it. + if h := response.Header.Get("X-Envd-Handover"); h != "" { + var hr EnvdHandoverResult + if err := json.Unmarshal([]byte(h), &hr); err == nil { + s.setHandoverResult(&hr) + } + } body, err := io.ReadAll(response.Body) if err != nil { return fmt.Errorf("failed to read envd init response body: %w", err) diff --git a/packages/orchestrator/pkg/sandbox/envd_upgrade_test.go b/packages/orchestrator/pkg/sandbox/envd_upgrade_test.go new file mode 100644 index 0000000000..c8c8886483 --- /dev/null +++ b/packages/orchestrator/pkg/sandbox/envd_upgrade_test.go @@ -0,0 +1,44 @@ +//go:build linux + +package sandbox + +import ( + "context" + "errors" + "io" + "net" + "syscall" + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestIsUpgradeDeliveryFailure guards the distinction CallEnvdUpgrade relies on: +// a request that never reached a running envd (so no upgrade happened) is a +// failure, while the expected post-send connection drop when envd execs +// mid-response is a success. Misclassifying the former as success would record a +// false upgrade in the rollout metrics. +func TestIsUpgradeDeliveryFailure(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + err error + want bool + }{ + {"connection refused -> failure", syscall.ECONNREFUSED, true}, + {"dialing connection refused -> failure", &net.OpError{Op: "dial", Err: syscall.ECONNREFUSED}, true}, + {"deadline exceeded -> not a delivery failure (ambiguous; confirmed by version)", context.DeadlineExceeded, false}, + {"dial timeout -> failure", &net.OpError{Op: "dial", Err: errors.New("i/o timeout")}, true}, + {"post-send reset -> success (envd exec'd)", &net.OpError{Op: "read", Err: syscall.ECONNRESET}, false}, + {"EOF after body -> success (envd exec'd)", io.EOF, false}, + {"generic error -> success (assume exec'd)", errors.New("unexpected"), false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.want, isUpgradeDeliveryFailure(tt.err)) + }) + } +} diff --git a/packages/orchestrator/pkg/sandbox/reboot.go b/packages/orchestrator/pkg/sandbox/reboot.go index ae32b92843..d9c7d0f747 100644 --- a/packages/orchestrator/pkg/sandbox/reboot.go +++ b/packages/orchestrator/pkg/sandbox/reboot.go @@ -45,6 +45,7 @@ func (f *Factory) RebootSandbox( runtime RuntimeMetadata, endAt time.Time, apiConfigToStore *orchestrator.SandboxConfig, + deferMarkRunning bool, procOpts ...func(*fc.ProcessOptions), ) (*Sandbox, error) { ctx, span := tracer.Start(ctx, "reboot sandbox") @@ -157,9 +158,15 @@ func (f *Factory) RebootSandbox( return nil, errors.Join(fmt.Errorf("wait for envd after reboot: %w", err), closeErr) } - f.Sandboxes.MarkRunning(ctx, sbx) + // deferMarkRunning: the caller promotes the sandbox to live itself after a + // post-resume step (the resume-time envd live-upgrade's post-/init), so it is + // not routable during the upgrade's pre-init auth window. Mirrors the resume + // path's WithDeferredLiveRegistration. + if !deferMarkRunning { + f.Sandboxes.MarkRunning(ctx, sbx) - go sbx.Checks.Start(context.WithoutCancel(ctx)) + go sbx.Checks.Start(context.WithoutCancel(ctx)) + } return sbx, nil } diff --git a/packages/orchestrator/pkg/sandbox/sandbox.go b/packages/orchestrator/pkg/sandbox/sandbox.go index c674ae175e..af8467305a 100644 --- a/packages/orchestrator/pkg/sandbox/sandbox.go +++ b/packages/orchestrator/pkg/sandbox/sandbox.go @@ -8,6 +8,7 @@ import ( "fmt" "net/http" "sync" + "sync/atomic" "time" "github.com/google/uuid" @@ -289,6 +290,16 @@ type Sandbox struct { Template template.Template + // liveEnvdVersion is the version the running envd reported on its most recent + // /init response (X-Envd-Version). Empty until first captured. The resume-time + // upgrade trigger uses this ground truth — not the template built-with, which + // never changes across live upgrades — to decide, label, and confirm upgrades. + liveEnvdVersion atomic.Pointer[string] + // handoverResult is the live-upgrade handover outcome the running envd + // reported on its most recent /init (X-Envd-Handover header), or nil if the + // running envd did not boot from a handover. + handoverResult atomic.Pointer[EnvdHandoverResult] + Checks *Checks hostStatsCollector *HostStatsCollector @@ -303,13 +314,16 @@ type Sandbox struct { stop utils.Lazy[error] - // startupStatsOnce guards the orchestrator.sandbox.uffd.startup.* recording - // so it fires only on the first WaitForEnvd — the actual sandbox start. - // ServeStats() is lifetime-cumulative on the UFFD handler, so a later - // WaitForEnvd on the same handler (e.g. the envd-binary swap + restart in a - // template build) would otherwise emit a sample inflated with post-startup - // faults rather than that init's working set. - startupStatsOnce sync.Once + // startupRecorded guards ALL first-WaitForEnvd recording — the envd-init + // duration + uffd.startup.* histograms, the envd-init call counter (in + // initEnvd), and SetStartedAt — so they fire only on the actual sandbox + // start. A later WaitForEnvd on the same handler (the post-upgrade readiness + // re-check, or the envd-binary swap + restart in a template build) re-runs + // /init to re-capture state but must not re-record these: ServeStats() is + // lifetime-cumulative, the duration/counter would double-count the resume + // KPI, and SetStartedAt would overwrite the real start with a later time. + // CAS'd true by the first WaitForEnvd; later calls see false and skip. + startupRecorded atomic.Bool // skipStartupMetrics suppresses the per-start KPI histograms (envd-init // duration, uffd startup pages/source-pages/bytes) for a throwaway resume, @@ -724,6 +738,14 @@ type resumeOptions struct { // (not addressable, not counted, no health checks) for throwaways the caller // reaps itself. skipLiveRegistration bool + // deferMarkRunning skips only MarkRunning and health-check startup inside + // ResumeSandbox, leaving everything else (metrics, host stats, network + // assignment) intact, so the caller can promote the sandbox to live itself + // once a post-resume step has completed. Used by the resume-time envd + // live-upgrade path so the sandbox is not routable during the sub-second + // pre-/init auth window after the upgrade re-exec. Unlike skipLiveRegistration + // the sandbox IS meant to go live — just later. + deferMarkRunning bool } // ResumeOption customizes a ResumeSandbox call. @@ -749,6 +771,18 @@ func WithoutLiveRegistration() ResumeOption { return func(o *resumeOptions) { o.skipLiveRegistration = true } } +// WithDeferredLiveRegistration resumes the sandbox but defers MarkRunning and +// health-check startup to the caller, so the sandbox is not addressable via the +// sandbox map until the caller promotes it with Sandboxes.MarkRunning + +// Checks.Start. Used by the resume-time envd live-upgrade path to keep the +// sandbox out of routing during the sub-second pre-/init window after the +// upgrade re-exec. Everything else (metrics, host stats, network assignment) +// runs as normal, so — unlike WithoutLiveRegistration — the sandbox is a real, +// soon-to-be-live sandbox, not a throwaway. +func WithDeferredLiveRegistration() ResumeOption { + return func(o *resumeOptions) { o.deferMarkRunning = true } +} + // ThrowawayResumeOptions are the resume options for a caller-reaped throwaway // (e.g. the pause-resume prefetch harvest): network-isolated and kept out of the // live registry. It is the single source of truth for that option set so callers @@ -1148,13 +1182,13 @@ func (f *Factory) ResumeSandbox( // live sandbox: keep it out of the live registry so it is not addressable and // does not inflate the node's reported allocation or emit per-sandbox metrics, // and skip health checks it would never need. - if !ropts.skipLiveRegistration { + if !ropts.skipLiveRegistration && !ropts.deferMarkRunning { f.Sandboxes.MarkRunning(ctx, sbx) } telemetry.ReportEvent(execCtx, "envd initialized") - if !ropts.skipLiveRegistration { + if !ropts.skipLiveRegistration && !ropts.deferMarkRunning { go sbx.Checks.Start(execCtx) } @@ -1929,7 +1963,17 @@ func (s *Sandbox) WaitForEnvd( ctx, span := tracer.Start(ctx, "sandbox-wait-for-start") defer span.End() + // Record the per-start KPIs, the envd-init counter, and StartedAt only on the + // FIRST WaitForEnvd for this handler (see startupRecorded). A later call — the + // post-upgrade readiness re-check, or the envd-binary swap during a template + // build — re-runs /init to re-capture state but must not re-record. + firstStart := s.startupRecorded.CompareAndSwap(false, true) + defer func() { + if !firstStart { + return + } + // A throwaway (the pause-resume prefetch harvest) is warm by construction // and must not pollute the customer resume KPIs (envd-init duration, // startup pages/source-pages — the consume-side payoff signals) or even be @@ -1947,24 +1991,18 @@ func (s *Sandbox) WaitForEnvd( attribute.String("exit_type", string(classifyEnvdInitExit(e))), )) - // Record the demand-fault working set the guest needed to reach this - // point. Only on the first WaitForEnvd: it is the actual start, and + // The demand-fault working set the guest needed to reach this point. // ServeStats() is cumulative since resume, so at this instant it equals - // the startup counts. A later WaitForEnvd on the same handler (e.g. the - // envd-binary swap + restart during a template build) would otherwise - // re-report a cumulative total polluted with intervening faults. - // Recorded for both outcomes (success label) so slow/failed starts can - // be correlated with page volume. - s.startupStatsOnce.Do(func() { - stats := s.memory.ServeStats() - startupAttrs := metric.WithAttributes( - attribute.String("start_type", string(startType)), - attribute.Bool("success", e == nil), - ) - uffdStartupPagesHistogram.Record(ctx, stats.Pages, startupAttrs) - uffdStartupSourcePagesHistogram.Record(ctx, stats.SourcePages, startupAttrs) - uffdStartupBytesHistogram.Record(ctx, stats.Bytes, startupAttrs) - }) + // the startup counts. Recorded for both outcomes (success label) so + // slow/failed starts can be correlated with page volume. + stats := s.memory.ServeStats() + startupAttrs := metric.WithAttributes( + attribute.String("start_type", string(startType)), + attribute.Bool("success", e == nil), + ) + uffdStartupPagesHistogram.Record(ctx, stats.Pages, startupAttrs) + uffdStartupSourcePagesHistogram.Record(ctx, stats.SourcePages, startupAttrs) + uffdStartupBytesHistogram.Record(ctx, stats.Bytes, startupAttrs) } if e != nil { @@ -1991,7 +2029,7 @@ func (s *Sandbox) WaitForEnvd( } }() - if err := s.initEnvd(ctx, startType); err != nil { + if err := s.initEnvd(ctx, startType, firstStart); err != nil { return fmt.Errorf("failed to init new envd: %w", err) } diff --git a/packages/orchestrator/pkg/server/main.go b/packages/orchestrator/pkg/server/main.go index 22670a6f64..f1df92018d 100644 --- a/packages/orchestrator/pkg/server/main.go +++ b/packages/orchestrator/pkg/server/main.go @@ -82,6 +82,10 @@ type Server struct { sandboxPauseDuration metric.Int64Histogram sandboxKilledCounter metric.Int64Counter uploadFailedCounter metric.Int64Counter + envdUpgradeAttempts metric.Int64Counter + envdUpgradeGated metric.Int64Counter + envdUpgradeHandover metric.Int64Counter + envdUpgradeDuration metric.Int64Histogram // uploadsWG tracks in-flight async snapshot uploads so a graceful shutdown // can wait for them to finish instead of dropping them. uploadsInFlight is @@ -165,6 +169,30 @@ func New(ctx context.Context, cfg ServiceConfig) (*Server, error) { } server.uploadFailedCounter = uploadFailedCounter + envdUpgradeAttempts, err := telemetry.GetCounter(meter, telemetry.OrchestratorEnvdUpgradeAttempts) + if err != nil { + return nil, fmt.Errorf("failed to register envd upgrade attempts counter: %w", err) + } + server.envdUpgradeAttempts = envdUpgradeAttempts + + envdUpgradeGated, err := telemetry.GetCounter(meter, telemetry.OrchestratorEnvdUpgradeGated) + if err != nil { + return nil, fmt.Errorf("failed to register envd upgrade gated counter: %w", err) + } + server.envdUpgradeGated = envdUpgradeGated + + envdUpgradeHandover, err := telemetry.GetCounter(meter, telemetry.OrchestratorEnvdUpgradeHandover) + if err != nil { + return nil, fmt.Errorf("failed to register envd upgrade handover counter: %w", err) + } + server.envdUpgradeHandover = envdUpgradeHandover + + envdUpgradeDuration, err := telemetry.GetHistogram(meter, telemetry.OrchestratorEnvdUpgradeDurationName) + if err != nil { + return nil, fmt.Errorf("failed to register envd upgrade duration histogram: %w", err) + } + server.envdUpgradeDuration = envdUpgradeDuration + _, err = telemetry.GetObservableUpDownCounter(meter, telemetry.OrchestratorSandboxCountMeterName, func(_ context.Context, observer metric.Int64Observer) error { observer.Observe(int64(server.sandboxFactory.Sandboxes.Count())) diff --git a/packages/orchestrator/pkg/server/sandboxes.go b/packages/orchestrator/pkg/server/sandboxes.go index f9ceec21ae..ecb9994f0a 100644 --- a/packages/orchestrator/pkg/server/sandboxes.go +++ b/packages/orchestrator/pkg/server/sandboxes.go @@ -27,6 +27,7 @@ import ( "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox" "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/fc" sbxtemplate "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/template" + buildenvd "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/envd" "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/metadata" "github.com/e2b-dev/infra/packages/shared/pkg/events" "github.com/e2b-dev/infra/packages/shared/pkg/featureflags" @@ -89,12 +90,16 @@ func (s *Server) Create(ctx context.Context, req *orchestrator.SandboxCreateRequ // memory resume; resume=true,fs_only=true → filesystem-only reboot. var fsOnly bool createStart := time.Now() + // Set by maybeUpgradeEnvd below; labels the resume-latency histogram so the + // treated (upgraded) vs untreated cohorts can be compared during the rollout. + var envdUpgraded bool defer func() { s.sandboxCreateDuration.Record(ctx, time.Since(createStart).Milliseconds(), metric.WithAttributes( attribute.Bool("sandbox.resume", isResume), attribute.Bool("fs_only", fsOnly), attribute.Bool("success", createErr == nil), + attribute.Bool("envd.upgraded", envdUpgraded), ), ) }() @@ -240,6 +245,10 @@ func (s *Server) Create(ctx context.Context, req *orchestrator.SandboxCreateRequ runtime, req.GetEndTime().AsTime(), req.GetSandbox(), + // Defer routing until after the resume-time envd upgrade's + // post-/init, so the sandbox isn't reachable during its pre-init + // auth window. Promoted below via markSandboxLive. + true, ) } else { sbx, err = s.sandboxFactory.ResumeSandbox( @@ -250,6 +259,9 @@ func (s *Server) Create(ctx context.Context, req *orchestrator.SandboxCreateRequ req.GetStartTime().AsTime(), req.GetEndTime().AsTime(), req.GetSandbox(), + // Defer routing until after the resume-time envd upgrade's + // post-/init (see markSandboxLive below). + sandbox.WithDeferredLiveRegistration(), ) } if err != nil { @@ -276,6 +288,32 @@ func (s *Server) Create(ctx context.Context, req *orchestrator.SandboxCreateRequ s.setupSandboxLifecycle(ctx, sbx) + // Resume-time envd live-upgrade. The API /resume maps to Create + // with snapshot=true, so this is the real resume path. Flag-driven, + // best-effort + recover-wrapped (see maybeUpgradeEnvd) so it can't disrupt + // resume. ctx already carries the LD context (envd-version/team/template). + if req.GetSandbox().GetSnapshot() { + var upErr error + envdUpgraded, upErr = s.maybeUpgradeEnvd(ctx, sbx) + if upErr != nil { + // Only an unrecoverable post-execve failure (new envd left + // uninitialized) returns an error; fail the resume rather than hand + // back a bricked sandbox. MarkRunning is deferred until markSandboxLive + // below, so the sandbox is not yet in the live registry — MarkStopping + // is a no-op here and stopSandboxAsync does the physical teardown. + s.sandboxFactory.Sandboxes.MarkStopping(ctx, sbx.Runtime.SandboxID, sbx.LifecycleID) + s.stopSandboxAsync(context.WithoutCancel(ctx), sbx) + + return nil, upErr + } + } + + // Promote to the live registry only now — after any resume-time envd upgrade + // has run its post-/init and restored the access token — so the sandbox is + // never routable during the upgrade's sub-second pre-init auth window. Both + // the resume and reboot paths above defer this. + s.markSandboxLive(ctx, sbx) + // Read scheduling metadata after the sandbox resumed so the template's // memfile/rootfs devices (and their headers) are resolved. var schedulingMetadata *orchestrator.SchedulingMetadata @@ -860,6 +898,8 @@ func (s *Server) Checkpoint(ctx context.Context, in *orchestrator.SandboxCheckpo sbx.GetStartedAt(), sbx.GetEndAt(), sbx.APIStoredConfig, + // Defer routing until after the upgrade's post-/init (markSandboxLive below). + sandbox.WithDeferredLiveRegistration(), ) if err != nil { telemetry.ReportCriticalError(ctx, "error resuming sandbox after checkpoint", err, telemetry.WithSandboxID(in.GetSandboxId())) @@ -876,6 +916,25 @@ func (s *Server) Checkpoint(ctx context.Context, in *orchestrator.SandboxCheckpo // Setup lifecycle for the resumed sandbox s.setupSandboxLifecycle(ctx, resumedSbx) + // resume-time envd live-upgrade. Best-effort and tightly gated so + // it can never disrupt the universal resume path — except an unrecoverable + // post-execve failure (new envd left uninitialized), which fails the + // checkpoint rather than leave a bricked sandbox. + if _, upErr := s.maybeUpgradeEnvd(ctx, resumedSbx); upErr != nil { + // Bricked past the execve — tear the resumed sandbox down. MarkRunning is + // deferred until markSandboxLive below, so the sandbox is not yet in the + // live registry: MarkStopping is a no-op and stopSandboxAsync does the + // physical teardown. + s.sandboxFactory.Sandboxes.MarkStopping(ctx, resumedSbx.Runtime.SandboxID, resumedSbx.LifecycleID) + s.stopSandboxAsync(context.WithoutCancel(ctx), resumedSbx) + + return nil, upErr + } + + // Promote to the live registry now that any resume-time upgrade's post-/init + // has restored auth — the sandbox was resumed with routing deferred. + s.markSandboxLive(ctx, resumedSbx) + // Embed prefetch data into the metadata so it's uploaded with the snapshot files in a single pass. if prefetchErr == nil { prefetchMapping := metadata.PrefetchEntriesToMapping(slices.Collect(maps.Values(prefetchData.BlockEntries)), prefetchData.BlockSize) @@ -1116,6 +1175,18 @@ func (s *Server) uploadSnapshotAsync(ctx context.Context, sbx *sandbox.Sandbox, } // setupSandboxLifecycle sets up the cleanup goroutine for a sandbox. +// markSandboxLive promotes a resumed sandbox to the live registry and starts its +// health checks. It is the counterpart to WithDeferredLiveRegistration (resume) +// and RebootSandbox's deferMarkRunning: callers on the resume-time upgrade path +// resume with routing deferred and call this only after maybeUpgradeEnvd has +// completed its post-/init, so the sandbox never appears in routing during the +// upgrade's pre-init auth window. Idempotent — MarkRunning is InsertIfAbsent. +func (s *Server) markSandboxLive(ctx context.Context, sbx *sandbox.Sandbox) { + s.sandboxFactory.Sandboxes.MarkRunning(ctx, sbx) + + go sbx.Checks.Start(context.WithoutCancel(ctx)) +} + func (s *Server) setupSandboxLifecycle(ctx context.Context, sbx *sandbox.Sandbox) { go func() { ctx, childSpan := tracer.Start(context.WithoutCancel(ctx), "stop sandbox-lifecycle", trace.WithNewRoot()) @@ -1176,3 +1247,212 @@ func (s *Server) publishSandboxEvent(ctx context.Context, sbx *sandbox.Sandbox, }, ) } + +// maybeUpgradeEnvd is the orchestrator's resume-time envd live-upgrade trigger +// . At resume it asks EnvdUpgradeTargetFlag whether the sandbox's envd +// should be swapped for a newer node-local build and, if so, delivers that +// binary into the guest and triggers envd's same-PID self-upgrade so the +// workload is preserved. +// +// Fully best-effort: recover()-wrapped, bounded timeouts, and every failure is +// logged-and-swallowed so it can never disrupt the universal resume path. The +// flag fallback is "off", so with no LaunchDarkly (e.g. dev) this is inert. +// +// It returns whether an upgrade actually completed (for the resume-latency +// label) and emits the rollout metrics: orchestrator.envd.upgrade.attempts +// {result,from_version,to_version}, .duration{result}, and .gated{reason}. +func (s *Server) maybeUpgradeEnvd(ctx context.Context, sbx *sandbox.Sandbox) (upgraded bool, fatalErr error) { + // Decide, label, and confirm against the version the running envd actually + // reports (captured on the resume-path /init) — not the template built-with, + // which never changes across live upgrades and would otherwise re-trigger the + // handover on every resume. Fall back to built-with only when no live version + // was captured (an envd too old to report it — which the gate then rejects). + from := sbx.LiveEnvdVersion() + if from == "" { + from = sbx.Config.Envd.Version + } + + var ( + attempted bool + toVersion string + result = "success" + start = time.Now() + ) + defer func() { + if r := recover(); r != nil { + sbxlogger.I(sbx).Error(ctx, "envd auto-upgrade panic (recovered)", zap.Any("panic", r)) + result = "panic" + attempted = true + upgraded = false + } + if attempted { + s.envdUpgradeAttempts.Add(ctx, 1, metric.WithAttributes( + attribute.String("result", result), + attribute.String("from_version", from), + attribute.String("to_version", toVersion), + )) + s.envdUpgradeDuration.Record(ctx, time.Since(start).Milliseconds(), + metric.WithAttributes(attribute.String("result", result))) + } + }() + + // Flag-driven resolver, keyed on the LIVE version. "" path => no upgrade, with + // a reason: off / same_version are the expected per-resume no-op (a re-resume + // of an already-upgraded sandbox), deliberately not counted as noise; the rest + // (not_staged — e.g. a bad SHA / rubbish flag value, getversion_failed, + // downgrade) are misconfigurations worth a counted, logged signal so a broken + // target is distinguishable from "already current". + path, tv, reason := featureflags.ResolveEnvdUpgrade(ctx, s.featureFlags, from, s.config.HostEnvdPath, buildenvd.GetEnvdVersion) + toVersion = tv + if path == "" { + switch reason { + case "off", "same_version": + // expected no-op — not counted + default: + s.envdUpgradeGated.Add(ctx, 1, metric.WithAttributes(attribute.String("reason", reason))) + sbxlogger.I(sbx).Warn(ctx, "envd auto-upgrade: target not resolved", + zap.String("reason", reason), zap.String("from", from)) + } + + return false, nil + } + + // The *running* envd must already have the /upgrade endpoint + handover code, + // else the delivery POST would 404 or hang. Count the skip so a ramp can see + // the gated population. + if ok, err := utils.IsGTEVersion(from, utils.MinEnvdVersionForUpgrade); err != nil || !ok { + s.envdUpgradeGated.Add(ctx, 1, metric.WithAttributes(attribute.String("reason", "old_envd"))) + + return false, nil + } + + attempted = true + start = time.Now() + + upCtx, span := tracer.Start(ctx, "envd-upgrade", trace.WithAttributes( + attribute.String("envd.from_version", from), + attribute.String("envd.to_version", toVersion), + )) + defer span.End() + + // Delivery and readiness get INDEPENDENT budgets, not a shared cap: a slow + // binary upload must not eat into the time envd has to re-adopt and answer + // /init (which would mislabel a would-be success as delivery_failed/not_ready). + const ( + deliverTimeout = 30 * time.Second + readyTimeout = 15 * time.Second + ) + + sbxlogger.I(sbx).Info(upCtx, "envd auto-upgrade: delivering+triggering", + zap.String("from", from), zap.String("to", toVersion), zap.String("path", path)) + + // Stream the new binary over the authenticated /upgrade endpoint (delivery + // + trigger in one call) and let envd same-PID re-exec into it. NB: not the + // build-time /files CopyFile path — a live, post-/init sandbox rejects that. + execConfirmed, err := sbx.CallEnvdUpgrade(upCtx, path, "/usr/bin/envd.next", deliverTimeout) + if err != nil { + result = "delivery_failed" + span.RecordError(err) + sbxlogger.I(sbx).Error(upCtx, "envd auto-upgrade: trigger failed", zap.Error(err)) + + // Delivery/trigger failed BEFORE execve, so the old envd is still running + // and serving — best-effort: let the resume proceed on the old version. + return false, nil + } + // WaitForEnvd re-runs /init, which re-captures the now-running version. + // + // Detach from upCtx (WithoutCancel) but keep the bounded readyTimeout: the + // exec is a fait accompli by this point, so if the parent resume is cancelled + // in this window we must still drive /init to completion. Running it on the + // cancellable parent would let an ambiguous cancellation skip /init yet still + // return recoverably, publishing a promoted-but-uninitialized sandbox (the + // exec'd envd never gets its auth/env restored). Version confirmation below + // then correctly distinguishes an actual exec from an untouched old envd. + readyCtx := context.WithoutCancel(upCtx) + if err := sbx.WaitForEnvd(readyCtx, sandbox.StartTypeResume, readyTimeout); err != nil { + result = "not_ready" + span.RecordError(err) + sbxlogger.I(sbx).Error(upCtx, "envd auto-upgrade: envd not ready after upgrade", zap.Error(err)) + + if !execConfirmed { + // The delivery deadline fired without a confirmed exec — envd may + // still be mid-handover on the OLD binary, which will thaw and keep + // serving. Don't tear down a recoverable sandbox: best-effort, let the + // resume proceed (a false-positive brick is worse than a missed + // upgrade). + return false, nil + } + + // The exec is confirmed (the old envd is gone) but the new envd never + // completed /init — its access token is unrestored and the + // WithAuthorization handover gate fail-closes every RPC. It can't be made + // both usable and secure without /init, so fail the resume (unrecoverable) + // rather than return a live-but-permanently-bricked sandbox. + return false, fmt.Errorf("envd live-upgrade left the sandbox uninitialized (post-upgrade /init failed): %w", err) + } + + // Confirm by ground truth: the running envd must now report the target + // version. This is the arbiter — a transport quirk (e.g. a slow exec that + // never answered) can't mislabel a non-swap as success. + if now := sbx.LiveEnvdVersion(); now != toVersion { + result = "version_mismatch" + span.SetAttributes(attribute.String("envd.observed_version", now)) + sbxlogger.I(sbx).Error(upCtx, "envd auto-upgrade: version did not flip", + zap.String("observed", now), zap.String("expected", toVersion)) + + // /init succeeded (envd is initialized and serving), it just isn't the + // expected version — usable, not bricked, so don't fail the resume. + return false, nil + } + + // The version flipped, but if the in-guest handover itself failed post-exec + // (ResumeFromHandover errored/panicked, so the workload was never re-adopted — + // orphaned and unreaped), the old envd is gone and the sandbox is broken. Fail + // the resume so the caller tears it down rather than hand back a live-but- + // broken sandbox that merely reports the target version. + if h := sbx.HandoverResult(); h != nil && h.Failed { + result = "handover_failed" + span.SetAttributes(attribute.Bool("envd.handover.failed", true)) + sbxlogger.I(sbx).Error(upCtx, "envd auto-upgrade: in-guest handover failed post-exec; workload not re-adopted") + + return false, errors.New("envd live-upgrade handover failed post-exec (workload not re-adopted)") + } + + span.SetAttributes(attribute.Bool("envd.upgraded", true)) + + // Record the handover outcome the new envd reported on /init (fleet + // visibility into what it re-adopted). Per item (proc|retained|watcher) as + // ok/failed so failed/(ok+failed) is the handover error rate — a non-zero + // failed means the swap dropped or degraded something (a lost watch, an + // unrecoverable exit code, a bad process config), which envd otherwise only + // logs in-guest. + if h := sbx.HandoverResult(); h != nil { + recordItem := func(item string, ok, failed int) { + s.envdUpgradeHandover.Add(upCtx, int64(ok), metric.WithAttributes( + attribute.String("item", item), attribute.String("result", "ok"))) + s.envdUpgradeHandover.Add(upCtx, int64(failed), metric.WithAttributes( + attribute.String("item", item), attribute.String("result", "failed"))) + } + recordItem("proc", h.Procs-h.ProcsFailed, h.ProcsFailed) + recordItem("retained", h.Retained-h.RetainedFailed, h.RetainedFailed) + recordItem("watcher", h.Watchers-h.WatchersFailed, h.WatchersFailed) + + span.SetAttributes( + attribute.Int("envd.handover.procs", h.Procs), + attribute.Int("envd.handover.procs_failed", h.ProcsFailed), + attribute.Int("envd.handover.retained", h.Retained), + attribute.Int("envd.handover.retained_failed", h.RetainedFailed), + attribute.Int("envd.handover.watchers", h.Watchers), + attribute.Int("envd.handover.watchers_failed", h.WatchersFailed), + ) + sbxlogger.I(sbx).Info(upCtx, "envd auto-upgrade: complete", + zap.String("to", toVersion), + zap.Int("procs", h.Procs), zap.Int("procs_failed", h.ProcsFailed), + zap.Int("retained", h.Retained), zap.Int("retained_failed", h.RetainedFailed), + zap.Int("watchers", h.Watchers), zap.Int("watchers_failed", h.WatchersFailed)) + } else { + sbxlogger.I(sbx).Info(upCtx, "envd auto-upgrade: complete", zap.String("to", toVersion)) + } + + return true, nil +} diff --git a/packages/shared/pkg/featureflags/envd_upgrade_resolver_test.go b/packages/shared/pkg/featureflags/envd_upgrade_resolver_test.go new file mode 100644 index 0000000000..f4afc7c923 --- /dev/null +++ b/packages/shared/pkg/featureflags/envd_upgrade_resolver_test.go @@ -0,0 +1,91 @@ +package featureflags + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestResolveEnvdUpgradePath exercises the resume-time upgrade decision without a +// LaunchDarkly client: the flag value is passed directly, binaries are real temp +// files (for the os.Stat check), and version resolution is injected. +func TestResolveEnvdUpgradePath(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + promoted := filepath.Join(dir, "envd") // HOST_ENVD_PATH + shaBin := filepath.Join(dir, "envd.1f95888") // versioned binary + require.NoError(t, os.WriteFile(promoted, []byte("x"), 0o755)) + require.NoError(t, os.WriteFile(shaBin, []byte("x"), 0o755)) + + // Version map: promoted binary is 0.6.12, the SHA binary is 0.7.0. An + // unknown path returns an error (unreadable binary). + versions := map[string]string{promoted: "0.6.12", shaBin: "0.7.0"} + getVersion := func(_ context.Context, path string) (string, error) { + v, ok := versions[path] + if !ok { + return "", fmt.Errorf("unknown binary %s", path) + } + + return v, nil + } + + tests := []struct { + name string + target string // flag value + builtWith string + wantPath string + wantVersion string + wantReason string + }{ + {"off returns empty", "off", "0.6.11", "", "", "off"}, + {"empty (unset) returns empty", "", "0.6.11", "", "", "off"}, + {"promoted, newer -> promoted path", "promoted", "0.6.11", promoted, "0.6.12", ""}, + {"promoted, same version -> empty (idempotent)", "promoted", "0.6.12", "", "", "same_version"}, + {"sha, exists and newer -> sha path", "1f95888", "0.6.11", shaBin, "0.7.0", ""}, + {"sha, same version -> empty", "1f95888", "0.7.0", "", "", "same_version"}, + {"sha, missing binary -> not_staged", "deadbee", "0.6.11", "", "", "not_staged"}, + // Upgrade-only: an older staged target must not trigger a downgrade. + {"sha, older target -> downgrade refused", "1f95888", "0.8.0", "", "", "downgrade"}, + {"promoted, older target -> downgrade refused", "promoted", "0.7.0", "", "", "downgrade"}, + // A target that isn't a bare identifier must be rejected before it is + // joined into a path (no traversal out of the staging dir / arbitrary exec). + {"target with .. -> invalid_target", "../../etc/passwd", "0.6.11", "", "", "invalid_target"}, + {"target with slash -> invalid_target", "sub/envd", "0.6.11", "", "", "invalid_target"}, + {"target with dot -> invalid_target", "1f95888.", "0.6.11", "", "", "invalid_target"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + gotPath, gotVersion, gotReason := resolveEnvdUpgradePath(t.Context(), tt.target, tt.builtWith, promoted, getVersion) + assert.Equal(t, tt.wantPath, gotPath) + assert.Equal(t, tt.wantVersion, gotVersion) + assert.Equal(t, tt.wantReason, gotReason) + }) + } +} + +// TestResolveEnvdUpgradePath_VersionError verifies an unreadable target is +// treated as "no upgrade" rather than propagating an error into the resume path. +func TestResolveEnvdUpgradePath_VersionError(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + promoted := filepath.Join(dir, "envd") + require.NoError(t, os.WriteFile(promoted, []byte("x"), 0o755)) + + getVersion := func(_ context.Context, _ string) (string, error) { + return "", assert.AnError + } + + gotPath, gotVersion, gotReason := resolveEnvdUpgradePath(t.Context(), "promoted", "0.6.11", promoted, getVersion) + assert.Empty(t, gotPath) + assert.Empty(t, gotVersion) + assert.Equal(t, "getversion_failed", gotReason) +} diff --git a/packages/shared/pkg/featureflags/flags.go b/packages/shared/pkg/featureflags/flags.go index 2362778d4a..a64e7f8513 100644 --- a/packages/shared/pkg/featureflags/flags.go +++ b/packages/shared/pkg/featureflags/flags.go @@ -6,6 +6,8 @@ import ( "net" "net/url" "os" + "path/filepath" + "regexp" "strings" "time" @@ -16,6 +18,7 @@ import ( "go.opentelemetry.io/otel/metric" "github.com/e2b-dev/infra/packages/shared/pkg/env" + "github.com/e2b-dev/infra/packages/shared/pkg/utils" ) // kinds @@ -515,9 +518,22 @@ var FirecrackerVersionMap = map[string]string{ // BuildIoEngine Sync is used by default as there seems to be a bad interaction between Async and a lot of io operations. var ( - BuildFirecrackerVersion = NewStringFlag("build-firecracker-version", env.GetEnv("DEFAULT_FIRECRACKER_VERSION", DefaultFirecrackerVersion)) - BuildKernelVersion = NewStringFlag("build-kernel-version", env.GetEnv("DEFAULT_KERNEL_VERSION", DefaultKernelVersion)) - BuildIoEngine = NewStringFlag("build-io-engine", "Sync") + BuildFirecrackerVersion = NewStringFlag("build-firecracker-version", env.GetEnv("DEFAULT_FIRECRACKER_VERSION", DefaultFirecrackerVersion)) + BuildKernelVersion = NewStringFlag("build-kernel-version", env.GetEnv("DEFAULT_KERNEL_VERSION", DefaultKernelVersion)) + BuildIoEngine = NewStringFlag("build-io-engine", "Sync") + + // EnvdUpgradeTargetFlag drives the resume-time envd live-upgrade. + // Multivariate string: + // "off" (fallback) — no upgrade; dev has no LD so this is inert & safe. + // "promoted" — track the node-local promoted envd (HOST_ENVD_PATH); upgrade + // whenever it differs from the sandbox's built-with version + // (no per-publish flag edits needed). + // "" — pin a specific versioned binary (/fc-envd/envd.). + // The resume-site LD context carries envd-version/team/template, so %-ramp + // and cohort canaries come for free. The fallback is env-overridable + // (ENVD_UPGRADE_TARGET) so it can be exercised where there is no LD (dev), + // mirroring build-firecracker-version's DEFAULT_FIRECRACKER_VERSION. + EnvdUpgradeTargetFlag = NewStringFlag("envd-upgrade-target", env.GetEnv("ENVD_UPGRADE_TARGET", "off")) DefaultPersistentVolumeType = NewStringFlag("default-persistent-volume-type", "") BuildNodeInfo = NewJSONFlag("preferred-build-node", ldvalue.Null()) FirecrackerVersions = NewJSONFlag("firecracker-versions", ldvalue.FromJSONMarshal(FirecrackerVersionMap)) @@ -832,6 +848,95 @@ func ResolveFirecrackerVersion(ctx context.Context, ff *Client, buildVersion str return buildVersion } +// ResolveEnvdUpgrade decides whether a resuming sandbox's envd should be swapped +// for a newer node-local build, per EnvdUpgradeTargetFlag, and returns the local +// path of the target binary ("" = no upgrade). It is the resume-time analog of +// ResolveFirecrackerVersion. +// +// hostEnvdPath is the promoted binary (cfg HostEnvdPath, e.g. /fc-envd/envd); +// versioned binaries live beside it as envd.. getVersion resolves a +// binary's baked version (orchestrator's build/core/envd.GetEnvdVersion) — it is +// injected so this shared package does not depend on the orchestrator. +// +// The "should we upgrade?" test compares baked version *strings* (built-with vs +// the target's version). This is sufficient because CLAUDE.md mandates bumping +// packages/envd/pkg/version.go on every behavioral change; if that ever stops +// holding, a same-version binary swap would be skipped and this must switch to +// comparing by git SHA. +// It returns the target binary's path and baked version ("" path = no upgrade), +// plus a reason for the no-upgrade case — off | not_staged | getversion_failed | +// same_version | downgrade, and "" when an upgrade IS returned — so the caller +// can tell a benign no-op (off / same_version) from a misconfigured target +// (not_staged from a bad SHA, getversion_failed, a refused downgrade). +func ResolveEnvdUpgrade( + ctx context.Context, + ff *Client, + builtWithVersion string, + hostEnvdPath string, + getVersion func(context.Context, string) (string, error), +) (path, version, reason string) { + return resolveEnvdUpgradePath(ctx, ff.StringFlag(ctx, EnvdUpgradeTargetFlag), builtWithVersion, hostEnvdPath, getVersion) +} + +// resolveEnvdUpgradePath is the pure decision, split out so it can be unit-tested +// without a LaunchDarkly client (the flag value is passed directly). It returns +// the target path and its baked version, or ("", "", ) for no upgrade. +// envdUpgradeTargetRe constrains a concrete-SHA EnvdUpgradeTargetFlag value to a +// bare alphanumeric identifier (git SHAs are hex, but any staged-binary suffix +// is safe) so it can't traverse out of the envd staging directory when joined +// into the candidate path. +var envdUpgradeTargetRe = regexp.MustCompile(`^[a-zA-Z0-9]+$`) + +func resolveEnvdUpgradePath( + ctx context.Context, + target string, + builtWithVersion string, + hostEnvdPath string, + getVersion func(context.Context, string) (string, error), +) (path, version, reason string) { + var candidate string + switch target { + case "", "off": + return "", "", "off" + case "promoted": + candidate = hostEnvdPath + default: + // A concrete git SHA -> the versioned binary next to the promoted one. + // The flag value becomes both a filesystem path and an exec target + // (version probing runs ` -version`), so reject anything that + // isn't a bare alphanumeric identifier: a value with path separators or + // ".." (e.g. "../../bin/sh") would otherwise escape the staging directory + // and run an arbitrary host binary. + if !envdUpgradeTargetRe.MatchString(target) { + return "", "", "invalid_target" + } + candidate = filepath.Join(filepath.Dir(hostEnvdPath), "envd."+target) + } + + if _, err := os.Stat(candidate); err != nil { + // Not staged on this node — e.g. a bad SHA / rubbish flag value, or a + // node that has not fetched the target yet. + return "", "", "not_staged" + } + + targetVersion, err := getVersion(ctx, candidate) + if err != nil || targetVersion == "" { + return "", "", "getversion_failed" + } + if targetVersion == builtWithVersion { + return "", "", "same_version" // already on the target (idempotent re-resume) + } + // Upgrade only: refuse to swap in an older envd. A staged binary that is not + // strictly newer than the sandbox's built-with version would otherwise be a + // live downgrade on resume. (Rollback, if ever needed, must be an explicit + // separate mechanism.) + if newer, verr := utils.IsGTEVersion(targetVersion, builtWithVersion); verr != nil || !newer { + return "", "", "downgrade" + } + + return candidate, targetVersion, "" +} + // defaultTrackedTemplates is the default map of template aliases tracked for metrics. // This is used to reduce metric cardinality. // JSON format: {"base": true, "code-interpreter-v1": true, ...} diff --git a/packages/shared/pkg/grpc/envd/upgrade/handover.pb.go b/packages/shared/pkg/grpc/envd/upgrade/handover.pb.go new file mode 100644 index 0000000000..d119adf5f8 --- /dev/null +++ b/packages/shared/pkg/grpc/envd/upgrade/handover.pb.go @@ -0,0 +1,798 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc (unknown) +// source: upgrade/handover.proto + +package upgrade + +import ( + filesystem "github.com/e2b-dev/infra/packages/shared/pkg/grpc/envd/filesystem" + process "github.com/e2b-dev/infra/packages/shared/pkg/grpc/envd/process" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// HandoverState is the entire cross-version compatibility contract for an envd +// live-upgrade (design §5.2, §6.4): the outgoing envd serializes it to a tmpfs +// file, the incoming (new-binary) envd decodes it after execve to reconstruct +// its world. It is an additive protobuf — an older envd omits fields a newer one +// added, and the reader treats absent fields as defaults. +// +// Schema is the versioning discipline §6.4 requires: it is bumped on any change, +// and a reader MUST refuse a blob whose schema exceeds the maximum it understands +// (decode every schema <= N, abort on schema > N) rather than mis-read a +// newer-than-known layout — the outgoing envd keeps running the old binary. +type HandoverState struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Schema uint32 `protobuf:"varint,1,opt,name=schema,proto3" json:"schema,omitempty"` + FromVer string `protobuf:"bytes,2,opt,name=from_ver,json=fromVer,proto3" json:"from_ver,omitempty"` + Processes []*HandoverProc `protobuf:"bytes,3,rep,name=processes,proto3" json:"processes,omitempty"` + // terminated carries the retention cache (recently-exited terminal events not + // yet drained). + Terminated []*HandoverExit `protobuf:"bytes,4,rep,name=terminated,proto3" json:"terminated,omitempty"` + // watchers is the filesystem service's active-watcher set. The filesystem + // service owns building/consuming these; the process handover only carries + // them. + Watchers []*HandoverWatcher `protobuf:"bytes,5,rep,name=watchers,proto3" json:"watchers,omitempty"` + // mounts is the API service's NFS mount ledger (path -> lifecycle), carried so + // the new envd's post-upgrade /init skips re-mounting a still-live mount. + Mounts []*MountEntry `protobuf:"bytes,6,rep,name=mounts,proto3" json:"mounts,omitempty"` + // forwards is the port forwarder's active socat set, carried so the new envd + // re-adopts the running socat children instead of spawning duplicates. + Forwards []*ForwardedPort `protobuf:"bytes,7,rep,name=forwards,proto3" json:"forwards,omitempty"` +} + +func (x *HandoverState) Reset() { + *x = HandoverState{} + if protoimpl.UnsafeEnabled { + mi := &file_upgrade_handover_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *HandoverState) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HandoverState) ProtoMessage() {} + +func (x *HandoverState) ProtoReflect() protoreflect.Message { + mi := &file_upgrade_handover_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HandoverState.ProtoReflect.Descriptor instead. +func (*HandoverState) Descriptor() ([]byte, []int) { + return file_upgrade_handover_proto_rawDescGZIP(), []int{0} +} + +func (x *HandoverState) GetSchema() uint32 { + if x != nil { + return x.Schema + } + return 0 +} + +func (x *HandoverState) GetFromVer() string { + if x != nil { + return x.FromVer + } + return "" +} + +func (x *HandoverState) GetProcesses() []*HandoverProc { + if x != nil { + return x.Processes + } + return nil +} + +func (x *HandoverState) GetTerminated() []*HandoverExit { + if x != nil { + return x.Terminated + } + return nil +} + +func (x *HandoverState) GetWatchers() []*HandoverWatcher { + if x != nil { + return x.Watchers + } + return nil +} + +func (x *HandoverState) GetMounts() []*MountEntry { + if x != nil { + return x.Mounts + } + return nil +} + +func (x *HandoverState) GetForwards() []*ForwardedPort { + if x != nil { + return x.Forwards + } + return nil +} + +// MountEntry is one NFS volume mount the outgoing envd had set up, keyed by the +// lifecycle it was mounted for. The kernel mount survives execve; carrying the +// ledger lets the new envd recognize a matching-lifecycle mount and leave it +// in place rather than force-unmounting and remounting it (ESTALE risk). +type MountEntry struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` + LifecycleId string `protobuf:"bytes,2,opt,name=lifecycle_id,json=lifecycleId,proto3" json:"lifecycle_id,omitempty"` +} + +func (x *MountEntry) Reset() { + *x = MountEntry{} + if protoimpl.UnsafeEnabled { + mi := &file_upgrade_handover_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MountEntry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MountEntry) ProtoMessage() {} + +func (x *MountEntry) ProtoReflect() protoreflect.Message { + mi := &file_upgrade_handover_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MountEntry.ProtoReflect.Descriptor instead. +func (*MountEntry) Descriptor() ([]byte, []int) { + return file_upgrade_handover_proto_rawDescGZIP(), []int{1} +} + +func (x *MountEntry) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *MountEntry) GetLifecycleId() string { + if x != nil { + return x.LifecycleId + } + return "" +} + +// ForwardedPort is one active port-forward. The socat child survives execve; +// carrying its pid lets the new forwarder re-adopt it (suppressing a duplicate +// socat and reaping it when the port closes) instead of respawning. +type ForwardedPort struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` // the forwarder map key: "-" + Port uint32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"` + ListenerPid int32 `protobuf:"varint,3,opt,name=listener_pid,json=listenerPid,proto3" json:"listener_pid,omitempty"` // pid of the guest process listening on the port + Family uint32 `protobuf:"varint,4,opt,name=family,proto3" json:"family,omitempty"` // IP version (4 or 6) + SocatPid int32 `protobuf:"varint,5,opt,name=socat_pid,json=socatPid,proto3" json:"socat_pid,omitempty"` // pid of the running socat child to re-adopt +} + +func (x *ForwardedPort) Reset() { + *x = ForwardedPort{} + if protoimpl.UnsafeEnabled { + mi := &file_upgrade_handover_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ForwardedPort) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ForwardedPort) ProtoMessage() {} + +func (x *ForwardedPort) ProtoReflect() protoreflect.Message { + mi := &file_upgrade_handover_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ForwardedPort.ProtoReflect.Descriptor instead. +func (*ForwardedPort) Descriptor() ([]byte, []int) { + return file_upgrade_handover_proto_rawDescGZIP(), []int{2} +} + +func (x *ForwardedPort) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *ForwardedPort) GetPort() uint32 { + if x != nil { + return x.Port + } + return 0 +} + +func (x *ForwardedPort) GetListenerPid() int32 { + if x != nil { + return x.ListenerPid + } + return 0 +} + +func (x *ForwardedPort) GetFamily() uint32 { + if x != nil { + return x.Family + } + return 0 +} + +func (x *ForwardedPort) GetSocatPid() int32 { + if x != nil { + return x.SocatPid + } + return 0 +} + +// HandoverProc is a live child re-adopted across the same-PID execve. The fd +// fields carry the *numbers* (kept valid across execve, CLOEXEC cleared); the +// kernel objects ride the fd table. +type HandoverProc struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Pid uint32 `protobuf:"varint,1,opt,name=pid,proto3" json:"pid,omitempty"` // K-anchor: live child, survives same-PID execve + Tag string `protobuf:"bytes,2,opt,name=tag,proto3" json:"tag,omitempty"` // K-metadata: not in /proc, unreconstructable if lost + HasTag bool `protobuf:"varint,3,opt,name=has_tag,json=hasTag,proto3" json:"has_tag,omitempty"` // distinguishes an empty tag from no tag + CgType string `protobuf:"bytes,4,opt,name=cg_type,json=cgType,proto3" json:"cg_type,omitempty"` // user | pty + Config *process.ProcessConfig `protobuf:"bytes,5,opt,name=config,proto3" json:"config,omitempty"` + StdoutFd int32 `protobuf:"varint,6,opt,name=stdout_fd,json=stdoutFd,proto3" json:"stdout_fd,omitempty"` + StderrFd int32 `protobuf:"varint,7,opt,name=stderr_fd,json=stderrFd,proto3" json:"stderr_fd,omitempty"` + StdinFd int32 `protobuf:"varint,8,opt,name=stdin_fd,json=stdinFd,proto3" json:"stdin_fd,omitempty"` + TtyFd int32 `protobuf:"varint,9,opt,name=tty_fd,json=ttyFd,proto3" json:"tty_fd,omitempty"` + TimeoutMs int64 `protobuf:"varint,10,opt,name=timeout_ms,json=timeoutMs,proto3" json:"timeout_ms,omitempty"` // per-process kill deadline remaining, re-armed on readopt +} + +func (x *HandoverProc) Reset() { + *x = HandoverProc{} + if protoimpl.UnsafeEnabled { + mi := &file_upgrade_handover_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *HandoverProc) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HandoverProc) ProtoMessage() {} + +func (x *HandoverProc) ProtoReflect() protoreflect.Message { + mi := &file_upgrade_handover_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HandoverProc.ProtoReflect.Descriptor instead. +func (*HandoverProc) Descriptor() ([]byte, []int) { + return file_upgrade_handover_proto_rawDescGZIP(), []int{3} +} + +func (x *HandoverProc) GetPid() uint32 { + if x != nil { + return x.Pid + } + return 0 +} + +func (x *HandoverProc) GetTag() string { + if x != nil { + return x.Tag + } + return "" +} + +func (x *HandoverProc) GetHasTag() bool { + if x != nil { + return x.HasTag + } + return false +} + +func (x *HandoverProc) GetCgType() string { + if x != nil { + return x.CgType + } + return "" +} + +func (x *HandoverProc) GetConfig() *process.ProcessConfig { + if x != nil { + return x.Config + } + return nil +} + +func (x *HandoverProc) GetStdoutFd() int32 { + if x != nil { + return x.StdoutFd + } + return 0 +} + +func (x *HandoverProc) GetStderrFd() int32 { + if x != nil { + return x.StderrFd + } + return 0 +} + +func (x *HandoverProc) GetStdinFd() int32 { + if x != nil { + return x.StdinFd + } + return 0 +} + +func (x *HandoverProc) GetTtyFd() int32 { + if x != nil { + return x.TtyFd + } + return 0 +} + +func (x *HandoverProc) GetTimeoutMs() int64 { + if x != nil { + return x.TimeoutMs + } + return 0 +} + +// HandoverExit is a recently-terminated process whose terminal event has not yet +// been drained by a client — carried so an exit code is not lost across the swap. +type HandoverExit struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Pid uint32 `protobuf:"varint,1,opt,name=pid,proto3" json:"pid,omitempty"` + Tag string `protobuf:"bytes,2,opt,name=tag,proto3" json:"tag,omitempty"` + HasTag bool `protobuf:"varint,3,opt,name=has_tag,json=hasTag,proto3" json:"has_tag,omitempty"` + End *process.ProcessEvent_EndEvent `protobuf:"bytes,4,opt,name=end,proto3" json:"end,omitempty"` + RemainingMs int64 `protobuf:"varint,5,opt,name=remaining_ms,json=remainingMs,proto3" json:"remaining_ms,omitempty"` // retention TTL remaining +} + +func (x *HandoverExit) Reset() { + *x = HandoverExit{} + if protoimpl.UnsafeEnabled { + mi := &file_upgrade_handover_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *HandoverExit) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HandoverExit) ProtoMessage() {} + +func (x *HandoverExit) ProtoReflect() protoreflect.Message { + mi := &file_upgrade_handover_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HandoverExit.ProtoReflect.Descriptor instead. +func (*HandoverExit) Descriptor() ([]byte, []int) { + return file_upgrade_handover_proto_rawDescGZIP(), []int{4} +} + +func (x *HandoverExit) GetPid() uint32 { + if x != nil { + return x.Pid + } + return 0 +} + +func (x *HandoverExit) GetTag() string { + if x != nil { + return x.Tag + } + return "" +} + +func (x *HandoverExit) GetHasTag() bool { + if x != nil { + return x.HasTag + } + return false +} + +func (x *HandoverExit) GetEnd() *process.ProcessEvent_EndEvent { + if x != nil { + return x.End + } + return nil +} + +func (x *HandoverExit) GetRemainingMs() int64 { + if x != nil { + return x.RemainingMs + } + return 0 +} + +// HandoverWatcher re-arms a persistent CreateWatcher on the incoming envd under +// the preserved id (design §6.9, Option C: re-arm from metadata, do NOT carry an +// inotify fd). +type HandoverWatcher struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Path string `protobuf:"bytes,2,opt,name=path,proto3" json:"path,omitempty"` + Recursive bool `protobuf:"varint,3,opt,name=recursive,proto3" json:"recursive,omitempty"` + IncludeEntryInfo bool `protobuf:"varint,4,opt,name=include_entry_info,json=includeEntryInfo,proto3" json:"include_entry_info,omitempty"` + PendingEvents []*filesystem.FilesystemEvent `protobuf:"bytes,5,rep,name=pending_events,json=pendingEvents,proto3" json:"pending_events,omitempty"` // un-polled GetWatcherEvents buffer +} + +func (x *HandoverWatcher) Reset() { + *x = HandoverWatcher{} + if protoimpl.UnsafeEnabled { + mi := &file_upgrade_handover_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *HandoverWatcher) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HandoverWatcher) ProtoMessage() {} + +func (x *HandoverWatcher) ProtoReflect() protoreflect.Message { + mi := &file_upgrade_handover_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HandoverWatcher.ProtoReflect.Descriptor instead. +func (*HandoverWatcher) Descriptor() ([]byte, []int) { + return file_upgrade_handover_proto_rawDescGZIP(), []int{5} +} + +func (x *HandoverWatcher) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *HandoverWatcher) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *HandoverWatcher) GetRecursive() bool { + if x != nil { + return x.Recursive + } + return false +} + +func (x *HandoverWatcher) GetIncludeEntryInfo() bool { + if x != nil { + return x.IncludeEntryInfo + } + return false +} + +func (x *HandoverWatcher) GetPendingEvents() []*filesystem.FilesystemEvent { + if x != nil { + return x.PendingEvents + } + return nil +} + +var File_upgrade_handover_proto protoreflect.FileDescriptor + +var file_upgrade_handover_proto_rawDesc = []byte{ + 0x0a, 0x16, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x2f, 0x68, 0x61, 0x6e, 0x64, 0x6f, 0x76, + 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x07, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, + 0x65, 0x1a, 0x15, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x63, 0x65, + 0x73, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1b, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x79, + 0x73, 0x74, 0x65, 0x6d, 0x2f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xc5, 0x02, 0x0a, 0x0d, 0x48, 0x61, 0x6e, 0x64, 0x6f, 0x76, + 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, + 0x19, 0x0a, 0x08, 0x66, 0x72, 0x6f, 0x6d, 0x5f, 0x76, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x66, 0x72, 0x6f, 0x6d, 0x56, 0x65, 0x72, 0x12, 0x33, 0x0a, 0x09, 0x70, 0x72, + 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, + 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x2e, 0x48, 0x61, 0x6e, 0x64, 0x6f, 0x76, 0x65, 0x72, + 0x50, 0x72, 0x6f, 0x63, 0x52, 0x09, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, 0x73, 0x12, + 0x35, 0x0a, 0x0a, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x64, 0x18, 0x04, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x2e, 0x48, 0x61, + 0x6e, 0x64, 0x6f, 0x76, 0x65, 0x72, 0x45, 0x78, 0x69, 0x74, 0x52, 0x0a, 0x74, 0x65, 0x72, 0x6d, + 0x69, 0x6e, 0x61, 0x74, 0x65, 0x64, 0x12, 0x34, 0x0a, 0x08, 0x77, 0x61, 0x74, 0x63, 0x68, 0x65, + 0x72, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x75, 0x70, 0x67, 0x72, 0x61, + 0x64, 0x65, 0x2e, 0x48, 0x61, 0x6e, 0x64, 0x6f, 0x76, 0x65, 0x72, 0x57, 0x61, 0x74, 0x63, 0x68, + 0x65, 0x72, 0x52, 0x08, 0x77, 0x61, 0x74, 0x63, 0x68, 0x65, 0x72, 0x73, 0x12, 0x2b, 0x0a, 0x06, + 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x75, + 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x2e, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x06, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x12, 0x32, 0x0a, 0x08, 0x66, 0x6f, 0x72, + 0x77, 0x61, 0x72, 0x64, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x75, 0x70, + 0x67, 0x72, 0x61, 0x64, 0x65, 0x2e, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x65, 0x64, 0x50, + 0x6f, 0x72, 0x74, 0x52, 0x08, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x73, 0x22, 0x43, 0x0a, + 0x0a, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x70, + 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, + 0x21, 0x0a, 0x0c, 0x6c, 0x69, 0x66, 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6c, 0x69, 0x66, 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, + 0x49, 0x64, 0x22, 0x8d, 0x01, 0x0a, 0x0d, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x65, 0x64, + 0x50, 0x6f, 0x72, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x6c, 0x69, + 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x5f, 0x70, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x0b, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x50, 0x69, 0x64, 0x12, 0x16, 0x0a, + 0x06, 0x66, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x66, + 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x6f, 0x63, 0x61, 0x74, 0x5f, 0x70, + 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x73, 0x6f, 0x63, 0x61, 0x74, 0x50, + 0x69, 0x64, 0x22, 0x9f, 0x02, 0x0a, 0x0c, 0x48, 0x61, 0x6e, 0x64, 0x6f, 0x76, 0x65, 0x72, 0x50, + 0x72, 0x6f, 0x63, 0x12, 0x10, 0x0a, 0x03, 0x70, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x03, 0x70, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x61, 0x67, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x74, 0x61, 0x67, 0x12, 0x17, 0x0a, 0x07, 0x68, 0x61, 0x73, 0x5f, 0x74, + 0x61, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x68, 0x61, 0x73, 0x54, 0x61, 0x67, + 0x12, 0x17, 0x0a, 0x07, 0x63, 0x67, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x63, 0x67, 0x54, 0x79, 0x70, 0x65, 0x12, 0x2e, 0x0a, 0x06, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x63, + 0x65, 0x73, 0x73, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x74, 0x64, + 0x6f, 0x75, 0x74, 0x5f, 0x66, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x73, 0x74, + 0x64, 0x6f, 0x75, 0x74, 0x46, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x74, 0x64, 0x65, 0x72, 0x72, + 0x5f, 0x66, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x73, 0x74, 0x64, 0x65, 0x72, + 0x72, 0x46, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x74, 0x64, 0x69, 0x6e, 0x5f, 0x66, 0x64, 0x18, + 0x08, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x73, 0x74, 0x64, 0x69, 0x6e, 0x46, 0x64, 0x12, 0x15, + 0x0a, 0x06, 0x74, 0x74, 0x79, 0x5f, 0x66, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, + 0x74, 0x74, 0x79, 0x46, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, + 0x5f, 0x6d, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x6f, + 0x75, 0x74, 0x4d, 0x73, 0x22, 0xa0, 0x01, 0x0a, 0x0c, 0x48, 0x61, 0x6e, 0x64, 0x6f, 0x76, 0x65, + 0x72, 0x45, 0x78, 0x69, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x70, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x03, 0x70, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x61, 0x67, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x74, 0x61, 0x67, 0x12, 0x17, 0x0a, 0x07, 0x68, 0x61, 0x73, + 0x5f, 0x74, 0x61, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x68, 0x61, 0x73, 0x54, + 0x61, 0x67, 0x12, 0x30, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, + 0x73, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, + 0x03, 0x65, 0x6e, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x72, 0x65, 0x6d, 0x61, 0x69, 0x6e, 0x69, 0x6e, + 0x67, 0x5f, 0x6d, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x72, 0x65, 0x6d, 0x61, + 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x4d, 0x73, 0x22, 0xc5, 0x01, 0x0a, 0x0f, 0x48, 0x61, 0x6e, 0x64, + 0x6f, 0x76, 0x65, 0x72, 0x57, 0x61, 0x74, 0x63, 0x68, 0x65, 0x72, 0x12, 0x0e, 0x0a, 0x02, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x70, + 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, + 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x09, 0x72, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x12, 0x2c, 0x0a, + 0x12, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x5f, 0x69, + 0x6e, 0x66, 0x6f, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x69, 0x6e, 0x63, 0x6c, 0x75, + 0x64, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x42, 0x0a, 0x0e, 0x70, + 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x05, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, + 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, + 0x52, 0x0d, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x42, + 0x98, 0x01, 0x0a, 0x0b, 0x63, 0x6f, 0x6d, 0x2e, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x42, + 0x0d, 0x48, 0x61, 0x6e, 0x64, 0x6f, 0x76, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, + 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x32, 0x62, + 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x69, 0x6e, 0x66, 0x72, 0x61, 0x2f, 0x70, 0x61, 0x63, 0x6b, 0x61, + 0x67, 0x65, 0x73, 0x2f, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x67, + 0x72, 0x70, 0x63, 0x2f, 0x65, 0x6e, 0x76, 0x64, 0x2f, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, + 0xa2, 0x02, 0x03, 0x55, 0x58, 0x58, 0xaa, 0x02, 0x07, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, + 0xca, 0x02, 0x07, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0xe2, 0x02, 0x13, 0x55, 0x70, 0x67, + 0x72, 0x61, 0x64, 0x65, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0xea, 0x02, 0x07, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, +} + +var ( + file_upgrade_handover_proto_rawDescOnce sync.Once + file_upgrade_handover_proto_rawDescData = file_upgrade_handover_proto_rawDesc +) + +func file_upgrade_handover_proto_rawDescGZIP() []byte { + file_upgrade_handover_proto_rawDescOnce.Do(func() { + file_upgrade_handover_proto_rawDescData = protoimpl.X.CompressGZIP(file_upgrade_handover_proto_rawDescData) + }) + return file_upgrade_handover_proto_rawDescData +} + +var file_upgrade_handover_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_upgrade_handover_proto_goTypes = []interface{}{ + (*HandoverState)(nil), // 0: upgrade.HandoverState + (*MountEntry)(nil), // 1: upgrade.MountEntry + (*ForwardedPort)(nil), // 2: upgrade.ForwardedPort + (*HandoverProc)(nil), // 3: upgrade.HandoverProc + (*HandoverExit)(nil), // 4: upgrade.HandoverExit + (*HandoverWatcher)(nil), // 5: upgrade.HandoverWatcher + (*process.ProcessConfig)(nil), // 6: process.ProcessConfig + (*process.ProcessEvent_EndEvent)(nil), // 7: process.ProcessEvent.EndEvent + (*filesystem.FilesystemEvent)(nil), // 8: filesystem.FilesystemEvent +} +var file_upgrade_handover_proto_depIdxs = []int32{ + 3, // 0: upgrade.HandoverState.processes:type_name -> upgrade.HandoverProc + 4, // 1: upgrade.HandoverState.terminated:type_name -> upgrade.HandoverExit + 5, // 2: upgrade.HandoverState.watchers:type_name -> upgrade.HandoverWatcher + 1, // 3: upgrade.HandoverState.mounts:type_name -> upgrade.MountEntry + 2, // 4: upgrade.HandoverState.forwards:type_name -> upgrade.ForwardedPort + 6, // 5: upgrade.HandoverProc.config:type_name -> process.ProcessConfig + 7, // 6: upgrade.HandoverExit.end:type_name -> process.ProcessEvent.EndEvent + 8, // 7: upgrade.HandoverWatcher.pending_events:type_name -> filesystem.FilesystemEvent + 8, // [8:8] is the sub-list for method output_type + 8, // [8:8] is the sub-list for method input_type + 8, // [8:8] is the sub-list for extension type_name + 8, // [8:8] is the sub-list for extension extendee + 0, // [0:8] is the sub-list for field type_name +} + +func init() { file_upgrade_handover_proto_init() } +func file_upgrade_handover_proto_init() { + if File_upgrade_handover_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_upgrade_handover_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HandoverState); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_upgrade_handover_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MountEntry); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_upgrade_handover_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ForwardedPort); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_upgrade_handover_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HandoverProc); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_upgrade_handover_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HandoverExit); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_upgrade_handover_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HandoverWatcher); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_upgrade_handover_proto_rawDesc, + NumEnums: 0, + NumMessages: 6, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_upgrade_handover_proto_goTypes, + DependencyIndexes: file_upgrade_handover_proto_depIdxs, + MessageInfos: file_upgrade_handover_proto_msgTypes, + }.Build() + File_upgrade_handover_proto = out.File + file_upgrade_handover_proto_rawDesc = nil + file_upgrade_handover_proto_goTypes = nil + file_upgrade_handover_proto_depIdxs = nil +} diff --git a/packages/shared/pkg/telemetry/envd_upgrade_meters_test.go b/packages/shared/pkg/telemetry/envd_upgrade_meters_test.go new file mode 100644 index 0000000000..0b91ed84d7 --- /dev/null +++ b/packages/shared/pkg/telemetry/envd_upgrade_meters_test.go @@ -0,0 +1,33 @@ +package telemetry + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/metric/noop" +) + +// TestEnvdUpgradeMetricsRegistered guards the rollout metrics: each must have a +// description and unit map entry (an easy omission — a missing entry silently +// ships an unlabelled metric) and must construct without error. +func TestEnvdUpgradeMetricsRegistered(t *testing.T) { + t.Parallel() + + for _, c := range []CounterType{OrchestratorEnvdUpgradeAttempts, OrchestratorEnvdUpgradeGated, OrchestratorEnvdUpgradeHandover} { + assert.NotEmptyf(t, counterDesc[c], "missing description for counter %s", c) + assert.NotEmptyf(t, counterUnits[c], "missing unit for counter %s", c) + } + assert.NotEmpty(t, histogramDesc[OrchestratorEnvdUpgradeDurationName], "missing histogram description") + assert.NotEmpty(t, histogramUnits[OrchestratorEnvdUpgradeDurationName], "missing histogram unit") + + m := noop.NewMeterProvider().Meter("github.com/e2b-dev/infra/packages/shared/pkg/telemetry") + _, err := GetCounter(m, OrchestratorEnvdUpgradeAttempts) + require.NoError(t, err) + _, err = GetCounter(m, OrchestratorEnvdUpgradeGated) + require.NoError(t, err) + _, err = GetCounter(m, OrchestratorEnvdUpgradeHandover) + require.NoError(t, err) + _, err = GetHistogram(m, OrchestratorEnvdUpgradeDurationName) + require.NoError(t, err) +} diff --git a/packages/shared/pkg/telemetry/meters.go b/packages/shared/pkg/telemetry/meters.go index 736bd69d25..7567bbb9ff 100644 --- a/packages/shared/pkg/telemetry/meters.go +++ b/packages/shared/pkg/telemetry/meters.go @@ -44,6 +44,22 @@ const ( // A non-zero rate means lost snapshots. OrchestratorSnapshotUploadFailedCounterName CounterType = "orchestrator.snapshot.upload.failed" + // OrchestratorEnvdUpgradeAttempts counts resume-time envd live-upgrade + // attempts, by result (success|delivery_failed|not_ready|panic) and + // from_version/to_version. success/total is the rollout success rate; + // attempts/resumes is the fire rate. + OrchestratorEnvdUpgradeAttempts CounterType = "orchestrator.envd.upgrade.attempts" + // OrchestratorEnvdUpgradeGated counts resumes the envd-upgrade-target flag + // targeted but the min-version gate skipped (reason=old_envd) — a silent + // no-op worth watching during a ramp. + OrchestratorEnvdUpgradeGated CounterType = "orchestrator.envd.upgrade.gated" + // OrchestratorEnvdUpgradeHandover counts live-upgrade handover items by item + // (proc|retained|watcher) and result (ok|failed), reported back by the new + // envd on /init. failed/(ok+failed) is the handover error rate — the + // fleet-visible signal (otherwise only logged in-guest) that the new envd + // dropped or degraded something it re-adopted across the swap. + OrchestratorEnvdUpgradeHandover CounterType = "orchestrator.envd.upgrade.handover" + // PauseResumePrefetchHarvestAttempts counts pause-resume prefetch harvest // attempts, by result (success|resume_failed|collect_failed|skipped). The // throwaway is absent from Prometheus otherwise (registration-skip), so this @@ -110,6 +126,11 @@ const ( SnapshotProcessMemoryDurationName HistogramType = "orchestrator.sandbox.snapshot.process_memory.duration" SnapshotProcessRootfsDurationName HistogramType = "orchestrator.sandbox.snapshot.process_rootfs.duration" + // OrchestratorEnvdUpgradeDurationName is the wall-time of a resume-time envd + // live-upgrade (delivery + trigger + WaitForEnvd) = overhead added to the + // resume. Labeled by result. + OrchestratorEnvdUpgradeDurationName HistogramType = "orchestrator.envd.upgrade.duration" + // Pre-pause envd heap collapse round-trip duration (the pause-path cost of // POST /collapse: network plus envd's madvise work), recorded once per pause // when the collapse-envd-heap flag is on. @@ -237,6 +258,9 @@ var counterDesc = map[CounterType]string{ EnvdCollapseChunks: "2 MiB chunks the pre-pause envd heap collapse attempted, by result", OrchestratorSandboxKilledCounterName: "Number of sandboxes killed, labeled by kill reason", OrchestratorSnapshotUploadFailedCounterName: "Number of pause-snapshot uploads that never landed durably", + OrchestratorEnvdUpgradeAttempts: "Resume-time envd live-upgrade attempts, by result and from/to version", + OrchestratorEnvdUpgradeGated: "Resumes the envd-upgrade-target flag targeted but the min-version gate skipped", + OrchestratorEnvdUpgradeHandover: "Live-upgrade handover items by item (proc|retained|watcher) and result (ok|failed)", PauseResumePrefetchHarvestAttempts: "Pause-resume prefetch harvest attempts, by result", TCPFirewallConnectionsTotal: "Total number of TCP firewall connections processed", TCPFirewallErrorsTotal: "Total number of TCP firewall errors", @@ -272,6 +296,9 @@ var counterUnits = map[CounterType]string{ EnvdCollapseChunks: "{chunk}", OrchestratorSandboxKilledCounterName: "{sandbox}", OrchestratorSnapshotUploadFailedCounterName: "{snapshot}", + OrchestratorEnvdUpgradeAttempts: "{attempt}", + OrchestratorEnvdUpgradeGated: "{sandbox}", + OrchestratorEnvdUpgradeHandover: "{item}", PauseResumePrefetchHarvestAttempts: "{attempt}", TCPFirewallConnectionsTotal: "{connection}", TCPFirewallErrorsTotal: "{error}", @@ -451,6 +478,7 @@ var histogramDesc = map[HistogramType]string{ BuildStepDurationHistogramName: "Time taken to build each step of a template", BuildRootfsSizeHistogramName: "Size of the built template rootfs in bytes", OrchestratorSandboxCreateDurationName: "Time taken to create a sandbox", + OrchestratorEnvdUpgradeDurationName: "Wall-time of a resume-time envd upgrade (delivery + trigger + WaitForEnvd)", WaitForEnvdDurationHistogramName: "Time taken for Envd to initialize successfully", EnvdCollapseDurationHistogramName: "Time taken for the pre-pause envd heap collapse round-trip", GuestSyncDurationHistogramName: "Time taken for the mandatory pre-pause guest sync (filesystem-only pause)", @@ -504,6 +532,7 @@ var histogramUnits = map[HistogramType]string{ BuildStepDurationHistogramName: "ms", BuildRootfsSizeHistogramName: "{By}", OrchestratorSandboxCreateDurationName: "ms", + OrchestratorEnvdUpgradeDurationName: "ms", WaitForEnvdDurationHistogramName: "ms", EnvdCollapseDurationHistogramName: "ms", GuestSyncDurationHistogramName: "ms", diff --git a/packages/shared/pkg/utils/version.go b/packages/shared/pkg/utils/version.go index 793889205e..0331946a5f 100644 --- a/packages/shared/pkg/utils/version.go +++ b/packages/shared/pkg/utils/version.go @@ -27,6 +27,14 @@ const MinEnvdVersionForHeapCollapse = "0.6.5" // filesystem-only pause. Older envds fall back to a plain guest sync. const MinEnvdVersionForFsFreeze = "0.6.6" +// MinEnvdVersionForUpgrade is the first envd that both exposes the live-upgrade +// POST /upgrade endpoint and writes the protobuf handover blob the incoming envd +// decodes. The resume-time auto-upgrade trigger delivers to the *running* (old) +// envd, which serializes the handover, so anything older either lacks /upgrade +// or writes the pre-proto (JSON) format the new envd can't read — both must be +// skipped. +const MinEnvdVersionForUpgrade = "0.6.12" + func sanitizeVersion(version string) string { if len(version) > 0 && version[0] != 'v' { version = "v" + version