diff --git a/packages/orchestrator/pkg/sandbox/reclaim.go b/packages/orchestrator/pkg/sandbox/reclaim.go index 2d65b8cd5a..435b66f823 100644 --- a/packages/orchestrator/pkg/sandbox/reclaim.go +++ b/packages/orchestrator/pkg/sandbox/reclaim.go @@ -11,7 +11,6 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/metric" - "go.opentelemetry.io/otel/trace" "go.uber.org/zap" "github.com/e2b-dev/infra/packages/shared/pkg/featureflags" @@ -171,25 +170,29 @@ func (s *Sandbox) guestPrepareFsForPause(ctx context.Context, cleanup *Cleanup) timeout := s.guestSyncTimeout(ctx) start := time.Now() - ctx, span := tracer.Start( - ctx, - "envd-guest-fs-pause", - trace.WithAttributes(attribute.Bool("fsfreeze", supportsFsFreeze)), - ) + // method records how the rootfs was quiesced: native "fsfreeze", "fsfreeze-exec" + // (old envd, via the exec API), or "sync" (fallback). Updated as we proceed. + method := "sync" + + ctx, span := tracer.Start(ctx, "envd-guest-fs-pause") defer span.End() // Record on every exit so slow and timed-out syncs are captured too. defer func() { + frozen := method != "sync" + span.SetAttributes(attribute.String("method", method), attribute.Bool("fsfreeze", frozen)) guestSyncDurationHistogram.Record(ctx, time.Since(start).Milliseconds(), metric.WithAttributes( attribute.Bool("success", e == nil), - attribute.Bool("fsfreeze", supportsFsFreeze), + attribute.Bool("fsfreeze", frozen), + attribute.String("method", method), attribute.Int64("timeout_ms", timeout.Milliseconds()), ), ) }() if supportsFsFreeze { + method = "fsfreeze" // fsfreeze flushes the rootfs AND blocks further writes until thaw, // closing the sync->pause race. FIFREEZE already syncs as part of // freezing, so a separate guest sync would be redundant. @@ -205,12 +208,55 @@ func (s *Sandbox) guestPrepareFsForPause(ctx context.Context, cleanup *Cleanup) if err := s.callEnvdFsfreeze(ctx, timeout); err != nil { return fmt.Errorf("fsfreeze before filesystem-only pause: %w", err) } - } else { - if err := s.guestSync(ctx, timeout); err != nil { - return fmt.Errorf("guest sync before filesystem-only pause: %w", err) + + return nil + } + + // Old envd, no native /fsfreeze. When enabled, freeze the rootfs via the exec + // API so the snapshot is captured on a quiesced, consistent filesystem instead + // of a merely sync'd one (which leaves the sync->pause write race open). + if s.featureFlags.BoolFlag(ctx, featureflags.FsFreezeViaExecFlag, sandboxLDContext(s.Runtime, s.Config)) { + // Probe for the fsfreeze binary first. `command -v` only inspects PATH and + // never touches the rootfs, so if it's missing (or the probe itself errors) + // the filesystem is definitely not frozen and it's safe to fall back to a + // plain sync — the "never fail a pause just because fsfreeze is missing" + // case. Only once we know the binary exists do we risk a freeze. + hasFsfreeze, err := s.guestHasFsfreeze(ctx, timeout) + if err != nil { + logger.L().Warn(ctx, "probing guest for fsfreeze failed; falling back to guest sync", + logger.WithSandboxID(s.Runtime.SandboxID), zap.Error(err)) + } else if hasFsfreeze { + // Register the rollback thaw before freezing so an aborted freeze can't + // leave the live VM frozen; thawing a non-frozen fs is a harmless no-op. + cleanup.Add(ctx, func(ctx context.Context) error { + s.bestEffortFsthawViaExec(ctx) + + return nil + }) + // Set method before the freeze — as the native path sets "fsfreeze" + // before its call — so the deferred metric attributes an attempted but + // failed/aborted freeze to fsfreeze-exec, not to the sync fallback. + method = "fsfreeze-exec" + // A freeze error here may leave the rootfs frozen: FIFREEZE persists + // after the command exits, so a timeout or stream error that races a + // freeze which already engaged still leaves it frozen. A fallback sync + // would then block on the frozen fs, so abort the pause like the native + // path does and let the registered cleanup thaw it — do not sync. + if err := s.guestFsfreezeViaExec(ctx, timeout); err != nil { + return fmt.Errorf("fsfreeze via exec before filesystem-only pause: %w", err) + } + + logger.L().Info(ctx, "froze guest rootfs via envd exec API before filesystem-only pause", + logger.WithSandboxID(s.Runtime.SandboxID)) + + return nil } } + if err := s.guestSync(ctx, timeout); err != nil { + return fmt.Errorf("guest sync before filesystem-only pause: %w", err) + } + return nil } @@ -220,13 +266,28 @@ func (s *Sandbox) guestPrepareFsForPause(ctx context.Context, cleanup *Cleanup) // error instead of persisting a rootfs missing acknowledged writes. Unlike // bestEffortReclaim's sync step (LD-flag gated, best-effort), this always runs // and always reports failure. -func (s *Sandbox) guestSync(ctx context.Context, syncTimeout time.Duration) (e error) { - rcCtx, cancel := context.WithTimeout(ctx, syncTimeout+reclaimOuterSlack) +func (s *Sandbox) guestSync(ctx context.Context, syncTimeout time.Duration) error { + exitCode, err := s.runGuestShellCommand(ctx, syncTimeout, "sync") + if err != nil { + return err + } + if exitCode != 0 { + return fmt.Errorf("guest sync exited with code %d", exitCode) + } + + return nil +} + +// runGuestShellCommand runs `sh -c