From 35ff85035069dd45c071c455fc05f0ef71a0c650 Mon Sep 17 00:00:00 2001 From: Babis Chalios Date: Tue, 28 Jul 2026 09:03:25 +0200 Subject: [PATCH] feat(metrics): label pause/resume telemetry by fs_only Make filesystem-only pause and resume distinguishable in metrics: - New `orchestrator.sandbox.pause.duration` histogram, recorded in the Pause handler with `fs_only` and `success` attributes. - `fs_only` attribute on `orchestrator.snapshot.upload.failed` (threaded through snapshotResult). - `fs_only` attribute on `orchestrator.sandbox.create.duration`, set at the reboot/resume fork, so e2e create/resume latency splits filesystem-only reboot vs memory restore (combined with the existing sandbox.resume bool). Previously fs-only-ness was only a span attribute, so pause/resume e2e latency, error rate, and upload failures could not be scoped to fs-only in metrics. (Reboot vs resume was already available on the envd-init/uffd metrics via start_type; this adds it to the e2e create.duration too.) Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/orchestrator/pkg/server/main.go | 7 ++++ packages/orchestrator/pkg/server/sandboxes.go | 34 +++++++++++++++---- packages/shared/pkg/telemetry/meters.go | 3 ++ 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/packages/orchestrator/pkg/server/main.go b/packages/orchestrator/pkg/server/main.go index 7164823386..22670a6f64 100644 --- a/packages/orchestrator/pkg/server/main.go +++ b/packages/orchestrator/pkg/server/main.go @@ -79,6 +79,7 @@ type Server struct { uploadedBuilds *ttlcache.Cache[string, struct{}] uploads *sandbox.Uploads sandboxCreateDuration metric.Int64Histogram + sandboxPauseDuration metric.Int64Histogram sandboxKilledCounter metric.Int64Counter uploadFailedCounter metric.Int64Counter @@ -146,6 +147,12 @@ func New(ctx context.Context, cfg ServiceConfig) (*Server, error) { } server.sandboxCreateDuration = sandboxCreateDuration + sandboxPauseDuration, err := telemetry.GetHistogram(meter, telemetry.PauseDurationHistogramName) + if err != nil { + return nil, fmt.Errorf("failed to register sandbox pause duration histogram: %w", err) + } + server.sandboxPauseDuration = sandboxPauseDuration + sandboxKilledCounter, err := telemetry.GetCounter(meter, telemetry.OrchestratorSandboxKilledCounterName) if err != nil { return nil, fmt.Errorf("failed to register sandbox kills counter: %w", err) diff --git a/packages/orchestrator/pkg/server/sandboxes.go b/packages/orchestrator/pkg/server/sandboxes.go index 687efa8993..f9ceec21ae 100644 --- a/packages/orchestrator/pkg/server/sandboxes.go +++ b/packages/orchestrator/pkg/server/sandboxes.go @@ -82,15 +82,19 @@ func (s *Server) Create(ctx context.Context, req *orchestrator.SandboxCreateRequ defer childSpan.End() isResume := req.GetSandbox().GetSnapshot() + // fsOnly is set at the resume fork below when this takes the filesystem-only + // reboot path (vs a memory restore), so create/resume e2e latency can be + // split reboot vs memory — mirroring the fs_only pause label. Combined with + // sandbox.resume: resume=false → fresh create; resume=true,fs_only=false → + // memory resume; resume=true,fs_only=true → filesystem-only reboot. + var fsOnly bool createStart := time.Now() defer func() { - if createErr != nil { - return - } - s.sandboxCreateDuration.Record(ctx, time.Since(createStart).Milliseconds(), metric.WithAttributes( attribute.Bool("sandbox.resume", isResume), + attribute.Bool("fs_only", fsOnly), + attribute.Bool("success", createErr == nil), ), ) }() @@ -228,6 +232,7 @@ func (s *Server) Create(ctx context.Context, req *orchestrator.SandboxCreateRequ var sbx *sandbox.Sandbox if meta.IsFilesystemOnly() { + fsOnly = true sbx, err = s.sandboxFactory.RebootSandbox( ctx, template, @@ -645,10 +650,23 @@ func recordSandboxKill(ctx context.Context, counter metric.Int64Counter, killRea counter.Add(ctx, 1, metric.WithAttributes(attribute.String("kill_reason", killReason))) } -func (s *Server) Pause(ctx context.Context, in *orchestrator.SandboxPauseRequest) (*orchestrator.SandboxPauseResponse, error) { +func (s *Server) Pause(ctx context.Context, in *orchestrator.SandboxPauseRequest) (resp *orchestrator.SandboxPauseResponse, err error) { ctx, childSpan := tracer.Start(ctx, "sandbox-pause") defer childSpan.End() + // Record pause duration split by fs_only vs memory (the gRPC RPC metric + // can't distinguish them) and success, so dashboards can scope pause + // call-count / error-rate / latency to filesystem-only pauses. + pauseStart := time.Now() + defer func() { + s.sandboxPauseDuration.Record(ctx, time.Since(pauseStart).Milliseconds(), + metric.WithAttributes( + attribute.Bool("fs_only", in.GetFilesystemOnly()), + attribute.Bool("success", err == nil), + ), + ) + }() + childSpan.SetAttributes( telemetry.WithSandboxID(in.GetSandboxId()), telemetry.WithTemplateID(in.GetTemplateId()), @@ -948,6 +966,9 @@ type snapshotResult struct { // with. The prefetch harvest reuses it verbatim when re-uploading the // metadata object, so the two can never drift. objectMetadata storage.ObjectMetadata + // filesystemOnly records whether this was a filesystem-only (memoryless) + // pause, so the async upload can label its failure counter with fs_only. + filesystemOnly bool } // snapshotAndCacheSandbox creates a snapshot of a sandbox and adds it to the @@ -1050,6 +1071,7 @@ func (s *Server) snapshotAndCacheSandbox( upload: upload, completeUpload: completeUpload, objectMetadata: objectMetadata, + filesystemOnly: filesystemOnly, }, nil } @@ -1084,7 +1106,7 @@ func (s *Server) uploadSnapshotAsync(ctx context.Context, sbx *sandbox.Sandbox, ) if err != nil { sbxlogger.I(sbx).Error(spanCtx, "snapshot upload did not durably land", zap.Error(err)) - s.uploadFailedCounter.Add(spanCtx, 1) + s.uploadFailedCounter.Add(spanCtx, 1, metric.WithAttributes(attribute.Bool("fs_only", res.filesystemOnly))) } else { sbxlogger.I(sbx).Info(spanCtx, "snapshot finished uploading successfully") } diff --git a/packages/shared/pkg/telemetry/meters.go b/packages/shared/pkg/telemetry/meters.go index 5af6999af0..24a1664315 100644 --- a/packages/shared/pkg/telemetry/meters.go +++ b/packages/shared/pkg/telemetry/meters.go @@ -106,6 +106,7 @@ const ( OrchestratorSandboxCreateDurationName HistogramType = "orchestrator.sandbox.create.duration" WaitForEnvdDurationHistogramName HistogramType = "orchestrator.sandbox.envd.init.duration" GuestSyncDurationHistogramName HistogramType = "orchestrator.sandbox.guest_sync.duration" + PauseDurationHistogramName HistogramType = "orchestrator.sandbox.pause.duration" // Pre-pause envd heap collapse round-trip duration (the pause-path cost of // POST /collapse: network plus envd's madvise work), recorded once per pause @@ -451,6 +452,7 @@ var histogramDesc = map[HistogramType]string{ WaitForEnvdDurationHistogramName: "Time taken for Envd to initialize successfully", EnvdCollapseDurationHistogramName: "Time taken for the pre-pause envd heap collapse round-trip", GuestSyncDurationHistogramName: "Time taken for the mandatory pre-pause guest sync (filesystem-only pause)", + PauseDurationHistogramName: "Time taken to pause a sandbox, labeled by fs_only (filesystem-only vs memory) and success", PauseResumePrefetchHarvestDurationName: "Time taken for a pause-resume prefetch harvest run (slot-hold cost)", PauseResumePrefetchHarvestPagesName: "Harvested resume-prefetch trace size in 2 MiB blocks, per successful harvest", @@ -501,6 +503,7 @@ var histogramUnits = map[HistogramType]string{ WaitForEnvdDurationHistogramName: "ms", EnvdCollapseDurationHistogramName: "ms", GuestSyncDurationHistogramName: "ms", + PauseDurationHistogramName: "ms", PauseResumePrefetchHarvestDurationName: "ms", PauseResumePrefetchHarvestPagesName: "{page}", UffdStartupPagesHistogramName: "{page}",