fix(orchestrator): preserve stale pre-init envd logs' true timing

Sandboxes resumed from a memory snapshot boot with the guest clock
still set to the template's snapshot time. envd's log exporter is
restored along with the rest of the VM, so it can send startup logs
stamped with that old time before /init corrects the clock.

Each sandbox now records a fresh host timestamp when its lifecycle
is created. The hyperloop /logs handler compares incoming envd
timestamps against it and, if one is more than a minute older,
replaces it with the lifecycle start time, keeping the original
under original_timestamp instead of dropping it. The one-minute
margin avoids touching logs that are only off by ordinary host/guest
clock skew.

Author: rguliyev

packages/orchestrator/pkg/hyperloopserver/handlers/logs.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"time"
 
 	"github.com/gin-gonic/gin"
 	"go.uber.org/zap"
@@ -52,6 +53,8 @@ func (h *APIStore) Logs(c *gin.Context) {
 	payload["envID"] = sbx.Runtime.TemplateID
 	payload["teamID"] = sbx.Runtime.TeamID
 
+	correctStaleTimestamp(payload, sbx.LifecycleStartedAt)
+
 	logs, err := json.Marshal(payload)
 	if err != nil {
 		h.sendAPIStoreError(c, http.StatusInternalServerError, "Error when parsing logs payload")
@@ -98,3 +101,42 @@ func (h *APIStore) validatePayloadSandboxID(payload map[string]any, sbxID string
 
 	return nil
 }
+
+// envdTimestampLayout matches zerolog's TimeFieldFormat in envd's logger.
+const envdTimestampLayout = time.RFC3339Nano
+
+// staleTimestampSlack tolerates normal host/guest clock skew so we only
+// override timestamps that are actually stale, not ones a second or two off.
+const staleTimestampSlack = time.Minute
+
+// correctStaleTimestamp fixes up envd log records still carrying the guest's
+// pre-resume clock. envd's log exporter survives the memory snapshot and can
+// flush pre-init logs before /init corrects the guest clock, so those
+// records land with a stale timestamp.
+//
+// lifecycleStart (Sandbox.LifecycleStartedAt) is set fresh per lifecycle
+// before the sandbox is reachable, so unlike GetStartedAt() it can't itself
+// be stale. Anything older than lifecycleStart (minus slack) gets overridden
+// with it, and the original value is kept under "original_timestamp".
+//
+// No-op if lifecycleStart is zero or "timestamp" isn't a parseable string.
+func correctStaleTimestamp(payload map[string]any, lifecycleStart time.Time) {
+	if lifecycleStart.IsZero() {
+		return
+	}
+
+	raw, ok := payload["timestamp"].(string)
+	if !ok {
+		return
+	}
+
+	ts, err := time.Parse(envdTimestampLayout, raw)
+	if err != nil {
+		return
+	}
+
+	if ts.Before(lifecycleStart.Add(-staleTimestampSlack)) {
+		payload["original_timestamp"] = raw
+		payload["timestamp"] = lifecycleStart.UTC().Format(envdTimestampLayout)
+	}
+}

packages/orchestrator/pkg/hyperloopserver/handlers/logs_test.go

@@ -0,0 +1,137 @@
+//go:build linux
+
+package handlers
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCorrectStaleTimestamp(t *testing.T) {
+	t.Parallel()
+
+	lifecycleStart, err := time.Parse(envdTimestampLayout, "2026-07-16T10:00:00Z")
+	require.NoError(t, err)
+
+	t.Run("overrides a timestamp from before the corrected clock", func(t *testing.T) {
+		t.Parallel()
+
+		staleRaw := "2026-06-09T08:00:00.123456789Z"
+		payload := map[string]any{"timestamp": staleRaw}
+
+		correctStaleTimestamp(payload, lifecycleStart)
+
+		assert.Equal(t, lifecycleStart.Format(envdTimestampLayout), payload["timestamp"])
+		assert.Equal(t, staleRaw, payload["original_timestamp"])
+	})
+
+	t.Run("leaves a post-correction timestamp untouched", func(t *testing.T) {
+		t.Parallel()
+
+		freshRaw := lifecycleStart.Add(time.Second).Format(envdTimestampLayout)
+		payload := map[string]any{"timestamp": freshRaw}
+
+		correctStaleTimestamp(payload, lifecycleStart)
+
+		assert.Equal(t, freshRaw, payload["timestamp"])
+		assert.NotContains(t, payload, "original_timestamp")
+	})
+
+	t.Run("leaves a timestamp exactly at lifecycleStart untouched", func(t *testing.T) {
+		t.Parallel()
+
+		boundaryRaw := lifecycleStart.Format(envdTimestampLayout)
+		payload := map[string]any{"timestamp": boundaryRaw}
+
+		correctStaleTimestamp(payload, lifecycleStart)
+
+		assert.Equal(t, boundaryRaw, payload["timestamp"])
+		assert.NotContains(t, payload, "original_timestamp")
+	})
+
+	t.Run("leaves a timestamp exactly at the slack cutoff untouched", func(t *testing.T) {
+		t.Parallel()
+
+		// Before is strict, so the cutoff itself must not be overridden.
+		cutoffRaw := lifecycleStart.Add(-staleTimestampSlack).Format(envdTimestampLayout)
+		payload := map[string]any{"timestamp": cutoffRaw}
+
+		correctStaleTimestamp(payload, lifecycleStart)
+
+		assert.Equal(t, cutoffRaw, payload["timestamp"])
+		assert.NotContains(t, payload, "original_timestamp")
+	})
+
+	t.Run("leaves a slightly earlier timestamp within slack untouched", func(t *testing.T) {
+		t.Parallel()
+
+		// Ordinary clock skew, not a stale guest clock - shouldn't override.
+		withinSlackRaw := lifecycleStart.Add(-staleTimestampSlack / 2).Format(envdTimestampLayout)
+		payload := map[string]any{"timestamp": withinSlackRaw}
+
+		correctStaleTimestamp(payload, lifecycleStart)
+
+		assert.Equal(t, withinSlackRaw, payload["timestamp"])
+		assert.NotContains(t, payload, "original_timestamp")
+	})
+
+	t.Run("overrides a timestamp just beyond the slack window", func(t *testing.T) {
+		t.Parallel()
+
+		staleRaw := lifecycleStart.Add(-staleTimestampSlack - time.Second).Format(envdTimestampLayout)
+		payload := map[string]any{"timestamp": staleRaw}
+
+		correctStaleTimestamp(payload, lifecycleStart)
+
+		assert.Equal(t, lifecycleStart.Format(envdTimestampLayout), payload["timestamp"])
+		assert.Equal(t, staleRaw, payload["original_timestamp"])
+	})
+
+	t.Run("is a no-op when lifecycleStart is zero", func(t *testing.T) {
+		t.Parallel()
+
+		staleRaw := "2026-06-09T08:00:00Z"
+		payload := map[string]any{"timestamp": staleRaw}
+
+		correctStaleTimestamp(payload, time.Time{})
+
+		assert.Equal(t, staleRaw, payload["timestamp"])
+		assert.NotContains(t, payload, "original_timestamp")
+	})
+
+	t.Run("is a no-op when timestamp is missing", func(t *testing.T) {
+		t.Parallel()
+
+		payload := map[string]any{"message": "hello"}
+
+		correctStaleTimestamp(payload, lifecycleStart)
+
+		assert.NotContains(t, payload, "timestamp")
+		assert.NotContains(t, payload, "original_timestamp")
+	})
+
+	t.Run("is a no-op when timestamp is not a string", func(t *testing.T) {
+		t.Parallel()
+
+		payload := map[string]any{"timestamp": 12345}
+
+		correctStaleTimestamp(payload, lifecycleStart)
+
+		assert.Equal(t, 12345, payload["timestamp"])
+		assert.NotContains(t, payload, "original_timestamp")
+	})
+
+	t.Run("is a no-op when timestamp is unparseable", func(t *testing.T) {
+		t.Parallel()
+
+		payload := map[string]any{"timestamp": "not-a-time"}
+
+		correctStaleTimestamp(payload, lifecycleStart)
+
+		assert.Equal(t, "not-a-time", payload["timestamp"])
+		assert.NotContains(t, payload, "original_timestamp")
+	})
+}

packages/orchestrator/pkg/sandbox/sandbox.go

@@ -270,6 +270,15 @@ type Sandbox struct {
 	// every time a new Firecracker VM is started.
 	LifecycleID string
 
+	// LifecycleStartedAt is the host-clock time (UTC) this lifecycle's
+	// Sandbox was constructed, set once before it's registered in the
+	// network map. Unlike Metadata.startedAt, which is caller-seeded and
+	// can carry over the previous lifecycle's value across a checkpoint,
+	// this is always fresh. Used by correctStaleTimestamp in
+	// hyperloopserver/handlers/logs.go to catch envd logs still carrying
+	// the pre-resume guest clock.
+	LifecycleStartedAt time.Time
+
 	config  cfg.BuilderConfig
 	files   *storage.SandboxFiles
 	cleanup *Cleanup
@@ -599,7 +608,8 @@ func (f *Factory) CreateSandbox(
 	}
 
 	sbx := &Sandbox{
-		LifecycleID: lifecycleID,
+		LifecycleID:        lifecycleID,
+		LifecycleStartedAt: time.Now().UTC(),
 
 		Resources:    resources,
 		Metadata:     metadata,
@@ -1010,7 +1020,8 @@ func (f *Factory) ResumeSandbox(
 	}
 
 	sbx := &Sandbox{
-		LifecycleID: lifecycleID,
+		LifecycleID:        lifecycleID,
+		LifecycleStartedAt: time.Now().UTC(),
 
 		Resources:    resources,
 		Metadata:     metadata,