feat(ateapi): track actor_template_uid per snapshot on ExternalSnapshot

cmd/ateapi/internal/controlapi/actor.go

@@ -140,6 +140,7 @@ func (s *ServiceImpl) CreateActor(ctx context.Context, inActor *ateapipb.Actor)
 		// repoint before that first resume reads as "no guest state" instead of
 		// "replaced template", and the resume restores the old template's
 		// memory and rootfs in full instead of the volume data alone.
+		outActor.Status.ExternalSnapshot.ActorTemplateUid = sourceTag.GetStatus().GetActorTemplateUid()
 		outActor.Status.CurrentActorTemplateUid = sourceTag.GetStatus().GetActorTemplateUid()
 	}
 	if errs := validateActorUpdate(ctx, field.NewPath("actor"), outActor, inActor, true); len(errs) > 0 {

cmd/ateapi/internal/controlapi/functionaltest/actor_test.go

@@ -72,7 +72,7 @@ func TestCreateActor_Success(t *testing.T) {
 		Status: &ateapipb.ActorStatus{
 			State:                   ateapipb.ActorState_ACTOR_STATE_SUSPENDED,
 			CurrentActorTemplateUid: tmpl.GetMetadata().GetUid(),
-			ExternalSnapshot:        &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL},
+			ExternalSnapshot:        &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ActorTemplateUid: tmpl.GetMetadata().GetUid()},
 		},
 		WorkerSelector: &ateapipb.Selector{MatchLabels: map[string]string{"tier": "free"}},
 	}
@@ -694,7 +694,7 @@ func TestUpdateActor_Success(t *testing.T) {
 		Status: &ateapipb.ActorStatus{
 			State:                   ateapipb.ActorState_ACTOR_STATE_SUSPENDED,
 			CurrentActorTemplateUid: tmpl.GetMetadata().GetUid(),
-			ExternalSnapshot:        &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL},
+			ExternalSnapshot:        &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ActorTemplateUid: tmpl.GetMetadata().GetUid()},
 		},
 		WorkerSelector: &ateapipb.Selector{
 			MatchLabels: map[string]string{"tier": "paid"},
@@ -841,7 +841,7 @@ func TestUpdateActor(t *testing.T) {
 		Status: &ateapipb.ActorStatus{
 			State:                   ateapipb.ActorState_ACTOR_STATE_SUSPENDED,
 			CurrentActorTemplateUid: tmpl.GetMetadata().GetUid(),
-			ExternalSnapshot:        &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL},
+			ExternalSnapshot:        &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ActorTemplateUid: tmpl.GetMetadata().GetUid()},
 		},
 		WorkerSelector: &ateapipb.Selector{
 			MatchLabels: map[string]string{"tier": "paid"},
@@ -1812,7 +1812,7 @@ func TestResumeActor(t *testing.T) {
 		Status: &ateapipb.ActorStatus{
 			State:                   ateapipb.ActorState_ACTOR_STATE_RUNNING,
 			CurrentActorTemplateUid: tmpl.GetMetadata().GetUid(),
-			ExternalSnapshot:        &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL},
+			ExternalSnapshot:        &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ActorTemplateUid: tmpl.GetMetadata().GetUid()},
 			WorkerAssignment: &ateapipb.WorkerAssignment{
 				Worker:          &ateapipb.ObjectRef{Name: podUID},
 				WorkerNamespace: ns,
@@ -2332,8 +2332,12 @@ func TestSuspendActor(t *testing.T) {
 		Metadata:      &ateapipb.ResourceMetadata{Name: name, Atespace: testAtespace},
 		ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"},
 		Status: &ateapipb.ActorStatus{
-			State:                   ateapipb.ActorState_ACTOR_STATE_SUSPENDED,
-			ExternalSnapshot:        &ateapipb.ExternalSnapshot{SnapshotUri: snapshotURI, ContentScope: sourceActor.GetStatus().GetExternalSnapshot().GetContentScope()},
+			State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED,
+			ExternalSnapshot: &ateapipb.ExternalSnapshot{
+				SnapshotUri:      snapshotURI,
+				ContentScope:     sourceActor.GetStatus().GetExternalSnapshot().GetContentScope(),
+				ActorTemplateUid: tmpl.GetMetadata().GetUid(),
+			},
 			CurrentActorTemplateUid: tmpl.GetMetadata().GetUid(),
 		},
 	}
@@ -2494,6 +2498,78 @@ func TestResumeActor_RepointTemplateBeforeResume(t *testing.T) {
 	}
 }
 
+// TestResumeActor_PausedAfterRepointUsesLocalProvenance verifies that an actor
+// paused after a template repoint restores from its pause checkpoint in FULL,
+// not DATA. At that point the actor holds an external snapshot captured on v1
+// and a local checkpoint captured on v2; judging the local restore by the
+// external snapshot's provenance would wrongly discard the v2 memory image.
+func TestResumeActor_PausedAfterRepointUsesLocalProvenance(t *testing.T) {
+	ns := namespaceForTest("ns-repoint-pause")
+	tc := setupTest(t, ns)
+	defer tc.cleanup()
+
+	ctx := context.Background()
+	tmpl := createTemplate(t, tc, ns)
+	tmpl2 := proto.Clone(tmpl).(*ateapipb.ActorTemplate)
+	tmpl2.Metadata = &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "tmpl2"}
+	tmpl2.Status = nil
+	if _, err := tc.client.CreateActorTemplate(ctx, &ateapipb.CreateActorTemplateRequest{ActorTemplate: tmpl2}); err != nil {
+		t.Fatalf("CreateActorTemplate(tmpl2) failed: %v", err)
+	}
+	worker := createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1")
+
+	const name = "actor-1"
+	actorRef := &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}
+	if _, err := tc.client.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
+		Metadata:      &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name},
+		ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"},
+	}}); err != nil {
+		t.Fatalf("CreateActor failed: %v", err)
+	}
+	if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil {
+		t.Fatalf("ResumeActor(v1) failed: %v", err)
+	}
+	suspended, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef})
+	if err != nil {
+		t.Fatalf("SuspendActor(v1) failed: %v", err)
+	}
+	waitForWorkerAvailable(t, tc, worker)
+
+	// Repoint at v2 while suspended; the external snapshot stays on v1.
+	toUpdate := proto.Clone(suspended.GetActor()).(*ateapipb.Actor)
+	toUpdate.ActorTemplate = &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl2"}
+	if _, err := tc.client.UpdateActor(ctx, &ateapipb.UpdateActorRequest{Actor: toUpdate}); err != nil {
+		t.Fatalf("UpdateActor(tmpl2) failed: %v", err)
+	}
+
+	// Resume on v2 (this restore is DATA, from the v1 external snapshot), then
+	// pause; the pause checkpoint is captured on v2 while the external
+	// snapshot still says v1.
+	if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil {
+		t.Fatalf("ResumeActor(v2 from v1 snapshot) failed: %v", err)
+	}
+	if got := tc.fakeAtelet.lastRestoreRequest().GetScope(); got != ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA {
+		t.Fatalf("first resume on v2 had scope = %v, want DATA", got)
+	}
+	if _, err := tc.client.PauseActor(ctx, &ateapipb.PauseActorRequest{Actor: actorRef}); err != nil {
+		t.Fatalf("PauseActor failed: %v", err)
+	}
+	waitForWorkerAvailable(t, tc, worker)
+
+	// Resume from PAUSED: the local checkpoint was captured on v2 and the
+	// actor's template is v2, so the restore must be FULL.
+	if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil {
+		t.Fatalf("ResumeActor(from v2 pause) failed: %v", err)
+	}
+	restoreReq := tc.fakeAtelet.lastRestoreRequest()
+	if got := restoreReq.GetType(); got != ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL {
+		t.Errorf("restore request type = %v, want LOCAL", got)
+	}
+	if got := restoreReq.GetScope(); got != ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL {
+		t.Errorf("restore request scope = %v, want FULL (local checkpoint was captured on v2)", got)
+	}
+}
+
 // TestPauseActor tests the full workflow of pausing a running actor.
 // Workflow:
 // 1. Creates a mock ActorTemplate.
@@ -2558,7 +2634,7 @@ func TestPauseActor(t *testing.T) {
 				ContentScope:              ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL,
 			},
 			CurrentActorTemplateUid: tmpl.GetMetadata().GetUid(),
-			ExternalSnapshot:        &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL},
+			ExternalSnapshot:        &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ActorTemplateUid: tmpl.GetMetadata().GetUid()},
 		},
 	}
 

cmd/ateapi/internal/controlapi/workflow_resume.go

@@ -48,8 +48,8 @@ type resumeSnapshotSource struct {
 	// selects the golden snapshot as the boot source for the pending restore:
 	// restore then combines the golden snapshot with the actor's data.
 	GoldenSnapshotURI resources.SnapshotURI
-	// TemplateReplaced is true when the snapshot's recorded template UID
-	// differs from the actor's current template.
+	// TemplateReplaced is true when the external snapshot's recorded template
+	// UID differs from the actor's current template.
 	TemplateReplaced bool
 }
 
@@ -184,13 +184,8 @@ func (w *ActorWorkflow) loadActorForResume(ctx context.Context, actorRef resourc
 			return nil, nil, src, status.Errorf(codes.DataLoss, "Actor %s external snapshot: %v", actorRef, err)
 		}
 		src.Scope = actor.GetStatus().GetExternalSnapshot().GetContentScope()
-		// The Actor records the template its guest state was built on; a
-		// different UID on its current template means it was repointed since
-		// the capture.
-		// TODO: Disallow updating the ActorTemplate ID for paused actors here
-		// as well; it is already disallowed at admission time.
-		builtOnTemplateUID := actor.GetStatus().GetCurrentActorTemplateUid()
-		src.TemplateReplaced = builtOnTemplateUID != "" && builtOnTemplateUID != actorTemplate.GetMetadata().GetUid()
+		capturedUnder := actor.GetStatus().GetExternalSnapshot().GetActorTemplateUid()
+		src.TemplateReplaced = capturedUnder != "" && capturedUnder != actorTemplate.GetMetadata().GetUid()
 	}
 
 	// The template's onResume configuration selects the boot source for the
@@ -701,8 +696,6 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou
 		// loadActorForResume resolved a golden URI per the template's onResume
 		// configuration, else what the pause captured.
 		switch {
-		case src.TemplateReplaced:
-			req.Scope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA
 		case !src.GoldenSnapshotURI.IsZero():
 			req.Scope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN
 			req.GoldenSnapshotUri = src.GoldenSnapshotURI.String()

cmd/ateapi/internal/controlapi/workflow_resume_test.go

@@ -1090,25 +1090,38 @@ func TestLoadActorForResume_DoesNotDefaultGolden(t *testing.T) {
 }
 
 // TestLoadActorForResume_TemplateReplaced covers the detection of a repointed
-// actor: the actor records the template UID its guest state was built on, and
-// a mismatch with its current template marks the source TemplateReplaced,
-// forcing the restore to data-only.
+// actor: when ExternalSnapshot.actor_template_uid differs from the actor's
+// current template UID, TemplateReplaced is set so the external restore
+// downgrades to data-only.
 func TestLoadActorForResume_TemplateReplaced(t *testing.T) {
 	actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"}
 
+	// "current" stands for the created template's own UID.
+	const current = "current"
+
 	tests := []struct {
-		name string
-		// builtOnTemplateUID seeds the template UID the actor's guest state
-		// was built on; "current" stands for the created template's own UID,
-		// "" leaves the field unset (an actor from before it was recorded).
-		builtOnTemplateUID string
-		noSnapshot         bool
-		want               bool
+		name        string
+		snapshotUID string
+		noSnapshot  bool
+		want        bool
 	}{
-		{name: "snapshot taken under the current template", builtOnTemplateUID: "current", want: false},
-		{name: "snapshot taken under a replaced template", builtOnTemplateUID: "some-other-uid", want: true},
-		{name: "snapshot without a recorded template UID", builtOnTemplateUID: "", want: false},
-		{name: "no durable snapshot", noSnapshot: true, want: false},
+		{
+			name:        "snapshot taken under the current template",
+			snapshotUID: current,
+		},
+		{
+			name:        "snapshot taken under a replaced template",
+			snapshotUID: "some-other-uid",
+			want:        true,
+		},
+		{
+			name:        "snapshot without a recorded template UID",
+			snapshotUID: "",
+		},
+		{
+			name:       "no durable snapshot",
+			noSnapshot: true,
+		},
 	}
 
 	for _, tt := range tests {
@@ -1124,21 +1137,23 @@ func TestLoadActorForResume_TemplateReplaced(t *testing.T) {
 				t.Fatalf("create template: %v", err)
 			}
 			if tmpl.GetMetadata().GetUid() == "" {
-				t.Fatal("created template has no UID; the matching case would be vacuous")
+				t.Fatal("created template has no UID; the matching cases would be vacuous")
+			}
+			resolve := func(uid string) string {
+				if uid == current {
+					return tmpl.GetMetadata().GetUid()
+				}
+				return uid
 			}
 
 			var seedOpts []func(*ateapipb.Actor)
 			if !tt.noSnapshot {
-				uid := tt.builtOnTemplateUID
-				if uid == "current" {
-					uid = tmpl.GetMetadata().GetUid()
-				}
 				seedOpts = append(seedOpts, func(a *ateapipb.Actor) {
 					a.Status.ExternalSnapshot = &ateapipb.ExternalSnapshot{
-						SnapshotUri:  someActorSnapshotURI(t, testStorageLocation, actorRef.Atespace, "snap-1"),
-						ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL,
+						SnapshotUri:      someActorSnapshotURI(t, testStorageLocation, actorRef.Atespace, "snap-1"),
+						ContentScope:     ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL,
+						ActorTemplateUid: resolve(tt.snapshotUID),
 					}
-					a.Status.CurrentActorTemplateUid = uid
 				})
 			}
 			seedWorkflowActor(t, ctx, persistence, actorRef, "ns", "tmpl1", ateapipb.ActorState_ACTOR_STATE_SUSPENDED, seedOpts...)
@@ -1310,11 +1325,11 @@ func TestResumeActor_AteletWireRequest(t *testing.T) {
 		localSnapshot *ateapipb.LocalSnapshotInfo
 		// externalSnapshot seeds Status.ExternalSnapshot (the durable snapshot).
 		externalSnapshot *ateapipb.ExternalSnapshot
-		// tmplUID seeds Status.CurrentActorTemplateUid, the template UID the
-		// guest state was built on: "current" stands for the created template's
-		// store-assigned UID (unknown until runtime), "" leaves the field unset
-		// (an actor from before it was recorded), anything else mismatches (a
-		// repointed actor).
+		// tmplUID seeds the template UID the snapshot's guest state was built
+		// on (stamped onto localSnapshot, externalSnapshot, and
+		// CurrentActorTemplateUid): "current" stands for the created template's
+		// store-assigned UID (unknown until runtime), "" leaves the field unset,
+		// anything else mismatches (a repointed actor).
 		tmplUID string
 	}
 	// templateSeed is the ActorTemplate configuration a row persists.
@@ -1680,9 +1695,11 @@ func TestResumeActor_AteletWireRequest(t *testing.T) {
 			},
 		},
 		{
-			// TemplateReplaced wins in the local branch too: the pause
-			// checkpoint restores as plain Data, the golden overlay dropped.
-			name: "29 template repoint beats the golden policy on the local path",
+			// An older external snapshot's template mismatch does not affect a
+			// local pause restore: templates can only be updated while
+			// SUSPENDED, so a pause checkpoint is always from the current
+			// template.
+			name: "29 local snapshot ignores an older external snapshot's template mismatch",
 			actor: actorSeed{
 				localSnapshot:    &ateapipb.LocalSnapshotInfo{SnapshotName: localSnapshotName, NodeVmsWithLocalSnapshots: []string{"node-1"}},
 				externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: actorURI, ContentScope: fullScope},
@@ -1696,7 +1713,8 @@ func TestResumeActor_AteletWireRequest(t *testing.T) {
 			want: restoreWant{
 				checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL,
 				snapshotName:   localSnapshotName,
-				scope:          ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA,
+				scope:          ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN,
+				goldenURI:      goldenURI,
 			},
 		},
 	}
@@ -1753,12 +1771,18 @@ func TestResumeActor_AteletWireRequest(t *testing.T) {
 			seedWorkflowActor(t, ctx, persistence, actorRef, "ns", "tmpl1", actorState, func(a *ateapipb.Actor) {
 				a.Status.WorkerAssignment = wireTestAssignment()
 				a.Status.LocalSnapshotInfo = tt.actor.localSnapshot
-				a.Status.ExternalSnapshot = tt.actor.externalSnapshot
 				uid := tt.actor.tmplUID
 				if uid == "current" {
 					uid = createdTmpl.GetMetadata().GetUid()
 				}
 				a.Status.CurrentActorTemplateUid = uid
+				if tt.actor.externalSnapshot != nil {
+					ext := proto.CloneOf(tt.actor.externalSnapshot)
+					if ext.ActorTemplateUid == "" {
+						ext.ActorTemplateUid = uid
+					}
+					a.Status.ExternalSnapshot = ext
+				}
 			})
 
 			actor, loadedTmpl, src, err := w.loadActorForResume(ctx, actorRef)

cmd/ateapi/internal/controlapi/workflow_suspend.go

@@ -384,8 +384,9 @@ func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef r
 	externalSnapshot := latestActor.GetStatus().GetExternalSnapshot()
 	if inProgressSnapshotURI != "" {
 		externalSnapshot = &ateapipb.ExternalSnapshot{
-			SnapshotUri:  inProgressSnapshotURI,
-			ContentScope: commitSnapshotScope(actorRef.Atespace, actorTemplate),
+			SnapshotUri:      inProgressSnapshotURI,
+			ContentScope:     commitSnapshotScope(actorRef.Atespace, actorTemplate),
+			ActorTemplateUid: latestActor.GetStatus().GetCurrentActorTemplateUid(),
 		}
 	}
 

cmd/ateapi/internal/controlapi/workflow_suspend_test.go

@@ -285,8 +285,9 @@ func TestEnsureSuspendedFinalized_NoAssignment(t *testing.T) {
 	actor := &ateapipb.Actor{
 		Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "actor-1"},
 		Status: &ateapipb.ActorStatus{
-			State:                 ateapipb.ActorState_ACTOR_STATE_SUSPENDING,
-			InProgressSnapshotUri: snapshotURI,
+			State:                   ateapipb.ActorState_ACTOR_STATE_SUSPENDING,
+			InProgressSnapshotUri:   snapshotURI,
+			CurrentActorTemplateUid: "tmpl-uid-1",
 			LocalSnapshotInfo: &ateapipb.LocalSnapshotInfo{
 				SnapshotName:              "actor-1-pause-snapshot",
 				NodeVmsWithLocalSnapshots: []string{"node1"},
@@ -308,6 +309,11 @@ func TestEnsureSuspendedFinalized_NoAssignment(t *testing.T) {
 	if got := stored.GetStatus().GetExternalSnapshot().GetSnapshotUri(); got != snapshotURI {
 		t.Errorf("SnapshotUri = %q, want %q", got, snapshotURI)
 	}
+	// The snapshot carries the template it was captured under, so a later
+	// repoint can tell that its guest state no longer matches.
+	if got := stored.GetStatus().GetExternalSnapshot().GetActorTemplateUid(); got != "tmpl-uid-1" {
+		t.Errorf("ExternalSnapshot.ActorTemplateUid = %q, want %q", got, "tmpl-uid-1")
+	}
 	if got := stored.GetStatus().GetInProgressSnapshotUri(); got != "" {
 		t.Errorf("InProgressSnapshotUri = %q, want cleared", got)
 	}