From c79f31bf31132569c481ba8bec49cb8c2bd7a7d8 Mon Sep 17 00:00:00 2001 From: shrutiyam-glitch Date: Thu, 17 Sep 2026 01:10:04 -0700 Subject: [PATCH 1/3] Document RevertActor and drop "terminal" from CRASHED RevertActor returns a RUNNING, PAUSED, or CRASHED actor to SUSPENDED at its last external snapshot, so CRASHED is no longer a dead end that only DeleteActor can clear. Docs and code comments still described it as terminal and told operators to delete and recreate the actor, losing its state. Update the api-guide, architecture, upgrade guide, and kubectl-ate README to cover the new verb, and correct the comments that justified keeping a partial external snapshot by naming actor deletion as the only remaining collector -- revert collects it too. --- benchmarking/locust/common/ateapi_pb2_grpc.py | 3 ++- cmd/ateapi/internal/controlapi/crash.go | 2 +- cmd/ateapi/internal/controlapi/crash_test.go | 2 +- .../internal/controlapi/workflow_delete_test.go | 7 ++++--- .../internal/controlapi/workflow_worker_delete.go | 5 ++--- .../controlapi/workflow_worker_delete_test.go | 2 +- cmd/kubectl-ate/README.md | 5 ++++- docs/api-guide.md | 9 ++++++++- docs/architecture.md | 4 +++- docs/metrics/registry/metrics.yaml | 10 +++++----- docs/upgrade.md | 13 +++++++------ pkg/proto/ateapipb/ateapi.proto | 3 ++- pkg/proto/ateapipb/ateapi_grpc.pb.go | 6 ++++-- 13 files changed, 44 insertions(+), 27 deletions(-) diff --git a/benchmarking/locust/common/ateapi_pb2_grpc.py b/benchmarking/locust/common/ateapi_pb2_grpc.py index 97ef55f6fb..6e0f54016f 100644 --- a/benchmarking/locust/common/ateapi_pb2_grpc.py +++ b/benchmarking/locust/common/ateapi_pb2_grpc.py @@ -270,7 +270,8 @@ def ResumeActor(self, request, context): raise NotImplementedError('Method not implemented!') def DeleteActor(self, request, context): - """Delete an actor. Only suspended actors can be deleted. + """Delete an actor. Only suspended or crashed actors can be deleted unless + any_state is set. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') diff --git a/cmd/ateapi/internal/controlapi/crash.go b/cmd/ateapi/internal/controlapi/crash.go index 34956a40a5..8333209cf0 100644 --- a/cmd/ateapi/internal/controlapi/crash.go +++ b/cmd/ateapi/internal/controlapi/crash.go @@ -74,7 +74,7 @@ func crashActor(ctx context.Context, st crashActorStore, actorRef resources.Acto reason = ateattr.ReasonUnknown } - // Release the worker before moving the actor to the terminal CRASHED state. + // Release the worker before moving the actor to CRASHED state. // If the release fails we must not clear the actor's worker assignment or // mark it CRASHED: doing so would strand the still-assigned worker with no // actor referencing it, so nothing would ever retry the release and the diff --git a/cmd/ateapi/internal/controlapi/crash_test.go b/cmd/ateapi/internal/controlapi/crash_test.go index 1b9a977211..6a5c80908d 100644 --- a/cmd/ateapi/internal/controlapi/crash_test.go +++ b/cmd/ateapi/internal/controlapi/crash_test.go @@ -532,7 +532,7 @@ func (f failingReleaseStore) ReleaseActorFromWorker(context.Context, string, str } // A transient failure releasing the worker must not move the actor to the -// terminal CRASHED state: doing so would strand the still-assigned worker with +// CRASHED state: doing so would strand the still-assigned worker with // no actor left to drive a retry, permanently consuming the worker slot. // crashActor must return the error with the actor and worker left intact so the // caller retries and the worker is reclaimed. diff --git a/cmd/ateapi/internal/controlapi/workflow_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_delete_test.go index fa95d62fea..492f6a26ec 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete_test.go @@ -364,8 +364,9 @@ func TestDeleteActor_CollectsInFlightSnapshotWithoutTemplate(t *testing.T) { // TestDeleteActor_CollectsSnapshotsAfterWorkerDelete verifies that // deleting an actor whose suspend a worker delete crashed mid-finalize reclaims -// every object that suspend wrote. CRASHED is terminal, so the actor delete is -// the only collector left: whatever it cannot name is leaked for good. +// every object that suspend wrote. When an actor crashes mid-suspend, only +// delete or revert can collect the partial snapshot: whatever they cannot name +// is leaked for good. func TestDeleteActor_CollectsSnapshotsAfterWorkerDelete(t *testing.T) { tests := []struct { name string @@ -447,7 +448,7 @@ func TestDeleteActor_CollectsSnapshotsAfterWorkerDelete(t *testing.T) { t.Fatalf("DeleteWorker: %v", err) } - // The actor is CRASHED and can only be deleted from here. + // The actor is CRASHED; delete reclaims its partial snapshot. stored, err := persistence.GetActor(ctx, actorRef) if err != nil { t.Fatalf("GetActor: %v", err) diff --git a/cmd/ateapi/internal/controlapi/workflow_worker_delete.go b/cmd/ateapi/internal/controlapi/workflow_worker_delete.go index 677d85cfbc..9ae4f76fb5 100644 --- a/cmd/ateapi/internal/controlapi/workflow_worker_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_worker_delete.go @@ -214,9 +214,8 @@ func (w *WorkerWorkflow) releaseBoundActor(ctx context.Context, worker *ateapipb toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_CRASHED toUpdate.Status.WorkerAssignment = nil // Local in-progress checkpoint dies with the worker: it lived on the node - // that went away. The external in-progress checkpoint is kept. It'll be deleted - // with the actor when the actor is deleted (only possible outcome from CRASHED - // state). + // that went away. The external in-progress checkpoint is kept so delete + // or revert can collect it. toUpdate.Status.InProgressLocalSnapshotName = "" return nil }) diff --git a/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go index 1cfecf6ae0..0dc6966737 100644 --- a/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go @@ -135,7 +135,7 @@ func TestDeleteWorkerWorkflow_ReleasesBoundActor(t *testing.T) { t.Errorf("in-progress local checkpoint not cleared: %v", got.GetStatus()) } // The durable one is kept: it names the prefix whatever atelet already - // uploaded lives under, which the actor's delete needs to collect it. + // uploaded lives under, which delete or revert needs to collect it. if want := someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "partial-snapshot"); got.GetStatus().GetInProgressSnapshotUri() != want { t.Errorf("in-progress external checkpoint not preserved: %v", got.GetStatus()) } diff --git a/cmd/kubectl-ate/README.md b/cmd/kubectl-ate/README.md index 94a9c00437..0fbbcd880a 100644 --- a/cmd/kubectl-ate/README.md +++ b/cmd/kubectl-ate/README.md @@ -109,7 +109,7 @@ kubectl ate get workers -l | `ATESPACE` | The atespace the actor belongs to. Part of the actor's identity; folded into the storage key as `actor::`. | | `NAME` | The actor's name. User-provided for application actors; UUID for the golden actor that each template materialises while building its golden tag. | | `TEMPLATE` | The `ActorTemplate` the actor was created from, displayed as `/`. | -| `STATE` | One of `ACTOR_STATE_RESUMING`, `ACTOR_STATE_RUNNING`, `ACTOR_STATE_SUSPENDING`, `ACTOR_STATE_SUSPENDED`. | +| `STATE` | Current lifecycle state (`ACTOR_STATE_RESUMING`, `ACTOR_STATE_RUNNING`, `ACTOR_STATE_SUSPENDING`, `ACTOR_STATE_SUSPENDED`, `ACTOR_STATE_PAUSING`, `ACTOR_STATE_PAUSED`, `ACTOR_STATE_CRASHED`, `ACTOR_STATE_DELETING`, `ACTOR_STATE_REVERTING`). | | `WORKER POD` | The worker pod (namespace/name) currently hosting the actor. Empty while suspended. | | `WORKER IP` | The pod IP of that worker. Empty while suspended. | | `VERSION` | Monotonic integer that increments on every state transition (resume / suspend / checkpoint). Useful for distinguishing snapshots. | @@ -202,6 +202,9 @@ kubectl ate resume actor my-actor -a # Suspend an actor (snapshots its state to storage and frees the worker) kubectl ate suspend actor my-actor -a +# Revert an actor to its last external snapshot (discards live, paused, or crashed state and returns to SUSPENDED) +kubectl ate revert actor my-actor -a + # Delete an actor (by default, requires the actor to be SUSPENDED or CRASHED). kubectl ate delete actor my-actor -a diff --git a/docs/api-guide.md b/docs/api-guide.md index b6d185006b..a424343220 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -450,7 +450,7 @@ Once a template is `Ready`, creating an actor logically (via `kubectl ate create * **Startup Logic:** Place expensive initialization (loading large models, establishing baseline connections) in your application's entry point. These will be captured in the Golden Snapshot and won't need to be repeated on every resumption. * **Placement:** Ensure your `ActorTemplate`'s `sandboxClass` matches your `WorkerPool`'s, and use the template's `workerSelector` to target specific pools — pool selection is by label match, not by namespace or RBAC. * **Version Management:** When updating code, create a new `ActorTemplate` (e.g. `v2`). Substrate treats each template as an immutable state root. -* **Eviction:** When its worker pod is evicted, an actor gets `SIGTERM` and 30 minutes to be suspended. After that it is killed and moves to `ACTOR_STATE_CRASHED`, and everything since its last snapshot is lost. So an actor that runs for more than 30 minutes without a suspend can lose data. +* **Eviction:** When its worker pod is evicted, an actor gets `SIGTERM` and 30 minutes to be suspended. After that it is killed and moves to `ACTOR_STATE_CRASHED`, and everything since its last snapshot is lost. So an actor that runs for more than 30 minutes without a suspend can lose data. A `CRASHED` actor can be recovered back to `ACTOR_STATE_SUSPENDED` at its last external snapshot using `RevertActor` (`kubectl ate revert`). --- @@ -504,6 +504,13 @@ Deletion always runs before the database reference is dropped, and a failure fai > **Do not delete a tag while actors created from it exist.** A clone borrows the tag's snapshot rather than copying it, and only stops borrowing at its own first suspend (its `status.externalSnapshot.snapshotUri` still names the tag's prefix while it is). Deleting the tag leaves such a clone unable to resume. This is not prevented today. +#### `RevertActor` +Discards an actor's live or crashed execution and transitions it to `ACTOR_STATE_SUSPENDED` at its last completed external snapshot (`status.externalSnapshot`). +* **Request:** `RevertActorRequest` + * `actor`: `ObjectRef` of the actor to revert. Accepted from `ACTOR_STATE_RUNNING`, `ACTOR_STATE_PAUSED`, and `ACTOR_STATE_CRASHED` (plus `ACTOR_STATE_REVERTING` for idempotent retries). Calling `RevertActor` on an already `ACTOR_STATE_SUSPENDED` actor returns `FAILED_PRECONDITION`. +* **Response:** `RevertActorResponse` containing the reverted `Actor` in `ACTOR_STATE_SUSPENDED`. +* Reverting terminates any bound worker sandbox, clears node-local pause checkpoints (`localSnapshotInfo`), and garbage-collects any partial external snapshot left by an interrupted suspend while preserving the last committed `externalSnapshot`. + #### `DeleteActor` Removes an actor from the registry and cleans up associated resources. * **Request:** `DeleteActorRequest` diff --git a/docs/architecture.md b/docs/architecture.md index e0b06a1343..dbe0a70b34 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -471,7 +471,9 @@ Actor that took it, and publishing it permits reuse from other Atespaces without `atespace/name` address. Deleting a tag deletes that copy; an Atespace with tags cannot be deleted until they are. -### Phase 4: Deletion +### Phase 4: Recovery (`RevertActor`) and Deletion (`DeleteActor`) + +An actor in `ACTOR_STATE_RUNNING`, `ACTOR_STATE_PAUSED`, or `ACTOR_STATE_CRASHED` can be reverted back to `ACTOR_STATE_SUSPENDED` via `RevertActor`. Reverting terminates any active sandbox, discards any local pause checkpoint or partial in-progress snapshot, and preserves the actor's last completed external snapshot so a subsequent `ResumeActor` restores from that checkpoint. By default, only actors in `ACTOR_STATE_SUSPENDED` or `ACTOR_STATE_CRASHED` state can be deleted from the Control Plane. With the `any_state` flag enabled, an actor in any state (such as `ACTOR_STATE_RUNNING` or `ACTOR_STATE_PAUSED`) can be deleted directly; the workflow terminates the running containers on the worker, detaches mounted volumes, and frees the worker assignment before deleting the record. diff --git a/docs/metrics/registry/metrics.yaml b/docs/metrics/registry/metrics.yaml index 45c6317310..ad5e5ff0f0 100644 --- a/docs/metrics/registry/metrics.yaml +++ b/docs/metrics/registry/metrics.yaml @@ -617,11 +617,11 @@ groups: brief: The number of actors that went to the ACTOR_STATE_CRASHED state, with the reasons. note: > This counter counts the moves into the CRASHED state of the state machine. - It does not count the crashes of a process. The CRASHED state is a - permanent loss of the actor. Substrate can also lose the data that it did - not write. Some moves into this state are not a loss of data. They are - careful responses to a control plane problem. The ate.failure.reason key - keeps these groups separate. + It does not count the crashes of a process. The CRASHED state is a loss of + the actor's live execution (recoverable back to SUSPENDED via RevertActor). + Substrate can also lose the data that it did not write. Some moves into + this state are not a loss of data. They are careful responses to a control + plane problem. The ate.failure.reason key keeps these groups separate. ate.sandbox.class is unknown when ateapi could not read the class of the worker: the worker record is already gone, or its assignment is already clear. diff --git a/docs/upgrade.md b/docs/upgrade.md index 966826b349..36a11f6870 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -211,12 +211,13 @@ Each warning comes back at the step where the mistake becomes possible. > inside that window saves its state and stays resumable. Handling > `SIGTERM` by exiting cleanly is not enough on its own; the suspend has > to reach the control plane and finish. An actor still awake when the -> window closes moves to `ACTOR_STATE_CRASHED`, which is terminal: -> `resume` and `suspend` are both refused, there is no recover verb, -> and the snapshot the actor still holds cannot be used to start it. It -> has to be deleted and recreated, losing its state. Scaling a serving -> pool down removes pods the same way, without suspending the actors on -> them. (Step 4 clones the pool; it never edits it.) +> window closes moves to `ACTOR_STATE_CRASHED`: `resume` and `suspend` +> are both refused, and everything since its last snapshot is lost. Call +> `RevertActor` (`kubectl ate revert`) to discard the crashed run and +> return the actor to `ACTOR_STATE_SUSPENDED` at its last external snapshot +> so it can be resumed. Scaling a serving pool down removes pods the same +> way, without suspending the actors on them. (Step 4 clones the pool; it +> never edits it.) > [!WARNING] > **On GKE, do not touch the node pool's label until every node is diff --git a/pkg/proto/ateapipb/ateapi.proto b/pkg/proto/ateapipb/ateapi.proto index 0022537716..0f67760856 100644 --- a/pkg/proto/ateapipb/ateapi.proto +++ b/pkg/proto/ateapipb/ateapi.proto @@ -43,7 +43,8 @@ service Control { // Resume an actor from its latest snapshot. rpc ResumeActor(ResumeActorRequest) returns (ResumeActorResponse) {} - // Delete an actor. Only suspended actors can be deleted. + // Delete an actor. Only suspended or crashed actors can be deleted unless + // any_state is set. rpc DeleteActor(DeleteActorRequest) returns (Actor) {} // Get the egress policy resource nested under an Actor. diff --git a/pkg/proto/ateapipb/ateapi_grpc.pb.go b/pkg/proto/ateapipb/ateapi_grpc.pb.go index f88dbd6700..977303fe86 100644 --- a/pkg/proto/ateapipb/ateapi_grpc.pb.go +++ b/pkg/proto/ateapipb/ateapi_grpc.pb.go @@ -89,7 +89,8 @@ type ControlClient interface { PauseActor(ctx context.Context, in *PauseActorRequest, opts ...grpc.CallOption) (*PauseActorResponse, error) // Resume an actor from its latest snapshot. ResumeActor(ctx context.Context, in *ResumeActorRequest, opts ...grpc.CallOption) (*ResumeActorResponse, error) - // Delete an actor. Only suspended actors can be deleted. + // Delete an actor. Only suspended or crashed actors can be deleted unless + // any_state is set. DeleteActor(ctx context.Context, in *DeleteActorRequest, opts ...grpc.CallOption) (*Actor, error) // Get the egress policy resource nested under an Actor. GetActorEgressPolicy(ctx context.Context, in *GetActorEgressPolicyRequest, opts ...grpc.CallOption) (*EgressPolicy, error) @@ -529,7 +530,8 @@ type ControlServer interface { PauseActor(context.Context, *PauseActorRequest) (*PauseActorResponse, error) // Resume an actor from its latest snapshot. ResumeActor(context.Context, *ResumeActorRequest) (*ResumeActorResponse, error) - // Delete an actor. Only suspended actors can be deleted. + // Delete an actor. Only suspended or crashed actors can be deleted unless + // any_state is set. DeleteActor(context.Context, *DeleteActorRequest) (*Actor, error) // Get the egress policy resource nested under an Actor. GetActorEgressPolicy(context.Context, *GetActorEgressPolicyRequest) (*EgressPolicy, error) From e164f2a96130178ee89199b169eaf5a7401e95fa Mon Sep 17 00:00:00 2001 From: shrutiyam-glitch Date: Fri, 18 Sep 2026 09:14:37 -0700 Subject: [PATCH 2/3] Update regarding external volumes --- cmd/ateapi/internal/controlapi/crash.go | 8 +++++--- docs/api-guide.md | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cmd/ateapi/internal/controlapi/crash.go b/cmd/ateapi/internal/controlapi/crash.go index 8333209cf0..746012acf5 100644 --- a/cmd/ateapi/internal/controlapi/crash.go +++ b/cmd/ateapi/internal/controlapi/crash.go @@ -94,9 +94,11 @@ func crashActor(ctx context.Context, st crashActorStore, actorRef resources.Acto _, err = st.UpdateActor(ctx, actorRef, store.PreconditionFrom(actor), func(toUpdate *ateapipb.Actor) error { toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_CRASHED - // InProgressSnapshotUri and InProgressLocalSnapshotName are kept for - // debugging; failed workflow steps must never promote either of them to an - // ActorSnapshot or to LocalSnapshotInfo. + // InProgressSnapshotUri and InProgressLocalSnapshotName are kept so a + // later DeleteActor or RevertActor can collect what they name: each is + // the only pointer to it, so clearing them here would leak the objects + // for good; failed workflow steps must never promote either of them to an + // ExternalSnapshot or to LocalSnapshotInfo. toUpdate.Status.WorkerAssignment = nil return nil }) diff --git a/docs/api-guide.md b/docs/api-guide.md index a424343220..778fbcc7ee 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -510,6 +510,7 @@ Discards an actor's live or crashed execution and transitions it to `ACTOR_STATE * `actor`: `ObjectRef` of the actor to revert. Accepted from `ACTOR_STATE_RUNNING`, `ACTOR_STATE_PAUSED`, and `ACTOR_STATE_CRASHED` (plus `ACTOR_STATE_REVERTING` for idempotent retries). Calling `RevertActor` on an already `ACTOR_STATE_SUSPENDED` actor returns `FAILED_PRECONDITION`. * **Response:** `RevertActorResponse` containing the reverted `Actor` in `ACTOR_STATE_SUSPENDED`. * Reverting terminates any bound worker sandbox, clears node-local pause checkpoints (`localSnapshotInfo`), and garbage-collects any partial external snapshot left by an interrupted suspend while preserving the last committed `externalSnapshot`. +* External volumes are not reverted. Their contents are never part of a snapshot, so a reverted actor comes back with its memory and root filesystem rewound but its volumes exactly as the discarded execution left them. #### `DeleteActor` Removes an actor from the registry and cleans up associated resources. From b52ffc17c999f8164bc27956ec5d557bfb430624 Mon Sep 17 00:00:00 2001 From: shrutiyam-glitch Date: Fri, 18 Sep 2026 10:07:52 -0700 Subject: [PATCH 3/3] clarify comments on deleting in-progress snapshots after crash --- cmd/ateapi/internal/controlapi/crash.go | 2 +- cmd/ateapi/internal/controlapi/workflow_delete_test.go | 9 +++++---- cmd/ateapi/internal/controlapi/workflow_worker_delete.go | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/cmd/ateapi/internal/controlapi/crash.go b/cmd/ateapi/internal/controlapi/crash.go index 746012acf5..2c314094a3 100644 --- a/cmd/ateapi/internal/controlapi/crash.go +++ b/cmd/ateapi/internal/controlapi/crash.go @@ -95,7 +95,7 @@ func crashActor(ctx context.Context, st crashActorStore, actorRef resources.Acto toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_CRASHED // InProgressSnapshotUri and InProgressLocalSnapshotName are kept so a - // later DeleteActor or RevertActor can collect what they name: each is + // later DeleteActor or RevertActor can delete what they name: each is // the only pointer to it, so clearing them here would leak the objects // for good; failed workflow steps must never promote either of them to an // ExternalSnapshot or to LocalSnapshotInfo. diff --git a/cmd/ateapi/internal/controlapi/workflow_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_delete_test.go index 492f6a26ec..83217a33a9 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete_test.go @@ -363,10 +363,10 @@ func TestDeleteActor_CollectsInFlightSnapshotWithoutTemplate(t *testing.T) { } // TestDeleteActor_CollectsSnapshotsAfterWorkerDelete verifies that -// deleting an actor whose suspend a worker delete crashed mid-finalize reclaims +// deleting an actor whose suspend a worker delete crashed mid-finalize deletes // every object that suspend wrote. When an actor crashes mid-suspend, only -// delete or revert can collect the partial snapshot: whatever they cannot name -// is leaked for good. +// DeleteActor or RevertActor can delete the in-progress snapshot +// (in_progress_snapshot_uri): whatever they cannot name is leaked for good. func TestDeleteActor_CollectsSnapshotsAfterWorkerDelete(t *testing.T) { tests := []struct { name string @@ -448,7 +448,8 @@ func TestDeleteActor_CollectsSnapshotsAfterWorkerDelete(t *testing.T) { t.Fatalf("DeleteWorker: %v", err) } - // The actor is CRASHED; delete reclaims its partial snapshot. + // The actor is CRASHED; DeleteActor deletes the in-progress snapshot + // (in_progress_snapshot_uri). stored, err := persistence.GetActor(ctx, actorRef) if err != nil { t.Fatalf("GetActor: %v", err) diff --git a/cmd/ateapi/internal/controlapi/workflow_worker_delete.go b/cmd/ateapi/internal/controlapi/workflow_worker_delete.go index 9ae4f76fb5..c24c932719 100644 --- a/cmd/ateapi/internal/controlapi/workflow_worker_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_worker_delete.go @@ -215,7 +215,7 @@ func (w *WorkerWorkflow) releaseBoundActor(ctx context.Context, worker *ateapipb toUpdate.Status.WorkerAssignment = nil // Local in-progress checkpoint dies with the worker: it lived on the node // that went away. The external in-progress checkpoint is kept so delete - // or revert can collect it. + // or revert can delete it. toUpdate.Status.InProgressLocalSnapshotName = "" return nil })