docs: Document RevertActor and drop "terminal" from CRASHED

benchmarking/locust/common/ateapi_pb2_grpc.py

@@ -270,7 +270,8 @@ def ResumeActor(self, request, context):
         raise NotImplementedError('Method not implemented!')
 
     def DeleteActor(self, request, context):
-        """Delete an actor. Only suspended actors can be deleted.
+        """Delete an actor. Only suspended or crashed actors can be deleted unless
+        any_state is set.
         """
         context.set_code(grpc.StatusCode.UNIMPLEMENTED)
         context.set_details('Method not implemented!')

cmd/ateapi/internal/controlapi/crash.go

@@ -74,7 +74,7 @@ func crashActor(ctx context.Context, st crashActorStore, actorRef resources.Acto
 		reason = ateattr.ReasonUnknown
 	}
 
-	// Release the worker before moving the actor to the terminal CRASHED state.
+	// Release the worker before moving the actor to CRASHED state.
 	// If the release fails we must not clear the actor's worker assignment or
 	// mark it CRASHED: doing so would strand the still-assigned worker with no
 	// actor referencing it, so nothing would ever retry the release and the
@@ -94,9 +94,11 @@ func crashActor(ctx context.Context, st crashActorStore, actorRef resources.Acto
 	_, err = st.UpdateActor(ctx, actorRef, store.PreconditionFrom(actor), func(toUpdate *ateapipb.Actor) error {
 		toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_CRASHED
 
-		// InProgressSnapshotUri and InProgressLocalSnapshotName are kept for
-		// debugging; failed workflow steps must never promote either of them to an
-		// ActorSnapshot or to LocalSnapshotInfo.
+		// InProgressSnapshotUri and InProgressLocalSnapshotName are kept so a
+		// later DeleteActor or RevertActor can delete what they name: each is
+		// the only pointer to it, so clearing them here would leak the objects
+		// for good; failed workflow steps must never promote either of them to an
+		// ExternalSnapshot or to LocalSnapshotInfo.
 		toUpdate.Status.WorkerAssignment = nil
 		return nil
 	})

cmd/ateapi/internal/controlapi/crash_test.go

@@ -532,7 +532,7 @@ func (f failingReleaseStore) ReleaseActorFromWorker(context.Context, string, str
 }
 
 // A transient failure releasing the worker must not move the actor to the
-// terminal CRASHED state: doing so would strand the still-assigned worker with
+// CRASHED state: doing so would strand the still-assigned worker with
 // no actor left to drive a retry, permanently consuming the worker slot.
 // crashActor must return the error with the actor and worker left intact so the
 // caller retries and the worker is reclaimed.

cmd/ateapi/internal/controlapi/workflow_delete_test.go

@@ -363,9 +363,10 @@ func TestDeleteActor_CollectsInFlightSnapshotWithoutTemplate(t *testing.T) {
 }
 
 // TestDeleteActor_CollectsSnapshotsAfterWorkerDelete verifies that
-// deleting an actor whose suspend a worker delete crashed mid-finalize reclaims
-// every object that suspend wrote. CRASHED is terminal, so the actor delete is
-// the only collector left: whatever it cannot name is leaked for good.
+// deleting an actor whose suspend a worker delete crashed mid-finalize deletes
+// every object that suspend wrote. When an actor crashes mid-suspend, only
+// DeleteActor or RevertActor can delete the in-progress snapshot
+// (in_progress_snapshot_uri): whatever they cannot name is leaked for good.
 func TestDeleteActor_CollectsSnapshotsAfterWorkerDelete(t *testing.T) {
 	tests := []struct {
 		name string
@@ -447,7 +448,8 @@ func TestDeleteActor_CollectsSnapshotsAfterWorkerDelete(t *testing.T) {
 				t.Fatalf("DeleteWorker: %v", err)
 			}
 
-			// The actor is CRASHED and can only be deleted from here.
+			// The actor is CRASHED; DeleteActor deletes the in-progress snapshot
+			// (in_progress_snapshot_uri).
 			stored, err := persistence.GetActor(ctx, actorRef)
 			if err != nil {
 				t.Fatalf("GetActor: %v", err)

cmd/ateapi/internal/controlapi/workflow_worker_delete.go

@@ -214,9 +214,8 @@ func (w *WorkerWorkflow) releaseBoundActor(ctx context.Context, worker *ateapipb
 		toUpdate.Status.State = ateapipb.ActorState_ACTOR_STATE_CRASHED
 		toUpdate.Status.WorkerAssignment = nil
 		// Local in-progress checkpoint dies with the worker: it lived on the node
-		// that went away. The external in-progress checkpoint is kept. It'll be deleted
-		// with the actor when the actor is deleted (only possible outcome from CRASHED
-		// state).
+		// that went away. The external in-progress checkpoint is kept so delete
+		// or revert can delete it.
 		toUpdate.Status.InProgressLocalSnapshotName = ""
 		return nil
 	})

cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go

@@ -135,7 +135,7 @@ func TestDeleteWorkerWorkflow_ReleasesBoundActor(t *testing.T) {
 		t.Errorf("in-progress local checkpoint not cleared: %v", got.GetStatus())
 	}
 	// The durable one is kept: it names the prefix whatever atelet already
-	// uploaded lives under, which the actor's delete needs to collect it.
+	// uploaded lives under, which delete or revert needs to collect it.
 	if want := someActorSnapshotURI(t, testStorageLocation, apiActorRef.Atespace, "partial-snapshot"); got.GetStatus().GetInProgressSnapshotUri() != want {
 		t.Errorf("in-progress external checkpoint not preserved: %v", got.GetStatus())
 	}

cmd/kubectl-ate/README.md

@@ -109,7 +109,7 @@ kubectl ate get workers -l <label-selector>
 | `ATESPACE` | The atespace the actor belongs to. Part of the actor's identity; folded into the storage key as `actor:<atespace>:<name>`. |
 | `NAME` | The actor's name. User-provided for application actors; UUID for the golden actor that each template materialises while building its golden tag. |
 | `TEMPLATE` | The `ActorTemplate` the actor was created from, displayed as `<atespace>/<name>`. |
-| `STATE` | One of `ACTOR_STATE_RESUMING`, `ACTOR_STATE_RUNNING`, `ACTOR_STATE_SUSPENDING`, `ACTOR_STATE_SUSPENDED`. |
+| `STATE` | Current lifecycle state (`ACTOR_STATE_RESUMING`, `ACTOR_STATE_RUNNING`, `ACTOR_STATE_SUSPENDING`, `ACTOR_STATE_SUSPENDED`, `ACTOR_STATE_PAUSING`, `ACTOR_STATE_PAUSED`, `ACTOR_STATE_CRASHED`, `ACTOR_STATE_DELETING`, `ACTOR_STATE_REVERTING`). |
 | `WORKER POD` | The worker pod (namespace/name) currently hosting the actor. Empty while suspended. |
 | `WORKER IP` | The pod IP of that worker. Empty while suspended. |
 | `VERSION` | Monotonic integer that increments on every state transition (resume / suspend / checkpoint). Useful for distinguishing snapshots. |
@@ -202,6 +202,9 @@ kubectl ate resume actor my-actor -a <atespace>
 # Suspend an actor (snapshots its state to storage and frees the worker)
 kubectl ate suspend actor my-actor -a <atespace>
 
+# Revert an actor to its last external snapshot (discards live, paused, or crashed state and returns to SUSPENDED)
+kubectl ate revert actor my-actor -a <atespace>
+
 # Delete an actor (by default, requires the actor to be SUSPENDED or CRASHED).
 kubectl ate delete actor my-actor -a <atespace>
 

docs/api-guide.md

@@ -450,7 +450,7 @@ Once a template is `Ready`, creating an actor logically (via `kubectl ate create
 *   **Startup Logic:** Place expensive initialization (loading large models, establishing baseline connections) in your application's entry point. These will be captured in the Golden Snapshot and won't need to be repeated on every resumption.
 *   **Placement:** Ensure your `ActorTemplate`'s `sandboxClass` matches your `WorkerPool`'s, and use the template's `workerSelector` to target specific pools — pool selection is by label match, not by namespace or RBAC.
 *   **Version Management:** When updating code, create a new `ActorTemplate` (e.g. `v2`). Substrate treats each template as an immutable state root.
-*   **Eviction:** When its worker pod is evicted, an actor gets `SIGTERM` and 30 minutes to be suspended. After that it is killed and moves to `ACTOR_STATE_CRASHED`, and everything since its last snapshot is lost. So an actor that runs for more than 30 minutes without a suspend can lose data.
+*   **Eviction:** When its worker pod is evicted, an actor gets `SIGTERM` and 30 minutes to be suspended. After that it is killed and moves to `ACTOR_STATE_CRASHED`, and everything since its last snapshot is lost. So an actor that runs for more than 30 minutes without a suspend can lose data. A `CRASHED` actor can be recovered back to `ACTOR_STATE_SUSPENDED` at its last external snapshot using `RevertActor` (`kubectl ate revert`).
 
 ---
 
@@ -504,6 +504,14 @@ Deletion always runs before the database reference is dropped, and a failure fai
 
 > **Do not delete a tag while actors created from it exist.** A clone borrows the tag's snapshot rather than copying it, and only stops borrowing at its own first suspend (its `status.externalSnapshot.snapshotUri` still names the tag's prefix while it is). Deleting the tag leaves such a clone unable to resume. This is not prevented today.
 
+#### `RevertActor`
+Discards an actor's live or crashed execution and transitions it to `ACTOR_STATE_SUSPENDED` at its last completed external snapshot (`status.externalSnapshot`).
+*   **Request:** `RevertActorRequest`
+    *   `actor`: `ObjectRef` of the actor to revert. Accepted from `ACTOR_STATE_RUNNING`, `ACTOR_STATE_PAUSED`, and `ACTOR_STATE_CRASHED` (plus `ACTOR_STATE_REVERTING` for idempotent retries). Calling `RevertActor` on an already `ACTOR_STATE_SUSPENDED` actor returns `FAILED_PRECONDITION`.
+*   **Response:** `RevertActorResponse` containing the reverted `Actor` in `ACTOR_STATE_SUSPENDED`.
+*   Reverting terminates any bound worker sandbox, clears node-local pause checkpoints (`localSnapshotInfo`), and garbage-collects any partial external snapshot left by an interrupted suspend while preserving the last committed `externalSnapshot`.
+*   External volumes are not reverted. Their contents are never part of a snapshot, so a reverted actor comes back with its memory and root filesystem rewound but its volumes exactly as the discarded execution left them.
+
 #### `DeleteActor`
 Removes an actor from the registry and cleans up associated resources.
 *   **Request:** `DeleteActorRequest`

docs/architecture.md

@@ -471,7 +471,9 @@ Actor that took it, and publishing it permits reuse from other Atespaces without
 `atespace/name` address. Deleting a tag deletes that copy; an Atespace with
 tags cannot be deleted until they are.
 
-### Phase 4: Deletion
+### Phase 4: Recovery (`RevertActor`) and Deletion (`DeleteActor`)
+
+An actor in `ACTOR_STATE_RUNNING`, `ACTOR_STATE_PAUSED`, or `ACTOR_STATE_CRASHED` can be reverted back to `ACTOR_STATE_SUSPENDED` via `RevertActor`. Reverting terminates any active sandbox, discards any local pause checkpoint or partial in-progress snapshot, and preserves the actor's last completed external snapshot so a subsequent `ResumeActor` restores from that checkpoint.
 
 By default, only actors in `ACTOR_STATE_SUSPENDED` or `ACTOR_STATE_CRASHED` state can be deleted from the Control Plane. With the `any_state` flag enabled, an actor in any state (such as `ACTOR_STATE_RUNNING` or `ACTOR_STATE_PAUSED`) can be deleted directly; the workflow terminates the running containers on the worker, detaches mounted volumes, and frees the worker assignment before deleting the record.
 

docs/metrics/registry/metrics.yaml

@@ -617,11 +617,11 @@ groups:
     brief: The number of actors that went to the ACTOR_STATE_CRASHED state, with the reasons.
     note: >
       This counter counts the moves into the CRASHED state of the state machine.
-      It does not count the crashes of a process. The CRASHED state is a
-      permanent loss of the actor. Substrate can also lose the data that it did
-      not write. Some moves into this state are not a loss of data. They are
-      careful responses to a control plane problem. The ate.failure.reason key
-      keeps these groups separate.
+      It does not count the crashes of a process. The CRASHED state is a loss of
+      the actor's live execution (recoverable back to SUSPENDED via RevertActor).
+      Substrate can also lose the data that it did not write. Some moves into
+      this state are not a loss of data. They are careful responses to a control
+      plane problem. The ate.failure.reason key keeps these groups separate.
       ate.sandbox.class is unknown when ateapi could not read the class of the
       worker: the worker record is already gone, or its assignment is already
       clear.

docs/upgrade.md

@@ -211,12 +211,13 @@ Each warning comes back at the step where the mistake becomes possible.
 > inside that window saves its state and stays resumable. Handling
 > `SIGTERM` by exiting cleanly is not enough on its own; the suspend has
 > to reach the control plane and finish. An actor still awake when the
-> window closes moves to `ACTOR_STATE_CRASHED`, which is terminal:
-> `resume` and `suspend` are both refused, there is no recover verb,
-> and the snapshot the actor still holds cannot be used to start it. It
-> has to be deleted and recreated, losing its state. Scaling a serving
-> pool down removes pods the same way, without suspending the actors on
-> them. (Step 4 clones the pool; it never edits it.)
+> window closes moves to `ACTOR_STATE_CRASHED`: `resume` and `suspend`
+> are both refused, and everything since its last snapshot is lost. Call
+> `RevertActor` (`kubectl ate revert`) to discard the crashed run and
+> return the actor to `ACTOR_STATE_SUSPENDED` at its last external snapshot
+> so it can be resumed. Scaling a serving pool down removes pods the same
+> way, without suspending the actors on them. (Step 4 clones the pool; it
+> never edits it.)
 
 > [!WARNING]
 > **On GKE, do not touch the node pool's label until every node is

pkg/proto/ateapipb/ateapi.proto

@@ -43,7 +43,8 @@ service Control {
   // Resume an actor from its latest snapshot.
   rpc ResumeActor(ResumeActorRequest) returns (ResumeActorResponse) {}
 
-  // Delete an actor. Only suspended actors can be deleted.
+  // Delete an actor. Only suspended or crashed actors can be deleted unless
+  // any_state is set.
   rpc DeleteActor(DeleteActorRequest) returns (Actor) {}
 
   // Get the egress policy resource nested under an Actor.