feat(runtime): pause on Kubernetes, unbounded checkpoints, cluster CI

.github/workflows/ci.yml

@@ -3,6 +3,9 @@ name: ci
 on:
   push:
   pull_request:
+  # The cluster-backed jobs are gated on this so they can be run against a
+  # branch without paying for a cluster on every push.
+  workflow_dispatch:
 
 permissions:
   contents: read
@@ -172,3 +175,60 @@ jobs:
       - name: Tear down
         if: always()
         run: docker compose --profile sample --profile e2e --profile e2e-images --profile ha down -v
+
+  # Kubernetes coverage needs a cluster, so it is separated from the fast jobs.
+  # It runs on the default branch and on demand rather than on every push,
+  # because building a cluster and loading images costs several minutes.
+  kubernetes:
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch'
+    needs: [go, helm]
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: helm/kind-action@v1
+        with:
+          cluster_name: agent-platform
+      - uses: azure/setup-helm@v4
+      # Deploys the platform with the chart and drives the suite through it, so
+      # Agent Sessions run as Pods and reach the Control Plane as a Service.
+      - name: End-to-end flows on Kubernetes
+        run: bash test/e2e/kubernetes/run.sh
+      - name: Cluster state on failure
+        if: failure()
+        run: |
+          kubectl -n agent-platform get pods -o wide || true
+          kubectl -n agent-platform logs -l app.kubernetes.io/name=agent-platform-control-plane --tail 200 || true
+          kubectl -n agent-platform logs -l app.kubernetes.io/name=agent-platform-runtime-worker --tail 200 || true
+          kubectl -n agent-platform-runtime get pods -o wide || true
+
+  # The Agent egress policy is inert on a CNI that ignores NetworkPolicy, so
+  # this builds its own cluster with Calico rather than reusing the kind default.
+  egress-policy:
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch'
+    needs: [go]
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+      # kind-action is not used here: it would create a cluster with kindnet,
+      # which accepts NetworkPolicy and ignores it. policy-check.sh builds its
+      # own cluster with Calico, so only the binary is needed.
+      - name: Install kind
+        run: |
+          curl -fsSLo /usr/local/bin/kind \
+            https://kind.sigs.k8s.io/dl/v0.27.0/kind-linux-amd64
+          chmod +x /usr/local/bin/kind
+      - name: Verify the egress policy is enforced
+        run: bash test/e2e/kubernetes/policy-check.sh

README.md

@@ -195,7 +195,7 @@ launched by the runtime worker, never by the Control Plane. OIDC single sign-on
 is implemented; SAML, SCIM, and per-organization identity providers are not. Runtime HTTP/HTTPS egress enforcement and checkpoint-based
 child Session resume are implemented. Several Control Plane instances can run over one PostgreSQL, NATS, and object
 store; see [High availability](docs/high-availability.md). Agent Sessions run as
-Docker containers or Kubernetes Pods, including stdio MCP servers on both; see
-[Kubernetes](docs/kubernetes.md) for the remaining backend differences. A Helm
-chart deploys the platform itself onto a cluster. Hosted tenant provisioning remains a future
+Docker containers or Kubernetes Pods, with the same controls on both; see
+[Kubernetes](docs/kubernetes.md) for how pause and checkpoint restore differ
+underneath. A Helm chart deploys the platform itself onto a cluster. Hosted tenant provisioning remains a future
 milestone.

api/openapi.yaml

@@ -372,6 +372,27 @@ paths:
       responses:
         "201": { description: Artifact stored }
         "413": { description: Artifact exceeds the configured size limit }
+  /agent/v1/sessions/{sessionId}/checkpoint:
+    parameters:
+      - $ref: "#/components/parameters/SessionId"
+    get:
+      tags: [agent]
+      security:
+        - agentToken: []
+      summary: Download the checkpoint this Session was created from
+      description: >-
+        Lets a runtime stream a restored checkpoint into the workspace instead of
+        carrying the bytes itself, which the Kubernetes runtime needs because a
+        Secret caps at 1 MB. Serves only the one checkpoint named by this
+        Session. The SHA-256 is returned in X-Checkpoint-Sha256 so the runtime
+        can verify the transfer.
+      responses:
+        "200":
+          description: Checkpoint content
+          headers:
+            X-Checkpoint-Sha256:
+              schema: { type: string }
+        "404": { description: "This Session was not created from a checkpoint" }
   /agent/v1/sessions/{sessionId}/checkpoints:
     parameters:
       - $ref: "#/components/parameters/SessionId"

deploy/helm/README.md

@@ -54,8 +54,11 @@ updates stop crossing instances. See [high availability](../../docs/high-availab
 ## What the worker gets
 
 A ServiceAccount with a namespace-scoped Role over Pods, Secrets,
-`pods/attach`, and NetworkPolicies in `runtimeWorker.runtimeNamespace`. Nothing
-cluster-scoped.
+`pods/attach`, `pods/exec`, and NetworkPolicies in
+`runtimeWorker.runtimeNamespace`. Nothing cluster-scoped.
+
+`pods/attach` drives stdio MCP servers; `pods/exec` is how pause and resume
+signal the Agent's processes, since Kubernetes has no cgroup freezer.
 
 The Control Plane Pod label `app.kubernetes.io/name: agent-platform-control-plane`
 is not cosmetic: the egress NetworkPolicy the worker writes selects Control Plane
@@ -82,3 +85,6 @@ Three flows stay Compose-only because they drive Docker itself —
 `durable_command_flow` detaches a container network and `recovery_flow` restarts
 Compose services — and `oidc_flow` needs an identity provider configured at
 install time.
+
+CI runs this on the default branch and on demand rather than on every push,
+since building a cluster and loading images costs several minutes.

deploy/helm/agent-platform/templates/runtime-worker.yaml

@@ -34,6 +34,11 @@ rules:
   - apiGroups: [""]
     resources: ["pods/attach"]
     verbs: ["create", "get"]
+  # Pause and resume signal the Agent's processes, since Kubernetes has no
+  # equivalent of the cgroup freezer.
+  - apiGroups: [""]
+    resources: ["pods/exec"]
+    verbs: ["create", "get"]
   # The Agent egress policy the worker writes on start-up.
   - apiGroups: ["networking.k8s.io"]
     resources: ["networkpolicies"]

docs/agent-protocol.md

@@ -131,6 +131,12 @@ variables:
 Python Agents can call `session.restored_checkpoint()` to read the bytes. It
 returns `None` for a Session that was not started from a checkpoint.
 
+How the bytes arrive depends on the runtime. Docker streams them through an
+init container; Kubernetes fetches them from
+`GET /agent/v1/sessions/{sessionId}/checkpoint` with the Session's Agent token
+and verifies the digest before the Agent starts. Either way the Agent sees the
+same file at the same path.
+
 ## Runtime network egress
 
 The worker configures standard `HTTP_PROXY`, `HTTPS_PROXY`, `http_proxy`, and

docs/examples/kubernetes-rbac.yaml

@@ -27,6 +27,11 @@ rules:
   - apiGroups: [""]
     resources: ["pods/attach"]
     verbs: ["create", "get"]
+  # Pause and resume signal the Agent's processes, since Kubernetes has no
+  # equivalent of the cgroup freezer.
+  - apiGroups: [""]
+    resources: ["pods/exec"]
+    verbs: ["create", "get"]
   # The egress policy the worker writes on start-up. Effective only on a CNI
   # that enforces NetworkPolicy; see docs/kubernetes.md.
   - apiGroups: ["networking.k8s.io"]

docs/kubernetes.md

@@ -16,6 +16,9 @@ Select it with `RUNTIME_PROVIDER=kubernetes` on the worker.
 | `KUBERNETES_CA_CERT` | PEM CA bundle when running outside the cluster |
 | `KUBERNETES_INSECURE_SKIP_VERIFY` | Local testing only; makes the connection interceptable |
 
+The worker needs `create` on `pods`, `secrets`, `pods/attach`, `pods/exec`, and
+`networkpolicies` in its runtime namespace. Nothing cluster-scoped.
+
 In-cluster the worker reads its credentials from the projected service account
 and needs no configuration beyond `RUNTIME_PROVIDER` and the namespace.
 
@@ -81,19 +84,33 @@ MCP servers, so there is no declared user to impose.
 
 This needs `create` on `pods/attach` in the runtime namespace.
 
-## What Kubernetes cannot do
+## Pause and resume
+
+Kubernetes has no equivalent of Docker's cgroup freezer, so a pause is signalled
+instead: `SIGSTOP` to every process in the Agent container's PID namespace,
+`SIGCONT` to resume. From the Agent's point of view that is what a freeze
+amounts to.
+
+The signalling walks `/proc` rather than using `kill -1`, so the helper shell
+does not stop itself before it has finished with the rest. It needs `create` on
+`pods/exec`, and it needs the Agent image to have a shell and `/proc` — a
+distroless image cannot be paused this way. A pause that fails is reported
+rather than leaving the Session stuck in `pausing`.
 
-The worker asks the runtime what it supports rather than assuming, so these are
-refused visibly instead of hanging.
+## Checkpoints
 
-**Pause and resume.** There is no API to freeze a running Pod; Docker uses the
-cgroup freezer. A pause request against a Kubernetes Session returns it to
-`running` with a message saying so, rather than leaving it stuck in `pausing`.
+A restored checkpoint is fetched by an init container from
+`GET /agent/v1/sessions/{id}/checkpoint`, authenticated with the Session's own
+Agent token, and its SHA-256 is re-checked before the Agent starts so a
+truncated transfer fails the Pod rather than handing over a partial file.
 
-**Checkpoints above 900 KB.** They travel in a per-Session Secret, which
-Kubernetes caps at 1 MB. Docker streams them through an init container's stdin
-and has no such limit. An oversized checkpoint is refused at Start rather than
-failing partway through.
+Carrying the bytes in a per-Session Secret would cap them at Kubernetes' 1 MB
+object limit. The endpoint exposes nothing new: the same bytes are mounted into
+the Agent's workspace either way, and it serves only the one checkpoint the
+Session was created from.
+
+The init container image is `busybox:1.37` by default and needs a shell, `wget`,
+and `sha256sum`.
 
 ## RBAC
 
@@ -104,6 +121,9 @@ rules:
   - apiGroups: [""]
     resources: ["pods", "secrets"]
     verbs: ["create", "get", "list", "delete"]
+  - apiGroups: [""]
+    resources: ["pods/attach", "pods/exec"]
+    verbs: ["create", "get"]
   - apiGroups: ["networking.k8s.io"]
     resources: ["networkpolicies"]
     verbs: ["create", "get", "list"]
@@ -128,9 +148,10 @@ go test ./internal/worker -run Integration -v
 
 Those cover starting a Session to completion, reporting a non-zero exit,
 distinguishing a missing workload from a failed one, mounting Session secrets
-read-only, restoring a checkpoint into the workspace, refusing an oversized one,
-stopping a running Session, writing the egress policy idempotently, and running
-a stdio MCP tool through attach. The secret and checkpoint cases assert from
+read-only, stopping a running Session, writing the egress policy idempotently,
+running a stdio MCP tool through attach, and pausing and resuming an Agent —
+that last one by watching a counter inside the container stop and start again,
+so a pause that reported success without freezing anything would fail. The secret and checkpoint cases assert from
 inside the container, so a mount that silently did not arrive fails the test.
 
 The stdio case uses the same fixture server as the Docker end-to-end suite,

examples/research-report-agent/main.py

@@ -56,14 +56,21 @@ async def main() -> None:
         if isinstance(checkpoint_test, dict):
             restored = session.restored_checkpoint()
             if restored is not None:
+                # Report the size rather than megabytes of padding.
+                head = restored.decode(errors="replace").split("\n", 1)[0]
                 await session.message(
-                    "Restored checkpoint: " + restored.decode(errors="replace")
+                    f"Restored checkpoint: {head} ({len(restored)} bytes)"
                 )
                 await session.complete(
                     {"checkpointRestored": True, "checkpointId": session.checkpoint_id}
                 )
                 return
             content = str(checkpoint_test.get("content", "checkpoint state"))
+            # Padding lets a test request a checkpoint larger than a Kubernetes
+            # Secret can carry, which is what the fetch path has to handle.
+            padding = int(checkpoint_test.get("sizeBytes", 0))
+            if padding > len(content):
+                content += "\n" + "x" * (padding - len(content) - 1)
             checkpoint_path = Path("/workspace/validation.checkpoint")
             checkpoint_path.write_text(content, encoding="utf-8")
             checkpoint = await session.upload_checkpoint(

internal/httpapi/server.go

@@ -135,6 +135,7 @@ func (s *Server) routes() {
 	s.mux.HandleFunc("GET /agent/v1/sessions/{sessionID}/connect", s.agentConnect)
 	s.mux.HandleFunc("POST /agent/v1/sessions/{sessionID}/artifacts", s.agentArtifactUpload)
 	s.mux.HandleFunc("POST /agent/v1/sessions/{sessionID}/checkpoints", s.agentCheckpointUpload)
+	s.mux.HandleFunc("GET /agent/v1/sessions/{sessionID}/checkpoint", s.agentCheckpointDownload)
 	s.mux.HandleFunc("POST /agent/v1/sessions/{sessionID}/models/{binding}/chat/completions", s.agentModelCompletion)
 	s.mux.HandleFunc("POST /agent/v1/sessions/{sessionID}/mcp/{serverID}/tools/{toolName}", s.agentMCPToolCall)
 
@@ -1032,6 +1033,45 @@ func (s *Server) downloadArtifact(w http.ResponseWriter, r *http.Request) {
 	_, _ = io.Copy(w, content)
 }
 
+// agentCheckpointDownload serves the checkpoint a Session was created from,
+// authenticated by that Session's own Agent token.
+//
+// It exists so a runtime can stream a restored checkpoint into the workspace
+// instead of carrying the bytes itself. The Kubernetes runtime used a per-
+// Session Secret, which caps at 1 MB; fetching removes that ceiling.
+//
+// This exposes nothing new to the Agent: the same bytes are mounted into its
+// workspace either way. It serves only the one checkpoint named by this
+// Session, so a compromised Agent cannot walk the organization's checkpoints.
+func (s *Server) agentCheckpointDownload(w http.ResponseWriter, r *http.Request) {
+	session, ok := s.authenticateAgentRequest(w, r)
+	if !ok {
+		return
+	}
+	if session.CheckpointID == "" {
+		writeError(w, http.StatusNotFound, "no_checkpoint", "This Session was not created from a checkpoint.")
+		return
+	}
+	checkpoint, err := s.store.Checkpoint(r.Context(), session.CheckpointID)
+	if err != nil || checkpoint.AgentVersionID != session.AgentVersionID {
+		writeError(w, http.StatusNotFound, "checkpoint_not_found", "Checkpoint not found.")
+		return
+	}
+	content, err := s.objects.Get(r.Context(), checkpoint.ObjectKey)
+	if err != nil {
+		writeError(w, http.StatusNotFound, "checkpoint_content_missing", "Checkpoint content is unavailable.")
+		return
+	}
+	defer content.Close()
+	w.Header().Set("Content-Type", "application/octet-stream")
+	// The runtime verifies this before handing the bytes to the Agent.
+	w.Header().Set("X-Checkpoint-Sha256", checkpoint.SHA256)
+	w.Header().Set("Cache-Control", "private, max-age=0")
+	if _, err := io.Copy(w, io.LimitReader(content, s.config.MaxArtifactBytes)); err != nil {
+		s.log.Warn("stream checkpoint to agent", "sessionId", session.ID, "error", err)
+	}
+}
+
 func (s *Server) downloadCheckpoint(w http.ResponseWriter, r *http.Request) {
 	checkpoint, err := s.store.Checkpoint(r.Context(), r.PathValue("checkpointID"))
 	if err != nil {