From 2f6a1f854cf307e1f89823eb3be689263df7f115 Mon Sep 17 00:00:00 2001 From: "fengjianhui.fjh" Date: Wed, 8 Apr 2026 07:51:47 +0000 Subject: [PATCH] feat(server,k8s): implement pause/resume with rootfs snapshot support - Add PausePolicy schema and API endpoints (/pause, /resume) - Add image_pull_policy configuration for sandbox containers - Support pausePolicy in pool-based BatchSandbox creation - Add SandboxSnapshot CRD and controller for snapshot lifecycle - Add image-committer Job for container rootfs snapshot/commit - Update AGENTS.md and Makefile for new components Implements OSEP-0008 pause/resume functionality using SandboxSnapshot CRD and nerdctl-based image committer for Kubernetes runtime. --- kubernetes/AGENTS.md | 71 ++ kubernetes/Dockerfile.image-committer | 56 + kubernetes/Makefile | 23 +- .../sandbox/v1alpha1/batchsandbox_types.go | 26 + .../sandbox/v1alpha1/sandboxsnapshot_types.go | 175 +++ .../sandbox/v1alpha1/zz_generated.deepcopy.go | 144 +++ kubernetes/cmd/controller/main.go | 19 + kubernetes/cmd/image-committer/main.go | 484 +++++++++ ...sandbox.opensandbox.io_batchsandboxes.yaml | 24 + ...ndbox.opensandbox.io_sandboxsnapshots.yaml | 215 ++++ kubernetes/config/crd/kustomization.yaml | 1 + kubernetes/config/rbac/role.yaml | 23 + .../controller/sandboxsnapshot_controller.go | 995 ++++++++++++++++++ kubernetes/test/e2e/e2e_suite_test.go | 23 + kubernetes/test/e2e/pause_resume_test.go | 723 +++++++++++++ .../testdata/batchsandbox-pooled-pause.yaml | 12 + .../batchsandbox-with-pause-policy.yaml | 17 + .../e2e/testdata/pool-with-pause-policy.yaml | 17 + .../e2e/testdata/registry-deployment.yaml | 52 + .../e2e/testdata/sandboxsnapshot-minimal.yaml | 11 + .../test/e2e/testdata/sandboxsnapshot.yaml | 27 + kubernetes/test/utils/image.go | 4 + server/opensandbox_server/api/schema.py | 33 + server/opensandbox_server/config.py | 35 + .../opensandbox_server/services/constants.py | 8 + .../services/k8s/agent_sandbox_provider.py | 1 + .../services/k8s/batchsandbox_provider.py | 19 +- .../services/k8s/kubernetes_service.py | 463 ++++++-- .../services/k8s/sandboxsnapshot_provider.py | 174 +++ .../services/k8s/workload_provider.py | 1 + server/tests/k8s/fixtures/k8s_fixtures.py | 14 +- server/tests/k8s/test_sandbox_pause_resume.py | 678 ++++++++++++ 32 files changed, 4484 insertions(+), 84 deletions(-) create mode 100644 kubernetes/AGENTS.md create mode 100644 kubernetes/Dockerfile.image-committer create mode 100644 kubernetes/apis/sandbox/v1alpha1/sandboxsnapshot_types.go create mode 100644 kubernetes/cmd/image-committer/main.go create mode 100644 kubernetes/config/crd/bases/sandbox.opensandbox.io_sandboxsnapshots.yaml create mode 100644 kubernetes/internal/controller/sandboxsnapshot_controller.go create mode 100644 kubernetes/test/e2e/pause_resume_test.go create mode 100644 kubernetes/test/e2e/testdata/batchsandbox-pooled-pause.yaml create mode 100644 kubernetes/test/e2e/testdata/batchsandbox-with-pause-policy.yaml create mode 100644 kubernetes/test/e2e/testdata/pool-with-pause-policy.yaml create mode 100644 kubernetes/test/e2e/testdata/registry-deployment.yaml create mode 100644 kubernetes/test/e2e/testdata/sandboxsnapshot-minimal.yaml create mode 100644 kubernetes/test/e2e/testdata/sandboxsnapshot.yaml create mode 100644 server/opensandbox_server/services/k8s/sandboxsnapshot_provider.py create mode 100644 server/tests/k8s/test_sandbox_pause_resume.py diff --git a/kubernetes/AGENTS.md b/kubernetes/AGENTS.md new file mode 100644 index 000000000..5ed33fb16 --- /dev/null +++ b/kubernetes/AGENTS.md @@ -0,0 +1,71 @@ +# Kubernetes Operator + +## Overview + +Kubernetes operator managing sandbox environments via custom resources. Provides BatchSandbox (O(1) batch delivery), Pool (resource pooling for fast provisioning), and optional task orchestration. Built with controller-runtime (Kubebuilder). + +## Structure + +``` +kubernetes/ +├── apis/sandbox/v1alpha1/ # CRD type definitions +│ ├── batchsandbox_types.go # BatchSandbox spec + status +│ ├── pool_types.go # Pool spec + status +│ └── sandboxsnapshot_types.go +├── cmd/ +│ ├── controller/main.go # Controller manager entry point +│ ├── image-committer/main.go # Image committer binary (runs as commit Job) +│ └── task-executor/main.go # Task executor binary (runs as sidecar) +├── internal/ +│ ├── controller/ # Reconciliation loops +│ ├── scheduler/ # Pool allocation logic (bufferMin/Max, poolMax) +│ └── utils/ # Utility functions +├── config/ +│ ├── crd/bases/ # Generated CRD YAML manifests +│ ├── rbac/ # ClusterRole, ClusterRoleBinding +│ ├── manager/ # Controller deployment manifest +│ └── samples/ # Example CRD instances +├── charts/ # Helm charts (opensandbox-controller, opensandbox-server, opensandbox) +├── test/e2e/ # End-to-end tests + testdata +└── Dockerfile # Controller image build + Dockerfile.image-committer # Image-committer image build +``` + +## Where to Look + +| Task | File | Notes | +|------|------|-------| +| Add CRD field | `apis/sandbox/v1alpha1/*_types.go` | Run `make install` to update CRDs | +| Controller logic | `internal/controller/` | BatchSandbox + Pool reconciliation | +| Pool allocation | `internal/scheduler/` | Buffer management, sandbox→pool assignment | +| Task execution | `cmd/task-executor/`, `internal/task-executor/` | Process-based tasks in sandboxes | +| Helm values | `charts/opensandbox-controller/values.yaml` | Controller + task-executor image refs | +| RBAC permissions | `config/rbac/` | ClusterRole rules | +| E2E tests | `test/e2e/` | Ginkgo/Gomega test framework | + +## Conventions + +- **Framework**: Kubebuilder with `controller-runtime` v0.21. +- **Go version**: 1.24. Own `go.mod` (`github.com/alibaba/opensandbox/sandbox-k8s`). +- **Concurrency**: BatchSandbox controller concurrency=32, Pool controller concurrency=1. +- **CRD version**: `v1alpha1` under group `sandbox.opensandbox.io`. +- **Helm charts**: Umbrella chart (`opensandbox`) wraps controller + server subcharts. +- **Logging**: `klog/v2` + `zap`. Log level configurable via `--zap-log-level` flag. + +## Anti-Patterns + +- `pause`/`resume` lifecycle uses SandboxSnapshot CRD + image-committer Job to snapshot and restore containers. +- BatchSandbox deletion waits for running tasks to terminate before removing the resource. +- Task-executor requires `shareProcessNamespace: true` and `SYS_PTRACE` capability in pod spec. +- Pool template changes do not affect already-allocated sandboxes. + +## Commands + +```bash +make install # install CRDs into cluster +make deploy CONTROLLER_IMG=... TASK_EXECUTOR_IMG=... # deploy controller +make docker-build # build controller image +make docker-build-task-executor # build task-executor image +make docker-build-image-committer # build image-committer image +make test # run tests +``` diff --git a/kubernetes/Dockerfile.image-committer b/kubernetes/Dockerfile.image-committer new file mode 100644 index 000000000..a71d8d583 --- /dev/null +++ b/kubernetes/Dockerfile.image-committer @@ -0,0 +1,56 @@ +# Copyright 2025 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build stage +FROM golang:1.24-alpine AS builder + +# Use Aliyun mirror for faster downloads in China +RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories + +WORKDIR /workspace + +# Copy go mod files +COPY go.mod go.sum ./ +RUN GOPROXY=https://goproxy.cn,direct go mod download + +# Copy source code +COPY cmd/image-committer/ cmd/image-committer/ + +# Build binary +RUN CGO_ENABLED=0 GOOS=linux go build -o /usr/local/bin/image-committer ./cmd/image-committer/ + +# Runtime stage +FROM alpine:3.19 + +# Use Aliyun mirror for faster downloads in China +RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories + +# Install containerd CLI tools +RUN apk add --no-cache \ + containerd-ctr \ + cri-tools \ + curl \ + jq \ + nerdctl + +# Create directories for socket mounts +RUN mkdir -p /var/run/containerd /run/k8s/containerd + +# Copy the built binary from builder stage +COPY --from=builder /usr/local/bin/image-committer /usr/local/bin/image-committer +RUN chmod +x /usr/local/bin/image-committer + +WORKDIR /workspace + +ENTRYPOINT ["/usr/local/bin/image-committer"] diff --git a/kubernetes/Makefile b/kubernetes/Makefile index b26a52286..a4b7e8cd4 100644 --- a/kubernetes/Makefile +++ b/kubernetes/Makefile @@ -54,6 +54,8 @@ OPERATOR_SDK_VERSION ?= v1.42.0 CONTROLLER_IMG ?= controller:dev # TASK_EXECUTOR_IMG defines the image for the task-executor service. TASK_EXECUTOR_IMG ?= task-executor:dev +# IMAGE_COMMITTER_IMG defines the image for the image-committer service. +IMAGE_COMMITTER_IMG ?= image-committer:dev # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) @@ -122,7 +124,7 @@ test: manifests generate fmt vet setup-envtest ## Run tests. # To use a different vendor for e2e tests, modify the setup under 'tests/e2e'. # The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. KIND_CLUSTER ?= sandbox-k8s-test-e2e -KIND_K8S_VERSION ?= v1.22.4 +KIND_K8S_VERSION ?= v1.27.3 GINKGO_ARGS ?= .PHONY: install-kind @@ -165,6 +167,17 @@ test-e2e: setup-test-e2e manifests generate fmt vet ## Run the e2e tests. Expect cleanup-test-e2e: ## Tear down the Kind cluster used for e2e tests @$(KIND) delete cluster --name $(KIND_CLUSTER) +# Pause/Resume E2E test variables +REGISTRY_IMAGE ?= registry:2 +REGISTRY_NODE_PORT ?= 30500 +REGISTRY_USERNAME ?= testuser +REGISTRY_PASSWORD ?= testpass + +.PHONY: test-e2e-pause-resume +test-e2e-pause-resume: setup-test-e2e ## Run pause/resume E2E tests + CONTROLLER_IMG=$(CONTROLLER_IMG) TASK_EXECUTOR_IMG=$(TASK_EXECUTOR_IMG) \ + KIND_CLUSTER=$(KIND_CLUSTER) go test ./test/e2e/ -v -ginkgo.v -ginkgo.focus="PauseResume" + # Common E2E setup targets - install CRDs and controller for any Kind cluster .PHONY: install-e2e-deps install-e2e-deps: @@ -278,6 +291,14 @@ docker-build-controller: ## Build docker image with the manager. docker-build-task-executor: ## Build docker image with task-executor. $(CONTAINER_TOOL) build $(DOCKER_BUILD_ARGS) --build-arg PACKAGE=cmd/task-executor/main.go --build-arg USERID=0 -t ${TASK_EXECUTOR_IMG} . +.PHONY: docker-build-image-committer +docker-build-image-committer: ## Build docker image for image commit operations. + $(CONTAINER_TOOL) build $(DOCKER_BUILD_ARGS) -f Dockerfile.image-committer -t ${IMAGE_COMMITTER_IMG} . + +.PHONY: docker-push-image-committer +docker-push-image-committer: ## Push docker image for image-committer. + $(CONTAINER_TOOL) push ${IMAGE_COMMITTER_IMG} + .PHONY: docker-push # docker-push: ## Push docker image with the manager. # $(CONTAINER_TOOL) push ${CONTROLLER_IMG} diff --git a/kubernetes/apis/sandbox/v1alpha1/batchsandbox_types.go b/kubernetes/apis/sandbox/v1alpha1/batchsandbox_types.go index 234c10061..436122e5c 100644 --- a/kubernetes/apis/sandbox/v1alpha1/batchsandbox_types.go +++ b/kubernetes/apis/sandbox/v1alpha1/batchsandbox_types.go @@ -70,6 +70,9 @@ type BatchSandboxSpec struct { // +kubebuilder:default=Retain // +kubebuilder:validation:Optional TaskResourcePolicyWhenCompleted *TaskResourcePolicy `json:"taskResourcePolicyWhenCompleted,omitempty"` + // PausePolicy defines the pause/resume policy for this sandbox. + // +optional + PausePolicy *PausePolicy `json:"pausePolicy,omitempty"` } type TaskResourcePolicy string @@ -79,6 +82,29 @@ const ( TaskResourcePolicyRelease TaskResourcePolicy = "Release" ) +// PausePolicy defines the policy for pause/resume operations. +type PausePolicy struct { + // SnapshotRegistry is the OCI registry for snapshot images. + // +kubebuilder:validation:Required + SnapshotRegistry string `json:"snapshotRegistry"` + + // SnapshotType indicates the type of snapshot (default: Rootfs). + // +optional + // +kubebuilder:validation:Optional + // +kubebuilder:default=Rootfs + SnapshotType SnapshotType `json:"snapshotType,omitempty"` + + // SnapshotPushSecret is the Secret name for pushing snapshots. + // +optional + // +kubebuilder:validation:Optional + SnapshotPushSecret string `json:"snapshotPushSecret,omitempty"` + + // ResumeImagePullSecret is the Secret name for pulling snapshots during resume. + // +optional + // +kubebuilder:validation:Optional + ResumeImagePullSecret string `json:"resumeImagePullSecret,omitempty"` +} + // BatchSandboxStatus defines the observed state of BatchSandbox. type BatchSandboxStatus struct { // ObservedGeneration is the most recent generation observed for this BatchSandbox. It corresponds to the diff --git a/kubernetes/apis/sandbox/v1alpha1/sandboxsnapshot_types.go b/kubernetes/apis/sandbox/v1alpha1/sandboxsnapshot_types.go new file mode 100644 index 000000000..e5b44cc9f --- /dev/null +++ b/kubernetes/apis/sandbox/v1alpha1/sandboxsnapshot_types.go @@ -0,0 +1,175 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// SnapshotType defines the type of snapshot. +type SnapshotType string + +const ( + SnapshotTypeRootfs SnapshotType = "Rootfs" +) + +// +kubebuilder:validation:Enum=Pending;Committing;Ready;Failed +// SandboxSnapshotPhase defines the phase of a snapshot. +type SandboxSnapshotPhase string + +const ( + SandboxSnapshotPhasePending SandboxSnapshotPhase = "Pending" + SandboxSnapshotPhaseCommitting SandboxSnapshotPhase = "Committing" + SandboxSnapshotPhaseReady SandboxSnapshotPhase = "Ready" + SandboxSnapshotPhaseFailed SandboxSnapshotPhase = "Failed" +) + +// ContainerSnapshot represents a snapshot of a single container. +type ContainerSnapshot struct { + // ContainerName is the name of the container. + ContainerName string `json:"containerName"` + // ImageURI is the target image URI for this container's snapshot. + ImageURI string `json:"imageUri"` + // ImageDigest is the digest of the pushed snapshot image. + // +optional + ImageDigest string `json:"imageDigest,omitempty"` +} + +// SandboxSnapshotSpec defines the desired state of SandboxSnapshot. +type SandboxSnapshotSpec struct { + // SandboxID is the stable public identifier for the sandbox. + SandboxID string `json:"sandboxId"` + + // SnapshotType indicates the type of snapshot (default: Rootfs). + // +optional + // +kubebuilder:validation:Optional + // +kubebuilder:default=Rootfs + SnapshotType SnapshotType `json:"snapshotType,omitempty"` + + // SourceBatchSandboxName is the name of the source BatchSandbox. + SourceBatchSandboxName string `json:"sourceBatchSandboxName"` + + // SourcePodName is the name of the source Pod. + // +optional + // +kubebuilder:validation:Optional + SourcePodName string `json:"sourcePodName,omitempty"` + + // SourceNodeName is the node where the source Pod runs. + // +optional + // +kubebuilder:validation:Optional + SourceNodeName string `json:"sourceNodeName,omitempty"` + + // SnapshotPushSecret is the Secret name for pushing to registry. + // +optional + SnapshotPushSecret string `json:"snapshotPushSecret,omitempty"` + + // ResumeImagePullSecret is the Secret name for pulling snapshot during resume. + // +optional + ResumeImagePullSecret string `json:"resumeImagePullSecret,omitempty"` + + // ResumeTemplate contains enough information to reconstruct BatchSandbox. + // +optional + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Schemaless + ResumeTemplate *runtime.RawExtension `json:"resumeTemplate,omitempty"` + + // SnapshotRegistry is the OCI registry for snapshot images. + // +optional + // +kubebuilder:validation:Optional + SnapshotRegistry string `json:"snapshotRegistry,omitempty"` + + // ContainerSnapshots holds per-container snapshot information. + // The controller fills this during resolution. + // +optional + // +kubebuilder:validation:Optional + ContainerSnapshots []ContainerSnapshot `json:"containerSnapshots,omitempty"` + + // PausedAt is the timestamp when pause was initiated. + PausedAt metav1.Time `json:"pausedAt"` + + // PauseVersion is incremented by the server to request a pause. + // Controller ACKs by setting status.pauseVersion to match when entering Committing phase. + PauseVersion int `json:"pauseVersion"` + + // ResumeVersion is incremented by the server to request a resume. + // Controller ACKs by setting status.resumeVersion to match when starting resume. + ResumeVersion int `json:"resumeVersion"` +} + +// SnapshotRecord represents a single pause or resume event in the snapshot history. +type SnapshotRecord struct { + // Action is "Pause" or "Resume". + Action string `json:"action"` + // Version is the pauseVersion or resumeVersion that triggered this action. + Version int `json:"version"` + // Timestamp is when this record was created. + Timestamp metav1.Time `json:"timestamp"` + // Message is a human-readable description of the event. + Message string `json:"message"` +} + +// SandboxSnapshotStatus defines the observed state of SandboxSnapshot. +type SandboxSnapshotStatus struct { + // Phase indicates the current phase of the snapshot. + Phase SandboxSnapshotPhase `json:"phase,omitempty"` + + // Message provides human-readable status information. + Message string `json:"message,omitempty"` + + // ReadyAt is the timestamp when the snapshot became Ready. + ReadyAt *metav1.Time `json:"readyAt,omitempty"` + + // ContainerSnapshots holds per-container snapshot results (filled by controller after push). + // +optional + ContainerSnapshots []ContainerSnapshot `json:"containerSnapshots,omitempty"` + + // PauseVersion is ACKed by the controller when entering Committing phase. + PauseVersion int `json:"pauseVersion"` + + // ResumeVersion is ACKed by the controller when starting resume. + ResumeVersion int `json:"resumeVersion"` + + // History records each pause/resume cycle. + // +optional + History []SnapshotRecord `json:"history,omitempty"` +} + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:shortName=sbxsnap +// +kubebuilder:printcolumn:name="PHASE",type="string",JSONPath=".status.phase" +// +kubebuilder:printcolumn:name="SANDBOX_ID",type="string",JSONPath=".spec.sandboxId" +// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp" +type SandboxSnapshot struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec SandboxSnapshotSpec `json:"spec,omitempty"` + Status SandboxSnapshotStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true +type SandboxSnapshotList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []SandboxSnapshot `json:"items"` +} + +func init() { + SchemeBuilder.Register(&SandboxSnapshot{}, &SandboxSnapshotList{}) +} diff --git a/kubernetes/apis/sandbox/v1alpha1/zz_generated.deepcopy.go b/kubernetes/apis/sandbox/v1alpha1/zz_generated.deepcopy.go index 05024c26e..a926d68df 100644 --- a/kubernetes/apis/sandbox/v1alpha1/zz_generated.deepcopy.go +++ b/kubernetes/apis/sandbox/v1alpha1/zz_generated.deepcopy.go @@ -124,6 +124,11 @@ func (in *BatchSandboxSpec) DeepCopyInto(out *BatchSandboxSpec) { *out = new(TaskResourcePolicy) **out = **in } + if in.PausePolicy != nil { + in, out := &in.PausePolicy, &out.PausePolicy + *out = new(PausePolicy) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BatchSandboxSpec. @@ -166,6 +171,36 @@ func (in *CapacitySpec) DeepCopy() *CapacitySpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContainerSnapshot) DeepCopyInto(out *ContainerSnapshot) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContainerSnapshot. +func (in *ContainerSnapshot) DeepCopy() *ContainerSnapshot { + if in == nil { + return nil + } + out := new(ContainerSnapshot) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PausePolicy) DeepCopyInto(out *PausePolicy) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PausePolicy. +func (in *PausePolicy) DeepCopy() *PausePolicy { + if in == nil { + return nil + } + out := new(PausePolicy) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Pool) DeepCopyInto(out *Pool) { *out = *in @@ -318,6 +353,115 @@ func (in *ScaleStrategy) DeepCopy() *ScaleStrategy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SandboxSnapshot) DeepCopyInto(out *SandboxSnapshot) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxSnapshot. +func (in *SandboxSnapshot) DeepCopy() *SandboxSnapshot { + if in == nil { + return nil + } + out := new(SandboxSnapshot) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SandboxSnapshot) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SandboxSnapshotList) DeepCopyInto(out *SandboxSnapshotList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]SandboxSnapshot, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxSnapshotList. +func (in *SandboxSnapshotList) DeepCopy() *SandboxSnapshotList { + if in == nil { + return nil + } + out := new(SandboxSnapshotList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SandboxSnapshotList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SandboxSnapshotSpec) DeepCopyInto(out *SandboxSnapshotSpec) { + *out = *in + if in.ResumeTemplate != nil { + in, out := &in.ResumeTemplate, &out.ResumeTemplate + *out = new(runtime.RawExtension) + (*in).DeepCopyInto(*out) + } + if in.ContainerSnapshots != nil { + in, out := &in.ContainerSnapshots, &out.ContainerSnapshots + *out = make([]ContainerSnapshot, len(*in)) + copy(*out, *in) + } + in.PausedAt.DeepCopyInto(&out.PausedAt) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxSnapshotSpec. +func (in *SandboxSnapshotSpec) DeepCopy() *SandboxSnapshotSpec { + if in == nil { + return nil + } + out := new(SandboxSnapshotSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SandboxSnapshotStatus) DeepCopyInto(out *SandboxSnapshotStatus) { + *out = *in + if in.ReadyAt != nil { + in, out := &in.ReadyAt, &out.ReadyAt + *out = (*in).DeepCopy() + } + if in.ContainerSnapshots != nil { + in, out := &in.ContainerSnapshots, &out.ContainerSnapshots + *out = make([]ContainerSnapshot, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxSnapshotStatus. +func (in *SandboxSnapshotStatus) DeepCopy() *SandboxSnapshotStatus { + if in == nil { + return nil + } + out := new(SandboxSnapshotStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TaskSpec) DeepCopyInto(out *TaskSpec) { *out = *in diff --git a/kubernetes/cmd/controller/main.go b/kubernetes/cmd/controller/main.go index 1e95cc281..038c152a1 100644 --- a/kubernetes/cmd/controller/main.go +++ b/kubernetes/cmd/controller/main.go @@ -19,6 +19,7 @@ import ( "flag" "os" "path/filepath" + "time" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -105,6 +106,14 @@ func main() { flag.Float64Var(&kubeClientQPS, "kube-client-qps", 100, "QPS for Kubernetes client rate limiter.") flag.IntVar(&kubeClientBurst, "kube-client-burst", 200, "Burst for Kubernetes client rate limiter.") + // Image committer + var imageCommitterImage string + flag.StringVar(&imageCommitterImage, "image-committer-image", "image-committer:dev", "The image used for commit operations (contains ctr/crictl tools).") + + // Commit job timeout + var commitJobTimeout time.Duration + flag.DurationVar(&commitJobTimeout, "commit-job-timeout", 10*time.Minute, "The timeout duration for commit jobs.") + opts := zap.Options{} opts.BindFlags(flag.CommandLine) @@ -268,6 +277,16 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "Pool") os.Exit(1) } + if err := (&controller.SandboxSnapshotReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("sandboxsnapshot-controller"), + ImageCommitterImage: imageCommitterImage, + CommitJobTimeout: commitJobTimeout, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "SandboxSnapshot") + os.Exit(1) + } // +kubebuilder:scaffold:builder if metricsCertWatcher != nil { diff --git a/kubernetes/cmd/image-committer/main.go b/kubernetes/cmd/image-committer/main.go new file mode 100644 index 000000000..80332405c --- /dev/null +++ b/kubernetes/cmd/image-committer/main.go @@ -0,0 +1,484 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + + "encoding/base64" + "io/ioutil" + "os/signal" +) + +// containerdSocket returns the containerd socket address from env or default +func containerdSocket() string { + if v := os.Getenv("CONTAINERD_SOCKET"); v != "" { + return v + } + return "/run/containerd/containerd.sock" +} + +// containerdNamespace returns the containerd namespace from env or default +func containerdNamespace() string { + if v := os.Getenv("CONTAINERD_NAMESPACE"); v != "" { + return v + } + return "k8s.io" +} + +// nerdctlBaseArgs returns the base arguments for nerdctl commands +func nerdctlBaseArgs() []string { + return []string{"--address", containerdSocket(), "--namespace", containerdNamespace()} +} + +type ContainerSpec struct { + Name string + URI string +} + +// Global tracking of paused containers for cleanup +var pausedContainerIds []string + +func main() { + args := os.Args[1:] + + // Set up signal handler to ensure all paused containers are resumed on exit + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + go func() { + sig := <-c + fmt.Fprintf(os.Stderr, "Received signal %v, cleaning up paused containers...\n", sig) + resumeAllPausedContainers() + os.Exit(1) + }() + + // Defer cleanup in case of panic or early termination + defer func() { + if r := recover(); r != nil { + fmt.Fprintf(os.Stderr, "Panic occurred: %v\n", r) + resumeAllPausedContainers() + panic(r) + } + }() + + // Parse arguments using unified format: + // [container2:uri2...] + var podName, namespace string + var containerSpecs []ContainerSpec + + if len(args) < 3 { + fmt.Fprintln(os.Stderr, "ERROR: Missing required parameters") + fmt.Fprintln(os.Stderr, "Usage: commit-snapshot [container2:uri2...]") + os.Exit(1) + } + + podName = args[0] + namespace = args[1] + + for i := 2; i < len(args); i++ { + spec, err := parseContainerSpec(args[i]) + if err != nil { + fmt.Fprintf(os.Stderr, "ERROR: %v\n", err) + os.Exit(1) + } + containerSpecs = append(containerSpecs, spec) + } + + // Validate required inputs + if len(podName) == 0 { + fmt.Fprintln(os.Stderr, "ERROR: Pod name is required") + os.Exit(1) + } + + if len(namespace) == 0 { + fmt.Fprintln(os.Stderr, "ERROR: Namespace is required") + os.Exit(1) + } + + if len(containerSpecs) == 0 { + fmt.Fprintln(os.Stderr, "ERROR: At least one container specification is required") + fmt.Fprintln(os.Stderr, "Usage: commit-snapshot [container2:uri2...]") + os.Exit(1) + } + + fmt.Println("=== Commit Snapshot Go Program ===") + fmt.Printf("Pod: %s\n", podName) + fmt.Printf("Namespace: %s\n", namespace) + for _, spec := range containerSpecs { + fmt.Printf("Container spec: %s -> %s\n", spec.Name, spec.URI) + } + + // Step 1: Discover pod sandbox + fmt.Println("\n=== Step 1: Find pod sandbox ===") + podSandboxID, err := getPodSandboxID(podName, namespace) + if err != nil { + fmt.Fprintf(os.Stderr, "ERROR: Failed to find pod: %v\n", err) + os.Exit(1) + } + fmt.Printf("Pod sandbox ID: %s\n", podSandboxID) + + // Step 2: Find container IDs and validate + fmt.Println("\n=== Step 2: Find container IDs ===") + containerMap := make(map[string]string) // Maps container name to container ID + for _, spec := range containerSpecs { + containerID, err := getContainerID(podSandboxID, spec.Name) + if err != nil { + resumeAllPausedContainers() + fmt.Fprintf(os.Stderr, "ERROR: Failed to find container '%s': %v\n", spec.Name, err) + os.Exit(1) + } + + fmt.Printf("Container '%s' -> ID: %s\n", spec.Name, containerID) + containerMap[spec.Name] = containerID + } + + // Step 3: Pause all containers + fmt.Println("\n=== Step 3: Pause all containers ===") + pauseErrors := 0 + for _, spec := range containerSpecs { + containerID := containerMap[spec.Name] + if err := pauseContainer(containerID); err != nil { + // On pause failure, we still try to continue since commit might work anyway (as in shell script) + fmt.Fprintf(os.Stderr, "WARNING: Could not pause '%s'. Will attempt commit anyway (container may be stopped).\n", spec.Name) + pauseErrors++ + } else { + // Track successfully paused containers for cleanup + pausedContainerIds = append(pausedContainerIds, containerID) + } + } + + // Step 4: Commit all containers + fmt.Println("\n=== Step 4: Commit all containers ===") + committedImages := make(map[string]string) // Maps container name to committed image URI + commitErrors := 0 + for _, spec := range containerSpecs { + containerID := containerMap[spec.Name] + if err := commitContainer(containerID, spec.URI); err != nil { + fmt.Fprintf(os.Stderr, "ERROR: Failed to commit container '%s': %v\n", spec.Name, err) + commitErrors++ + } else { + committedImages[spec.Name] = spec.URI + fmt.Printf("Successfully committed: %s -> %s\n", containerID, spec.URI) + } + } + + // Step 5: Resume all paused containers (regardless of commit success/failure) + fmt.Println("\n=== Step 5: Resume all paused containers ===") + resumeAllPausedContainers() + + // If there were commit errors, exit with failure after cleanup + if commitErrors > 0 { + fmt.Fprintf(os.Stderr, "ERROR: %d container(s) failed to commit. All containers have been resumed.\n", commitErrors) + os.Exit(1) + } + + // Step 6: Push all committed images + fmt.Println("\n=== Step 6: Push all images ===") + pushErrors := 0 + for _, spec := range containerSpecs { + if _, ok := committedImages[spec.Name]; ok { + if err := pushImage(spec.URI); err != nil { + fmt.Fprintf(os.Stderr, "ERROR: Failed to push image for container '%s': %v\n", spec.Name, err) + pushErrors++ + } else { + fmt.Printf("Successfully pushed: %s\n", spec.URI) + } + } + } + + if pushErrors > 0 { + fmt.Fprintf(os.Stderr, "ERROR: %d image(s) failed to push.\n", pushErrors) + os.Exit(1) + } + + // Step 7: Extract digests and output results + fmt.Println("\n=== Step 7: Extract digests ===") + digests := make(map[string]string) // Maps container name to digest + firstDigest := "" + + for _, spec := range containerSpecs { + if _, ok := committedImages[spec.Name]; ok { + digest, err := getImageDigest(spec.URI) + if err != nil { + fmt.Fprintf(os.Stderr, "WARN: Failed to extract digest for %s: %v\n", spec.URI, err) + digest = "sha256:placeholder" // fallback digest + } + + digests[spec.Name] = digest + fmt.Printf("Container '%s' digest: %s\n", spec.Name, digest) + + // Capture first digest for legacy output + if firstDigest == "" { + firstDigest = digest + } + } + } + + // Final output - SNAPSHOT_DIGEST_ variables for each container + fmt.Println("\n=== Snapshot completed successfully ===") + for _, spec := range containerSpecs { + if digest, ok := digests[spec.Name]; ok { + upperName := strings.ToUpper(strings.ReplaceAll(spec.Name, "-", "_")) + fmt.Printf("SNAPSHOT_DIGEST_%s=%s\n", upperName, digest) + fmt.Printf(" Image: %s\n", spec.URI) + fmt.Printf(" Digest: %s\n", digest) + } + } + + // Legacy single-digest output for backward compatibility + fmt.Printf("SNAPSHOT_DIGEST=%s\n", firstDigest) +} + +// parseContainerSpec parses a "container:uri" string into ContainerSpec +func parseContainerSpec(specStr string) (ContainerSpec, error) { + parts := strings.SplitN(specStr, ":", 2) + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return ContainerSpec{}, fmt.Errorf("invalid container spec '%s'. Expected format: container_name:uri", specStr) + } + + return ContainerSpec{ + Name: parts[0], + URI: parts[1], + }, nil +} + +// getPodSandboxID uses crictl to find the pod sandbox ID +func getPodSandboxID(podName, namespace string) (string, error) { + cmd := exec.Command("crictl", "pods", "--name", podName, "--namespace", namespace, "-q") + output, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("failed to find pod %s in namespace %s: %v", podName, namespace, err) + } + + // Handle multiple sandbox IDs - take the first one (most recent) + lines := strings.Split(strings.TrimSpace(string(output)), "\n") + sandboxID := strings.TrimSpace(lines[0]) + if sandboxID == "" { + return "", fmt.Errorf("pod sandbox not found for %s in namespace %s", podName, namespace) + } + + return sandboxID, nil +} + +// getContainerID uses crictl to find the container ID within a pod sandbox +func getContainerID(podSandboxID, containerName string) (string, error) { + cmd := exec.Command("crictl", "ps", "--pod", podSandboxID, "--name", containerName, "-q") + output, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("failed to find container %s in pod sandbox %s: %v", containerName, podSandboxID, err) + } + + containerID := strings.TrimSpace(string(output)) + if containerID == "" { + return "", fmt.Errorf("container %s not found in pod sandbox %s", containerName, podSandboxID) + } + + return containerID, nil +} + +// pauseContainer uses nerdctl to pause a container +func pauseContainer(containerID string) error { + fmt.Printf("Pausing container %s...\n", containerID) + args := append(nerdctlBaseArgs(), "pause", containerID) + cmd := exec.Command("nerdctl", args...) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to pause container %s: %v, output: %s", containerID, err, string(output)) + } + fmt.Printf("Paused successfully: %s\n", containerID) + return nil +} + +// resumeContainer uses nerdctl to resume a container +func resumeContainer(containerID string) error { + fmt.Printf("Resuming container %s...\n", containerID) + args := append(nerdctlBaseArgs(), "unpause", containerID) + cmd := exec.Command("nerdctl", args...) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to resume container %s: %v, output: %s", containerID, err, string(output)) + } + fmt.Printf("Resumed successfully: %s\n", containerID) + return nil +} + +// resumeAllPausedContainers resumes all paused containers that were tracked +func resumeAllPausedContainers() { + if len(pausedContainerIds) == 0 { + return + } + + fmt.Println("\n=== Cleanup: Resuming all paused containers ===") + + // Process in reverse order to match pause order + for i := len(pausedContainerIds) - 1; i >= 0; i-- { + containerID := pausedContainerIds[i] + err := resumeContainer(containerID) + if err != nil { + fmt.Fprintf(os.Stderr, "WARNING: Could not resume container %s: %v\n", containerID, err) + } + } + + // Clear the paused containers list after cleanup + pausedContainerIds = []string{} +} + +// commitContainer uses nerdctl to commit a container to an image +func commitContainer(containerID, targetImage string) error { + fmt.Printf("Committing container %s to image %s...\n", containerID, targetImage) + args := append(nerdctlBaseArgs(), "commit", containerID, targetImage) + cmd := exec.Command("nerdctl", args...) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to commit container %s to %s: %v, output: %s", containerID, targetImage, err, string(output)) + } + return nil +} + +// pushImage uses nerdctl to push the image to the registry. +// nerdctl push does not support --username/--password flags, so we use +// nerdctl login first, then nerdctl push with --insecure-registry. +func pushImage(targetImage string) error { + fmt.Printf("Pushing image %s...\n", targetImage) + + // Parse registry host from target image + imageParts := strings.Split(targetImage, "/") + if len(imageParts) == 0 { + return fmt.Errorf("invalid target image: %s", targetImage) + } + registryHost := imageParts[0] + + isInsecure := strings.Contains(registryHost, "local") || + strings.Contains(registryHost, "localhost") || + strings.HasPrefix(registryHost, "127.") || + strings.HasPrefix(registryHost, "10.") || + strings.HasPrefix(registryHost, "192.168.") + + // Try to login using credentials from mounted secret + credDir := "/var/run/opensandbox/registry" + configPath := filepath.Join(credDir, "config.json") + if _, err := os.Stat(configPath); err == nil { + fmt.Printf("Found registry credentials at %s\n", configPath) + if err := nerdctlLogin(configPath, registryHost, isInsecure); err != nil { + fmt.Fprintf(os.Stderr, "WARNING: nerdctl login failed: %v (will attempt push anyway)\n", err) + } + } else { + fmt.Println("No registry credentials found, assuming insecure or pre-authenticated registry") + } + + // Build push options + pushOpts := append(nerdctlBaseArgs(), "push") + if isInsecure { + pushOpts = append(pushOpts, "--insecure-registry") + } + pushOpts = append(pushOpts, targetImage) + + cmd := exec.Command("nerdctl", pushOpts...) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to push image %s: %v, output: %s", targetImage, err, string(output)) + } + + return nil +} + +// nerdctlLogin extracts credentials from a Docker config.json and runs nerdctl login. +func nerdctlLogin(configPath, registryHost string, insecure bool) error { + data, err := ioutil.ReadFile(configPath) + if err != nil { + return fmt.Errorf("failed to read config: %w", err) + } + + var creds map[string]interface{} + if err := json.Unmarshal(data, &creds); err != nil { + return fmt.Errorf("failed to parse config: %w", err) + } + + auths, ok := creds["auths"].(map[string]interface{}) + if !ok || auths[registryHost] == nil { + return fmt.Errorf("no auth entry for registry %s", registryHost) + } + + authEntry, ok := auths[registryHost].(map[string]interface{}) + if !ok { + return fmt.Errorf("invalid auth entry for registry %s", registryHost) + } + + // Try "auth" field first (base64 encoded), then fall back to username/password fields + var username, password string + if authVal, ok := authEntry["auth"].(string); ok && authVal != "" { + decoded, err := base64.StdEncoding.DecodeString(authVal) + if err != nil { + return fmt.Errorf("failed to decode auth: %w", err) + } + parts := strings.SplitN(string(decoded), ":", 2) + if len(parts) != 2 { + return fmt.Errorf("invalid auth format") + } + username = parts[0] + password = parts[1] + } else { + if u, ok := authEntry["username"].(string); ok { + username = u + } + if p, ok := authEntry["password"].(string); ok { + password = p + } + } + + if username == "" || password == "" { + return fmt.Errorf("empty username or password for registry %s", registryHost) + } + + fmt.Printf("Logging in to registry %s as %s\n", registryHost, username) + + loginOpts := append(nerdctlBaseArgs(), "login", "-u", username, "-p", password) + if insecure { + loginOpts = append(loginOpts, "--insecure-registry") + } + loginOpts = append(loginOpts, registryHost) + + cmd := exec.Command("nerdctl", loginOpts...) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("nerdctl login failed: %v, output: %s", err, string(output)) + } + + fmt.Printf("Login succeeded for %s\n", registryHost) + return nil +} + +// getImageDigest uses nerdctl to get the digest of the image +func getImageDigest(imageRef string) (string, error) { + args := append(nerdctlBaseArgs(), "inspect", "--format", "{{.Id}}", imageRef) + cmd := exec.Command("nerdctl", args...) + output, err := cmd.Output() + if err == nil { + digest := strings.TrimSpace(string(output)) + if digest != "" { + return digest, nil + } + } + + // If primary method fails (due to formatting, API changes, etc), return placeholder + // The shell script had more complex fallback mechanisms but this covers the essential use case + return "sha256:placeholder", nil +} diff --git a/kubernetes/config/crd/bases/sandbox.opensandbox.io_batchsandboxes.yaml b/kubernetes/config/crd/bases/sandbox.opensandbox.io_batchsandboxes.yaml index 72c43bdda..5a46c6e15 100644 --- a/kubernetes/config/crd/bases/sandbox.opensandbox.io_batchsandboxes.yaml +++ b/kubernetes/config/crd/bases/sandbox.opensandbox.io_batchsandboxes.yaml @@ -95,6 +95,30 @@ spec: If a time in the past is provided, the batch-sandbox will be deleted immediately. format: date-time type: string + pausePolicy: + description: PausePolicy defines the pause/resume policy for this + sandbox. + properties: + resumeImagePullSecretName: + description: ResumeImagePullSecretName is the Secret name for + pulling snapshots during resume. + type: string + snapshotPushSecretName: + description: SnapshotPushSecretName is the Secret name for pushing + snapshots. + type: string + snapshotRegistry: + description: SnapshotRegistry is the OCI registry for snapshot + images. + type: string + snapshotType: + default: Rootfs + description: 'SnapshotType indicates the type of snapshot (default: + Rootfs).' + type: string + required: + - snapshotRegistry + type: object poolRef: description: |- PoolRef references the Pool resource name for pooled sandbox creation. diff --git a/kubernetes/config/crd/bases/sandbox.opensandbox.io_sandboxsnapshots.yaml b/kubernetes/config/crd/bases/sandbox.opensandbox.io_sandboxsnapshots.yaml new file mode 100644 index 000000000..c04a8b850 --- /dev/null +++ b/kubernetes/config/crd/bases/sandbox.opensandbox.io_sandboxsnapshots.yaml @@ -0,0 +1,215 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: sandboxsnapshots.sandbox.opensandbox.io +spec: + group: sandbox.opensandbox.io + names: + kind: SandboxSnapshot + listKind: SandboxSnapshotList + plural: sandboxsnapshots + shortNames: + - sbxsnap + singular: sandboxsnapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.phase + name: PHASE + type: string + - jsonPath: .spec.sandboxId + name: SANDBOX_ID + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: SandboxSnapshotSpec defines the desired state of SandboxSnapshot. + properties: + containerSnapshots: + description: |- + ContainerSnapshots holds per-container snapshot information. + The controller fills this during resolution. + items: + description: ContainerSnapshot represents a snapshot of a single + container. + properties: + containerName: + description: ContainerName is the name of the container. + type: string + imageDigest: + description: ImageDigest is the digest of the pushed snapshot + image. + type: string + imageUri: + description: ImageURI is the target image URI for this container's + snapshot. + type: string + required: + - containerName + - imageUri + type: object + type: array + pauseVersion: + description: |- + PauseVersion is incremented by the server to request a pause. + Controller ACKs by setting status.pauseVersion to match when entering Committing phase. + type: integer + pausedAt: + description: PausedAt is the timestamp when pause was initiated. + format: date-time + type: string + resumeImagePullSecretName: + description: ResumeImagePullSecretName is the Secret name for pulling + snapshot during resume. + type: string + resumeTemplate: + description: ResumeTemplate contains enough information to reconstruct + BatchSandbox. + x-kubernetes-preserve-unknown-fields: true + resumeVersion: + description: |- + ResumeVersion is incremented by the server to request a resume. + Controller ACKs by setting status.resumeVersion to match when starting resume. + type: integer + sandboxId: + description: SandboxID is the stable public identifier for the sandbox. + type: string + snapshotPushSecretName: + description: SnapshotPushSecretName is the Secret name for pushing + to registry. + type: string + snapshotRegistry: + description: SnapshotRegistry is the OCI registry for snapshot images. + type: string + snapshotType: + default: Rootfs + description: 'SnapshotType indicates the type of snapshot (default: + Rootfs).' + type: string + sourceBatchSandboxName: + description: SourceBatchSandboxName is the name of the source BatchSandbox. + type: string + sourceNodeName: + description: SourceNodeName is the node where the source Pod runs. + type: string + sourcePodName: + description: SourcePodName is the name of the source Pod. + type: string + required: + - pauseVersion + - pausedAt + - resumeVersion + - sandboxId + - sourceBatchSandboxName + type: object + status: + description: SandboxSnapshotStatus defines the observed state of SandboxSnapshot. + properties: + containerSnapshots: + description: ContainerSnapshots holds per-container snapshot results + (filled by controller after push). + items: + description: ContainerSnapshot represents a snapshot of a single + container. + properties: + containerName: + description: ContainerName is the name of the container. + type: string + imageDigest: + description: ImageDigest is the digest of the pushed snapshot + image. + type: string + imageUri: + description: ImageURI is the target image URI for this container's + snapshot. + type: string + required: + - containerName + - imageUri + type: object + type: array + history: + description: History records each pause/resume cycle. + items: + description: SnapshotRecord represents a single pause or resume + event in the snapshot history. + properties: + action: + description: Action is "Pause" or "Resume". + type: string + message: + description: Message is a human-readable description of the + event. + type: string + timestamp: + description: Timestamp is when this record was created. + format: date-time + type: string + version: + description: Version is the pauseVersion or resumeVersion that + triggered this action. + type: integer + required: + - action + - message + - timestamp + - version + type: object + type: array + message: + description: Message provides human-readable status information. + type: string + pauseVersion: + description: PauseVersion is ACKed by the controller when entering + Committing phase. + type: integer + phase: + description: Phase indicates the current phase of the snapshot. + enum: + - Pending + - Committing + - Ready + - Failed + type: string + readyAt: + description: ReadyAt is the timestamp when the snapshot became Ready. + format: date-time + type: string + resumeVersion: + description: ResumeVersion is ACKed by the controller when starting + resume. + type: integer + required: + - pauseVersion + - resumeVersion + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/kubernetes/config/crd/kustomization.yaml b/kubernetes/config/crd/kustomization.yaml index 158214bc9..4c4bc7acd 100644 --- a/kubernetes/config/crd/kustomization.yaml +++ b/kubernetes/config/crd/kustomization.yaml @@ -4,6 +4,7 @@ resources: - bases/sandbox.opensandbox.io_batchsandboxes.yaml - bases/sandbox.opensandbox.io_pools.yaml +- bases/sandbox.opensandbox.io_sandboxsnapshots.yaml # +kubebuilder:scaffold:crdkustomizeresource patches: diff --git a/kubernetes/config/rbac/role.yaml b/kubernetes/config/rbac/role.yaml index 87fb96026..bb7c26a7b 100644 --- a/kubernetes/config/rbac/role.yaml +++ b/kubernetes/config/rbac/role.yaml @@ -25,11 +25,32 @@ rules: - get - patch - update +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - batch + resources: + - jobs/status + verbs: + - get + - patch + - update - apiGroups: - sandbox.opensandbox.io resources: - batchsandboxes - pools + - sandboxsnapshots verbs: - create - delete @@ -43,6 +64,7 @@ rules: resources: - batchsandboxes/finalizers - pools/finalizers + - sandboxsnapshots/finalizers verbs: - update - apiGroups: @@ -50,6 +72,7 @@ rules: resources: - batchsandboxes/status - pools/status + - sandboxsnapshots/status verbs: - get - patch diff --git a/kubernetes/internal/controller/sandboxsnapshot_controller.go b/kubernetes/internal/controller/sandboxsnapshot_controller.go new file mode 100644 index 000000000..bdb696a82 --- /dev/null +++ b/kubernetes/internal/controller/sandboxsnapshot_controller.go @@ -0,0 +1,995 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/retry" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + sandboxv1alpha1 "github.com/alibaba/OpenSandbox/sandbox-k8s/apis/sandbox/v1alpha1" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils" +) + +const ( + // SandboxSnapshotFinalizer is the finalizer for SandboxSnapshot cleanup + SandboxSnapshotFinalizer = "sandboxsnapshot.sandbox.opensandbox.io/cleanup" + + // DefaultCommitJobTimeout is the default timeout for commit jobs + DefaultCommitJobTimeout = 10 * time.Minute + + DefaultTTLSecondsAfterFinished = 300 + + // CommitJobContainerName is the container name in commit job + CommitJobContainerName = "commit" + + // ContainerdSocketPath is the default containerd socket path + ContainerdSocketPath = "/var/run/containerd/containerd.sock" + + // CrictlSocketPath is the CRI socket path for crictl + CrictlSocketPath = "/run/containerd/containerd.sock" + + // LabelSandboxSnapshotName is the label key for sandbox snapshot name + LabelSandboxSnapshotName = "sandbox.opensandbox.io/sandbox-snapshot-name" + + // AnnotationResumedFromSnapshot marks a BatchSandbox as resumed from a snapshot + AnnotationResumedFromSnapshot = "sandbox.opensandbox.io/resumed-from-snapshot" +) + +// SandboxSnapshotReconciler reconciles a SandboxSnapshot object +type SandboxSnapshotReconciler struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder + + // ImageCommitterImage is the image for image-committer (contains ctr/crictl) + ImageCommitterImage string + + // CommitJobTimeout is the timeout for commit jobs (default: 10 minutes) + CommitJobTimeout time.Duration +} + +// +kubebuilder:rbac:groups=sandbox.opensandbox.io,resources=sandboxsnapshots,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=sandbox.opensandbox.io,resources=sandboxsnapshots/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=sandbox.opensandbox.io,resources=sandboxsnapshots/finalizers,verbs=update +// +kubebuilder:rbac:groups=sandbox.opensandbox.io,resources=batchsandboxes,verbs=get;list;watch;delete +// +kubebuilder:rbac:groups=sandbox.opensandbox.io,resources=pools,verbs=get;list;watch +// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=batch,resources=jobs/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch +// +kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;update;patch;delete + +// Reconcile is part of the main kubernetes reconciliation loop +func (r *SandboxSnapshotReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := logf.FromContext(ctx) + + // Fetch the SandboxSnapshot instance + snapshot := &sandboxv1alpha1.SandboxSnapshot{} + if err := r.Get(ctx, req.NamespacedName, snapshot); err != nil { + if errors.IsNotFound(err) { + log.Info("SandboxSnapshot resource not found") + return ctrl.Result{}, nil + } + log.Error(err, "Failed to get SandboxSnapshot") + return ctrl.Result{}, err + } + + // Handle deletion + if !snapshot.DeletionTimestamp.IsZero() { + return r.handleDeletion(ctx, snapshot) + } + + // Add finalizer if not present + if !controllerutil.ContainsFinalizer(snapshot, SandboxSnapshotFinalizer) { + if err := utils.UpdateFinalizer(r.Client, snapshot, utils.AddFinalizerOpType, SandboxSnapshotFinalizer); err != nil { + log.Error(err, "Failed to add finalizer", "finalizer", SandboxSnapshotFinalizer) + return ctrl.Result{}, err + } + log.Info("Added finalizer", "finalizer", SandboxSnapshotFinalizer) + return ctrl.Result{RequeueAfter: time.Millisecond * 100}, nil + } + + // Version-based dispatch: check pause and resume versions + specPV := snapshot.Spec.PauseVersion + statusPV := snapshot.Status.PauseVersion + specRV := snapshot.Spec.ResumeVersion + statusRV := snapshot.Status.ResumeVersion + + log.Info("Reconciling SandboxSnapshot", + "snapshot", snapshot.Name, + "phase", snapshot.Status.Phase, + "specPV", specPV, "statusPV", statusPV, + "specRV", specRV, "statusRV", statusRV, + ) + + // 1. Pause requested: spec.pauseVersion > status.pauseVersion + if specPV > statusPV { + phase := snapshot.Status.Phase + if phase == "" || phase == sandboxv1alpha1.SandboxSnapshotPhaseReady || phase == sandboxv1alpha1.SandboxSnapshotPhaseFailed { + // Initialize or re-initialize for a new pause cycle + log.Info("Pause version mismatch, resetting to Pending", + "specPV", specPV, "statusPV", statusPV) + if err := r.updateSnapshotStatus(ctx, snapshot, sandboxv1alpha1.SandboxSnapshotPhasePending, "Pause requested"); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: time.Millisecond * 100}, nil + } + + // Normal phase state machine + switch phase { + case sandboxv1alpha1.SandboxSnapshotPhasePending: + return r.handlePending(ctx, snapshot) + case sandboxv1alpha1.SandboxSnapshotPhaseCommitting: + return r.handleCommitting(ctx, snapshot) + default: + log.Info("Unexpected phase during pause, treating as Pending", "phase", phase) + return r.handlePending(ctx, snapshot) + } + } + + // 2. Resume requested: spec.resumeVersion > status.resumeVersion + if specRV > statusRV { + return r.handleResume(ctx, snapshot) + } + + // 3. Idle — versions match, dispatch by phase for cleanup + phase := snapshot.Status.Phase + switch phase { + case sandboxv1alpha1.SandboxSnapshotPhaseReady: + return r.handleReady(ctx, snapshot) + case sandboxv1alpha1.SandboxSnapshotPhaseFailed: + return r.handleFailed(ctx, snapshot) + default: + log.Info("Idle with no pending work", "phase", phase) + return ctrl.Result{}, nil + } +} + +// ensureResolved resolves the template and fills spec.ContainerSnapshots with per-container +// image URIs along with pause policy info. It looks up the source BatchSandbox and +// fills in missing spec fields from the BatchSandbox, including pausePolicy, template +// for container snapshots, and ResumeTemplate for resuming after pause. +func (r *SandboxSnapshotReconciler) ensureResolved(ctx context.Context, snapshot *sandboxv1alpha1.SandboxSnapshot) error { + log := logf.FromContext(ctx) + + // If ContainerSnapshots already have all values populated, re-generate image URIs + // with current pauseVersion (they may be stale from a previous pause cycle). + if len(snapshot.Spec.ContainerSnapshots) > 0 { + allResolved := true + for _, cs := range snapshot.Spec.ContainerSnapshots { + if cs.ContainerName != "" && cs.ImageURI != "" { + continue + } + allResolved = false + break + } + + // Check also if essential pause policy fields are populated + if allResolved && snapshot.Spec.SnapshotType != "" && snapshot.Spec.SnapshotRegistry != "" { + // Re-generate image URIs to reflect current pauseVersion + registry := snapshot.Spec.SnapshotRegistry + needsUpdate := false + for i := range snapshot.Spec.ContainerSnapshots { + cs := &snapshot.Spec.ContainerSnapshots[i] + expectedURI := fmt.Sprintf("%s/%s-%s:snapshot-v%d", registry, snapshot.Spec.SandboxID, cs.ContainerName, snapshot.Spec.PauseVersion) + if cs.ImageURI != expectedURI { + log.Info("Updating stale image URI for re-pause", "container", cs.ContainerName, "old", cs.ImageURI, "new", expectedURI) + cs.ImageURI = expectedURI + needsUpdate = true + } + } + // Persist the updated image URIs to etcd + if needsUpdate { + if err := r.Update(ctx, snapshot); err != nil { + return fmt.Errorf("failed to update image URIs for re-pause: %w", err) + } + log.Info("Persisted updated image URIs for re-pause") + } + log.Info("Snapshot already resolved, skipping full resolution") + return nil + } + } + + // Look up the source BatchSandbox + bs := &sandboxv1alpha1.BatchSandbox{} + if err := r.Get(ctx, types.NamespacedName{ + Name: snapshot.Spec.SourceBatchSandboxName, + Namespace: snapshot.Namespace, + }, bs); err != nil { + return fmt.Errorf("failed to get source BatchSandbox %s: %w", snapshot.Spec.SourceBatchSandboxName, err) + } + + // If SourcePodName is empty, find the running pod for this sandbox + if snapshot.Spec.SourcePodName == "" { + pod, err := r.findPodForSandbox(ctx, bs, snapshot.Namespace) + if err != nil { + return fmt.Errorf("failed to find running pod for sandbox: %w", err) + } + snapshot.Spec.SourcePodName = pod.Name + snapshot.Spec.SourceNodeName = pod.Spec.NodeName + log.Info("Resolved pod info", "pod", pod.Name, "node", pod.Spec.NodeName) + } + + // Fill in pause policy fields from BatchSandbox + if bs.Spec.PausePolicy != nil { + // Extract pause policy fields + snapshot.Spec.SnapshotType = bs.Spec.PausePolicy.SnapshotType + snapshot.Spec.SnapshotRegistry = bs.Spec.PausePolicy.SnapshotRegistry + snapshot.Spec.SnapshotPushSecret = bs.Spec.PausePolicy.SnapshotPushSecret + snapshot.Spec.ResumeImagePullSecret = bs.Spec.PausePolicy.ResumeImagePullSecret + } else { + return fmt.Errorf("BatchSandbox %s has no pausePolicy configured", bs.Name) + } + + // Resolve the template: prefer spec.Template, otherwise look up Pool CR + var template *corev1.PodTemplateSpec + if bs.Spec.Template != nil { + template = bs.Spec.Template + log.Info("Resolved template directly from BatchSandbox spec") + } else if bs.Spec.PoolRef != "" { + // PoolRef mode: look up the Pool CR to get template + pool := &sandboxv1alpha1.Pool{} + if err := r.Get(ctx, types.NamespacedName{ + Name: bs.Spec.PoolRef, + Namespace: snapshot.Namespace, + }, pool); err != nil { + return fmt.Errorf("failed to look up Pool CR %s to get template: %w", bs.Spec.PoolRef, err) + } + if pool.Spec.Template == nil { + return fmt.Errorf("Pool %s has no template defined", bs.Spec.PoolRef) + } + template = pool.Spec.Template + log.Info("Resolved template via Pool CR", "pool", bs.Spec.PoolRef) + } else { + return fmt.Errorf("BatchSandbox %s has neither template nor poolRef, cannot resolve", bs.Name) + } + + // Build ResumeTemplate from the template with resolved fields + resumeTemplateData := map[string]interface{}{ + "template": convertPodTemplateSpecToMap(template), // Convert the template to map[string]interface{} + } + + // Add or update BatchSandbox-level fields to ResumeTemplate if they exist + if bs.Spec.ExpireTime != nil { + resumeTemplateData["expireTime"] = bs.Spec.ExpireTime // Copy the expireTime + } + if bs.Spec.PausePolicy != nil { + // We add the original pause policy back to the ResumeTemplate + // So that resumed sandboxes retain the same pause capability + resumeTemplateData["pausePolicy"] = map[string]interface{}{ + "snapshotType": bs.Spec.PausePolicy.SnapshotType, + "snapshotRegistry": bs.Spec.PausePolicy.SnapshotRegistry, + "snapshotPushSecretName": bs.Spec.PausePolicy.SnapshotPushSecret, + "resumeImagePullSecretName": bs.Spec.PausePolicy.ResumeImagePullSecret, + } + } + + // Convert the entire resume template to RawExtension + resumeTemplateRaw, err := convertToRawExtension(resumeTemplateData) + if err != nil { + return fmt.Errorf("failed to convert resume template to raw extension: %w", err) + } + snapshot.Spec.ResumeTemplate = &resumeTemplateRaw + + // Resolve snapshot registry + registry := snapshot.Spec.SnapshotRegistry + if registry == "" { + return fmt.Errorf("snapshotRegistry not resolved in pausePolicy") + } + + // Build ContainerSnapshots from the template containers + containerSnapshots := make([]sandboxv1alpha1.ContainerSnapshot, 0, len(template.Spec.Containers)) + for _, c := range template.Spec.Containers { + // Include pauseVersion in image tag to distinguish between multiple pauses + imageURI := fmt.Sprintf("%s/%s-%s:snapshot-v%d", registry, snapshot.Spec.SandboxID, c.Name, snapshot.Spec.PauseVersion) + containerSnapshots = append(containerSnapshots, sandboxv1alpha1.ContainerSnapshot{ + ContainerName: c.Name, + ImageURI: imageURI, + }) + } + + if len(containerSnapshots) == 0 { + return fmt.Errorf("no containers found in template for BatchSandbox %s", bs.Name) + } + + // Update the snapshot spec with resolved fields + snapshot.Spec.ContainerSnapshots = containerSnapshots + + if err := r.Update(ctx, snapshot); err != nil { + return fmt.Errorf("failed to update snapshot with resolved fields: %w", err) + } + + log.Info("Resolved and updated snapshot fields", "count", len(containerSnapshots), "snapshot", snapshot.Name) + return nil +} + +// findPodForSandbox finds the running pod belonging to a BatchSandbox. +// It first tries to parse the alloc-status annotation, then falls back to label selector. +func (r *SandboxSnapshotReconciler) findPodForSandbox(ctx context.Context, bs *sandboxv1alpha1.BatchSandbox, namespace string) (*corev1.Pod, error) { + log := logf.FromContext(ctx) + + // Try alloc-status annotation first (pool-based allocation) + alloc, err := parseSandboxAllocation(bs) + if err == nil && len(alloc.Pods) > 0 { + // Get the first allocated pod + pod := &corev1.Pod{} + if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: alloc.Pods[0]}, pod); err == nil { + if pod.Status.Phase == corev1.PodRunning { + return pod, nil + } + log.Info("Allocated pod not running, trying others", "pod", pod.Name, "phase", pod.Status.Phase) + } + // Try other pods in the allocation + for _, podName := range alloc.Pods[1:] { + p := &corev1.Pod{} + if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: podName}, p); err == nil { + if p.Status.Phase == corev1.PodRunning { + return p, nil + } + } + } + } + + // Fallback: list pods owned by this BatchSandbox + podList := &corev1.PodList{} + if err := r.List(ctx, podList, + client.InNamespace(namespace), + client.MatchingLabels{LabelBatchSandboxPodIndexKey: "0"}, + ); err != nil { + return nil, fmt.Errorf("failed to list pods: %w", err) + } + + // Filter pods owned by this BatchSandbox + for i := range podList.Items { + pod := &podList.Items[i] + for _, owner := range pod.OwnerReferences { + if owner.Kind == "BatchSandbox" && owner.Name == bs.Name && pod.Status.Phase == corev1.PodRunning { + return pod, nil + } + } + } + + // Last resort: find by naming convention {batchSandboxName}-0 + podName := fmt.Sprintf("%s-0", bs.Name) + pod := &corev1.Pod{} + if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: podName}, pod); err == nil { + return pod, nil + } + + return nil, fmt.Errorf("no running pod found for BatchSandbox %s", bs.Name) +} + +// handlePending creates the commit Job after ensuring resolution of container snapshots +func (r *SandboxSnapshotReconciler) handlePending(ctx context.Context, snapshot *sandboxv1alpha1.SandboxSnapshot) (ctrl.Result, error) { + log := logf.FromContext(ctx) + + // Ensure container snapshots are resolved before creating the commit job + if err := r.ensureResolved(ctx, snapshot); err != nil { + log.Error(err, "Failed to resolve container snapshots") + if updateErr := r.updateSnapshotStatus(ctx, snapshot, sandboxv1alpha1.SandboxSnapshotPhaseFailed, err.Error()); updateErr != nil { + return ctrl.Result{}, updateErr + } + return ctrl.Result{}, nil + } + + // Build and create the commit Job + job, err := r.buildCommitJob(snapshot) + if err != nil { + log.Error(err, "Failed to build commit job") + if updateErr := r.updateSnapshotStatus(ctx, snapshot, sandboxv1alpha1.SandboxSnapshotPhaseFailed, err.Error()); updateErr != nil { + return ctrl.Result{}, updateErr + } + return ctrl.Result{}, nil + } + + // Check if job already exists + existingJob := &batchv1.Job{} + err = r.Get(ctx, types.NamespacedName{Namespace: job.Namespace, Name: job.Name}, existingJob) + if err == nil { + // Job already exists, update phase to Committing + log.Info("Commit job already exists", "job", job.Name) + if updateErr := r.updateSnapshotStatus(ctx, snapshot, sandboxv1alpha1.SandboxSnapshotPhaseCommitting, "Commit job created"); updateErr != nil { + return ctrl.Result{}, updateErr + } + return ctrl.Result{RequeueAfter: time.Second}, nil + } + + if !errors.IsNotFound(err) { + log.Error(err, "Failed to check existing job") + return ctrl.Result{}, err + } + + // Create the job + if err := r.Create(ctx, job); err != nil { + log.Error(err, "Failed to create commit job") + r.Recorder.Eventf(snapshot, corev1.EventTypeWarning, "FailedCreateJob", "Failed to create commit job: %v", err) + return ctrl.Result{}, err + } + + log.Info("Created commit job", "job", job.Name) + r.Recorder.Eventf(snapshot, corev1.EventTypeNormal, "CreatedJob", "Created commit job: %s", job.Name) + + // Update phase to Committing + if err := r.updateSnapshotStatus(ctx, snapshot, sandboxv1alpha1.SandboxSnapshotPhaseCommitting, "Commit job created"); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{RequeueAfter: time.Second}, nil +} + +// handleCommitting checks the commit Job status +func (r *SandboxSnapshotReconciler) handleCommitting(ctx context.Context, snapshot *sandboxv1alpha1.SandboxSnapshot) (ctrl.Result, error) { + log := logf.FromContext(ctx) + + jobName := r.getJobName(snapshot) + job := &batchv1.Job{} + err := r.Get(ctx, types.NamespacedName{Namespace: snapshot.Namespace, Name: jobName}, job) + if err != nil { + if errors.IsNotFound(err) { + log.Info("Commit job not found, re-creating", "job", jobName) + return r.handlePending(ctx, snapshot) + } + log.Error(err, "Failed to get commit job") + return ctrl.Result{}, err + } + + // Check job status + if job.Status.Succeeded > 0 { + log.Info("Commit job succeeded", "job", jobName) + r.Recorder.Eventf(snapshot, corev1.EventTypeNormal, "JobSucceeded", "Commit job succeeded") + + // Populate status.ContainerSnapshots from spec.ContainerSnapshots + statusSnapshots := make([]sandboxv1alpha1.ContainerSnapshot, len(snapshot.Spec.ContainerSnapshots)) + copy(statusSnapshots, snapshot.Spec.ContainerSnapshots) + + // Transition to Ready and append pause history record + now := metav1.Now() + pauseRecord := sandboxv1alpha1.SnapshotRecord{ + Action: "Pause", + Version: snapshot.Spec.PauseVersion, + Timestamp: now, + Message: "Snapshot is ready", + } + if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + latestSnapshot := &sandboxv1alpha1.SandboxSnapshot{} + if err := r.Get(ctx, types.NamespacedName{Namespace: snapshot.Namespace, Name: snapshot.Name}, latestSnapshot); err != nil { + return err + } + latestSnapshot.Status.Phase = sandboxv1alpha1.SandboxSnapshotPhaseReady + latestSnapshot.Status.Message = "Snapshot is ready" + latestSnapshot.Status.ReadyAt = &now + latestSnapshot.Status.ContainerSnapshots = statusSnapshots + latestSnapshot.Status.PauseVersion = snapshot.Spec.PauseVersion + latestSnapshot.Status.History = append(latestSnapshot.Status.History, pauseRecord) + return r.Status().Update(ctx, latestSnapshot) + }); err != nil { + log.Error(err, "Failed to update snapshot status to Ready") + return ctrl.Result{}, err + } + + log.Info("Snapshot is ready", "snapshot", snapshot.Name) + r.Recorder.Eventf(snapshot, corev1.EventTypeNormal, "SnapshotReady", "Snapshot %s is ready", snapshot.Name) + + // Requeue to trigger handleReady for source BatchSandbox cleanup + return ctrl.Result{RequeueAfter: time.Second}, nil + } + + if job.Status.Failed > 0 { + log.Info("Commit job failed", "job", jobName) + r.Recorder.Eventf(snapshot, corev1.EventTypeWarning, "JobFailed", "Commit job failed") + + // Get failure message from job conditions + message := "Commit job failed" + for _, condition := range job.Status.Conditions { + if condition.Type == batchv1.JobFailed { + message = condition.Message + break + } + } + + if err := r.updateSnapshotStatus(ctx, snapshot, sandboxv1alpha1.SandboxSnapshotPhaseFailed, message); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + + // Job still running, requeue + log.Info("Commit job still running", "job", jobName) + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil +} + +// handleReady handles a ready snapshot. +// It deletes the original (paused) BatchSandbox after the snapshot is Ready, +// If the BatchSandbox has already been resumed (marked with annotation +// sandbox.opensandbox.io/resumed-from-snapshot), deletion is skipped. +func (r *SandboxSnapshotReconciler) handleReady(ctx context.Context, snapshot *sandboxv1alpha1.SandboxSnapshot) (ctrl.Result, error) { + log := logf.FromContext(ctx) + + bsName := snapshot.Spec.SourceBatchSandboxName + if bsName == "" { + log.Info("No source BatchSandbox specified, nothing to clean up") + return ctrl.Result{}, nil + } + + // Check if the source BatchSandbox still exists + bs := &sandboxv1alpha1.BatchSandbox{} + err := r.Get(ctx, types.NamespacedName{ + Name: bsName, + Namespace: snapshot.Namespace, + }, bs) + if err != nil { + if errors.IsNotFound(err) { + log.Info("Source BatchSandbox already deleted", "batchSandbox", bsName) + return ctrl.Result{}, nil + } + log.Error(err, "Failed to get source BatchSandbox") + return ctrl.Result{}, err + } + + // Only delete the BatchSandbox if the last history record is a Pause action. + // If it was a Resume, the BatchSandbox was just created by the controller and + // should not be deleted. + if len(snapshot.Status.History) > 0 { + lastRecord := snapshot.Status.History[len(snapshot.Status.History)-1] + if lastRecord.Action != "Pause" { + log.Info("Last action was not Pause, skipping BatchSandbox cleanup", + "batchSandbox", bsName, "lastAction", lastRecord.Action) + return ctrl.Result{}, nil + } + } + + // Delete the original (paused) BatchSandbox + if err := r.Delete(ctx, bs, client.PropagationPolicy(metav1.DeletePropagationBackground)); err != nil { + if errors.IsNotFound(err) { + log.Info("BatchSandbox already gone", "batchSandbox", bsName) + return ctrl.Result{}, nil + } + log.Error(err, "Failed to delete source BatchSandbox", "batchSandbox", bsName) + return ctrl.Result{}, err + } + + log.Info("Deleted original (paused) BatchSandbox", "batchSandbox", bsName) + r.Recorder.Eventf(snapshot, corev1.EventTypeNormal, "CleanedUpBatchSandbox", + "Deleted paused BatchSandbox %s after snapshot Ready", bsName) + + return ctrl.Result{}, nil +} + +// handleFailed handles a failed snapshot +func (r *SandboxSnapshotReconciler) handleFailed(ctx context.Context, snapshot *sandboxv1alpha1.SandboxSnapshot) (ctrl.Result, error) { + // Snapshot failed, nothing to do + return ctrl.Result{}, nil +} + +// handleDeletion handles the deletion of a SandboxSnapshot +func (r *SandboxSnapshotReconciler) handleDeletion(ctx context.Context, snapshot *sandboxv1alpha1.SandboxSnapshot) (ctrl.Result, error) { + log := logf.FromContext(ctx) + + // Clean up the commit job if it exists + jobName := r.getJobName(snapshot) + job := &batchv1.Job{} + err := r.Get(ctx, types.NamespacedName{Namespace: snapshot.Namespace, Name: jobName}, job) + if err == nil { + // Delete the job + if deleteErr := r.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationBackground)); deleteErr != nil && !errors.IsNotFound(deleteErr) { + log.Error(deleteErr, "Failed to delete commit job") + return ctrl.Result{}, deleteErr + } + log.Info("Deleted commit job", "job", jobName) + } + + // Remove finalizer + if controllerutil.ContainsFinalizer(snapshot, SandboxSnapshotFinalizer) { + if err := utils.UpdateFinalizer(r.Client, snapshot, utils.RemoveFinalizerOpType, SandboxSnapshotFinalizer); err != nil { + log.Error(err, "Failed to remove finalizer") + return ctrl.Result{}, err + } + log.Info("Removed finalizer", "finalizer", SandboxSnapshotFinalizer) + } + + return ctrl.Result{}, nil +} + +// buildCommitJob builds a Job for committing container snapshots. +// It supports multi-container sandboxes by creating init containers for each +// container snapshot that needs to be committed, followed by a main verification container. +func (r *SandboxSnapshotReconciler) buildCommitJob(snapshot *sandboxv1alpha1.SandboxSnapshot) (*batchv1.Job, error) { + jobName := r.getJobName(snapshot) + + // Use image-committer image (contains ctr and crictl tools) + imageCommitterImage := r.ImageCommitterImage + if imageCommitterImage == "" { + imageCommitterImage = "image-committer:dev" // Default fallback + } + + // Build volume mounts for containerd and CRI sockets + volumeMounts := []corev1.VolumeMount{ + { + Name: "containerd-sock", + MountPath: ContainerdSocketPath, + }, + { + Name: "cri-sock", + MountPath: CrictlSocketPath, + }, + } + + // Build volumes for host paths + volumes := []corev1.Volume{ + { + Name: "containerd-sock", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: ContainerdSocketPath, + }, + }, + }, + { + Name: "cri-sock", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: CrictlSocketPath, + }, + }, + }, + } + + // Add registry credentials from secret if specified + if snapshot.Spec.SnapshotPushSecret != "" { + volumes = append(volumes, corev1.Volume{ + Name: "registry-creds", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: snapshot.Spec.SnapshotPushSecret, + Items: []corev1.KeyToPath{ + { + Key: ".dockerconfigjson", + Path: "config.json", + }, + }, + }, + }, + }) + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "registry-creds", + MountPath: "/var/run/opensandbox/registry", + ReadOnly: true, + }) + } + + // Build commit command using new multi-container format: + // image-committer [ ...] + containerSnapshots := snapshot.Spec.ContainerSnapshots + + if len(containerSnapshots) == 0 { + return nil, fmt.Errorf("no container snapshots specified in snapshot spec") + } + + var containerSpecs []string + for _, cs := range containerSnapshots { + spec := fmt.Sprintf("%s:%s", cs.ContainerName, cs.ImageURI) + containerSpecs = append(containerSpecs, spec) + } + fullCommand := fmt.Sprintf("/usr/local/bin/image-committer %s %s %s", + snapshot.Spec.SourcePodName, + snapshot.Namespace, + strings.Join(containerSpecs, " "), + ) + + // Build the job + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: jobName, + Namespace: snapshot.Namespace, + Labels: map[string]string{ + LabelSandboxSnapshotName: snapshot.Name, + }, + }, + Spec: batchv1.JobSpec{ + TTLSecondsAfterFinished: ptrToInt32(int32(DefaultTTLSecondsAfterFinished)), + ActiveDeadlineSeconds: ptrToInt64(int64(r.getCommitJobTimeout().Seconds())), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + Containers: []corev1.Container{ + { + Name: CommitJobContainerName, + Image: imageCommitterImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/bin/sh", "-c"}, + Args: []string{fullCommand}, + VolumeMounts: volumeMounts, + Env: []corev1.EnvVar{ + { + Name: "CONTAINERD_SOCKET", + Value: ContainerdSocketPath, + }, + { + Name: "CRI_RUNTIME_ENDPOINT", + Value: CrictlSocketPath, + }, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptrToInt64(0), // Run as root to access containerd + }, + }, + }, + Volumes: volumes, + NodeName: snapshot.Spec.SourceNodeName, + }, + }, + }, + } + + // Set owner reference + if err := ctrl.SetControllerReference(snapshot, job, r.Scheme); err != nil { + return nil, fmt.Errorf("failed to set controller reference: %w", err) + } + + return job, nil +} + +// getJobName returns the job name for a snapshot +func (r *SandboxSnapshotReconciler) getJobName(snapshot *sandboxv1alpha1.SandboxSnapshot) string { + return fmt.Sprintf("%s-commit-v%d", snapshot.Name, snapshot.Spec.PauseVersion) +} + +// updateSnapshotStatus updates the snapshot status +func (r *SandboxSnapshotReconciler) updateSnapshotStatus(ctx context.Context, snapshot *sandboxv1alpha1.SandboxSnapshot, phase sandboxv1alpha1.SandboxSnapshotPhase, message string) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + latestSnapshot := &sandboxv1alpha1.SandboxSnapshot{} + if err := r.Get(ctx, types.NamespacedName{Namespace: snapshot.Namespace, Name: snapshot.Name}, latestSnapshot); err != nil { + return err + } + + latestSnapshot.Status.Phase = phase + latestSnapshot.Status.Message = message + + return r.Status().Update(ctx, latestSnapshot) + }) +} + +// getCommitJobTimeout returns the configured timeout or the default +func (r *SandboxSnapshotReconciler) getCommitJobTimeout() time.Duration { + if r.CommitJobTimeout > 0 { + return r.CommitJobTimeout + } + return DefaultCommitJobTimeout +} + +// ptrToInt64 returns a pointer to an int64 +func ptrToInt64(v int64) *int64 { + return &v +} +func ptrToInt32(v int32) *int32 { + return &v +} + +// handleResume creates a new BatchSandbox from the snapshot resumeTemplate. +// It ACKs resumeVersion and appends a resume history record. +func (r *SandboxSnapshotReconciler) handleResume(ctx context.Context, snapshot *sandboxv1alpha1.SandboxSnapshot) (ctrl.Result, error) { + log := logf.FromContext(ctx) + log.Info("Handling resume request", "snapshot", snapshot.Name, "resumeVersion", snapshot.Spec.ResumeVersion) + + // Validate prerequisites + if snapshot.Spec.ResumeTemplate == nil || snapshot.Spec.ResumeTemplate.Raw == nil { + log.Error(fmt.Errorf("resumeTemplate is empty"), "Cannot resume without resumeTemplate") + return ctrl.Result{}, nil + } + + if len(snapshot.Status.ContainerSnapshots) == 0 { + log.Error(fmt.Errorf("no containerSnapshots in status"), "Cannot resume without container snapshot images") + return ctrl.Result{}, nil + } + + // Parse resumeTemplate + var resumeTemplate map[string]interface{} + if err := json.Unmarshal(snapshot.Spec.ResumeTemplate.Raw, &resumeTemplate); err != nil { + log.Error(err, "Failed to parse resumeTemplate") + return ctrl.Result{}, nil + } + + template, ok := resumeTemplate["template"].(map[string]interface{}) + if !ok { + log.Error(fmt.Errorf("template not found in resumeTemplate"), "Invalid resumeTemplate format") + return ctrl.Result{}, nil + } + + // Replace container images from status.ContainerSnapshots + podSpec, ok := template["spec"].(map[string]interface{}) + if !ok { + log.Error(fmt.Errorf("spec not found in template"), "Invalid template format") + return ctrl.Result{}, nil + } + containers, ok := podSpec["containers"].([]interface{}) + if !ok { + log.Error(fmt.Errorf("containers not found in template spec"), "Invalid template format") + return ctrl.Result{}, nil + } + for _, cs := range snapshot.Status.ContainerSnapshots { + for i, c := range containers { + container, ok := c.(map[string]interface{}) + if !ok { + continue + } + if container["name"] == cs.ContainerName { + container["image"] = cs.ImageURI + containers[i] = container + break + } + } + } + + // Add imagePullSecrets from spec + if snapshot.Spec.ResumeImagePullSecret != "" { + podSpec["imagePullSecrets"] = []interface{}{ + map[string]interface{}{"name": snapshot.Spec.ResumeImagePullSecret}, + } + } + + // Build BatchSandbox manifest + bsSpec := map[string]interface{}{ + "replicas": 1, + "template": template, + } + + // Add expireTime from resumeTemplate if present + if expireTime, ok := resumeTemplate["expireTime"]; ok && expireTime != nil { + bsSpec["expireTime"] = expireTime + } + + // Add pausePolicy from resumeTemplate if present + if pausePolicy, ok := resumeTemplate["pausePolicy"]; ok && pausePolicy != nil { + bsSpec["pausePolicy"] = pausePolicy + } + + batchsandboxManifest := map[string]interface{}{ + "apiVersion": fmt.Sprintf("%s/%s", sandboxv1alpha1.GroupVersion.Group, sandboxv1alpha1.GroupVersion.Version), + "kind": "BatchSandbox", + "metadata": map[string]interface{}{ + "name": snapshot.Spec.SandboxID, + "namespace": snapshot.Namespace, + "labels": map[string]interface{}{ + "sandbox.opensandbox.io/sandbox-id": snapshot.Spec.SandboxID, + "sandbox.opensandbox.io/resumed-from-snapshot": "true", + }, + "annotations": map[string]interface{}{ + "sandbox.opensandbox.io/resumed-from-snapshot": "true", + }, + }, + "spec": bsSpec, + } + + // Create BatchSandbox using unstructured + bsJSON, err := json.Marshal(batchsandboxManifest) + if err != nil { + log.Error(err, "Failed to marshal BatchSandbox manifest") + return ctrl.Result{}, err + } + + unstructuredBS := &unstructured.Unstructured{} + if err := unstructuredBS.UnmarshalJSON(bsJSON); err != nil { + log.Error(err, "Failed to decode BatchSandbox manifest") + return ctrl.Result{}, err + } + + if err := r.Create(ctx, unstructuredBS); err != nil { + if errors.IsAlreadyExists(err) { + log.Info("BatchSandbox already exists, resume may have been processed", "name", snapshot.Spec.SandboxID) + } else { + log.Error(err, "Failed to create BatchSandbox") + return ctrl.Result{}, err + } + } + + log.Info("Created BatchSandbox from snapshot", "name", snapshot.Spec.SandboxID) + r.Recorder.Eventf(snapshot, corev1.EventTypeNormal, "ResumedBatchSandbox", + "Created BatchSandbox %s from snapshot", snapshot.Spec.SandboxID) + + // ACK resumeVersion and append resume history record + now := metav1.Now() + resumeRecord := sandboxv1alpha1.SnapshotRecord{ + Action: "Resume", + Version: snapshot.Spec.ResumeVersion, + Timestamp: now, + Message: fmt.Sprintf("Resumed to BatchSandbox %s", snapshot.Spec.SandboxID), + } + if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + latestSnapshot := &sandboxv1alpha1.SandboxSnapshot{} + if err := r.Get(ctx, types.NamespacedName{Namespace: snapshot.Namespace, Name: snapshot.Name}, latestSnapshot); err != nil { + return err + } + latestSnapshot.Status.ResumeVersion = snapshot.Spec.ResumeVersion + latestSnapshot.Status.History = append(latestSnapshot.Status.History, resumeRecord) + return r.Status().Update(ctx, latestSnapshot) + }); err != nil { + log.Error(err, "Failed to ACK resumeVersion") + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil +} + +// convertPodTemplateSpecToMap converts a PodTemplateSpec to a map[string]interface{} +func convertPodTemplateSpecToMap(template *corev1.PodTemplateSpec) map[string]interface{} { + if template == nil { + return nil + } + + result := make(map[string]interface{}) + + // Convert ObjectMeta + if !template.ObjectMeta.CreationTimestamp.IsZero() || len(template.ObjectMeta.Labels) > 0 || len(template.ObjectMeta.Annotations) > 0 { + meta := make(map[string]interface{}) + if len(template.ObjectMeta.Labels) > 0 { + meta["labels"] = template.ObjectMeta.Labels + } + if len(template.ObjectMeta.Annotations) > 0 { + meta["annotations"] = template.ObjectMeta.Annotations + } + result["metadata"] = meta + } + + // Convert PodSpec + podSpecBytes, _ := json.Marshal(template.Spec) + var podSpecMap map[string]interface{} + _ = json.Unmarshal(podSpecBytes, &podSpecMap) + if podSpecMap != nil { + result["spec"] = podSpecMap + } + + return result +} + +// convertToRawExtension converts a struct to RawExtension +func convertToRawExtension(data interface{}) (runtime.RawExtension, error) { + jsonBytes, err := json.Marshal(data) + if err != nil { + return runtime.RawExtension{}, err + } + + return runtime.RawExtension{ + Raw: jsonBytes, + }, nil +} + +// SetupWithManager sets up the controller with the Manager +func (r *SandboxSnapshotReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&sandboxv1alpha1.SandboxSnapshot{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). + Owns(&batchv1.Job{}). + Named("sandboxsnapshot"). + Complete(r) +} + +// Add the JSON import for marshaling/unmarshaling diff --git a/kubernetes/test/e2e/e2e_suite_test.go b/kubernetes/test/e2e/e2e_suite_test.go index 58998f053..7cc3b1ba6 100644 --- a/kubernetes/test/e2e/e2e_suite_test.go +++ b/kubernetes/test/e2e/e2e_suite_test.go @@ -56,6 +56,15 @@ var _ = BeforeSuite(func() { _, err = utils.Run(cmd) ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the task-executor image") + By("building the image-committer image") + makeArgs = []string{"docker-build-image-committer", fmt.Sprintf("IMAGE_COMMITTER_IMG=%s", utils.ImageCommitterImage)} + if dockerBuildArgs != "" { + makeArgs = append(makeArgs, fmt.Sprintf("DOCKER_BUILD_ARGS=%s", dockerBuildArgs)) + } + cmd = exec.Command("make", makeArgs...) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the image-committer image") + // If you want to change the e2e test vendor from Kind, ensure the image is // built and available before running the tests. Also, remove the following block. By("loading the manager(Operator) image on Kind") @@ -65,6 +74,20 @@ var _ = BeforeSuite(func() { By("loading the task-executor image on Kind") err = utils.LoadImageToKindClusterWithName(utils.TaskExecutorImage) ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the task-executor image into Kind") + + By("loading the image-committer image on Kind") + err = utils.LoadImageToKindClusterWithName(utils.ImageCommitterImage) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the image-committer image into Kind") + + // TODO docker pull + By("loading the registry:2 image on Kind (required for pause/resume tests)") + err = utils.LoadImageToKindClusterWithName("registry:2") + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the registry:2 image into Kind") + + //TODO ensure this + By("loading the alpine image on Kind (required for commit jobs)") + err = utils.LoadImageToKindClusterWithName("alpine:latest") + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the alpine image into Kind") }) var _ = AfterSuite(func() { diff --git a/kubernetes/test/e2e/pause_resume_test.go b/kubernetes/test/e2e/pause_resume_test.go new file mode 100644 index 000000000..19e424dec --- /dev/null +++ b/kubernetes/test/e2e/pause_resume_test.go @@ -0,0 +1,723 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/alibaba/OpenSandbox/sandbox-k8s/test/utils" +) + +const ( + pauseResumeNamespace = "default" + registryServiceAddr = "docker-registry.default.svc.cluster.local:5000" + registryUsername = "testuser" + registryPassword = "testpass" +) + +var _ = Describe("PauseResume", Ordered, func() { + SetDefaultEventuallyTimeout(3 * time.Minute) + SetDefaultEventuallyPollingInterval(time.Second) + + BeforeAll(func() { + By("creating manager namespace") + cmd := exec.Command("kubectl", "create", "ns", namespace) + _, err := utils.Run(cmd) + if err != nil { + Expect(err.Error()).To(ContainSubstring("AlreadyExists")) + } + + By("labeling the namespace to enforce the restricted security policy") + cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace, + "pod-security.kubernetes.io/enforce=restricted") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to label namespace with restricted policy") + + By("installing CRDs") + cmd = exec.Command("kubectl", "apply", "-f", "config/crd/bases") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs") + + By("deploying the controller-manager") + cmd = exec.Command("kubectl", "apply", "-k", "config/default") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") + + By("waiting for controller to be ready") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager", + "-n", namespace, "-o", "jsonpath={.items[0].status.phase}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Running")) + }, 2*time.Minute).Should(Succeed()) + + By("creating registry authentication secrets") + err = createHtpasswdSecret(pauseResumeNamespace) + Expect(err).NotTo(HaveOccurred()) + + err = createDockerRegistrySecrets(pauseResumeNamespace) + Expect(err).NotTo(HaveOccurred()) + + By("deploying Docker Registry") + registryYAML, err := renderTemplate("testdata/registry-deployment.yaml", nil) + Expect(err).NotTo(HaveOccurred()) + + registryFile := filepath.Join("/tmp", "test-registry.yaml") + err = os.WriteFile(registryFile, []byte(registryYAML), 0644) + Expect(err).NotTo(HaveOccurred()) + defer os.Remove(registryFile) + + cmd = exec.Command("kubectl", "apply", "-f", registryFile) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("waiting for registry to be ready") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "deployment", "docker-registry", + "-n", pauseResumeNamespace, "-o", "jsonpath={.status.availableReplicas}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("1")) + }, 2*time.Minute).Should(Succeed()) + }) + + AfterAll(func() { + By("cleaning up Docker Registry") + cmd := exec.Command("kubectl", "delete", "deployment", "docker-registry", "-n", pauseResumeNamespace, "--ignore-not-found=true") + utils.Run(cmd) + cmd = exec.Command("kubectl", "delete", "service", "docker-registry", "-n", pauseResumeNamespace, "--ignore-not-found=true") + utils.Run(cmd) + + By("cleaning up secrets") + for _, secret := range []string{"registry-auth", "registry-push-secret", "registry-pull-secret"} { + cmd = exec.Command("kubectl", "delete", "secret", secret, "-n", pauseResumeNamespace, "--ignore-not-found=true") + utils.Run(cmd) + } + + By("cleaning up any remaining sandboxsnapshots") + cmd = exec.Command("kubectl", "delete", "sandboxsnapshots", "--all", "-n", pauseResumeNamespace, "--ignore-not-found=true") + utils.Run(cmd) + + By("cleaning up any remaining batchsandboxes") + cmd = exec.Command("kubectl", "delete", "batchsandboxes", "--all", "-n", pauseResumeNamespace, "--ignore-not-found=true") + utils.Run(cmd) + + By("undeploying the controller-manager") + cmd = exec.Command("kubectl", "delete", "-k", "config/default", "--ignore-not-found=true") + utils.Run(cmd) + + By("uninstalling CRDs") + cmd = exec.Command("kubectl", "delete", "-f", "config/crd/bases", "--ignore-not-found=true") + utils.Run(cmd) + + By("removing manager namespace") + cmd = exec.Command("kubectl", "delete", "ns", namespace, "--ignore-not-found=true") + utils.Run(cmd) + }) + + Context("Pause and Resume", func() { + It("should complete the full pause-resume flow end-to-end", func() { + const sandboxName = "test-pause-resume" + const snapshotName = "test-pause-resume" + + // --- Step 1: Create BatchSandbox --- + By("creating BatchSandbox with pausePolicy") + bsYAML, err := renderTemplate("testdata/batchsandbox-with-pause-policy.yaml", map[string]interface{}{ + "BatchSandboxName": sandboxName, + "Namespace": pauseResumeNamespace, + "SandboxImage": utils.SandboxImage, + "SnapshotRegistry": registryServiceAddr, + "SnapshotPushSecretName": "registry-push-secret", + "ResumeImagePullSecretName": "registry-pull-secret", + }) + Expect(err).NotTo(HaveOccurred()) + + bsFile := filepath.Join("/tmp", "test-pause-resume-bs.yaml") + err = os.WriteFile(bsFile, []byte(bsYAML), 0644) + Expect(err).NotTo(HaveOccurred()) + defer os.Remove(bsFile) + + cmd := exec.Command("kubectl", "apply", "-f", bsFile) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("waiting for BatchSandbox to be Running") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "batchsandbox", sandboxName, + "-n", pauseResumeNamespace, "-o", "jsonpath={.status.ready}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("1")) + }, 2*time.Minute).Should(Succeed()) + + // --- Step 2: Get pod/node info --- + By("getting pod and node info from BatchSandbox") + cmd = exec.Command("kubectl", "get", "pods", "-n", pauseResumeNamespace, "-o", "json") + podsJSON, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + var podList struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + OwnerReferences []struct { + Kind string `json:"kind"` + Name string `json:"name"` + } `json:"ownerReferences"` + } `json:"metadata"` + Spec struct { + NodeName string `json:"nodeName"` + } `json:"spec"` + } `json:"items"` + } + err = json.Unmarshal([]byte(podsJSON), &podList) + Expect(err).NotTo(HaveOccurred()) + + var podName, nodeName string + for _, pod := range podList.Items { + for _, owner := range pod.Metadata.OwnerReferences { + if owner.Kind == "BatchSandbox" && owner.Name == sandboxName { + podName = pod.Metadata.Name + nodeName = pod.Spec.NodeName + break + } + } + if podName != "" { + break + } + } + Expect(podName).NotTo(BeEmpty(), "Should find a pod owned by BatchSandbox") + + // --- Step 2.5: Write marker file for rootfs verification --- + markerValue := fmt.Sprintf("pause-test-%d", time.Now().UnixNano()) + By("writing marker file into container for rootfs verification") + cmd = exec.Command("kubectl", "exec", podName, "-n", pauseResumeNamespace, + "-c", "sandbox", "--", "sh", "-c", fmt.Sprintf("echo '%s' > /tmp/pause-marker", markerValue)) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + // --- Step 3: Create SandboxSnapshot --- + By("creating SandboxSnapshot CR") + pausedAt := time.Now().UTC().Format(time.RFC3339) + snapshotYAML, err := renderTemplate("testdata/sandboxsnapshot.yaml", map[string]interface{}{ + "SnapshotName": snapshotName, + "Namespace": pauseResumeNamespace, + "SandboxId": sandboxName, + "SourceBatchSandboxName": sandboxName, + "SourcePodName": podName, + "SourceNodeName": nodeName, + "SnapshotRegistry": registryServiceAddr, + "ImageUri": fmt.Sprintf("%s/%s:snapshot", registryServiceAddr, sandboxName), + "SnapshotPushSecretName": "registry-push-secret", + "ResumeImagePullSecretName": "registry-pull-secret", + "SandboxImage": utils.SandboxImage, + "PausedAt": pausedAt, + }) + Expect(err).NotTo(HaveOccurred()) + + snapshotFile := filepath.Join("/tmp", "test-pause-resume-snapshot.yaml") + err = os.WriteFile(snapshotFile, []byte(snapshotYAML), 0644) + Expect(err).NotTo(HaveOccurred()) + defer os.Remove(snapshotFile) + + cmd = exec.Command("kubectl", "apply", "-f", snapshotFile) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + // --- Step 4: Wait for snapshot Ready --- + By("waiting for SandboxSnapshot to be Ready") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "sandboxsnapshot", snapshotName, + "-n", pauseResumeNamespace, "-o", "jsonpath={.status.phase}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Ready")) + }, 3*time.Minute).Should(Succeed()) + + // --- Step 5: Verify commit Job succeeded --- + By("verifying commit Job completed successfully") + cmd = exec.Command("kubectl", "get", "job", fmt.Sprintf("%s-commit-v1", snapshotName), + "-n", pauseResumeNamespace, "-o", "jsonpath={.status.succeeded}") + output, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + Expect(output).To(Equal("1")) + + // --- Step 6: Verify status.containerSnapshots populated --- + By("verifying snapshot status has containerSnapshots with imageUri") + cmd = exec.Command("kubectl", "get", "sandboxsnapshot", snapshotName, + "-n", pauseResumeNamespace, "-o", "jsonpath={.status.containerSnapshots[0].imageUri}") + output, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + Expect(output).NotTo(BeEmpty(), "Snapshot status should contain containerSnapshots with imageUri") + + // --- Step 7: Verify source BatchSandbox was auto-deleted by handleReady --- + By("verifying source BatchSandbox was auto-deleted after snapshot Ready") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "batchsandbox", sandboxName, "-n", pauseResumeNamespace) + output, err := utils.Run(cmd) + g.Expect(output).To(ContainSubstring("NotFound")) + g.Expect(err).To(HaveOccurred()) + }, 30*time.Second).Should(Succeed()) + + // --- Step 8: Resume - patch Snapshot CR to trigger controller resume --- + By("patching SandboxSnapshot resumeVersion to trigger resume") + cmd = exec.Command("kubectl", "patch", "sandboxsnapshot", snapshotName, + "-n", pauseResumeNamespace, "--type=merge", + "-p", `{"spec":{"resumeVersion":1}}`) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("waiting for controller to ACK resumeVersion") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "sandboxsnapshot", snapshotName, + "-n", pauseResumeNamespace, "-o", "jsonpath={.status.resumeVersion}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("1")) + }, 30*time.Second).Should(Succeed()) + + By("waiting for resumed BatchSandbox to be Running") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "batchsandbox", sandboxName, + "-n", pauseResumeNamespace, "-o", "jsonpath={.status.ready}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("1")) + }, 2*time.Minute).Should(Succeed()) + + // --- Step 8.5: Verify rootfs data persistence --- + By("getting resumed pod name") + cmd = exec.Command("kubectl", "get", "pods", "-n", pauseResumeNamespace, "-o", "json") + resumedPodsJSON, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + var resumedPodList struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + OwnerReferences []struct { + Kind string `json:"kind"` + Name string `json:"name"` + } `json:"ownerReferences"` + } `json:"metadata"` + } `json:"items"` + } + err = json.Unmarshal([]byte(resumedPodsJSON), &resumedPodList) + Expect(err).NotTo(HaveOccurred()) + + var resumedPodName string + for _, pod := range resumedPodList.Items { + for _, owner := range pod.Metadata.OwnerReferences { + if owner.Kind == "BatchSandbox" && owner.Name == sandboxName { + resumedPodName = pod.Metadata.Name + break + } + } + if resumedPodName != "" { + break + } + } + Expect(resumedPodName).NotTo(BeEmpty(), "Should find a pod owned by resumed BatchSandbox") + + By("reading marker file from resumed container to verify rootfs persistence") + cmd = exec.Command("kubectl", "exec", resumedPodName, "-n", pauseResumeNamespace, + "-c", "sandbox", "--", "cat", "/tmp/pause-marker") + output, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + Expect(strings.TrimSpace(output)).To(Equal(markerValue), + "Rootfs data should persist across pause/resume") + + By("verifying resumed-from-snapshot annotation on BatchSandbox") + cmd = exec.Command("kubectl", "get", "batchsandbox", sandboxName, + "-n", pauseResumeNamespace, "-o", "jsonpath={.metadata.annotations.sandbox\\.opensandbox\\.io/resumed-from-snapshot}") + output, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + Expect(output).To(Equal("true")) + + By("verifying snapshot history has Pause and Resume records") + cmd = exec.Command("kubectl", "get", "sandboxsnapshot", snapshotName, + "-n", pauseResumeNamespace, "-o", "jsonpath={.status.history[*].action}") + output, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + Expect(output).To(ContainSubstring("Pause")) + Expect(output).To(ContainSubstring("Resume")) + + // --- Cleanup --- + By("cleaning up") + cmd = exec.Command("kubectl", "delete", "batchsandbox", sandboxName, "-n", pauseResumeNamespace) + utils.Run(cmd) + cmd = exec.Command("kubectl", "delete", "sandboxsnapshot", snapshotName, "-n", pauseResumeNamespace) + utils.Run(cmd) + }) + + It("should complete pool-based pause-resume with rootfs verification", func() { + const poolName = "test-pool-pause" + const sandboxName = "test-pool-pause-resume" + const snapshotName = "test-pool-pause-snap" + + // --- Step 1: Create Pool CR --- + By("creating Pool CR") + poolYAML, err := renderTemplate("testdata/pool-with-pause-policy.yaml", map[string]interface{}{ + "PoolName": poolName, + "Namespace": pauseResumeNamespace, + "SandboxImage": utils.SandboxImage, + "BufferMax": 2, + "BufferMin": 1, + "PoolMax": 5, + "PoolMin": 1, + }) + Expect(err).NotTo(HaveOccurred()) + + poolFile := filepath.Join("/tmp", "test-pool-pause.yaml") + err = os.WriteFile(poolFile, []byte(poolYAML), 0644) + Expect(err).NotTo(HaveOccurred()) + defer os.Remove(poolFile) + + cmd := exec.Command("kubectl", "apply", "-f", poolFile) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("waiting for Pool to have available pods") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pool", poolName, + "-n", pauseResumeNamespace, "-o", "jsonpath={.status.available}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).NotTo(BeEmpty()) + g.Expect(output).NotTo(Equal("0")) + }, 2*time.Minute).Should(Succeed()) + + // --- Step 2: Create BatchSandbox with poolRef + pausePolicy --- + By("creating BatchSandbox with poolRef and pausePolicy") + bsYAML, err := renderTemplate("testdata/batchsandbox-pooled-pause.yaml", map[string]interface{}{ + "BatchSandboxName": sandboxName, + "Namespace": pauseResumeNamespace, + "PoolName": poolName, + "SnapshotRegistry": registryServiceAddr, + "SnapshotPushSecretName": "registry-push-secret", + "ResumeImagePullSecretName": "registry-pull-secret", + }) + Expect(err).NotTo(HaveOccurred()) + + bsFile := filepath.Join("/tmp", "test-pool-pause-bs.yaml") + err = os.WriteFile(bsFile, []byte(bsYAML), 0644) + Expect(err).NotTo(HaveOccurred()) + defer os.Remove(bsFile) + + cmd = exec.Command("kubectl", "apply", "-f", bsFile) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("waiting for BatchSandbox to be Running") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "batchsandbox", sandboxName, + "-n", pauseResumeNamespace, "-o", "jsonpath={.status.ready}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("1")) + }, 2*time.Minute).Should(Succeed()) + + // --- Step 3: Get pod name from alloc-status --- + By("getting pod name from alloc-status annotation") + var podName string + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "batchsandbox", sandboxName, + "-n", pauseResumeNamespace, + "-o", "jsonpath={.metadata.annotations.sandbox\\.opensandbox\\.io/alloc-status}") + allocStatusJSON, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(allocStatusJSON).NotTo(BeEmpty(), "alloc-status annotation should exist") + + var allocStatus struct { + Pods []string `json:"pods"` + } + err = json.Unmarshal([]byte(allocStatusJSON), &allocStatus) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(allocStatus.Pods)).To(BeNumerically(">=", 1)) + podName = allocStatus.Pods[0] + }).Should(Succeed()) + Expect(podName).NotTo(BeEmpty(), "Should have allocated pod name") + + // --- Step 4: Write marker file --- + markerValue := fmt.Sprintf("pool-pause-test-%d", time.Now().UnixNano()) + By("writing marker file into container for rootfs verification") + cmd = exec.Command("kubectl", "exec", podName, "-n", pauseResumeNamespace, + "-c", "sandbox", "--", "sh", "-c", fmt.Sprintf("echo '%s' > /tmp/pause-marker", markerValue)) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + // --- Step 5: Create minimal SandboxSnapshot (controller resolves via poolRef) --- + By("creating minimal SandboxSnapshot CR (controller resolves template from Pool CR)") + pausedAt := time.Now().UTC().Format(time.RFC3339) + snapshotYAML, err := renderTemplate("testdata/sandboxsnapshot-minimal.yaml", map[string]interface{}{ + "SnapshotName": snapshotName, + "Namespace": pauseResumeNamespace, + "SandboxId": sandboxName, + "SourceBatchSandboxName": sandboxName, + "PausedAt": pausedAt, + }) + Expect(err).NotTo(HaveOccurred()) + + snapshotFile := filepath.Join("/tmp", "test-pool-pause-snapshot.yaml") + err = os.WriteFile(snapshotFile, []byte(snapshotYAML), 0644) + Expect(err).NotTo(HaveOccurred()) + defer os.Remove(snapshotFile) + + cmd = exec.Command("kubectl", "apply", "-f", snapshotFile) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + // --- Step 6: Wait for snapshot Ready --- + By("waiting for SandboxSnapshot to be Ready") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "sandboxsnapshot", snapshotName, + "-n", pauseResumeNamespace, "-o", "jsonpath={.status.phase}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Ready")) + }, 3*time.Minute).Should(Succeed()) + + // --- Step 7: Verify source BatchSandbox was auto-deleted --- + By("verifying source BatchSandbox was auto-deleted after snapshot Ready") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "batchsandbox", sandboxName, "-n", pauseResumeNamespace) + output, err := utils.Run(cmd) + g.Expect(output).To(ContainSubstring("NotFound")) + g.Expect(err).To(HaveOccurred()) + }, 30*time.Second).Should(Succeed()) + + // --- Step 8: Resume --- + By("patching SandboxSnapshot resumeVersion to trigger resume") + cmd = exec.Command("kubectl", "patch", "sandboxsnapshot", snapshotName, + "-n", pauseResumeNamespace, "--type=merge", + "-p", `{"spec":{"resumeVersion":1}}`) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("waiting for controller to ACK resumeVersion") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "sandboxsnapshot", snapshotName, + "-n", pauseResumeNamespace, "-o", "jsonpath={.status.resumeVersion}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("1")) + }, 30*time.Second).Should(Succeed()) + + By("waiting for resumed BatchSandbox to be Running") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "batchsandbox", sandboxName, + "-n", pauseResumeNamespace, "-o", "jsonpath={.status.ready}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("1")) + }, 2*time.Minute).Should(Succeed()) + + // --- Step 9: Verify rootfs data persistence --- + By("getting resumed pod name") + cmd = exec.Command("kubectl", "get", "pods", "-n", pauseResumeNamespace, "-o", "json") + resumedPodsJSON, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + var resumedPodList struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + OwnerReferences []struct { + Kind string `json:"kind"` + Name string `json:"name"` + } `json:"ownerReferences"` + } `json:"metadata"` + } `json:"items"` + } + err = json.Unmarshal([]byte(resumedPodsJSON), &resumedPodList) + Expect(err).NotTo(HaveOccurred()) + + var resumedPodName string + for _, pod := range resumedPodList.Items { + for _, owner := range pod.Metadata.OwnerReferences { + if owner.Kind == "BatchSandbox" && owner.Name == sandboxName { + resumedPodName = pod.Metadata.Name + break + } + } + if resumedPodName != "" { + break + } + } + Expect(resumedPodName).NotTo(BeEmpty(), "Should find a pod owned by resumed BatchSandbox") + + By("reading marker file from resumed container to verify rootfs persistence") + cmd = exec.Command("kubectl", "exec", resumedPodName, "-n", pauseResumeNamespace, + "-c", "sandbox", "--", "cat", "/tmp/pause-marker") + output, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + Expect(strings.TrimSpace(output)).To(Equal(markerValue), + "Rootfs data should persist across pause/resume") + + // --- Step 10: Verify history records --- + By("verifying snapshot history has Pause and Resume records") + cmd = exec.Command("kubectl", "get", "sandboxsnapshot", snapshotName, + "-n", pauseResumeNamespace, "-o", "jsonpath={.status.history[*].action}") + output, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + Expect(output).To(ContainSubstring("Pause")) + Expect(output).To(ContainSubstring("Resume")) + + // --- Cleanup --- + By("cleaning up") + cmd = exec.Command("kubectl", "delete", "batchsandbox", sandboxName, "-n", pauseResumeNamespace, "--ignore-not-found=true") + utils.Run(cmd) + cmd = exec.Command("kubectl", "delete", "sandboxsnapshot", snapshotName, "-n", pauseResumeNamespace, "--ignore-not-found=true") + utils.Run(cmd) + cmd = exec.Command("kubectl", "delete", "pool", poolName, "-n", pauseResumeNamespace, "--ignore-not-found=true") + utils.Run(cmd) + }) + }) + + Context("Failure", func() { + It("should transition to Failed when source Pod does not exist", func() { + const snapshotName = "test-pause-fail" + + By("creating SandboxSnapshot with non-existent source") + pausedAt := time.Now().UTC().Format(time.RFC3339) + snapshotYAML, err := renderTemplate("testdata/sandboxsnapshot.yaml", map[string]interface{}{ + "SnapshotName": snapshotName, + "Namespace": pauseResumeNamespace, + "SandboxId": "nonexistent-sandbox", + "SourceBatchSandboxName": "nonexistent-sandbox", + "SourcePodName": "nonexistent-pod", + "SourceNodeName": "nonexistent-node", + "SnapshotRegistry": registryServiceAddr, + "ImageUri": fmt.Sprintf("%s/nonexistent:snapshot", registryServiceAddr), + "SnapshotPushSecretName": "registry-push-secret", + "ResumeImagePullSecretName": "registry-pull-secret", + "SandboxImage": utils.SandboxImage, + "PausedAt": pausedAt, + }) + Expect(err).NotTo(HaveOccurred()) + + snapshotFile := filepath.Join("/tmp", "test-pause-fail-snapshot.yaml") + err = os.WriteFile(snapshotFile, []byte(snapshotYAML), 0644) + Expect(err).NotTo(HaveOccurred()) + defer os.Remove(snapshotFile) + + cmd := exec.Command("kubectl", "apply", "-f", snapshotFile) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("waiting for SandboxSnapshot to reach Failed phase") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "sandboxsnapshot", snapshotName, + "-n", pauseResumeNamespace, "-o", "jsonpath={.status.phase}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Failed")) + }, 2*time.Minute).Should(Succeed()) + + By("cleaning up") + cmd = exec.Command("kubectl", "delete", "sandboxsnapshot", snapshotName, "-n", pauseResumeNamespace) + utils.Run(cmd) + }) + }) +}) + +// createHtpasswdSecret creates the htpasswd secret for registry authentication. +// Docker Registry v2 only supports bcrypt hashes, not MD5 ($apr1$) or SHA1. +func createHtpasswdSecret(namespace string) error { + htpasswdEntry := "" + pyCmd := exec.Command("python3", "-c", + fmt.Sprintf("import bcrypt; print('%s:' + bcrypt.hashpw(b'%s', bcrypt.gensalt(rounds=10)).decode())", + registryUsername, registryPassword)) + if output, err := pyCmd.Output(); err == nil { + htpasswdEntry = strings.TrimSpace(string(output)) + } + + if htpasswdEntry == "" { + return fmt.Errorf("failed to generate bcrypt htpasswd: python3 bcrypt not available") + } + + tmpFile := filepath.Join(os.TempDir(), "htpasswd") + if err := os.WriteFile(tmpFile, []byte(htpasswdEntry), 0644); err != nil { + return fmt.Errorf("failed to write htpasswd file: %w", err) + } + defer os.Remove(tmpFile) + + cmd := exec.Command("kubectl", "create", "secret", "generic", "registry-auth", + "--from-file=htpasswd="+tmpFile, "-n", namespace) + if _, err := utils.Run(cmd); err != nil { + cmd = exec.Command("kubectl", "delete", "secret", "registry-auth", "-n", namespace, "--ignore-not-found=true") + utils.Run(cmd) + cmd = exec.Command("kubectl", "create", "secret", "generic", "registry-auth", + "--from-file=htpasswd="+tmpFile, "-n", namespace) + if _, err := utils.Run(cmd); err != nil { + return fmt.Errorf("failed to create registry-auth secret: %w", err) + } + } + + return nil +} + +// createDockerRegistrySecrets creates docker-registry secrets for push/pull. +func createDockerRegistrySecrets(namespace string) error { + server := registryServiceAddr + + cmd := exec.Command("kubectl", "create", "secret", "docker-registry", "registry-push-secret", + "--docker-server="+server, + "--docker-username="+registryUsername, + "--docker-password="+registryPassword, + "-n", namespace) + if _, err := utils.Run(cmd); err != nil { + cmd = exec.Command("kubectl", "delete", "secret", "registry-push-secret", "-n", namespace, "--ignore-not-found=true") + utils.Run(cmd) + cmd = exec.Command("kubectl", "create", "secret", "docker-registry", "registry-push-secret", + "--docker-server="+server, + "--docker-username="+registryUsername, + "--docker-password="+registryPassword, + "-n", namespace) + if _, err := utils.Run(cmd); err != nil { + return fmt.Errorf("failed to create registry-push-secret: %w", err) + } + } + + cmd = exec.Command("kubectl", "create", "secret", "docker-registry", "registry-pull-secret", + "--docker-server="+server, + "--docker-username="+registryUsername, + "--docker-password="+registryPassword, + "-n", namespace) + if _, err := utils.Run(cmd); err != nil { + cmd = exec.Command("kubectl", "delete", "secret", "registry-pull-secret", "-n", namespace, "--ignore-not-found=true") + utils.Run(cmd) + cmd = exec.Command("kubectl", "create", "secret", "docker-registry", "registry-pull-secret", + "--docker-server="+server, + "--docker-username="+registryUsername, + "--docker-password="+registryPassword, + "-n", namespace) + if _, err := utils.Run(cmd); err != nil { + return fmt.Errorf("failed to create registry-pull-secret: %w", err) + } + } + + return nil +} diff --git a/kubernetes/test/e2e/testdata/batchsandbox-pooled-pause.yaml b/kubernetes/test/e2e/testdata/batchsandbox-pooled-pause.yaml new file mode 100644 index 000000000..ccbda11f7 --- /dev/null +++ b/kubernetes/test/e2e/testdata/batchsandbox-pooled-pause.yaml @@ -0,0 +1,12 @@ +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: BatchSandbox +metadata: + name: {{.BatchSandboxName}} + namespace: {{.Namespace}} +spec: + replicas: 1 + poolRef: {{.PoolName}} + pausePolicy: + snapshotRegistry: {{.SnapshotRegistry}} + snapshotPushSecretName: {{.SnapshotPushSecretName}} + resumeImagePullSecretName: {{.ResumeImagePullSecretName}} diff --git a/kubernetes/test/e2e/testdata/batchsandbox-with-pause-policy.yaml b/kubernetes/test/e2e/testdata/batchsandbox-with-pause-policy.yaml new file mode 100644 index 000000000..8a5bfa0bf --- /dev/null +++ b/kubernetes/test/e2e/testdata/batchsandbox-with-pause-policy.yaml @@ -0,0 +1,17 @@ +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: BatchSandbox +metadata: + name: {{.BatchSandboxName}} + namespace: {{.Namespace}} +spec: + replicas: 1 + template: + spec: + containers: + - name: sandbox + image: {{.SandboxImage}} + command: ["sh", "-c", "echo 'Hello from sandbox' && sleep 3600"] + pausePolicy: + snapshotRegistry: {{.SnapshotRegistry}} + snapshotPushSecretName: {{.SnapshotPushSecretName}} + resumeImagePullSecretName: {{.ResumeImagePullSecretName}} \ No newline at end of file diff --git a/kubernetes/test/e2e/testdata/pool-with-pause-policy.yaml b/kubernetes/test/e2e/testdata/pool-with-pause-policy.yaml new file mode 100644 index 000000000..f3e785a0b --- /dev/null +++ b/kubernetes/test/e2e/testdata/pool-with-pause-policy.yaml @@ -0,0 +1,17 @@ +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: Pool +metadata: + name: {{.PoolName}} + namespace: {{.Namespace}} +spec: + template: + spec: + containers: + - name: sandbox + image: {{.SandboxImage}} + command: ["sh", "-c", "echo 'Hello from pool sandbox' && sleep 3600"] + capacitySpec: + bufferMax: {{.BufferMax}} + bufferMin: {{.BufferMin}} + poolMax: {{.PoolMax}} + poolMin: {{.PoolMin}} diff --git a/kubernetes/test/e2e/testdata/registry-deployment.yaml b/kubernetes/test/e2e/testdata/registry-deployment.yaml new file mode 100644 index 000000000..b0f3dc7d7 --- /dev/null +++ b/kubernetes/test/e2e/testdata/registry-deployment.yaml @@ -0,0 +1,52 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: docker-registry + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: docker-registry + template: + metadata: + labels: + app: docker-registry + spec: + containers: + - name: registry + image: registry:2 + ports: + - containerPort: 5000 + env: + - name: REGISTRY_AUTH + value: htpasswd + - name: REGISTRY_AUTH_HTPASSWD_REALM + value: "Registry Realm" + - name: REGISTRY_AUTH_HTPASSWD_PATH + value: /auth/htpasswd + volumeMounts: + - name: auth + mountPath: /auth + - name: data + mountPath: /var/lib/registry + volumes: + - name: auth + secret: + secretName: registry-auth + - name: data + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: docker-registry + namespace: default +spec: + type: NodePort + ports: + - port: 5000 + targetPort: 5000 + nodePort: 30500 + selector: + app: docker-registry \ No newline at end of file diff --git a/kubernetes/test/e2e/testdata/sandboxsnapshot-minimal.yaml b/kubernetes/test/e2e/testdata/sandboxsnapshot-minimal.yaml new file mode 100644 index 000000000..daed53aa2 --- /dev/null +++ b/kubernetes/test/e2e/testdata/sandboxsnapshot-minimal.yaml @@ -0,0 +1,11 @@ +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: SandboxSnapshot +metadata: + name: {{.SnapshotName}} + namespace: {{.Namespace}} +spec: + sandboxId: {{.SandboxId}} + sourceBatchSandboxName: {{.SourceBatchSandboxName}} + pausedAt: {{.PausedAt}} + pauseVersion: 1 + resumeVersion: 0 diff --git a/kubernetes/test/e2e/testdata/sandboxsnapshot.yaml b/kubernetes/test/e2e/testdata/sandboxsnapshot.yaml new file mode 100644 index 000000000..96a5a35b4 --- /dev/null +++ b/kubernetes/test/e2e/testdata/sandboxsnapshot.yaml @@ -0,0 +1,27 @@ +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: SandboxSnapshot +metadata: + name: {{.SnapshotName}} + namespace: {{.Namespace}} +spec: + sandboxId: {{.SandboxId}} + snapshotType: Rootfs + sourceBatchSandboxName: {{.SourceBatchSandboxName}} + sourcePodName: {{.SourcePodName}} + sourceNodeName: {{.SourceNodeName}} + snapshotRegistry: {{.SnapshotRegistry}} + snapshotPushSecretName: {{.SnapshotPushSecretName}} + resumeImagePullSecretName: {{.ResumeImagePullSecretName}} + containerSnapshots: + - containerName: sandbox + imageUri: {{.ImageUri}} + resumeTemplate: + template: + spec: + containers: + - name: sandbox + image: {{.SandboxImage}} + command: ["sh", "-c", "echo 'Hello from sandbox' && sleep 3600"] + pausedAt: {{.PausedAt}} + pauseVersion: 1 + resumeVersion: 0 diff --git a/kubernetes/test/utils/image.go b/kubernetes/test/utils/image.go index 3dd6edd63..378771600 100644 --- a/kubernetes/test/utils/image.go +++ b/kubernetes/test/utils/image.go @@ -25,6 +25,10 @@ var ( // Can be overridden via TASK_EXECUTOR_IMG env var TaskExecutorImage = getEnv("TASK_EXECUTOR_IMG", "task-executor:dev") + // ImageCommitterImage is the image-committer image + // Can be overridden via IMAGE_COMMITTER_IMG env var + ImageCommitterImage = getEnv("IMAGE_COMMITTER_IMG", "image-committer:dev") + // SandboxImage is the image used for sandbox containers in tests // Always uses TaskExecutorImage to ensure the image is available in Kind SandboxImage = TaskExecutorImage diff --git a/server/opensandbox_server/api/schema.py b/server/opensandbox_server/api/schema.py index 831703cdb..d24dc0091 100644 --- a/server/opensandbox_server/api/schema.py +++ b/server/opensandbox_server/api/schema.py @@ -120,6 +120,34 @@ class Config: populate_by_name = True +class PausePolicy(BaseModel): + """Configuration for pause/resume with rootfs snapshot.""" + + snapshot_type: Literal["Rootfs"] = Field( + "Rootfs", + alias="snapshotType", + description="Snapshot type, currently only 'Rootfs' is supported", + ) + snapshot_registry: str = Field( + ..., + alias="snapshotRegistry", + description="OCI registry for snapshot images, e.g. registry.example.com/snapshots", + ) + snapshot_push_secret_name: Optional[str] = Field( + None, + alias="snapshotPushSecretName", + description="K8s Secret name for pushing snapshot to registry", + ) + resume_image_pull_secret_name: Optional[str] = Field( + None, + alias="resumeImagePullSecretName", + description="K8s Secret name for pulling snapshot image during resume", + ) + + class Config: + populate_by_name = True + + # ============================================================================ # Volume Definitions # ============================================================================ @@ -386,6 +414,11 @@ class CreateSandboxRequest(BaseModel): None, description="Opaque container for provider-specific or transient parameters not covered by the core API", ) + pause_policy: Optional[PausePolicy] = Field( + None, + alias="pausePolicy", + description="Optional pause policy for snapshot support", + ) class Config: populate_by_name = True diff --git a/server/opensandbox_server/config.py b/server/opensandbox_server/config.py index 0abc012e7..b4892c436 100644 --- a/server/opensandbox_server/config.py +++ b/server/opensandbox_server/config.py @@ -357,6 +357,14 @@ class KubernetesRuntimeConfig(BaseModel): "If unset, no resource constraints are applied." ), ) + image_pull_policy: Optional[str] = Field( + default="IfNotPresent", + description=( + "Image pull policy for sandbox containers. " + "Values: Always, IfNotPresent, Never. " + "Can be overridden per-sandbox via image.pull_policy in create request." + ), + ) class ExecdInitResources(BaseModel): @@ -427,6 +435,28 @@ class EgressConfig(BaseModel): ) +class PauseConfig(BaseModel): + """Pause/resume configuration for snapshot support.""" + + default_snapshot_registry: str = Field( + default="", + description="Default registry for snapshots when pausePolicy.snapshotRegistry is not set.", + ) + committer_image: str = Field( + default="containerd/containerd:1.7", + description="Image used by commit Job Pod for rootfs snapshot.", + ) + cleanup_snapshot_image_on_delete: bool = Field( + default=False, + description="Whether to delete snapshot image from registry when sandbox is deleted.", + ) + commit_timeout_seconds: int = Field( + default=600, + ge=1, + description="Timeout for commit job in seconds.", + ) + + class RuntimeConfig(BaseModel): """Runtime selection (docker, kubernetes, etc.).""" @@ -574,6 +604,10 @@ class AppConfig(BaseModel): default=None, description="Secure container runtime configuration (gVisor, Kata, Firecracker).", ) + pause: Optional[PauseConfig] = Field( + default=None, + description="Pause/resume configuration for snapshot support.", + ) @model_validator(mode="after") def validate_runtime_blocks(self) -> "AppConfig": @@ -698,6 +732,7 @@ def get_config_path() -> Path: "StorageConfig", "KubernetesRuntimeConfig", "EgressConfig", + "PauseConfig", "EGRESS_MODE_DNS", "EGRESS_MODE_DNS_NFT", "SecureRuntimeConfig", diff --git a/server/opensandbox_server/services/constants.py b/server/opensandbox_server/services/constants.py index 919c1488b..995272cc9 100644 --- a/server/opensandbox_server/services/constants.py +++ b/server/opensandbox_server/services/constants.py @@ -105,6 +105,14 @@ class SandboxErrorCodes: OSSFS_MOUNT_FAILED = "VOLUME::OSSFS_MOUNT_FAILED" OSSFS_UNMOUNT_FAILED = "VOLUME::OSSFS_UNMOUNT_FAILED" + # Pause/Resume error codes + PAUSE_POLICY_NOT_CONFIGURED = "KUBERNETES::PAUSE_POLICY_NOT_CONFIGURED" + SNAPSHOT_IN_PROGRESS = "KUBERNETES::SNAPSHOT_IN_PROGRESS" + SNAPSHOT_NOT_FOUND = "KUBERNETES::SNAPSHOT_NOT_FOUND" + SNAPSHOT_NOT_READY = "KUBERNETES::SNAPSHOT_NOT_READY" + UNSUPPORTED_REPLICAS = "KUBERNETES::UNSUPPORTED_REPLICAS" + INVALID_STATE = "KUBERNETES::INVALID_STATE" + __all__ = [ "RESERVED_LABEL_PREFIX", diff --git a/server/opensandbox_server/services/k8s/agent_sandbox_provider.py b/server/opensandbox_server/services/k8s/agent_sandbox_provider.py index 3e32151e7..eb46e9bb0 100644 --- a/server/opensandbox_server/services/k8s/agent_sandbox_provider.py +++ b/server/opensandbox_server/services/k8s/agent_sandbox_provider.py @@ -141,6 +141,7 @@ def create_workload( annotations: Optional[Dict[str, str]] = None, egress_auth_token: Optional[str] = None, egress_mode: str = EGRESS_MODE_DNS, + pause_policy: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """Create an agent-sandbox Sandbox CRD workload.""" if self.runtime_class: diff --git a/server/opensandbox_server/services/k8s/batchsandbox_provider.py b/server/opensandbox_server/services/k8s/batchsandbox_provider.py index cbec25d9a..a516da776 100644 --- a/server/opensandbox_server/services/k8s/batchsandbox_provider.py +++ b/server/opensandbox_server/services/k8s/batchsandbox_provider.py @@ -82,6 +82,7 @@ def __init__( if template_file_path: logger.info("Using BatchSandbox template file: %s", template_file_path) self.execd_init_resources = k8s_config.execd_init_resources if k8s_config else None + self.image_pull_policy = k8s_config.image_pull_policy if k8s_config else "IfNotPresent" # Initialize secure runtime resolver self.resolver = SecureRuntimeResolver(app_config) if app_config else None @@ -120,6 +121,7 @@ def create_workload( annotations: Optional[Dict[str, str]] = None, egress_auth_token: Optional[str] = None, egress_mode: str = EGRESS_MODE_DNS, + pause_policy: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """ Create a BatchSandbox workload. @@ -185,6 +187,7 @@ def create_workload( entrypoint=entrypoint, env=env, annotations=annotations, + pause_policy=pause_policy, ) # Extract extra pod spec fragments from template (volumes/volumeMounts only). @@ -250,6 +253,10 @@ def create_workload( "spec": pod_spec, }, } + + # Add pausePolicy if provided + if pause_policy: + spec["pausePolicy"] = pause_policy runtime_manifest = { "apiVersion": f"{self.group}/{self.version}", "kind": "BatchSandbox", @@ -347,6 +354,7 @@ def _create_workload_from_pool( entrypoint: List[str], env: Dict[str, str], annotations: Optional[Dict[str, str]] = None, + pause_policy: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """ Create BatchSandbox workload from a pre-warmed resource pool. @@ -363,7 +371,8 @@ def _create_workload_from_pool( expires_at: Expiration time entrypoint: Container entrypoint command (can be customized) env: Environment variables (can be customized) - + pause_policy: Optional pause/resume policy configuration + Returns: Dict with 'name' and 'uid' of created BatchSandbox @@ -377,6 +386,8 @@ def _create_workload_from_pool( } if expires_at is not None: spec["expireTime"] = expires_at.isoformat() + if pause_policy is not None: + spec["pausePolicy"] = pause_policy runtime_manifest = { "apiVersion": f"{self.group}/{self.version}", "kind": "BatchSandbox", @@ -636,6 +647,7 @@ def _build_main_container( return V1Container( name="sandbox", image=image_spec.uri, + image_pull_policy= self.image_pull_policy, command=wrapped_command, env=env_vars if env_vars else None, resources=resources, @@ -662,7 +674,10 @@ def _container_to_dict(self, container: V1Container) -> Dict[str, Any]: "name": container.name, "image": container.image, } - + + if container.image_pull_policy: + result["imagePullPolicy"] = container.image_pull_policy + if container.command: result["command"] = container.command diff --git a/server/opensandbox_server/services/k8s/kubernetes_service.py b/server/opensandbox_server/services/k8s/kubernetes_service.py index dba179163..6f43d50f0 100644 --- a/server/opensandbox_server/services/k8s/kubernetes_service.py +++ b/server/opensandbox_server/services/k8s/kubernetes_service.py @@ -68,6 +68,7 @@ ) from opensandbox_server.services.k8s.client import K8sClient from opensandbox_server.services.k8s.provider_factory import create_workload_provider +from opensandbox_server.services.k8s.sandboxsnapshot_provider import SandboxSnapshotProvider logger = logging.getLogger(__name__) @@ -138,7 +139,10 @@ def __init__(self, config: Optional[AppConfig] = None): "message": f"Invalid workload provider configuration: {str(e)}", }, ) from e - + + # Initialize snapshot provider for pause/resume + self.snapshot_provider = SandboxSnapshotProvider(self.k8s_client) + logger.info( "KubernetesSandboxService initialized: namespace=%s, execd_image=%s", self.namespace, @@ -215,7 +219,7 @@ async def _wait_for_sandbox_ready( ), }, ) - + except HTTPException: raise except Exception as e: @@ -250,12 +254,12 @@ def _normalize_create_status(status_info: Dict[str, Any]) -> Dict[str, Any]: "state": "Running", "message": "Pod has IP assigned and sandbox is ready for requests", } - + @staticmethod def _is_unschedulable_status(status_info: Dict[str, Any]) -> bool: reason = str(status_info.get("reason") or "") return reason == "POD_PLATFORM_UNSCHEDULABLE" - + def _ensure_network_policy_support(self, request: CreateSandboxRequest) -> None: """ Validate that network policy can be honored under the current runtime config. @@ -379,6 +383,7 @@ async def create_sandbox(self, request: CreateSandboxRequest) -> CreateSandboxRe egress_mode=egress_mode, volumes=request.volumes, platform=request.platform, + pause_policy=request.pause_policy.model_dump(by_alias=True) if request.pause_policy else None, ) logger.info( @@ -450,7 +455,9 @@ async def create_sandbox(self, request: CreateSandboxRequest) -> CreateSandboxRe def get_sandbox(self, sandbox_id: str) -> Sandbox: """ Get sandbox by ID. - + + Aggregates state from both BatchSandbox and SnapshotSnapshot resources. + Args: sandbox_id: Unique sandbox identifier @@ -461,12 +468,18 @@ def get_sandbox(self, sandbox_id: str) -> Sandbox: HTTPException: If sandbox not found """ try: - workload = self.workload_provider.get_workload( + # Get both BatchSandbox and Snapshot for state aggregation + batchsandbox = self.workload_provider.get_workload( sandbox_id=sandbox_id, namespace=self.namespace, ) - - if not workload: + snapshot = self.snapshot_provider.get_snapshot(sandbox_id, self.namespace) + + # Derive aggregated state + state, reason, message = self._derive_sandbox_state(batchsandbox, snapshot) + + # Handle not found case + if state == "NotFound": raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail={ @@ -474,9 +487,61 @@ def get_sandbox(self, sandbox_id: str) -> Sandbox: "message": f"Sandbox '{sandbox_id}' not found", }, ) - - return self._build_sandbox_from_workload(workload) - + + # Build Sandbox from BatchSandbox if available + if batchsandbox: + sandbox = self._build_sandbox_from_workload(batchsandbox) + # Override status with aggregated state + sandbox.status.state = state + sandbox.status.reason = reason + sandbox.status.message = message + return sandbox + + # Paused state: build from snapshot + if snapshot: + metadata = snapshot.get("metadata", {}) + labels = metadata.get("labels", {}) + creation_timestamp = metadata.get("creationTimestamp") + spec = snapshot.get("spec", {}) + + # Extract user metadata + user_metadata = { + k: v for k, v in labels.items() if not k.startswith("opensandbox.io/") + } + + # Get image URI from snapshot spec (from ContainerSnapshots for multi-container) + container_snapshots = spec.get("containerSnapshots", []) + if container_snapshots: + # Multi-container: use first container's image + image_uri = container_snapshots[0].get("imageUri", "unknown") + else: + image_uri = "unknown" + image_spec = ImageSpec(uri=image_uri) if image_uri else ImageSpec(uri="unknown") + + return Sandbox( + id=sandbox_id, + status=SandboxStatus( + state=state, + reason=reason, + message=message, + last_transition_at=creation_timestamp, + ), + created_at=creation_timestamp, + expires_at=None, + metadata=user_metadata if user_metadata else None, + image=image_spec, + entrypoint=[], + ) + + # Should not reach here due to NotFound check above + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={ + "code": SandboxErrorCodes.K8S_SANDBOX_NOT_FOUND, + "message": f"Sandbox '{sandbox_id}' not found", + }, + ) + except HTTPException: raise except Exception as e: @@ -510,11 +575,56 @@ def list_sandboxes(self, request: ListSandboxesRequest) -> ListSandboxesResponse ) # Convert to Sandbox objects - sandboxes = [ - self._build_sandbox_from_workload(w) - for w in workloads - ] - + sandboxes = [self._build_sandbox_from_workload(w) for w in workloads] + + # Include paused sandboxes (Ready snapshots without BatchSandbox) + try: + snapshots = self.snapshot_provider.list_snapshots( + namespace=self.namespace, + label_selector=f"sandbox.opensandbox.io/sandbox-id", + ) + + workload_ids = { + w.get("metadata", {}).get("labels", {}).get(SANDBOX_ID_LABEL) for w in workloads + } + + for snap in snapshots: + snap_id = snap.get("spec", {}).get("sandboxId", "") + phase = snap.get("status", {}).get("phase", "") + if snap_id and phase == "Ready" and snap_id not in workload_ids: + metadata = snap.get("metadata", {}) + labels = metadata.get("labels", {}) + spec = snap.get("spec", {}) + # Get image from ContainerSnapshots (multi-container support) + container_snapshots = spec.get("containerSnapshots", []) + if container_snapshots: + image_uri = container_snapshots[0].get("imageUri", "unknown") + else: + image_uri = "unknown" + user_metadata = { + k: v for k, v in labels.items() if not k.startswith("opensandbox.io/") + } + + paused_sandbox = Sandbox( + id=snap_id, + status=SandboxStatus( + state="Paused", + reason="SNAPSHOT_READY", + message="Sandbox paused", + last_transition_at=metadata.get("creationTimestamp"), + ), + created_at=metadata.get("creationTimestamp"), + expires_at=None, + metadata=user_metadata if user_metadata else None, + image=( + ImageSpec(uri=image_uri) if image_uri else ImageSpec(uri="unknown") + ), + entrypoint=[], + ) + sandboxes.append(paused_sandbox) + except Exception as e: + logger.warning("Failed to list paused sandboxes from snapshots: %s", e) + # Apply filters filtered = self._apply_filters(sandboxes, request.filter) @@ -556,76 +666,226 @@ def list_sandboxes(self, request: ListSandboxesRequest) -> ListSandboxesResponse def delete_sandbox(self, sandbox_id: str) -> None: """ - Delete a sandbox. - - Args: - sandbox_id: Unique sandbox identifier - - Raises: - HTTPException: If deletion fails + Delete sandbox and associated snapshot. """ + deleted = False + + # 1. Delete BatchSandbox try: - self.workload_provider.delete_workload( - sandbox_id=sandbox_id, - namespace=self.namespace, - ) - - logger.info(f"Deleted sandbox: {sandbox_id}") - + self.workload_provider.delete_workload(sandbox_id, self.namespace) + deleted = True + logger.info("Deleted BatchSandbox %s", sandbox_id) + except Exception as e: + logger.debug("BatchSandbox %s not found or already deleted: %s", sandbox_id, e) + + # 2. Delete SandboxSnapshot + try: + self.snapshot_provider.delete_snapshot(sandbox_id, self.namespace) + deleted = True + logger.info("Deleted SandboxSnapshot %s", sandbox_id) except Exception as e: - if "not found" in str(e).lower(): + logger.debug("SandboxSnapshot %s not found or already deleted: %s", sandbox_id, e) + + if not deleted: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={ + "code": SandboxErrorCodes.K8S_SANDBOX_NOT_FOUND, + "message": f"Sandbox '{sandbox_id}' not found", + }, + ) + + def pause_sandbox(self, sandbox_id: str) -> None: + """ + Pause sandbox by creating SandboxSnapshot CR. + + The controller handles Pod discovery, commit, push, and BatchSandbox cleanup. + """ + # 1. Get BatchSandbox + batchsandbox = self.workload_provider.get_workload(sandbox_id, self.namespace) + if not batchsandbox: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={ + "code": SandboxErrorCodes.K8S_SANDBOX_NOT_FOUND, + "message": f"Sandbox '{sandbox_id}' not found", + }, + ) + + # 2. Validate state + workload_status = self.workload_provider.get_status(batchsandbox) + if workload_status["state"] != "Running": + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "code": SandboxErrorCodes.INVALID_STATE, + "message": f"Cannot pause sandbox in state {workload_status['state']}", + }, + ) + + spec = batchsandbox.get("spec", {}) + if spec.get("replicas", 0) != 1: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "code": SandboxErrorCodes.UNSUPPORTED_REPLICAS, + "message": "Pause only supports replicas=1", + }, + ) + + if not spec.get("pausePolicy"): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "code": SandboxErrorCodes.PAUSE_POLICY_NOT_CONFIGURED, + "message": "Sandbox does not have pausePolicy configured", + }, + ) + + # 3. Check for in-flight snapshot or determine if re-pause + existing_snapshot = self.snapshot_provider.get_snapshot(sandbox_id, self.namespace) + + if existing_snapshot: + phase = existing_snapshot.get("status", {}).get("phase") + # In-flight pause: versions don't match means controller is still processing + spec_pv = existing_snapshot.get("spec", {}).get("pauseVersion", 0) + status_pv = existing_snapshot.get("status", {}).get("pauseVersion", 0) + if spec_pv > status_pv: raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, + status_code=status.HTTP_409_CONFLICT, detail={ - "code": SandboxErrorCodes.K8S_SANDBOX_NOT_FOUND, - "message": f"Sandbox '{sandbox_id}' not found", + "code": SandboxErrorCodes.SNAPSHOT_IN_PROGRESS, + "message": "Snapshot already in progress", }, - ) from e - - logger.error(f"Error deleting sandbox {sandbox_id}: {e}") + ) + + # Re-pause: patch existing Snapshot CR with incremented pauseVersion + current_pause_version = status_pv + new_pause_version = current_pause_version + 1 + try: + self.snapshot_provider.patch_snapshot_spec( + snapshot_name=sandbox_id, + namespace=self.namespace, + spec_patch={ + "pauseVersion": new_pause_version, + "pausedAt": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "sourceBatchSandboxName": batchsandbox["metadata"]["name"], + }, + ) + logger.info( + "Patched SandboxSnapshot %s pauseVersion=%d for re-pause", + sandbox_id, + new_pause_version, + ) + except Exception as e: + logger.error("Failed to patch SandboxSnapshot for re-pause: %s", e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={ + "code": SandboxErrorCodes.K8S_API_ERROR, + "message": f"Failed to re-pause snapshot: {str(e)}", + }, + ) + return + + # 4. First pause: create minimal SandboxSnapshot CR + batch_sandbox_name = batchsandbox["metadata"]["name"] + snapshot_body = { + "apiVersion": f"{SandboxSnapshotProvider.GROUP}/{SandboxSnapshotProvider.VERSION}", + "kind": "SandboxSnapshot", + "metadata": { + "name": sandbox_id, + "namespace": self.namespace, + "labels": { + "sandbox.opensandbox.io/sandbox-id": sandbox_id, + }, + }, + "spec": { + "sandboxId": sandbox_id, + "sourceBatchSandboxName": batch_sandbox_name, + "pausedAt": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "pauseVersion": 1, + "resumeVersion": 0, + }, + } + + try: + self.snapshot_provider.create_snapshot(self.namespace, snapshot_body) + logger.info("Created SandboxSnapshot %s for pause", sandbox_id) + except Exception as e: + logger.error("Failed to create SandboxSnapshot: %s", e) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={ "code": SandboxErrorCodes.K8S_API_ERROR, - "message": f"Failed to delete sandbox: {str(e)}", + "message": f"Failed to create snapshot: {str(e)}", }, - ) from e - - def pause_sandbox(self, sandbox_id: str) -> None: - """ - Pause sandbox (not supported in Kubernetes). - - Args: - sandbox_id: Unique sandbox identifier - - Raises: - HTTPException: Always raises 501 Not Implemented - """ - raise HTTPException( - status_code=status.HTTP_501_NOT_IMPLEMENTED, - detail={ - "code": SandboxErrorCodes.API_NOT_SUPPORTED, - "message": "Pause operation is not supported in Kubernetes runtime", - }, - ) - + ) + def resume_sandbox(self, sandbox_id: str) -> None: """ - Resume sandbox (not supported in Kubernetes). - - Args: - sandbox_id: Unique sandbox identifier - - Raises: - HTTPException: Always raises 501 Not Implemented + Resume sandbox by patching SandboxSnapshot CR to trigger controller-driven resume. + + The controller watches for spec.resumeVersion > status.resumeVersion and + creates the BatchSandbox from resumeTemplate. """ - raise HTTPException( - status_code=status.HTTP_501_NOT_IMPLEMENTED, - detail={ - "code": SandboxErrorCodes.API_NOT_SUPPORTED, - "message": "Resume operation is not supported in Kubernetes runtime", - }, - ) + # 1. Get SandboxSnapshot + snapshot = self.snapshot_provider.get_snapshot(sandbox_id, self.namespace) + if not snapshot: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={ + "code": SandboxErrorCodes.SNAPSHOT_NOT_FOUND, + "message": f"No snapshot found for sandbox {sandbox_id}", + }, + ) + + # 2. Validate snapshot is Ready + phase = snapshot.get("status", {}).get("phase") + if phase != "Ready": + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "code": SandboxErrorCodes.SNAPSHOT_NOT_READY, + "message": f"Snapshot is in phase {phase}, cannot resume", + }, + ) + + # 3. Check BatchSandbox doesn't already exist + existing = self.workload_provider.get_workload(sandbox_id, self.namespace) + if existing: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "code": SandboxErrorCodes.INVALID_STATE, + "message": "BatchSandbox already exists, cannot resume", + }, + ) + + # 4. Increment resumeVersion to trigger controller resume + current_resume_version = snapshot.get("status", {}).get("resumeVersion", 0) + new_resume_version = current_resume_version + 1 + + try: + self.snapshot_provider.patch_snapshot_spec( + snapshot_name=sandbox_id, + namespace=self.namespace, + spec_patch={"resumeVersion": new_resume_version}, + ) + logger.info( + "Patched SandboxSnapshot %s resumeVersion=%d to trigger resume", + sandbox_id, + new_resume_version, + ) + except Exception as e: + logger.error("Failed to patch SandboxSnapshot for resume: %s", e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={ + "code": SandboxErrorCodes.K8S_API_ERROR, + "message": f"Failed to trigger resume: {e}", + }, + ) def get_access_renew_extend_seconds(self, sandbox_id: str) -> Optional[int]: workload = self.workload_provider.get_workload( @@ -807,10 +1067,59 @@ def _get_egress_auth_token(self, workload: Any) -> Optional[str]: return annotations.get(SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY) return None - def _build_sandbox_from_workload( + def _derive_sandbox_state( self, - workload: Any, - ) -> Sandbox: + batchsandbox: Optional[Dict[str, Any]], + snapshot: Optional[Dict[str, Any]], + ) -> tuple[str, str, str]: + """ + Derive sandbox state from BatchSandbox and SnapshotSnapshot. + + Returns: + Tuple of (state, reason, message) + """ + # Snapshot failed + if snapshot and snapshot.get("status", {}).get("phase") == "Failed": + return ( + "Failed", + "SNAPSHOT_FAILED", + snapshot.get("status", {}).get("message", "Snapshot failed"), + ) + + # Pausing (both exist, snapshot in progress) + if batchsandbox and snapshot: + phase = snapshot.get("status", {}).get("phase") + # Check if BatchSandbox is resumed from snapshot + annotations = batchsandbox.get("metadata", {}).get("annotations", {}) + if annotations.get("sandbox.opensandbox.io/resumed-from-snapshot") == "true": + # Resumed sandbox - return actual workload status + status = self.workload_provider.get_status(batchsandbox) + return (status["state"], status["reason"], status["message"]) + if phase in ("Pending", "Committing"): + return ("Pausing", f"SNAPSHOT_{phase.upper()}", f"Snapshot is {phase.lower()}") + if phase == "Ready": + return ("Pausing", "SNAPSHOT_READY_CLEANUP", "Releasing resources") + + # Paused (no workload, snapshot ready) + if not batchsandbox and snapshot: + phase = snapshot.get("status", {}).get("phase") + if phase == "Ready": + return ("Paused", "SNAPSHOT_READY", "Sandbox paused") + if phase in ("Pending", "Committing"): + return ("Pausing", f"SNAPSHOT_{phase.upper()}", f"Snapshot is {phase.lower()}") + + # Resuming (workload from snapshot) + if batchsandbox: + status = self.workload_provider.get_status(batchsandbox) + annotations = batchsandbox.get("metadata", {}).get("annotations", {}) + if annotations.get("sandbox.opensandbox.io/resumed-from-snapshot") == "true": + if status["state"] != "Running": + return ("Resuming", "RESUMING", "Restoring from snapshot") + return (status["state"], status["reason"], status["message"]) + + return ("NotFound", "SANDBOX_NOT_FOUND", "Sandbox does not exist") + + def _build_sandbox_from_workload(self, workload: Any) -> Sandbox: """ Build Sandbox object from Kubernetes workload. @@ -868,7 +1177,7 @@ def _build_sandbox_from_workload( image_spec = ImageSpec(uri=image_uri) if image_uri else ImageSpec(uri="unknown") platform_spec = self._extract_platform_from_workload(workload) - + return Sandbox( id=sandbox_id, status=SandboxStatus( @@ -1028,7 +1337,7 @@ def _extract_platform_value_from_affinity( elif inferred != term_value: return None return inferred - + def _apply_filters(self, sandboxes: list[Sandbox], filter_spec: Any) -> list[Sandbox]: """ Apply filters to sandbox list. diff --git a/server/opensandbox_server/services/k8s/sandboxsnapshot_provider.py b/server/opensandbox_server/services/k8s/sandboxsnapshot_provider.py new file mode 100644 index 000000000..a12c23e25 --- /dev/null +++ b/server/opensandbox_server/services/k8s/sandboxsnapshot_provider.py @@ -0,0 +1,174 @@ +# Copyright 2025 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +SandboxSnapshot CRD provider for pause/resume operations. +""" + +import logging +from typing import Any, Dict, List, Optional + +from kubernetes.client import ApiException +from opensandbox_server.services.k8s.client import K8sClient + +logger = logging.getLogger(__name__) + + +class SandboxSnapshotProvider: + """Provider for SandboxSnapshot CRD operations.""" + + GROUP = "sandbox.opensandbox.io" + VERSION = "v1alpha1" + PLURAL = "sandboxsnapshots" + + def __init__(self, k8s_client: K8sClient): + """Initialize provider with K8sClient. + + Args: + k8s_client: Kubernetes client for API operations. + """ + self.k8s_client = k8s_client + + def get_snapshot( + self, + snapshot_name: str, + namespace: str, + ) -> Optional[Dict[str, Any]]: + """Get SandboxSnapshot by name. + + Args: + snapshot_name: Name of the SandboxSnapshot. + namespace: Kubernetes namespace. + + Returns: + SandboxSnapshot dict if found, None if not found. + """ + try: + return self.k8s_client.get_custom_object( + group=self.GROUP, + version=self.VERSION, + namespace=namespace, + plural=self.PLURAL, + name=snapshot_name, + ) + except ApiException as e: + if e.status == 404: + return None + logger.warning( + "Failed to get SandboxSnapshot %s/%s: %s", + namespace, + snapshot_name, + e, + ) + raise + + def create_snapshot( + self, + namespace: str, + body: Dict[str, Any], + ) -> Dict[str, Any]: + """Create a SandboxSnapshot. + + Args: + namespace: Kubernetes namespace. + body: SandboxSnapshot manifest. + + Returns: + Created SandboxSnapshot. + """ + return self.k8s_client.create_custom_object( + group=self.GROUP, + version=self.VERSION, + namespace=namespace, + plural=self.PLURAL, + body=body, + ) + + def patch_snapshot_spec( + self, + snapshot_name: str, + namespace: str, + spec_patch: Dict[str, Any], + ) -> Dict[str, Any]: + """Patch SandboxSnapshot spec. + + Args: + snapshot_name: Name of the SandboxSnapshot. + namespace: Kubernetes namespace. + spec_patch: Spec fields to patch (merged with existing spec). + + Returns: + Updated SandboxSnapshot. + """ + body = {"spec": spec_patch} + return self.k8s_client.patch_custom_object( + group=self.GROUP, + version=self.VERSION, + namespace=namespace, + plural=self.PLURAL, + name=snapshot_name, + body=body, + ) + + def delete_snapshot( + self, + snapshot_name: str, + namespace: str, + ) -> None: + """Delete a SandboxSnapshot. + + Args: + snapshot_name: Name of the SandboxSnapshot. + namespace: Kubernetes namespace. + """ + try: + self.k8s_client.delete_custom_object( + group=self.GROUP, + version=self.VERSION, + namespace=namespace, + plural=self.PLURAL, + name=snapshot_name, + ) + except ApiException as e: + if e.status == 404: + return # Already deleted + logger.warning( + "Failed to delete SandboxSnapshot %s/%s: %s", + namespace, + snapshot_name, + e, + ) + raise + + def list_snapshots( + self, + namespace: str, + label_selector: Optional[str] = None, + ) -> List[Dict[str, Any]]: + """List SandboxSnapshots in namespace. + + Args: + namespace: Kubernetes namespace. + label_selector: Optional label selector to filter results. + + Returns: + List of SandboxSnapshot dicts. + """ + return self.k8s_client.list_custom_objects( + group=self.GROUP, + version=self.VERSION, + namespace=namespace, + plural=self.PLURAL, + label_selector=label_selector, + ) \ No newline at end of file diff --git a/server/opensandbox_server/services/k8s/workload_provider.py b/server/opensandbox_server/services/k8s/workload_provider.py index e51f6606f..5bc4e43ac 100644 --- a/server/opensandbox_server/services/k8s/workload_provider.py +++ b/server/opensandbox_server/services/k8s/workload_provider.py @@ -52,6 +52,7 @@ def create_workload( annotations: Optional[Dict[str, str]] = None, egress_auth_token: Optional[str] = None, egress_mode: str = EGRESS_MODE_DNS, + pause_policy: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """ Create a new workload resource. diff --git a/server/tests/k8s/fixtures/k8s_fixtures.py b/server/tests/k8s/fixtures/k8s_fixtures.py index 2a07c035f..547a0a126 100644 --- a/server/tests/k8s/fixtures/k8s_fixtures.py +++ b/server/tests/k8s/fixtures/k8s_fixtures.py @@ -293,8 +293,9 @@ def k8s_service(k8s_app_config): from unittest.mock import patch, MagicMock with patch('opensandbox_server.services.k8s.kubernetes_service.K8sClient') as mock_k8s_client_cls, \ - patch('opensandbox_server.services.k8s.kubernetes_service.create_workload_provider') as mock_create_provider: - + patch('opensandbox_server.services.k8s.kubernetes_service.create_workload_provider') as mock_create_provider, \ + patch('opensandbox_server.services.k8s.kubernetes_service.SandboxSnapshotProvider') as mock_snapshot_provider_cls: + # Mock K8sClient instance mock_k8s_client = MagicMock() mock_k8s_client_cls.return_value = mock_k8s_client @@ -302,14 +303,19 @@ def k8s_service(k8s_app_config): # Mock WorkloadProvider instance mock_provider = MagicMock() mock_create_provider.return_value = mock_provider - + + # Mock SnapshotProvider instance + mock_snapshot_provider = MagicMock() + mock_snapshot_provider_cls.return_value = mock_snapshot_provider + from opensandbox_server.services.k8s.kubernetes_service import KubernetesSandboxService service = KubernetesSandboxService(k8s_app_config) # Save mock objects for access in tests service.k8s_client = mock_k8s_client service.workload_provider = mock_provider - + service.snapshot_provider = mock_snapshot_provider + yield service diff --git a/server/tests/k8s/test_sandbox_pause_resume.py b/server/tests/k8s/test_sandbox_pause_resume.py new file mode 100644 index 000000000..5b585b024 --- /dev/null +++ b/server/tests/k8s/test_sandbox_pause_resume.py @@ -0,0 +1,678 @@ +# Copyright 2025 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Unit tests for pause_sandbox and resume_sandbox functionality. + +Tests cover KubernetesSandboxService pause and resume operations including +SandboxSnapshot CR creation, state validation, and error handling. +""" + +import pytest +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch + +from fastapi import HTTPException + +from opensandbox_server.services.constants import SandboxErrorCodes + + +class TestPauseSandbox: + """pause_sandbox method tests""" + + def test_pause_sandbox_success(self, k8s_service): + """ + Test case: Successfully pause a running sandbox + + Purpose: Verify that minimal SandboxSnapshot CR is created without + Pod lookup, imageUri, or resumeTemplate. + """ + sandbox_id = "test-sandbox-123" + + # Mock BatchSandbox in Running state with pausePolicy + k8s_service.workload_provider.get_workload.return_value = { + "metadata": { + "name": sandbox_id, + "namespace": "test-namespace", + }, + "spec": { + "replicas": 1, + "pausePolicy": { + "snapshotRegistry": "registry.example.com", + "snapshotType": "Rootfs", + }, + "template": {"spec": {"containers": [{"name": "sandbox", "image": "python:3.11"}]}}, + }, + } + k8s_service.workload_provider.get_status.return_value = { + "state": "Running", + "reason": "POD_READY_WITH_IP", + "message": "Pod is ready", + "last_transition_at": datetime.now(timezone.utc), + } + + # Mock no existing snapshot + k8s_service.snapshot_provider.get_snapshot.return_value = None + + # Mock snapshot creation + k8s_service.snapshot_provider.create_snapshot.return_value = { + "metadata": {"name": sandbox_id} + } + + k8s_service.pause_sandbox(sandbox_id) + + # Verify SandboxSnapshot CR was created + k8s_service.snapshot_provider.create_snapshot.assert_called_once() + call_args = k8s_service.snapshot_provider.create_snapshot.call_args + assert call_args[0][0] == k8s_service.namespace + + snapshot_cr = call_args[0][1] + assert snapshot_cr["metadata"]["name"] == sandbox_id + assert snapshot_cr["metadata"]["labels"]["sandbox.opensandbox.io/sandbox-id"] == sandbox_id + assert snapshot_cr["spec"]["sandboxId"] == sandbox_id + assert snapshot_cr["spec"]["sourceBatchSandboxName"] == sandbox_id + assert "pausedAt" in snapshot_cr["spec"] + + for key in ( + "snapshotType", + "snapshotRegistry", + "imageUri", + "resumeTemplate", + "sourcePodName", + "sourceContainerName", + "sourceNodeName", + "snapshotPushSecretName", + "resumeImagePullSecretName", + ): + assert key not in snapshot_cr["spec"] + + def test_pause_sandbox_with_secrets(self, k8s_service): + """ + Test case: Pause sandbox with push/pull secrets in pausePolicy + + Purpose: Verify secrets are NOT included in minimal CR — controller handles this + """ + sandbox_id = "test-sandbox-secrets" + + k8s_service.workload_provider.get_workload.return_value = { + "metadata": { + "name": sandbox_id, + "namespace": "test-namespace", + }, + "spec": { + "replicas": 1, + "pausePolicy": { + "snapshotRegistry": "registry.example.com", + "snapshotType": "Rootfs", + "snapshotPushSecretName": "push-secret", + "resumeImagePullSecretName": "pull-secret", + }, + "template": {"spec": {"containers": [{"name": "sandbox", "image": "python:3.11"}]}}, + }, + } + k8s_service.workload_provider.get_status.return_value = { + "state": "Running", + "reason": "POD_READY_WITH_IP", + "message": "Pod is ready", + "last_transition_at": datetime.now(timezone.utc), + } + + k8s_service.snapshot_provider.get_snapshot.return_value = None + k8s_service.snapshot_provider.create_snapshot.return_value = { + "metadata": {"name": sandbox_id} + } + + k8s_service.pause_sandbox(sandbox_id) + + call_args = k8s_service.snapshot_provider.create_snapshot.call_args + snapshot_cr = call_args[0][1] + assert snapshot_cr["spec"]["sandboxId"] == sandbox_id + assert snapshot_cr["spec"]["sourceBatchSandboxName"] == sandbox_id + assert "pausedAt" in snapshot_cr["spec"] + assert "snapshotPushSecretName" not in snapshot_cr["spec"] + assert "resumeImagePullSecretName" not in snapshot_cr["spec"] + + def test_pause_sandbox_not_found(self, k8s_service): + """ + Test case: Pause sandbox that does not exist + + Purpose: Verify that 404 is returned when sandbox is not found + """ + sandbox_id = "nonexistent-sandbox" + + # Mock no BatchSandbox + k8s_service.workload_provider.get_workload.return_value = None + + # Execute and verify + with pytest.raises(HTTPException) as exc_info: + k8s_service.pause_sandbox(sandbox_id) + + assert exc_info.value.status_code == 404 + assert exc_info.value.detail["code"] == SandboxErrorCodes.K8S_SANDBOX_NOT_FOUND + + def test_pause_sandbox_invalid_state(self, k8s_service): + """ + Test case: Pause sandbox that is not in Running state + + Purpose: Verify that 409 is returned when sandbox is not in Running state + """ + sandbox_id = "pending-sandbox" + + # Mock BatchSandbox in Pending state + k8s_service.workload_provider.get_workload.return_value = { + "metadata": {"name": sandbox_id}, + "spec": { + "replicas": 1, + "pausePolicy": {"snapshotRegistry": "registry.example.com"}, + }, + } + k8s_service.workload_provider.get_status.return_value = { + "state": "Pending", + "reason": "POD_SCHEDULED", + "message": "Pod is pending", + "last_transition_at": datetime.now(timezone.utc), + } + + # Execute and verify + with pytest.raises(HTTPException) as exc_info: + k8s_service.pause_sandbox(sandbox_id) + + assert exc_info.value.status_code == 409 + assert exc_info.value.detail["code"] == SandboxErrorCodes.INVALID_STATE + + def test_pause_sandbox_no_pause_policy(self, k8s_service): + """ + Test case: Pause sandbox without pausePolicy configured + + Purpose: Verify that 400 is returned when sandbox has no pausePolicy + """ + sandbox_id = "sandbox-without-pause-policy" + + # Mock BatchSandbox in Running state without pausePolicy + k8s_service.workload_provider.get_workload.return_value = { + "metadata": {"name": sandbox_id}, + "spec": { + "replicas": 1, + # No pausePolicy + }, + } + k8s_service.workload_provider.get_status.return_value = { + "state": "Running", + "reason": "POD_READY_WITH_IP", + "message": "Pod is running", + "last_transition_at": datetime.now(timezone.utc), + } + + # Execute and verify + with pytest.raises(HTTPException) as exc_info: + k8s_service.pause_sandbox(sandbox_id) + + assert exc_info.value.status_code == 400 + assert exc_info.value.detail["code"] == SandboxErrorCodes.PAUSE_POLICY_NOT_CONFIGURED + + def test_pause_sandbox_snapshot_in_progress(self, k8s_service): + """ + Test case: Pause sandbox when snapshot is already in progress + + Purpose: Verify that 409 is returned when a snapshot is already being created + """ + sandbox_id = "sandbox-with-snapshot-in-progress" + + # Mock BatchSandbox in Running state with pausePolicy + k8s_service.workload_provider.get_workload.return_value = { + "metadata": {"name": sandbox_id}, + "spec": { + "replicas": 1, + "pausePolicy": {"snapshotRegistry": "registry.example.com"}, + }, + } + k8s_service.workload_provider.get_status.return_value = { + "state": "Running", + "reason": "POD_READY_WITH_IP", + "message": "Pod is running", + "last_transition_at": datetime.now(timezone.utc), + } + + # Mock existing snapshot in Committing phase + k8s_service.snapshot_provider.get_snapshot.return_value = { + "metadata": {"name": sandbox_id}, + "status": {"phase": "Committing"}, + } + + # Execute and verify + with pytest.raises(HTTPException) as exc_info: + k8s_service.pause_sandbox(sandbox_id) + + assert exc_info.value.status_code == 409 + assert exc_info.value.detail["code"] == SandboxErrorCodes.SNAPSHOT_IN_PROGRESS + + def test_pause_sandbox_unsupported_replicas(self, k8s_service): + """ + Test case: Pause sandbox with replicas != 1 + + Purpose: Verify that 400 is returned for unsupported replicas count + """ + sandbox_id = "multi-replica-sandbox" + + # Mock BatchSandbox with replicas=2 + k8s_service.workload_provider.get_workload.return_value = { + "metadata": {"name": sandbox_id}, + "spec": { + "replicas": 2, # Unsupported + "pausePolicy": {"snapshotRegistry": "registry.example.com"}, + }, + } + k8s_service.workload_provider.get_status.return_value = { + "state": "Running", + "reason": "POD_READY_WITH_IP", + "message": "Pod is running", + "last_transition_at": datetime.now(timezone.utc), + } + + # Execute and verify + with pytest.raises(HTTPException) as exc_info: + k8s_service.pause_sandbox(sandbox_id) + + assert exc_info.value.status_code == 400 + assert exc_info.value.detail["code"] == SandboxErrorCodes.UNSUPPORTED_REPLICAS + + +class TestResumeSandbox: + """resume_sandbox method tests""" + + def test_resume_sandbox_multi_container(self, k8s_service): + """ + Test case: Resume sandbox with multi-container snapshots + + Purpose: Verify that BatchSandbox is created with correct images + for each container from containerSnapshots in snapshot status. + """ + sandbox_id = "paused-sandbox-multi" + + # Mock Snapshot in Ready phase with multi-container snapshots + k8s_service.snapshot_provider.get_snapshot.return_value = { + "metadata": {"name": sandbox_id}, + "spec": { + "sandboxId": sandbox_id, + "resumeTemplate": { + "template": { + "spec": { + "containers": [ + {"name": "sandbox", "image": "old-image:latest"}, + {"name": "sidecar", "image": "old-sidecar:latest"}, + ] + } + }, + "expireTime": "2025-12-24T12:00:00Z", + "pausePolicy": {"snapshotRegistry": "registry.example.com"}, + }, + }, + "status": { + "phase": "Ready", + "containerSnapshots": [ + { + "containerName": "sandbox", + "imageURI": "registry.example.com/paused-sandbox-multi:sandbox-snapshot", + }, + { + "containerName": "sidecar", + "imageURI": "registry.example.com/paused-sandbox-multi:sidecar-snapshot", + }, + ], + }, + } + + # Mock no existing BatchSandbox + k8s_service.workload_provider.get_workload.return_value = None + + # Mock BatchSandbox creation + k8s_service.k8s_client.create_custom_object.return_value = { + "metadata": {"name": sandbox_id, "uid": "new-uid"} + } + + # Set provider attributes + k8s_service.workload_provider.group = "sandbox.opensandbox.io" + k8s_service.workload_provider.version = "v1alpha1" + k8s_service.workload_provider.plural = "batchsandboxes" + + # Execute + k8s_service.resume_sandbox(sandbox_id) + + # Verify BatchSandbox was created with correct images per container + k8s_service.k8s_client.create_custom_object.assert_called_once() + call_args = k8s_service.k8s_client.create_custom_object.call_args + body = call_args[1]["body"] + + assert body["metadata"]["name"] == sandbox_id + containers = body["spec"]["template"]["spec"]["containers"] + sandbox_container = next(c for c in containers if c["name"] == "sandbox") + sidecar_container = next(c for c in containers if c["name"] == "sidecar") + assert sandbox_container["image"] == ( + "registry.example.com/paused-sandbox-multi:sandbox-snapshot" + ) + assert sidecar_container["image"] == ( + "registry.example.com/paused-sandbox-multi:sidecar-snapshot" + ) + # Verify resumed-from-snapshot annotation + assert ( + body["metadata"]["annotations"]["sandbox.opensandbox.io/resumed-from-snapshot"] + == "true" + ) + + def test_resume_sandbox_no_container_snapshots(self, k8s_service): + """ + Test case: Resume sandbox when containerSnapshots is empty + + Purpose: Verify that when containerSnapshots is empty/missing, + original template images are preserved (no image replacement occurs). + The controller now requires containerSnapshots — there is no legacy + single-container fallback. + """ + sandbox_id = "paused-sandbox-no-cs" + + # Mock Snapshot in Ready phase without containerSnapshots + k8s_service.snapshot_provider.get_snapshot.return_value = { + "metadata": {"name": sandbox_id}, + "spec": { + "sandboxId": sandbox_id, + "resumeTemplate": { + "template": { + "spec": { + "containers": [{"name": "sandbox", "image": "original-image:latest"}] + } + }, + "expireTime": "2025-12-24T12:00:00Z", + "pausePolicy": {"snapshotRegistry": "registry.example.com"}, + }, + }, + "status": {"phase": "Ready"}, + } + + # Mock no existing BatchSandbox + k8s_service.workload_provider.get_workload.return_value = None + + # Mock BatchSandbox creation + k8s_service.k8s_client.create_custom_object.return_value = { + "metadata": {"name": sandbox_id, "uid": "new-uid"} + } + + # Set provider attributes + k8s_service.workload_provider.group = "sandbox.opensandbox.io" + k8s_service.workload_provider.version = "v1alpha1" + k8s_service.workload_provider.plural = "batchsandboxes" + + # Execute + k8s_service.resume_sandbox(sandbox_id) + + # Verify BatchSandbox was created with original images preserved + k8s_service.k8s_client.create_custom_object.assert_called_once() + call_args = k8s_service.k8s_client.create_custom_object.call_args + body = call_args[1]["body"] + + assert body["metadata"]["name"] == sandbox_id + containers = body["spec"]["template"]["spec"]["containers"] + # Original image preserved — no replacement since containerSnapshots is empty + assert containers[0]["image"] == "original-image:latest" + + def test_resume_sandbox_not_found(self, k8s_service): + """ + Test case: Resume sandbox when no snapshot exists + + Purpose: Verify that 404 is returned when snapshot is not found + """ + sandbox_id = "nonexistent-snapshot" + + # Mock no snapshot + k8s_service.snapshot_provider.get_snapshot.return_value = None + + # Execute and verify + with pytest.raises(HTTPException) as exc_info: + k8s_service.resume_sandbox(sandbox_id) + + assert exc_info.value.status_code == 404 + assert exc_info.value.detail["code"] == SandboxErrorCodes.SNAPSHOT_NOT_FOUND + + def test_resume_sandbox_not_ready(self, k8s_service): + """ + Test case: Resume sandbox when snapshot is not in Ready phase + + Purpose: Verify that 409 is returned when snapshot is not Ready + """ + sandbox_id = "snapshot-committing" + + # Mock Snapshot in Committing phase + k8s_service.snapshot_provider.get_snapshot.return_value = { + "metadata": {"name": sandbox_id}, + "spec": { + "sandboxId": sandbox_id, + "imageUri": "registry.example.com/snapshot-committing:snapshot", + }, + "status": {"phase": "Committing"}, + } + + # Execute and verify + with pytest.raises(HTTPException) as exc_info: + k8s_service.resume_sandbox(sandbox_id) + + assert exc_info.value.status_code == 409 + assert exc_info.value.detail["code"] == SandboxErrorCodes.SNAPSHOT_NOT_READY + + def test_resume_sandbox_already_exists(self, k8s_service): + """ + Test case: Resume sandbox when BatchSandbox already exists + + Purpose: Verify that 409 is returned when BatchSandbox already exists + """ + sandbox_id = "existing-sandbox" + + # Mock Snapshot in Ready phase + k8s_service.snapshot_provider.get_snapshot.return_value = { + "metadata": {"name": sandbox_id}, + "spec": { + "sandboxId": sandbox_id, + "imageUri": "registry.example.com/existing-sandbox:snapshot", + "resumeTemplate": { + "template": { + "spec": {"containers": [{"name": "sandbox", "image": "image:latest"}]} + } + }, + }, + "status": {"phase": "Ready"}, + } + + # Mock existing BatchSandbox + k8s_service.workload_provider.get_workload.return_value = { + "metadata": {"name": sandbox_id}, + "spec": {}, + } + + # Execute and verify + with pytest.raises(HTTPException) as exc_info: + k8s_service.resume_sandbox(sandbox_id) + + assert exc_info.value.status_code == 409 + assert exc_info.value.detail["code"] == SandboxErrorCodes.INVALID_STATE + + +class TestDeleteSandboxCleansSnapshot: + """delete_sandbox cleanup tests""" + + def test_delete_sandbox_cleans_snapshot(self, k8s_service): + """ + Test case: Delete sandbox cleans up both BatchSandbox and Snapshot + + Purpose: Verify that delete_sandbox removes both resources + """ + sandbox_id = "sandbox-with-snapshot" + + # Mock successful deletion of both resources + k8s_service.workload_provider.delete_workload.return_value = None + k8s_service.snapshot_provider.delete_snapshot.return_value = None + + # Execute + k8s_service.delete_sandbox(sandbox_id) + + # Verify both were deleted + k8s_service.workload_provider.delete_workload.assert_called_once_with( + sandbox_id, k8s_service.namespace + ) + k8s_service.snapshot_provider.delete_snapshot.assert_called_once_with( + sandbox_id, k8s_service.namespace + ) + + def test_delete_sandbox_only_workload_exists(self, k8s_service): + """ + Test case: Delete sandbox when only BatchSandbox exists (no snapshot) + + Purpose: Verify deletion succeeds when only BatchSandbox exists + """ + sandbox_id = "sandbox-no-snapshot" + + # Mock BatchSandbox deletion success + k8s_service.workload_provider.delete_workload.return_value = None + + # Mock snapshot deletion raises 404 (already deleted/doesn't exist) + k8s_service.snapshot_provider.delete_snapshot.return_value = None + + # Execute - should not raise + k8s_service.delete_sandbox(sandbox_id) + + # Verify both were attempted + k8s_service.workload_provider.delete_workload.assert_called_once() + k8s_service.snapshot_provider.delete_snapshot.assert_called_once() + + def test_delete_sandbox_only_snapshot_exists(self, k8s_service): + """ + Test case: Delete sandbox when only Snapshot exists (no BatchSandbox) + + Purpose: Verify deletion succeeds when only Snapshot exists + """ + sandbox_id = "paused-sandbox-no-workload" + + # Mock BatchSandbox deletion fails + k8s_service.workload_provider.delete_workload.side_effect = Exception("not found") + + # Mock snapshot deletion succeeds + k8s_service.snapshot_provider.delete_snapshot.return_value = None + + # Execute - should not raise since snapshot was deleted + k8s_service.delete_sandbox(sandbox_id) + + # Verify both were attempted + k8s_service.workload_provider.delete_workload.assert_called_once() + k8s_service.snapshot_provider.delete_snapshot.assert_called_once() + + def test_delete_sandbox_not_found_raises_404(self, k8s_service): + """ + Test case: Delete sandbox when neither resource exists + + Purpose: Verify 404 is raised when sandbox doesn't exist at all + """ + sandbox_id = "nonexistent-sandbox" + + # Mock both deletions fail + k8s_service.workload_provider.delete_workload.side_effect = Exception("not found") + k8s_service.snapshot_provider.delete_snapshot.side_effect = Exception("not found") + + # Execute and verify + with pytest.raises(HTTPException) as exc_info: + k8s_service.delete_sandbox(sandbox_id) + + assert exc_info.value.status_code == 404 + assert exc_info.value.detail["code"] == SandboxErrorCodes.K8S_SANDBOX_NOT_FOUND + + +class TestDeriveSandboxState: + """_derive_sandbox_state method tests""" + + def test_derive_state_running(self, k8s_service): + batchsandbox = { + "metadata": {"name": "test"}, + "status": {"replicas": 1, "ready": 1, "allocated": 1}, + } + k8s_service.workload_provider.get_status.return_value = { + "state": "Running", + "reason": "POD_READY", + "message": "Pod is running", + } + + state, reason, message = k8s_service._derive_sandbox_state(batchsandbox, None) + + assert state == "Running" + assert reason == "POD_READY" + + def test_derive_state_paused(self, k8s_service): + snapshot = { + "metadata": {"name": "test"}, + "status": {"phase": "Ready"}, + } + + state, reason, message = k8s_service._derive_sandbox_state(None, snapshot) + + assert state == "Paused" + assert reason == "SNAPSHOT_READY" + + def test_derive_state_pausing(self, k8s_service): + batchsandbox = { + "metadata": {"name": "test"}, + "status": {}, + } + snapshot = { + "metadata": {"name": "test"}, + "status": {"phase": "Committing"}, + } + k8s_service.workload_provider.get_status.return_value = { + "state": "Running", + "reason": "POD_READY", + "message": "Pod is running", + } + + state, reason, message = k8s_service._derive_sandbox_state(batchsandbox, snapshot) + + assert state == "Pausing" + assert "COMMITTING" in reason + + def test_derive_state_resuming(self, k8s_service): + batchsandbox = { + "metadata": { + "name": "test", + "annotations": {"sandbox.opensandbox.io/resumed-from-snapshot": "true"}, + }, + "status": {}, + } + k8s_service.workload_provider.get_status.return_value = { + "state": "Pending", + "reason": "POD_SCHEDULED", + "message": "Pod is pending", + } + + state, reason, message = k8s_service._derive_sandbox_state(batchsandbox, None) + + assert state == "Resuming" + + def test_derive_state_not_found(self, k8s_service): + state, reason, message = k8s_service._derive_sandbox_state(None, None) + + assert state == "NotFound" + + def test_derive_state_snapshot_failed(self, k8s_service): + snapshot = { + "metadata": {"name": "test"}, + "status": {"phase": "Failed", "message": "Push failed"}, + } + + state, reason, message = k8s_service._derive_sandbox_state(None, snapshot) + + assert state == "Failed" + assert reason == "SNAPSHOT_FAILED"