diff --git a/.github/labels.yml b/.github/labels.yml index 1df3760f..3285c30f 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -14,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# # GitHub label definitions for Skyhook (NodeWright). # Sync to GitHub with: make labels # diff --git a/agent/go/cmd/agent/main.go b/agent/go/cmd/agent/main.go index 844fb0f9..028d9ff7 100644 --- a/agent/go/cmd/agent/main.go +++ b/agent/go/cmd/agent/main.go @@ -2,6 +2,7 @@ * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/agent/go/internal/config/schemas/v1/skyhook-agent-schema.json b/agent/go/internal/config/schemas/v1/skyhook-agent-schema.json index a9765b7b..3f8188ef 100644 --- a/agent/go/internal/config/schemas/v1/skyhook-agent-schema.json +++ b/agent/go/internal/config/schemas/v1/skyhook-agent-schema.json @@ -111,4 +111,3 @@ "modes" ] } - diff --git a/agent/go/internal/interrupts/interrupts.go b/agent/go/internal/interrupts/interrupts.go index b2b0b0e4..8d6eccaa 100644 --- a/agent/go/internal/interrupts/interrupts.go +++ b/agent/go/internal/interrupts/interrupts.go @@ -1,18 +1,20 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package interrupts diff --git a/agent/go/internal/interrupts/interrupts_test.go b/agent/go/internal/interrupts/interrupts_test.go index e27e3874..35ac5a44 100644 --- a/agent/go/internal/interrupts/interrupts_test.go +++ b/agent/go/internal/interrupts/interrupts_test.go @@ -1,18 +1,20 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package interrupts diff --git a/chart/CHANGELOG.md b/chart/CHANGELOG.md index fcb3ad0c..48bb3c35 100644 --- a/chart/CHANGELOG.md +++ b/chart/CHANGELOG.md @@ -9,7 +9,7 @@ All notable changes to this project will be documented in this file. ### Bug Fixes -- *(chart)* Repair immutable Deployment selector on skyhook->nodewright upgrade +- *(chart)* Repair immutable Deployment selector on skyhook->nodewright upgrade ## [chart/v0.17.0] - 2026-06-12 @@ -34,7 +34,7 @@ fix(chart): agent container path pointing to skyhook not nodewright - Merge pull request #255 from NVIDIA/chore/chart-bump-v0.16.1 chore(chart): bump to v0.16.1 with operator webhook cert deadlock fix -- Bump chart versions +- Bump chart versions ## [chart/v0.16.1] - 2026-05-26 diff --git a/chart/templates/skyhook-crd.yaml b/chart/templates/skyhook-crd.yaml index a0ad1665..fb357832 100644 --- a/chart/templates/skyhook-crd.yaml +++ b/chart/templates/skyhook-crd.yaml @@ -133,6 +133,52 @@ spec: setting for this Skyhook type: boolean type: object + drainConfig: + description: |- + DrainConfig tunes how nodes are drained before running interrupt packages. + If unset, the operator preserves its existing drain behavior. + properties: + deleteEmptyDirData: + default: true + description: |- + DeleteEmptyDirData allows draining pods that use emptyDir volumes. + Defaults to true to preserve the operator's existing behavior. + nullable: true + type: boolean + disableEviction: + default: false + description: |- + DisableEviction bypasses the eviction API and deletes pods directly. + This bypasses PodDisruptionBudgets. + nullable: true + type: boolean + force: + default: true + description: |- + Force allows draining pods not managed by a controller. + Defaults to true to preserve the operator's existing behavior. + nullable: true + type: boolean + gracePeriod: + description: |- + GracePeriod overrides the grace period used on pod eviction/delete. + Unset uses each pod's own terminationGracePeriodSeconds. + nullable: true + type: string + ignoreDaemonSets: + default: true + description: |- + IgnoreDaemonSets skips DaemonSet-managed pods during drain. + Defaults to true to preserve the operator's existing behavior. + nullable: true + type: boolean + timeout: + description: |- + Timeout bounds how long the operator waits for a node to drain. + Zero or unset means no timeout. + nullable: true + type: string + type: object interruptionBudget: description: InterruptionBudget configures how many nodes that match node selectors that allowed to be interrupted at once. diff --git a/docs/interrupt_flow.md b/docs/interrupt_flow.md index 0ea70544..0ba384b6 100644 --- a/docs/interrupt_flow.md +++ b/docs/interrupt_flow.md @@ -81,6 +81,75 @@ The interrupt flow is managed by the `ProcessInterrupt` and `EnsureNodeIsReadyFo - Ensure the node is ready before proceeding with package operations - Handle the timing and sequencing of all stages +## Drain Configuration + +Interrupt-enabled Skyhooks can tune drain behavior with `spec.drainConfig`. +Unset fields preserve the operator's existing behavior: + +```yaml +apiVersion: skyhook.nvidia.com/v1alpha1 +kind: Skyhook +metadata: + name: gpu-mode-switch +spec: + drainConfig: + disableEviction: false + deleteEmptyDirData: true + force: true + ignoreDaemonSets: true + timeout: 10m + gracePeriod: 30s +``` + +The fields map to Kubernetes drain behavior: + +- `disableEviction`: when `true`, pods are deleted directly instead of evicted. This bypasses PodDisruptionBudgets. The default is `false`, so the eviction API is used. +- `deleteEmptyDirData`: when `false`, pods with `emptyDir` volumes block drain. The default is `true`. +- `force`: when `false`, pods without a managing controller block drain. The default is `true`. +- `ignoreDaemonSets`: when `true`, DaemonSet-managed pods are skipped during drain. The default is `true`. +- `timeout`: bounds how long a node may spend draining. Unset or zero means no timeout. When the timeout expires, the node is marked `erroring` and package stages do not proceed on that node. +- `gracePeriod`: overrides the grace period used for eviction or direct deletion. Unset uses each pod's own `terminationGracePeriodSeconds`. + +The operator also skips pods that are already terminating, pods that tolerate +the `node.kubernetes.io/unschedulable` taint, mirror/static pods, and pods in +`kube-system`. These exclusions are not user-configurable. + +Compared to earlier releases, the default drain filter now follows Kubernetes +matching more closely: the unschedulable toleration check uses Kubernetes +`ToleratesTaint` semantics, DaemonSet pods are identified from the controller +owner reference, and already-terminating or mirror/static pods are ignored. + +`podNonInterruptLabels` remains a pre-drain barrier. Matching pods must finish +or move away before the operator starts the configurable drain step. + +### Recovering From a Drain Timeout + +When `spec.drainConfig.timeout` expires, the operator records a `DrainTimeout` +warning event, marks the node and Skyhook `erroring`, and leaves the node +cordoned. The operator stops issuing further evict/delete actions while the +blocking condition remains, so package stages do not proceed on that node. + +To recover, remove the underlying blocker first, such as a PDB with zero allowed +disruptions, an unmanaged pod when `force: false`, or an `emptyDir` pod when +`deleteEmptyDirData: false`. Then reset the failed rollout metadata: + +```bash +kubectl skyhook reset --confirm +``` + +For a single node, use: + +```bash +kubectl skyhook node reset --skyhook --confirm +``` + +If the blocker clears after the timeout without a reset, a later reconcile can +observe the node as drained and continue from current cluster state. Reset is +still the recommended recovery workflow in production because it explicitly +clears the `erroring` status, drain-start metadata, cordon metadata, and batch +state before retrying. If the blocker is still present after reset, the drain +will time out again. + ## Best Practices - Always test interrupt-enabled packages in non-production environments first diff --git a/k8s-tests/chainsaw/skyhook/drain-config/chainsaw-test.yaml b/k8s-tests/chainsaw/skyhook/drain-config/chainsaw-test.yaml new file mode 100644 index 00000000..43e11f58 --- /dev/null +++ b/k8s-tests/chainsaw/skyhook/drain-config/chainsaw-test.yaml @@ -0,0 +1,138 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: drain-config + labels: + pool: interrupt +spec: + timeouts: + assert: 240s + exec: 90s + catch: + - get: + apiVersion: v1 + kind: Node + selector: skyhook.nvidia.com/test-node=skyhooke2e + format: yaml + - get: + apiVersion: skyhook.nvidia.com/v1alpha1 + kind: Skyhook + name: drain-config-disable-eviction + namespace: skyhook + format: yaml + - get: + apiVersion: skyhook.nvidia.com/v1alpha1 + kind: Skyhook + name: drain-config-timeout + namespace: skyhook + format: yaml + - get: + apiVersion: v1 + kind: Pod + namespace: skyhook + selector: app in (drain-config-pdb,drain-config-blocker) + format: yaml + steps: + - name: disable-eviction-bypasses-pdb + description: Verify disableEviction deletes through a zero-disruption PDB and lets the interrupt complete + try: + - script: + content: | + ../skyhook-cli reset drain-config-disable-eviction --confirm 2>/dev/null || true + kubectl -n skyhook delete skyhook drain-config-disable-eviction --ignore-not-found --wait=false + kubectl -n skyhook delete deployment drain-config-pdb --ignore-not-found --wait=false + kubectl -n skyhook delete pdb drain-config-pdb --ignore-not-found + kubectl -n skyhook delete configmap drain-config-original-pod --ignore-not-found + kubectl -n skyhook delete pod -l app=drain-config-pdb --ignore-not-found --wait=false + - apply: + file: disable-eviction.yaml + - script: + content: | + kubectl -n skyhook rollout status deployment/drain-config-pdb --timeout=60s + pod="$(kubectl -n skyhook get pod -l app=drain-config-pdb -o jsonpath='{.items[0].metadata.name}')" + kubectl -n skyhook wait --for=condition=Ready "pod/${pod}" --timeout=30s + kubectl -n skyhook create configmap drain-config-original-pod --from-literal=name="${pod}" --dry-run=client -o yaml | kubectl apply -f - + kubectl -n skyhook wait --for=jsonpath='{.status.disruptionsAllowed}'=0 pdb/drain-config-pdb --timeout=60s + - apply: + file: disable-eviction-skyhook.yaml + - script: + content: | + kubectl -n skyhook wait --for=condition=Ready skyhook/drain-config-disable-eviction --timeout=180s + - script: + content: | + pod="$(kubectl -n skyhook get configmap drain-config-original-pod -o jsonpath='{.data.name}')" + if kubectl -n skyhook get pod "${pod}" >/dev/null 2>&1; then + echo "expected original PDB-protected pod ${pod} to be deleted directly" + exit 1 + fi + finally: + - script: + content: | + ../skyhook-cli reset drain-config-disable-eviction --confirm 2>/dev/null || true + kubectl -n skyhook delete skyhook drain-config-disable-eviction --ignore-not-found --wait=false + kubectl -n skyhook delete deployment drain-config-pdb --ignore-not-found --wait=false + kubectl -n skyhook delete pdb drain-config-pdb --ignore-not-found + kubectl -n skyhook delete configmap drain-config-original-pod --ignore-not-found + kubectl -n skyhook delete pod -l app=drain-config-pdb --ignore-not-found --wait=false + kubectl get nodes -l skyhook.nvidia.com/test-node=skyhooke2e -o name | while read -r node; do kubectl uncordon "${node}" || true; done + - name: timeout-surfaces-erroring + description: Verify timeout marks the node erroring and emits a drain event when drain cannot proceed + try: + - script: + content: | + ../skyhook-cli reset drain-config-timeout --confirm 2>/dev/null || true + kubectl -n skyhook delete skyhook drain-config-timeout --ignore-not-found --wait=false + kubectl -n skyhook delete pod drain-config-blocker --ignore-not-found --wait=false + - apply: + file: timeout.yaml + - wait: + apiVersion: v1 + kind: Pod + name: drain-config-blocker + namespace: skyhook + timeout: 30s + for: + condition: + name: Ready + value: 'true' + - script: + content: | + kubectl -n skyhook wait --for=jsonpath='{.status.status}'=erroring skyhook/drain-config-timeout --timeout=180s + - assert: + file: timeout-assert.yaml + - script: + content: | + for _ in $(seq 1 30); do + events="$(kubectl get events -A -o jsonpath='{range .items[*]}{.metadata.namespace}{"\t"}{.reason}{"\t"}{.action}{"\t"}{.involvedObject.kind}{"\t"}{.involvedObject.name}{"\t"}{.regarding.kind}{"\t"}{.regarding.name}{"\t"}{.message}{"\t"}{.note}{"\n"}{end}' 2>/dev/null || true)" + if printf "%s\n" "${events}" | grep -F "Drain" | grep -F "drain-config-timeout" | grep -F "drain timed out"; then + exit 0 + fi + sleep 2 + done + printf "%s\n" "${events}" + echo "expected drain timeout event for drain-config-timeout" + exit 1 + finally: + - script: + content: | + ../skyhook-cli reset drain-config-timeout --confirm 2>/dev/null || true + kubectl -n skyhook delete skyhook drain-config-timeout --ignore-not-found --wait=false + kubectl -n skyhook delete pod drain-config-blocker --ignore-not-found --wait=false + kubectl get nodes -l skyhook.nvidia.com/test-node=skyhooke2e -o name | while read -r node; do kubectl uncordon "${node}" || true; done diff --git a/k8s-tests/chainsaw/skyhook/drain-config/disable-eviction-skyhook.yaml b/k8s-tests/chainsaw/skyhook/drain-config/disable-eviction-skyhook.yaml new file mode 100644 index 00000000..aa497e72 --- /dev/null +++ b/k8s-tests/chainsaw/skyhook/drain-config/disable-eviction-skyhook.yaml @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: skyhook.nvidia.com/v1alpha1 +kind: Skyhook +metadata: + name: drain-config-disable-eviction + namespace: skyhook +spec: + drainConfig: + disableEviction: true + gracePeriod: 0s + nodeSelectors: + matchLabels: + skyhook.nvidia.com/test-node: skyhooke2e + interruptionBudget: + count: 1 + packages: + drain-pdb: + version: "1.2.3" + image: ghcr.io/nvidia/skyhook/agentless + uninstall: + enabled: false + apply: false + interrupt: + type: service + services: [rsyslog] + env: + - name: SLEEP_LEN + value: "1" diff --git a/k8s-tests/chainsaw/skyhook/drain-config/disable-eviction.yaml b/k8s-tests/chainsaw/skyhook/drain-config/disable-eviction.yaml new file mode 100644 index 00000000..a058d00f --- /dev/null +++ b/k8s-tests/chainsaw/skyhook/drain-config/disable-eviction.yaml @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: drain-config-pdb + namespace: skyhook +spec: + replicas: 1 + selector: + matchLabels: + app: drain-config-pdb + template: + metadata: + labels: + app: drain-config-pdb + spec: + terminationGracePeriodSeconds: 5 + nodeSelector: + skyhook.nvidia.com/test-node: skyhooke2e + containers: + - name: workload + image: ghcr.io/nvidia/skyhook/agentless:3.2.3 + imagePullPolicy: IfNotPresent + command: ["sh", "-c", "sleep 1000"] +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: drain-config-pdb + namespace: skyhook +spec: + minAvailable: 1 + selector: + matchLabels: + app: drain-config-pdb diff --git a/k8s-tests/chainsaw/skyhook/drain-config/timeout-assert.yaml b/k8s-tests/chainsaw/skyhook/drain-config/timeout-assert.yaml new file mode 100644 index 00000000..8851414c --- /dev/null +++ b/k8s-tests/chainsaw/skyhook/drain-config/timeout-assert.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Node +metadata: + labels: + skyhook.nvidia.com/test-node: skyhooke2e + skyhook.nvidia.com/status_drain-config-timeout: erroring + annotations: + skyhook.nvidia.com/status_drain-config-timeout: erroring + (contains(keys(@), 'skyhook.nvidia.com/drainStart_drain-config-timeout')): true +spec: + taints: + - effect: NoSchedule + key: node.kubernetes.io/unschedulable +--- +apiVersion: skyhook.nvidia.com/v1alpha1 +kind: Skyhook +metadata: + name: drain-config-timeout +status: + status: erroring + nodeStatus: + (values(@)): + - erroring diff --git a/k8s-tests/chainsaw/skyhook/drain-config/timeout.yaml b/k8s-tests/chainsaw/skyhook/drain-config/timeout.yaml new file mode 100644 index 00000000..94f801ed --- /dev/null +++ b/k8s-tests/chainsaw/skyhook/drain-config/timeout.yaml @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v1 +kind: Pod +metadata: + name: drain-config-blocker + namespace: skyhook + labels: + app: drain-config-blocker +spec: + terminationGracePeriodSeconds: 5 + restartPolicy: Never + nodeSelector: + skyhook.nvidia.com/test-node: skyhooke2e + containers: + - name: workload + image: ghcr.io/nvidia/skyhook/agentless:3.2.3 + imagePullPolicy: IfNotPresent + command: ["sh", "-c", "sleep 1000"] +--- +apiVersion: skyhook.nvidia.com/v1alpha1 +kind: Skyhook +metadata: + name: drain-config-timeout + namespace: skyhook +spec: + drainConfig: + force: false + timeout: 5s + nodeSelectors: + matchLabels: + skyhook.nvidia.com/test-node: skyhooke2e + interruptionBudget: + count: 1 + packages: + drain-timeout: + version: "1.2.3" + image: ghcr.io/nvidia/skyhook/agentless + uninstall: + enabled: false + apply: false + interrupt: + type: service + services: [rsyslog] + env: + - name: SLEEP_LEN + value: "1" diff --git a/operator/RELEASE_NOTES.md b/operator/RELEASE_NOTES.md index 895b17c4..9ce85cba 100644 --- a/operator/RELEASE_NOTES.md +++ b/operator/RELEASE_NOTES.md @@ -41,6 +41,20 @@ For the full commit-level log see CHANGELOG.md. the reboot pending to be retried. Also fixes `Reset()` deleting the cordon annotation with a key missing the Skyhook name. +### New Features + +- Add `spec.drainConfig` so interrupt drains can tune eviction, direct deletion, + emptyDir handling, unmanaged-pod handling, DaemonSet skipping, timeout, and + grace-period behavior. + +### Changed + +- Align the default interrupt drain pod filter with Kubernetes drain semantics: + already-terminating pods and mirror/static pods are skipped, unschedulable + tolerations use Kubernetes `ToleratesTaint` matching, and DaemonSet pods are + identified from the controller owner reference instead of the previous owner + reference count heuristic. + ## operator/v0.16.1 - 2026-05-22 ### Bug Fixes diff --git a/operator/api/v1alpha1/skyhook_types.go b/operator/api/v1alpha1/skyhook_types.go index e37b04b1..cad8dffd 100644 --- a/operator/api/v1alpha1/skyhook_types.go +++ b/operator/api/v1alpha1/skyhook_types.go @@ -62,6 +62,11 @@ type SkyhookSpec struct { // InterruptionBudget configures how many nodes that match node selectors that allowed to be interrupted at once. InterruptionBudget InterruptionBudget `json:"interruptionBudget,omitempty"` + // DrainConfig tunes how nodes are drained before running interrupt packages. + // If unset, the operator preserves its existing drain behavior. + // +optional + DrainConfig *DrainConfig `json:"drainConfig,omitempty"` + // Packages are the DAG of packages to be applied to nodes. Packages Packages `json:"packages,omitempty"` @@ -205,6 +210,64 @@ func (i *InterruptionBudget) Validate() error { return nil } +type DrainConfig struct { + // DisableEviction bypasses the eviction API and deletes pods directly. + // This bypasses PodDisruptionBudgets. + // +optional + //+kubebuilder:default=false + //+nullable + DisableEviction *bool `json:"disableEviction,omitempty"` + + // DeleteEmptyDirData allows draining pods that use emptyDir volumes. + // Defaults to true to preserve the operator's existing behavior. + // +optional + //+kubebuilder:default=true + //+nullable + DeleteEmptyDirData *bool `json:"deleteEmptyDirData,omitempty"` + + // Force allows draining pods not managed by a controller. + // Defaults to true to preserve the operator's existing behavior. + // +optional + //+kubebuilder:default=true + //+nullable + Force *bool `json:"force,omitempty"` + + // IgnoreDaemonSets skips DaemonSet-managed pods during drain. + // Defaults to true to preserve the operator's existing behavior. + // +optional + //+kubebuilder:default=true + //+nullable + IgnoreDaemonSets *bool `json:"ignoreDaemonSets,omitempty"` + + // Timeout bounds how long the operator waits for a node to drain. + // Zero or unset means no timeout. + // +optional + //+nullable + Timeout *metav1.Duration `json:"timeout,omitempty"` + + // GracePeriod overrides the grace period used on pod eviction/delete. + // Unset uses each pod's own terminationGracePeriodSeconds. + // +optional + //+nullable + GracePeriod *metav1.Duration `json:"gracePeriod,omitempty"` +} + +func (d *DrainConfig) Validate() error { + if d == nil { + return nil + } + + if d.Timeout != nil && d.Timeout.Duration < 0 { + return errors.New("drainConfig.timeout must be greater than or equal to 0") + } + + if d.GracePeriod != nil && d.GracePeriod.Duration < 0 { + return errors.New("drainConfig.gracePeriod must be greater than or equal to 0") + } + + return nil +} + type PackageRef struct { // Name of the package. Do not set unless you know what your doing. Comes from map key. //+optional diff --git a/operator/api/v1alpha1/skyhook_webhook.go b/operator/api/v1alpha1/skyhook_webhook.go index d3c563ed..b91c351c 100644 --- a/operator/api/v1alpha1/skyhook_webhook.go +++ b/operator/api/v1alpha1/skyhook_webhook.go @@ -235,6 +235,10 @@ func (r *Skyhook) Validate() error { return err } + if err := r.Spec.DrainConfig.Validate(); err != nil { + return err + } + // DeploymentPolicy and InterruptionBudget are mutually exclusive if r.Spec.DeploymentPolicy != "" && (r.Spec.InterruptionBudget.Percent != nil || r.Spec.InterruptionBudget.Count != nil) { return fmt.Errorf("deploymentPolicy and interruptionBudget are mutually exclusive") diff --git a/operator/api/v1alpha1/skyhook_webhook_test.go b/operator/api/v1alpha1/skyhook_webhook_test.go index 58bd1635..447b473e 100644 --- a/operator/api/v1alpha1/skyhook_webhook_test.go +++ b/operator/api/v1alpha1/skyhook_webhook_test.go @@ -21,6 +21,7 @@ package v1alpha1 import ( "encoding/json" "strings" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -662,6 +663,26 @@ var _ = Describe("Skyhook Webhook", func() { res5.MemoryLimit = resource.MustParse("-1Mi") Expect(mkSkyhook(res5).Validate()).NotTo(Succeed()) }) + + It("should validate drain config durations", func() { + skyhook := &Skyhook{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: SkyhookSpec{ + DrainConfig: &DrainConfig{ + Timeout: &metav1.Duration{Duration: time.Minute}, + GracePeriod: &metav1.Duration{Duration: 30 * time.Second}, + }, + }, + } + Expect(skyhook.Validate()).To(Succeed()) + + skyhook.Spec.DrainConfig.Timeout = &metav1.Duration{Duration: -time.Second} + Expect(skyhook.Validate()).NotTo(Succeed()) + + skyhook.Spec.DrainConfig.Timeout = nil + skyhook.Spec.DrainConfig.GracePeriod = &metav1.Duration{Duration: -time.Second} + Expect(skyhook.Validate()).NotTo(Succeed()) + }) }) It("packages should UnmarshalJSON without rewriting the image", func() { diff --git a/operator/api/v1alpha1/zz_generated.deepcopy.go b/operator/api/v1alpha1/zz_generated.deepcopy.go index aca9129e..d131f605 100644 --- a/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -246,6 +246,51 @@ func (in *DeploymentStrategy) DeepCopy() *DeploymentStrategy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DrainConfig) DeepCopyInto(out *DrainConfig) { + *out = *in + if in.DisableEviction != nil { + in, out := &in.DisableEviction, &out.DisableEviction + *out = new(bool) + **out = **in + } + if in.DeleteEmptyDirData != nil { + in, out := &in.DeleteEmptyDirData, &out.DeleteEmptyDirData + *out = new(bool) + **out = **in + } + if in.Force != nil { + in, out := &in.Force, &out.Force + *out = new(bool) + **out = **in + } + if in.IgnoreDaemonSets != nil { + in, out := &in.IgnoreDaemonSets, &out.IgnoreDaemonSets + *out = new(bool) + **out = **in + } + if in.Timeout != nil { + in, out := &in.Timeout, &out.Timeout + *out = new(metav1.Duration) + **out = **in + } + if in.GracePeriod != nil { + in, out := &in.GracePeriod, &out.GracePeriod + *out = new(metav1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DrainConfig. +func (in *DrainConfig) DeepCopy() *DrainConfig { + if in == nil { + return nil + } + out := new(DrainConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ExponentialStrategy) DeepCopyInto(out *ExponentialStrategy) { *out = *in @@ -652,6 +697,11 @@ func (in *SkyhookSpec) DeepCopyInto(out *SkyhookSpec) { (*in).DeepCopyInto(*out) } in.InterruptionBudget.DeepCopyInto(&out.InterruptionBudget) + if in.DrainConfig != nil { + in, out := &in.DrainConfig, &out.DrainConfig + *out = new(DrainConfig) + (*in).DeepCopyInto(*out) + } if in.Packages != nil { in, out := &in.Packages, &out.Packages *out = make(Packages, len(*in)) diff --git a/operator/cmd/cli/RELEASE_NOTES.md b/operator/cmd/cli/RELEASE_NOTES.md index 492fa7fe..1f34c44f 100644 --- a/operator/cmd/cli/RELEASE_NOTES.md +++ b/operator/cmd/cli/RELEASE_NOTES.md @@ -2,3 +2,12 @@ Human-authored highlights, behavior changes, and upgrade steps for the CLI. For the full commit-level log see CHANGELOG.md. + +## Unreleased + +### Changed + +- `reset` and `node reset` now select nodes with any resettable Skyhook + metadata, including status, cordon, and drain-start metadata. Nodes with a + malformed `nodeState` annotation are reset with an empty package state instead + of being skipped. diff --git a/operator/cmd/cli/app/node/node_reset.go b/operator/cmd/cli/app/node/node_reset.go index 3d8da812..3f4b8c12 100644 --- a/operator/cmd/cli/app/node/node_reset.go +++ b/operator/cmd/cli/app/node/node_reset.go @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * @@ -40,6 +40,37 @@ type nodeResetOptions struct { confirm bool } +func resetAnnotationKeys(skyhookName string) []string { + return []string{ + nodeStateAnnotationPrefix + skyhookName, + statusAnnotationPrefix + skyhookName, + cordonAnnotationPrefix + skyhookName, + drainStartAnnotationPrefix + skyhookName, + versionAnnotationPrefix + skyhookName, + autoTaintAnnotationPrefix + skyhookName, + } +} + +func resetLabelKeys(skyhookName string) []string { + return []string{ + statusLabelPrefix + skyhookName, + } +} + +func hasResettableMetadata(annotations, labels map[string]string, annotationKeys, labelKeys []string) bool { + for _, annotationKey := range annotationKeys { + if _, ok := annotations[annotationKey]; ok { + return true + } + } + for _, labelKey := range labelKeys { + if _, ok := labels[labelKey]; ok { + return true + } + } + return false +} + // BindToCmd binds the options to the command flags func (o *nodeResetOptions) BindToCmd(cmd *cobra.Command) { cmd.Flags().StringVar(&o.skyhookName, "skyhook", "", "Name of the Skyhook CR (required)") @@ -124,6 +155,8 @@ func runNodeReset(ctx context.Context, cmd *cobra.Command, kubeClient *client.Cl // Find nodes that have the specified Skyhook annotation annotationKey := nodeStateAnnotationPrefix + opts.skyhookName + annotationKeys := resetAnnotationKeys(opts.skyhookName) + labelKeys := resetLabelKeys(opts.skyhookName) nodesToReset := make([]string, 0, len(matchedNodes)) nodeStates := make(map[string]v1alpha1.NodeState) @@ -131,17 +164,17 @@ func runNodeReset(ctx context.Context, cmd *cobra.Command, kubeClient *client.Cl idx := nodeMap[nodeName] node := &nodeList.Items[idx] - annotation, ok := node.Annotations[annotationKey] - if !ok { + if !hasResettableMetadata(node.Annotations, node.Labels, annotationKeys, labelKeys) { continue } - var nodeState v1alpha1.NodeState - if err := json.Unmarshal([]byte(annotation), &nodeState); err != nil { - if cliCtx.GlobalFlags.Verbose { - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Warning: skipping node %q - invalid annotation: %v\n", nodeName, err) + nodeState := v1alpha1.NodeState{} + if annotation, ok := node.Annotations[annotationKey]; ok { + if err := json.Unmarshal([]byte(annotation), &nodeState); err != nil { + if cliCtx.GlobalFlags.Verbose { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Warning: node %q has invalid node state annotation; resetting metadata with empty package state: %v\n", nodeName, err) + } } - continue } nodesToReset = append(nodesToReset, nodeName) @@ -191,10 +224,22 @@ func runNodeReset(ctx context.Context, cmd *cobra.Command, kubeClient *client.Cl successCount := 0 for _, nodeName := range nodesToReset { - if err := utils.RemoveNodeAnnotation(ctx, kubeClient.Kubernetes(), nodeName, annotationKey); err != nil { - updateErrors = append(updateErrors, fmt.Sprintf("%s: %v", nodeName, err)) + nodeHasError := false + for _, annotationKey := range annotationKeys { + if err := utils.RemoveNodeAnnotation(ctx, kubeClient.Kubernetes(), nodeName, annotationKey); err != nil { + updateErrors = append(updateErrors, fmt.Sprintf("%s: %v", nodeName, err)) + nodeHasError = true + break + } + } + if nodeHasError { continue } + for _, labelKey := range labelKeys { + if err := utils.RemoveNodeLabel(ctx, kubeClient.Kubernetes(), nodeName, labelKey); err != nil && cliCtx.GlobalFlags.Verbose { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Warning: failed to remove label %q from node %q: %v\n", labelKey, nodeName, err) + } + } successCount++ } diff --git a/operator/cmd/cli/app/node/node_reset_test.go b/operator/cmd/cli/app/node/node_reset_test.go index 5faa3c62..a108814d 100644 --- a/operator/cmd/cli/app/node/node_reset_test.go +++ b/operator/cmd/cli/app/node/node_reset_test.go @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * @@ -141,12 +141,14 @@ var _ = Describe("Node Reset Command", func() { } nodeStateJSON, _ := json.Marshal(nodeState) annotationKey := nodeStateAnnotationPrefix + "my-skyhook" + drainStartAnnotationKey := drainStartAnnotationPrefix + "my-skyhook" node := &corev1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "worker-1", Annotations: map[string]string{ - annotationKey: string(nodeStateJSON), + annotationKey: string(nodeStateJSON), + drainStartAnnotationKey: "2026-06-02T12:00:00Z", }, }, } @@ -163,6 +165,87 @@ var _ = Describe("Node Reset Command", func() { Expect(err).NotTo(HaveOccurred()) _, exists := updatedNode.Annotations[annotationKey] Expect(exists).To(BeFalse()) + _, exists = updatedNode.Annotations[drainStartAnnotationKey] + Expect(exists).To(BeFalse()) + }) + + It("should reset drain metadata before node state exists", func() { + drainStartAnnotationKey := drainStartAnnotationPrefix + "my-skyhook" + statusAnnotationKey := statusAnnotationPrefix + "my-skyhook" + + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-1", + Annotations: map[string]string{ + drainStartAnnotationKey: "2026-06-02T12:00:00Z", + statusAnnotationKey: "erroring", + }, + }, + } + _, err := mockKube.CoreV1().Nodes().Create(gocontext.Background(), node, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + opts := &nodeResetOptions{skyhookName: "my-skyhook", confirm: true} + err = runNodeReset(gocontext.Background(), cmd, kubeClient, []string{"worker-1"}, opts, cliCtx) + Expect(err).NotTo(HaveOccurred()) + Expect(output.String()).To(ContainSubstring("Successfully reset 1 node")) + + updatedNode, err := mockKube.CoreV1().Nodes().Get(gocontext.Background(), "worker-1", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + _, exists := updatedNode.Annotations[drainStartAnnotationKey] + Expect(exists).To(BeFalse()) + _, exists = updatedNode.Annotations[statusAnnotationKey] + Expect(exists).To(BeFalse()) + }) + + It("should reset a node with only a status label", func() { + statusLabelKey := statusLabelPrefix + "my-skyhook" + + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-1", + Labels: map[string]string{ + statusLabelKey: "erroring", + }, + }, + } + _, err := mockKube.CoreV1().Nodes().Create(gocontext.Background(), node, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + opts := &nodeResetOptions{skyhookName: "my-skyhook", confirm: true} + err = runNodeReset(gocontext.Background(), cmd, kubeClient, []string{"worker-1"}, opts, cliCtx) + Expect(err).NotTo(HaveOccurred()) + Expect(output.String()).To(ContainSubstring("Successfully reset 1 node")) + + updatedNode, err := mockKube.CoreV1().Nodes().Get(gocontext.Background(), "worker-1", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + _, exists := updatedNode.Labels[statusLabelKey] + Expect(exists).To(BeFalse()) + }) + + It("should reset a node with an invalid node state annotation", func() { + annotationKey := nodeStateAnnotationPrefix + "my-skyhook" + + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-1", + Annotations: map[string]string{ + annotationKey: "invalid-json", + }, + }, + } + _, err := mockKube.CoreV1().Nodes().Create(gocontext.Background(), node, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + opts := &nodeResetOptions{skyhookName: "my-skyhook", confirm: true} + err = runNodeReset(gocontext.Background(), cmd, kubeClient, []string{"worker-1"}, opts, cliCtx) + Expect(err).NotTo(HaveOccurred()) + Expect(output.String()).To(ContainSubstring("Successfully reset 1 node")) + + updatedNode, err := mockKube.CoreV1().Nodes().Get(gocontext.Background(), "worker-1", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + _, exists := updatedNode.Annotations[annotationKey] + Expect(exists).To(BeFalse()) }) It("should respect dry-run flag", func() { diff --git a/operator/cmd/cli/app/node/node_status.go b/operator/cmd/cli/app/node/node_status.go index a7f0e790..3a9ec202 100644 --- a/operator/cmd/cli/app/node/node_status.go +++ b/operator/cmd/cli/app/node/node_status.go @@ -35,7 +35,15 @@ import ( "github.com/NVIDIA/nodewright/operator/internal/cli/utils" ) -const nodeStateAnnotationPrefix = v1alpha1.METADATA_PREFIX + "/nodeState_" +const ( + nodeStateAnnotationPrefix = v1alpha1.METADATA_PREFIX + "/nodeState_" + statusAnnotationPrefix = v1alpha1.METADATA_PREFIX + "/status_" + cordonAnnotationPrefix = v1alpha1.METADATA_PREFIX + "/cordon_" + drainStartAnnotationPrefix = v1alpha1.METADATA_PREFIX + "/drainStart_" + versionAnnotationPrefix = v1alpha1.METADATA_PREFIX + "/version_" + autoTaintAnnotationPrefix = v1alpha1.METADATA_PREFIX + "/autoTaint_" + statusLabelPrefix = v1alpha1.METADATA_PREFIX + "/status_" +) // nodeStatusOptions holds the options for the node status command type nodeStatusOptions struct { diff --git a/operator/cmd/cli/app/reset.go b/operator/cmd/cli/app/reset.go index 4043a2fd..5d461ad3 100644 --- a/operator/cmd/cli/app/reset.go +++ b/operator/cmd/cli/app/reset.go @@ -26,6 +26,7 @@ import ( "strings" "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/NVIDIA/nodewright/operator/api/v1alpha1" "github.com/NVIDIA/nodewright/operator/internal/cli/client" @@ -34,12 +35,13 @@ import ( ) const ( - nodeStateAnnotationPrefix = v1alpha1.METADATA_PREFIX + "/nodeState_" - statusAnnotationPrefix = v1alpha1.METADATA_PREFIX + "/status_" - cordonAnnotationPrefix = v1alpha1.METADATA_PREFIX + "/cordon_" - versionAnnotationPrefix = v1alpha1.METADATA_PREFIX + "/version_" - autoTaintAnnotationPrefix = v1alpha1.METADATA_PREFIX + "/autoTaint_" - statusLabelPrefix = v1alpha1.METADATA_PREFIX + "/status_" + nodeStateAnnotationPrefix = v1alpha1.METADATA_PREFIX + "/nodeState_" + statusAnnotationPrefix = v1alpha1.METADATA_PREFIX + "/status_" + cordonAnnotationPrefix = v1alpha1.METADATA_PREFIX + "/cordon_" + drainStartAnnotationPrefix = v1alpha1.METADATA_PREFIX + "/drainStart_" + versionAnnotationPrefix = v1alpha1.METADATA_PREFIX + "/version_" + autoTaintAnnotationPrefix = v1alpha1.METADATA_PREFIX + "/autoTaint_" + statusLabelPrefix = v1alpha1.METADATA_PREFIX + "/status_" ) // resetOptions holds the options for the reset command @@ -49,6 +51,37 @@ type resetOptions struct { pkg string // --package [:] } +func resetAnnotationKeys(skyhookName string) []string { + return []string{ + nodeStateAnnotationPrefix + skyhookName, + statusAnnotationPrefix + skyhookName, + cordonAnnotationPrefix + skyhookName, + drainStartAnnotationPrefix + skyhookName, + versionAnnotationPrefix + skyhookName, + autoTaintAnnotationPrefix + skyhookName, + } +} + +func resetLabelKeys(skyhookName string) []string { + return []string{ + statusLabelPrefix + skyhookName, + } +} + +func hasResettableMetadata(annotations, labels map[string]string, annotationKeys, labelKeys []string) bool { + for _, annotationKey := range annotationKeys { + if _, ok := annotations[annotationKey]; ok { + return true + } + } + for _, labelKey := range labelKeys { + if _, ok := labels[labelKey]; ok { + return true + } + } + return false +} + // NewResetCmd creates the reset command func NewResetCmd(ctx *cliContext.CLIContext) *cobra.Command { opts := &resetOptions{} @@ -97,33 +130,53 @@ existing batch state.`, } func runReset(ctx context.Context, cmd *cobra.Command, kubeClient *client.Client, skyhookName string, opts *resetOptions, cliCtx *cliContext.CLIContext) error { - nodeStates, err := utils.ListNodesWithSkyhookState(ctx, kubeClient.Kubernetes(), skyhookName, "") - if err != nil { - // nil map signals a list-from-apiserver failure (e.g. RBAC denied, - // unreachable apiserver); the helper returns an initialized map - // even when only parse failures occurred, so we use map identity - // to distinguish hard failures from per-node parse warnings. - if nodeStates == nil { - return fmt.Errorf("listing nodes: %w", err) - } - if cliCtx.GlobalFlags.Verbose { - _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Warning: %v\n", err) - } - } - nodesToReset := make([]string, 0, len(nodeStates)) - for name := range nodeStates { - nodesToReset = append(nodesToReset, name) - } - sort.Strings(nodesToReset) - // why: branch on --package BEFORE the empty-nodes early-return so the // per-package path can version-gate on the operator. Otherwise a `reset // --package` against an unsupported old operator that happens to have // zero stateful nodes would silently succeed instead of erroring. if opts.pkg != "" { + nodeStates, err := utils.ListNodesWithSkyhookState(ctx, kubeClient.Kubernetes(), skyhookName, "") + if err != nil { + if nodeStates == nil { + return fmt.Errorf("listing nodes: %w", err) + } + if cliCtx.GlobalFlags.Verbose { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Warning: %v\n", err) + } + } return runPackageReset(ctx, cmd, kubeClient, skyhookName, nodeStates, opts, cliCtx) } + nodeList, err := kubeClient.Kubernetes().CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("listing nodes: %w", err) + } + + annotationKey := nodeStateAnnotationPrefix + skyhookName + annotationKeys := resetAnnotationKeys(skyhookName) + labelKeys := resetLabelKeys(skyhookName) + nodesToReset := make([]string, 0) + nodeStates := make(map[string]v1alpha1.NodeState) + + for _, node := range nodeList.Items { + if !hasResettableMetadata(node.Annotations, node.Labels, annotationKeys, labelKeys) { + continue + } + + nodeState := v1alpha1.NodeState{} + if annotation, ok := node.Annotations[annotationKey]; ok { + if err := json.Unmarshal([]byte(annotation), &nodeState); err != nil { + if cliCtx.GlobalFlags.Verbose { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Warning: node %q has invalid node state annotation; resetting metadata with empty package state: %v\n", node.Name, err) + } + } + } + + nodesToReset = append(nodesToReset, node.Name) + nodeStates[node.Name] = nodeState + } + sort.Strings(nodesToReset) + if len(nodesToReset) == 0 { _, _ = fmt.Fprintf(cmd.OutOrStdout(), "No nodes have state for Skyhook %q\n", skyhookName) return nil @@ -193,16 +246,8 @@ func resetNodeAnnotations(ctx context.Context, cmd *cobra.Command, kubeClient *c successCount := 0 for _, nodeName := range nodesToReset { - annotationsToRemove := []string{ - nodeStateAnnotationPrefix + skyhookName, - statusAnnotationPrefix + skyhookName, - cordonAnnotationPrefix + skyhookName, - versionAnnotationPrefix + skyhookName, - autoTaintAnnotationPrefix + skyhookName, - } - labelsToRemove := []string{ - statusLabelPrefix + skyhookName, - } + annotationsToRemove := resetAnnotationKeys(skyhookName) + labelsToRemove := resetLabelKeys(skyhookName) // Try to remove the main nodeState annotation first - this is the critical one mainAnnotationKey := nodeStateAnnotationPrefix + skyhookName diff --git a/operator/cmd/cli/app/reset_test.go b/operator/cmd/cli/app/reset_test.go index dd46f401..656a67a8 100644 --- a/operator/cmd/cli/app/reset_test.go +++ b/operator/cmd/cli/app/reset_test.go @@ -172,6 +172,7 @@ var _ = Describe("Reset Command", func() { annotationKey := nodeStateAnnotationPrefix + skyhookName statusAnnotationKey := statusAnnotationPrefix + skyhookName cordonAnnotationKey := cordonAnnotationPrefix + skyhookName + drainStartAnnotationKey := drainStartAnnotationPrefix + skyhookName versionAnnotationKey := versionAnnotationPrefix + skyhookName statusLabelKey := statusLabelPrefix + skyhookName @@ -179,10 +180,11 @@ var _ = Describe("Reset Command", func() { ObjectMeta: metav1.ObjectMeta{ Name: "worker-1", Annotations: map[string]string{ - annotationKey: string(nodeStateJSON), - statusAnnotationKey: "complete", - cordonAnnotationKey: "true", - versionAnnotationKey: "1.0.0", + annotationKey: string(nodeStateJSON), + statusAnnotationKey: "complete", + cordonAnnotationKey: "true", + drainStartAnnotationKey: "2026-06-02T12:00:00Z", + versionAnnotationKey: "1.0.0", }, Labels: map[string]string{ statusLabelKey: "complete", @@ -216,6 +218,8 @@ var _ = Describe("Reset Command", func() { Expect(exists).To(BeFalse()) _, exists = updatedNode1.Annotations[cordonAnnotationKey] Expect(exists).To(BeFalse()) + _, exists = updatedNode1.Annotations[drainStartAnnotationKey] + Expect(exists).To(BeFalse()) _, exists = updatedNode1.Annotations[versionAnnotationKey] Expect(exists).To(BeFalse()) _, exists = updatedNode1.Labels[statusLabelKey] @@ -228,6 +232,60 @@ var _ = Describe("Reset Command", func() { Expect(exists).To(BeFalse()) }) + It("should reset nodes with drain metadata before node state exists", func() { + drainStartAnnotationKey := drainStartAnnotationPrefix + skyhookName + statusAnnotationKey := statusAnnotationPrefix + skyhookName + + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-1", + Annotations: map[string]string{ + drainStartAnnotationKey: "2026-06-02T12:00:00Z", + statusAnnotationKey: "erroring", + }, + }, + } + _, err := mockKube.CoreV1().Nodes().Create(gocontext.Background(), node, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + opts := &resetOptions{confirm: true} + err = runReset(gocontext.Background(), cmd, kubeClient, skyhookName, opts, cliCtx) + Expect(err).NotTo(HaveOccurred()) + Expect(output.String()).To(ContainSubstring("Successfully reset 1 node")) + + updatedNode, err := mockKube.CoreV1().Nodes().Get(gocontext.Background(), "worker-1", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + _, exists := updatedNode.Annotations[drainStartAnnotationKey] + Expect(exists).To(BeFalse()) + _, exists = updatedNode.Annotations[statusAnnotationKey] + Expect(exists).To(BeFalse()) + }) + + It("should reset nodes with only a status label", func() { + statusLabelKey := statusLabelPrefix + skyhookName + + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-1", + Labels: map[string]string{ + statusLabelKey: "erroring", + }, + }, + } + _, err := mockKube.CoreV1().Nodes().Create(gocontext.Background(), node, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + opts := &resetOptions{confirm: true} + err = runReset(gocontext.Background(), cmd, kubeClient, skyhookName, opts, cliCtx) + Expect(err).NotTo(HaveOccurred()) + Expect(output.String()).To(ContainSubstring("Successfully reset 1 node")) + + updatedNode, err := mockKube.CoreV1().Nodes().Get(gocontext.Background(), "worker-1", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + _, exists := updatedNode.Labels[statusLabelKey] + Expect(exists).To(BeFalse()) + }) + It("should respect dry-run flag", func() { nodeState := v1alpha1.NodeState{ "pkg1|1.0": {Name: "pkg1", Version: "1.0", Stage: v1alpha1.StageApply, State: v1alpha1.StateComplete}, @@ -325,7 +383,12 @@ var _ = Describe("Reset Command", func() { opts := &resetOptions{confirm: true} err = runReset(gocontext.Background(), cmd, kubeClient, skyhookName, opts, cliCtx) Expect(err).NotTo(HaveOccurred()) - Expect(output.String()).To(ContainSubstring("No nodes have state")) + Expect(output.String()).To(ContainSubstring("Successfully reset 1 node")) + + updatedNode, err := mockKube.CoreV1().Nodes().Get(gocontext.Background(), "worker-1", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + _, exists := updatedNode.Annotations[annotationKey] + Expect(exists).To(BeFalse()) }) It("should continue even if some annotations/labels don't exist", func() { diff --git a/operator/config/crd/bases/skyhook.nvidia.com_skyhooks.yaml b/operator/config/crd/bases/skyhook.nvidia.com_skyhooks.yaml index f91a943c..f7d40685 100644 --- a/operator/config/crd/bases/skyhook.nvidia.com_skyhooks.yaml +++ b/operator/config/crd/bases/skyhook.nvidia.com_skyhooks.yaml @@ -138,6 +138,52 @@ spec: setting for this Skyhook type: boolean type: object + drainConfig: + description: |- + DrainConfig tunes how nodes are drained before running interrupt packages. + If unset, the operator preserves its existing drain behavior. + properties: + deleteEmptyDirData: + default: true + description: |- + DeleteEmptyDirData allows draining pods that use emptyDir volumes. + Defaults to true to preserve the operator's existing behavior. + nullable: true + type: boolean + disableEviction: + default: false + description: |- + DisableEviction bypasses the eviction API and deletes pods directly. + This bypasses PodDisruptionBudgets. + nullable: true + type: boolean + force: + default: true + description: |- + Force allows draining pods not managed by a controller. + Defaults to true to preserve the operator's existing behavior. + nullable: true + type: boolean + gracePeriod: + description: |- + GracePeriod overrides the grace period used on pod eviction/delete. + Unset uses each pod's own terminationGracePeriodSeconds. + nullable: true + type: string + ignoreDaemonSets: + default: true + description: |- + IgnoreDaemonSets skips DaemonSet-managed pods during drain. + Defaults to true to preserve the operator's existing behavior. + nullable: true + type: boolean + timeout: + description: |- + Timeout bounds how long the operator waits for a node to drain. + Zero or unset means no timeout. + nullable: true + type: string + type: object interruptionBudget: description: InterruptionBudget configures how many nodes that match node selectors that allowed to be interrupted at once. diff --git a/operator/internal/controller/skyhook_controller.go b/operator/internal/controller/skyhook_controller.go index 0ee3ba1b..cdb3ddbb 100644 --- a/operator/internal/controller/skyhook_controller.go +++ b/operator/internal/controller/skyhook_controller.go @@ -35,6 +35,7 @@ import ( "github.com/NVIDIA/nodewright/operator/api/v1alpha1" "github.com/NVIDIA/nodewright/operator/internal/dal" + "github.com/NVIDIA/nodewright/operator/internal/drain" "github.com/NVIDIA/nodewright/operator/internal/version" "github.com/NVIDIA/nodewright/operator/internal/wrapper" "github.com/go-logr/logr" @@ -1276,7 +1277,8 @@ func (r *SkyhookReconciler) HandleConfigUpdates(ctx context.Context, clusterStat for _, node := range skyhook.GetNodes() { exists, err := r.PodExists(ctx, node.GetNode().Name, skyhook.GetSkyhook().Name, &_package) if err != nil { - return false, false, err + return false, false, fmt.Errorf("checking package pod existence on node %s for package %s: %w", + node.GetNode().Name, _package.GetUniqueName(), err) } if !exists && node.IsPackageComplete(_package) { @@ -1303,14 +1305,16 @@ func (r *SkyhookReconciler) HandleConfigUpdates(ctx context.Context, clusterStat }, ) if err != nil { - return false, false, err + return false, false, fmt.Errorf("listing package pods on node %s for package %s: %w", + node.GetNode().Name, _package.GetUniqueName(), err) } if pods != nil { for _, pod := range pods.Items { err := r.Delete(ctx, &pod) if err != nil { - return false, false, err + return false, false, fmt.Errorf("deleting erroring pod %s/%s on node %s: %w", + pod.Namespace, pod.Name, node.GetNode().Name, err) } } } @@ -1485,47 +1489,16 @@ func (r *SkyhookReconciler) IsDrained(ctx context.Context, skyhookNode wrapper.S return true, nil } - // checking for any running or pending pods with no toleration to unschedulable - // if its has an unschedulable toleration we can ignore + options := drain.OptionsFromConfig(skyhookNode.GetSkyhook().Spec.DrainConfig) for _, pod := range pods.Items { - - if ShouldEvict(&pod) { + if drain.DecidePod(&pod, options).BlocksDrain() { return false, nil } - } return true, nil } -func ShouldEvict(pod *corev1.Pod) bool { - switch pod.Status.Phase { - case corev1.PodRunning, corev1.PodPending: - - for _, taint := range pod.Spec.Tolerations { - switch taint.Key { - case "node.kubernetes.io/unschedulable": // ignoring - return false - } - } - - if len(pod.ObjectMeta.OwnerReferences) > 1 { - for _, owner := range pod.ObjectMeta.OwnerReferences { - if owner.Kind == "DaemonSet" { // ignoring - return false - } - } - } - - if pod.GetNamespace() == "kube-system" { - return false - } - - return true - } - return false -} - // HandleFinalizer returns true only if we container is deleted and we handled it completely, else false. // For Skyhooks with UninstallEnabled packages, this uses a multi-reconcile flow: // Phase 1: trigger uninstall for enabled packages, return false to requeue @@ -1775,9 +1748,40 @@ func (r *SkyhookReconciler) DrainNode(ctx context.Context, skyhookNode wrapper.S return false, err } if drained { + skyhookNode.ClearDrainStart() return true, nil } + drainStartedAt, err := skyhookNode.DrainStartedAt() + if err != nil { + return false, fmt.Errorf("error reading drain start for node [%s]: %w", skyhookNode.GetNode().Name, err) + } + + drainConfig := skyhookNode.GetSkyhook().Spec.DrainConfig + now := metav1.Now() + if drainStartedAt == nil { + skyhookNode.StartDrain(now) + skyhookNode.SetStatus(v1alpha1.StatusInProgress) + } else if drainConfig != nil && drain.TimedOut(drainStartedAt, drainConfig.Timeout, now.Time) { + r.recorder.Eventf(skyhookNode.GetNode(), nil, corev1.EventTypeWarning, EventsReasonSkyhookDrain, "DrainTimeout", + "drain timed out after [%s] for node [%s] package [%s:%s] from [skyhook:%s]", + drainConfig.Timeout.Duration, + skyhookNode.GetNode().Name, + _package.Name, + _package.Version, + skyhookNode.GetSkyhook().Name, + ) + r.recorder.Eventf(skyhookNode.GetSkyhook().Skyhook, nil, corev1.EventTypeWarning, EventsReasonSkyhookDrain, "DrainTimeout", + "drain timed out after [%s] for node [%s] package [%s:%s]", + drainConfig.Timeout.Duration, + skyhookNode.GetNode().Name, + _package.Name, + _package.Version, + ) + skyhookNode.SetStatus(v1alpha1.StatusErroring) + return false, nil + } + pods, err := r.dal.GetPods(ctx, client.MatchingFields{ fieldSelectorNodeName: skyhookNode.GetNode().Name, }) @@ -1797,19 +1801,35 @@ func (r *SkyhookReconciler) DrainNode(ctx context.Context, skyhookNode wrapper.S skyhookNode.GetSkyhook().Name, ) + options := drain.OptionsFromConfig(skyhookNode.GetSkyhook().Spec.DrainConfig) errs := make([]error, 0) + waitingForPods := false for _, pod := range pods.Items { - - if ShouldEvict(&pod) { - eviction := policyv1.Eviction{} + decision := drain.DecidePod(&pod, options) + switch decision.Action { + case drain.ActionBlock: + waitingForPods = true + case drain.ActionEvict: + waitingForPods = true + eviction := policyv1.Eviction{DeleteOptions: options.EvictionDeleteOptions()} err := r.Client.SubResource("eviction").Create(ctx, &pod, &eviction) if err != nil { errs = append(errs, fmt.Errorf("error evicting pod [%s:%s]: %w", pod.Namespace, pod.Name, err)) } + case drain.ActionDelete: + waitingForPods = true + err := r.Delete(ctx, &pod, options.DeleteOptions()...) + if err != nil { + errs = append(errs, fmt.Errorf("error deleting pod [%s:%s]: %w", pod.Namespace, pod.Name, err)) + } } } - return len(errs) == 0, utilerrors.NewAggregate(errs) + if len(errs) > 0 { + return false, utilerrors.NewAggregate(errs) + } + + return !waitingForPods, nil } // Interrupt should not be called unless safe to do so, IE already cordoned and drained diff --git a/operator/internal/controller/skyhook_controller_test.go b/operator/internal/controller/skyhook_controller_test.go index f7c7ca22..cbb429e8 100644 --- a/operator/internal/controller/skyhook_controller_test.go +++ b/operator/internal/controller/skyhook_controller_test.go @@ -34,9 +34,15 @@ import ( . "github.com/onsi/gomega" "github.com/stretchr/testify/mock" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -520,6 +526,357 @@ var _ = Describe("skyhook controller tests", func() { Expect(envs).To(BeEquivalentTo(expected)) }) + Context("DrainNode", func() { + fakeDrainClient := func(objects ...client.Object) client.WithWatch { + scheme := runtime.NewScheme() + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + Expect(v1alpha1.AddToScheme(scheme)).To(Succeed()) + + return fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objects...). + WithIndex(&corev1.Pod{}, fieldSelectorNodeName, func(obj client.Object) []string { + pod, ok := obj.(*corev1.Pod) + if !ok { + return nil + } + return []string{pod.Spec.NodeName} + }). + Build() + } + + It("should delete pods directly when disableEviction is true", func() { + gracePeriodSeconds := int64(-1) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "workload", + Namespace: "default", + }, + Spec: corev1.PodSpec{ + NodeName: "node-a", + Containers: []corev1.Container{ + {Name: "workload", Image: "busybox"}, + }, + }, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + } + + baseClient := fakeDrainClient(pod) + testClient := interceptor.NewClient(baseClient, interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + deleteOptions := &client.DeleteOptions{} + deleteOptions.ApplyOptions(opts) + if deleteOptions.GracePeriodSeconds != nil { + gracePeriodSeconds = *deleteOptions.GracePeriodSeconds + } + return c.Delete(ctx, obj, opts...) + }, + }) + + r, err := NewSkyhookReconciler(testClient.Scheme(), testClient, events.NewFakeRecorder(10), opts) + Expect(err).ToNot(HaveOccurred()) + + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-a"}} + skyhook := &v1alpha1.Skyhook{ + ObjectMeta: metav1.ObjectMeta{Name: "drain-delete"}, + Spec: v1alpha1.SkyhookSpec{ + DrainConfig: &v1alpha1.DrainConfig{ + DisableEviction: ptr(true), + GracePeriod: &metav1.Duration{Duration: 7 * time.Second}, + }, + Packages: v1alpha1.Packages{}, + }, + } + skyhookNode, err := wrapper.NewSkyhookNode(node, skyhook) + Expect(err).ToNot(HaveOccurred()) + + drained, err := r.DrainNode(ctx, skyhookNode, &v1alpha1.Package{ + PackageRef: v1alpha1.PackageRef{Name: "pkg", Version: "1.0.0"}, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(drained).To(BeFalse()) + Expect(gracePeriodSeconds).To(Equal(int64(7))) + + deletedPod := &corev1.Pod{} + err = testClient.Get(ctx, types.NamespacedName{Namespace: "default", Name: "workload"}, deletedPod) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + + drained, err = r.DrainNode(ctx, skyhookNode, &v1alpha1.Package{ + PackageRef: v1alpha1.PackageRef{Name: "pkg", Version: "1.0.0"}, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(drained).To(BeTrue()) + }) + + It("should wait without deleting unmanaged pods when force is false", func() { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "workload", + Namespace: "default", + }, + Spec: corev1.PodSpec{ + NodeName: "node-a", + Containers: []corev1.Container{ + {Name: "workload", Image: "busybox"}, + }, + }, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + } + + deleteCalled := false + evictCalled := false + baseClient := fakeDrainClient(pod) + testClient := interceptor.NewClient(baseClient, interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + deleteCalled = true + return c.Delete(ctx, obj, opts...) + }, + SubResourceCreate: func(ctx context.Context, c client.Client, subResourceName string, obj client.Object, subResource client.Object, opts ...client.SubResourceCreateOption) error { + evictCalled = true + return nil + }, + }) + + r, err := NewSkyhookReconciler(testClient.Scheme(), testClient, events.NewFakeRecorder(10), opts) + Expect(err).ToNot(HaveOccurred()) + + force := false + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-a"}} + skyhook := &v1alpha1.Skyhook{ + ObjectMeta: metav1.ObjectMeta{Name: "drain-block"}, + Spec: v1alpha1.SkyhookSpec{ + DrainConfig: &v1alpha1.DrainConfig{ + Force: &force, + }, + Packages: v1alpha1.Packages{}, + }, + } + skyhookNode, err := wrapper.NewSkyhookNode(node, skyhook) + Expect(err).ToNot(HaveOccurred()) + + drained, err := r.DrainNode(ctx, skyhookNode, &v1alpha1.Package{ + PackageRef: v1alpha1.PackageRef{Name: "pkg", Version: "1.0.0"}, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(drained).To(BeFalse()) + Expect(deleteCalled).To(BeFalse()) + Expect(evictCalled).To(BeFalse()) + Expect(skyhookNode.Status()).To(Equal(v1alpha1.StatusInProgress)) + }) + + It("should block before drain when podNonInterruptLabels match a running pod", func() { + goldenPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "golden", + Namespace: "default", + Labels: map[string]string{ + "workload": "golden", + }, + }, + Spec: corev1.PodSpec{ + NodeName: "node-a", + Containers: []corev1.Container{ + {Name: "golden", Image: "busybox"}, + }, + }, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + } + evictablePod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "evictable", + Namespace: "default", + }, + Spec: corev1.PodSpec{ + NodeName: "node-a", + Containers: []corev1.Container{ + {Name: "evictable", Image: "busybox"}, + }, + }, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + } + + deleteCalled := false + evictCalled := false + baseClient := fakeDrainClient(goldenPod, evictablePod) + testClient := interceptor.NewClient(baseClient, interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + deleteCalled = true + return c.Delete(ctx, obj, opts...) + }, + SubResourceCreate: func(ctx context.Context, c client.Client, subResourceName string, obj client.Object, subResource client.Object, opts ...client.SubResourceCreateOption) error { + evictCalled = true + return nil + }, + }) + + r, err := NewSkyhookReconciler(testClient.Scheme(), testClient, events.NewFakeRecorder(10), opts) + Expect(err).ToNot(HaveOccurred()) + + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-a"}} + skyhook := &v1alpha1.Skyhook{ + ObjectMeta: metav1.ObjectMeta{Name: "drain-golden"}, + Spec: v1alpha1.SkyhookSpec{ + PodNonInterruptLabels: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "workload": "golden", + }, + }, + DrainConfig: &v1alpha1.DrainConfig{ + DisableEviction: ptr(true), + }, + Packages: v1alpha1.Packages{}, + }, + } + skyhookNode, err := wrapper.NewSkyhookNode(node, skyhook) + Expect(err).ToNot(HaveOccurred()) + + ready, err := r.EnsureNodeIsReadyForInterrupt(ctx, skyhookNode, &v1alpha1.Package{ + PackageRef: v1alpha1.PackageRef{Name: "pkg", Version: "1.0.0"}, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(ready).To(BeFalse()) + Expect(deleteCalled).To(BeFalse()) + Expect(evictCalled).To(BeFalse()) + Expect(skyhookNode.GetNode().Spec.Unschedulable).To(BeTrue()) + + drainStartedAt, err := skyhookNode.DrainStartedAt() + Expect(err).ToNot(HaveOccurred()) + Expect(drainStartedAt).To(BeNil()) + }) + + It("should mark the node erroring when drain timeout expires", func() { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "workload", + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "ReplicaSet", + Name: "workload-rs", + Controller: ptr(true), + }, + }, + }, + Spec: corev1.PodSpec{ + NodeName: "node-a", + Containers: []corev1.Container{ + {Name: "workload", Image: "busybox"}, + }, + }, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + } + + deleteCalled := false + baseClient := fakeDrainClient(pod) + testClient := interceptor.NewClient(baseClient, interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + deleteCalled = true + return c.Delete(ctx, obj, opts...) + }, + SubResourceCreate: func(ctx context.Context, c client.Client, subResourceName string, obj client.Object, subResource client.Object, opts ...client.SubResourceCreateOption) error { + Fail("drain timeout should not evict pods") + return nil + }, + }) + + r, err := NewSkyhookReconciler(testClient.Scheme(), testClient, events.NewFakeRecorder(10), opts) + Expect(err).ToNot(HaveOccurred()) + + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node-a", + Annotations: map[string]string{ + "skyhook.nvidia.com/drainStart_drain-timeout": time.Now().Add(-2 * time.Minute).Format(time.RFC3339Nano), + }, + }, + } + skyhook := &v1alpha1.Skyhook{ + ObjectMeta: metav1.ObjectMeta{Name: "drain-timeout"}, + Spec: v1alpha1.SkyhookSpec{ + DrainConfig: &v1alpha1.DrainConfig{ + Timeout: &metav1.Duration{Duration: time.Second}, + }, + Packages: v1alpha1.Packages{}, + }, + } + skyhookNode, err := wrapper.NewSkyhookNode(node, skyhook) + Expect(err).ToNot(HaveOccurred()) + + drained, err := r.DrainNode(ctx, skyhookNode, &v1alpha1.Package{ + PackageRef: v1alpha1.PackageRef{Name: "pkg", Version: "1.0.0"}, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(drained).To(BeFalse()) + Expect(deleteCalled).To(BeFalse()) + Expect(skyhookNode.Status()).To(Equal(v1alpha1.StatusErroring)) + }) + + It("should emit drain timeout events when the node is already erroring", func() { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "workload", + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "ReplicaSet", + Name: "workload-rs", + Controller: ptr(true), + }, + }, + }, + Spec: corev1.PodSpec{ + NodeName: "node-a", + Containers: []corev1.Container{ + {Name: "workload", Image: "busybox"}, + }, + }, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + } + + baseClient := fakeDrainClient(pod) + testClient := interceptor.NewClient(baseClient, interceptor.Funcs{ + SubResourceCreate: func(ctx context.Context, c client.Client, subResourceName string, obj client.Object, subResource client.Object, opts ...client.SubResourceCreateOption) error { + Fail("drain timeout should not evict pods") + return nil + }, + }) + + recorder := events.NewFakeRecorder(10) + r, err := NewSkyhookReconciler(testClient.Scheme(), testClient, recorder, opts) + Expect(err).ToNot(HaveOccurred()) + + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node-a", + Annotations: map[string]string{ + "skyhook.nvidia.com/drainStart_drain-timeout": time.Now().Add(-2 * time.Minute).Format(time.RFC3339Nano), + "skyhook.nvidia.com/status_drain-timeout": string(v1alpha1.StatusErroring), + }, + }, + } + skyhook := &v1alpha1.Skyhook{ + ObjectMeta: metav1.ObjectMeta{Name: "drain-timeout"}, + Spec: v1alpha1.SkyhookSpec{ + DrainConfig: &v1alpha1.DrainConfig{ + Timeout: &metav1.Duration{Duration: time.Second}, + }, + Packages: v1alpha1.Packages{}, + }, + } + skyhookNode, err := wrapper.NewSkyhookNode(node, skyhook) + Expect(err).ToNot(HaveOccurred()) + + drained, err := r.DrainNode(ctx, skyhookNode, &v1alpha1.Package{ + PackageRef: v1alpha1.PackageRef{Name: "pkg", Version: "1.0.0"}, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(drained).To(BeFalse()) + Expect(skyhookNode.Status()).To(Equal(v1alpha1.StatusErroring)) + Eventually(recorder.Events).Should(Receive(ContainSubstring("Warning Drain drain timed out after [1s] for node [node-a] package [pkg:1.0.0] from [skyhook:drain-timeout]"))) + Eventually(recorder.Events).Should(Receive(ContainSubstring("Warning Drain drain timed out after [1s] for node [node-a] package [pkg:1.0.0]"))) + }) + }) + It("should set monotonic SKYHOOK_NODE_ORDER across nodes and batches", func() { now := time.Now() testSkyhook := wrapper.NewSkyhookWrapper(&v1alpha1.Skyhook{ diff --git a/operator/internal/drain/drain.go b/operator/internal/drain/drain.go new file mode 100644 index 00000000..72e2a223 --- /dev/null +++ b/operator/internal/drain/drain.go @@ -0,0 +1,209 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package drain + +import ( + "time" + + "github.com/NVIDIA/nodewright/operator/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + MirrorPodAnnotationKey = "kubernetes.io/config.mirror" + + ReasonPhase = "phase" + ReasonTerminating = "terminating" + ReasonUnschedulableToleration = "unschedulable-toleration" + ReasonDaemonSet = "daemonset" + ReasonKubeSystem = "kube-system" + ReasonMirrorPod = "mirror-pod" + ReasonUnmanaged = "unmanaged" + ReasonEmptyDir = "emptydir" + ReasonEviction = "eviction" + ReasonDelete = "delete" +) + +type Action string + +const ( + ActionIgnore Action = "ignore" + ActionBlock Action = "block" + ActionEvict Action = "evict" + ActionDelete Action = "delete" +) + +type Options struct { + DisableEviction bool + DeleteEmptyDirData bool + Force bool + IgnoreDaemonSets bool + GracePeriodSeconds *int64 +} + +func DefaultOptions() Options { + return Options{ + DeleteEmptyDirData: true, + Force: true, + IgnoreDaemonSets: true, + } +} + +func OptionsFromConfig(config *v1alpha1.DrainConfig) Options { + options := DefaultOptions() + if config == nil { + return options + } + + if config.DisableEviction != nil { + options.DisableEviction = *config.DisableEviction + } + if config.DeleteEmptyDirData != nil { + options.DeleteEmptyDirData = *config.DeleteEmptyDirData + } + if config.Force != nil { + options.Force = *config.Force + } + if config.IgnoreDaemonSets != nil { + options.IgnoreDaemonSets = *config.IgnoreDaemonSets + } + if config.GracePeriod != nil { + seconds := int64(config.GracePeriod.Duration / time.Second) + if config.GracePeriod.Duration%time.Second != 0 { + seconds++ + } + options.GracePeriodSeconds = &seconds + } + + return options +} + +func TimedOut(startedAt *metav1.Time, timeout *metav1.Duration, now time.Time) bool { + if startedAt == nil || timeout == nil || timeout.Duration == 0 { + return false + } + return !now.Before(startedAt.Add(timeout.Duration)) +} + +func (o Options) DeleteOptions() []client.DeleteOption { + if o.GracePeriodSeconds == nil { + return nil + } + return []client.DeleteOption{client.GracePeriodSeconds(*o.GracePeriodSeconds)} +} + +func (o Options) EvictionDeleteOptions() *metav1.DeleteOptions { + if o.GracePeriodSeconds == nil { + return nil + } + return &metav1.DeleteOptions{GracePeriodSeconds: o.GracePeriodSeconds} +} + +type Decision struct { + Action Action + Reason string +} + +func (d Decision) BlocksDrain() bool { + return d.Action != ActionIgnore +} + +func (d Decision) RequiresAction() bool { + return d.Action == ActionEvict || d.Action == ActionDelete +} + +func DecidePod(pod *corev1.Pod, options Options) Decision { + if pod == nil { + return Decision{Action: ActionIgnore, Reason: ReasonPhase} + } + + switch pod.Status.Phase { + case corev1.PodRunning, corev1.PodPending: + default: + return Decision{Action: ActionIgnore, Reason: ReasonPhase} + } + + if pod.DeletionTimestamp != nil { + return Decision{Action: ActionIgnore, Reason: ReasonTerminating} + } + + if toleratesUnschedulable(pod) { + return Decision{Action: ActionIgnore, Reason: ReasonUnschedulableToleration} + } + + controllerRef := metav1.GetControllerOf(pod) + if options.IgnoreDaemonSets && controllerRef != nil && controllerRef.Kind == "DaemonSet" { + return Decision{Action: ActionIgnore, Reason: ReasonDaemonSet} + } + + if pod.Namespace == "kube-system" { + return Decision{Action: ActionIgnore, Reason: ReasonKubeSystem} + } + + if isMirrorPod(pod) { + return Decision{Action: ActionIgnore, Reason: ReasonMirrorPod} + } + + if controllerRef == nil && !options.Force { + return Decision{Action: ActionBlock, Reason: ReasonUnmanaged} + } + + if hasEmptyDir(pod) && !options.DeleteEmptyDirData { + return Decision{Action: ActionBlock, Reason: ReasonEmptyDir} + } + + if options.DisableEviction { + return Decision{Action: ActionDelete, Reason: ReasonDelete} + } + + return Decision{Action: ActionEvict, Reason: ReasonEviction} +} + +func toleratesUnschedulable(pod *corev1.Pod) bool { + unschedulableTaint := corev1.Taint{ + Key: corev1.TaintNodeUnschedulable, + Effect: corev1.TaintEffectNoSchedule, + } + for _, toleration := range pod.Spec.Tolerations { + if toleration.ToleratesTaint(klog.Background(), &unschedulableTaint, false) { + return true + } + } + return false +} + +func isMirrorPod(pod *corev1.Pod) bool { + if pod.Annotations == nil { + return false + } + _, ok := pod.Annotations[MirrorPodAnnotationKey] + return ok +} + +func hasEmptyDir(pod *corev1.Pod) bool { + for _, volume := range pod.Spec.Volumes { + if volume.EmptyDir != nil { + return true + } + } + return false +} diff --git a/operator/internal/drain/drain_suite_test.go b/operator/internal/drain/drain_suite_test.go new file mode 100644 index 00000000..d2796cfd --- /dev/null +++ b/operator/internal/drain/drain_suite_test.go @@ -0,0 +1,31 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package drain + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestDrain(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Drain Suite") +} diff --git a/operator/internal/drain/drain_test.go b/operator/internal/drain/drain_test.go new file mode 100644 index 00000000..c9f05f1b --- /dev/null +++ b/operator/internal/drain/drain_test.go @@ -0,0 +1,328 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package drain + +import ( + "time" + + "github.com/NVIDIA/nodewright/operator/api/v1alpha1" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func boolPtr(value bool) *bool { + return &value +} + +func durationPtr(value time.Duration) *metav1.Duration { + return &metav1.Duration{Duration: value} +} + +func int64Ptr(value int64) *int64 { + return &value +} + +var _ = Describe("DecidePod", func() { + controller := true + const daemonSetKind = "DaemonSet" + + basePod := func() *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "workload", + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "ReplicaSet", + Name: "workload-rs", + Controller: &controller, + }, + }, + }, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + } + } + + It("should preserve the current default behavior", func() { + pod := basePod() + + decision := DecidePod(pod, DefaultOptions()) + + Expect(decision).To(Equal(Decision{Action: ActionEvict, Reason: ReasonEviction})) + Expect(decision.BlocksDrain()).To(BeTrue()) + Expect(decision.RequiresAction()).To(BeTrue()) + }) + + It("should delete directly when eviction is disabled", func() { + pod := basePod() + options := DefaultOptions() + options.DisableEviction = true + + decision := DecidePod(pod, options) + + Expect(decision).To(Equal(Decision{Action: ActionDelete, Reason: ReasonDelete})) + Expect(decision.BlocksDrain()).To(BeTrue()) + Expect(decision.RequiresAction()).To(BeTrue()) + }) + + It("should ignore pods that are not running or pending", func() { + pod := basePod() + pod.Status.Phase = corev1.PodSucceeded + + decision := DecidePod(pod, DefaultOptions()) + + Expect(decision).To(Equal(Decision{Action: ActionIgnore, Reason: ReasonPhase})) + Expect(decision.BlocksDrain()).To(BeFalse()) + }) + + It("should ignore pods that are already terminating", func() { + pod := basePod() + deletionTimestamp := metav1.NewTime(time.Date(2026, time.June, 2, 12, 0, 0, 0, time.UTC)) + pod.DeletionTimestamp = &deletionTimestamp + + decision := DecidePod(pod, DefaultOptions()) + + Expect(decision).To(Equal(Decision{Action: ActionIgnore, Reason: ReasonTerminating})) + Expect(decision.BlocksDrain()).To(BeFalse()) + }) + + It("should never drain pods that tolerate the unschedulable taint", func() { + pod := basePod() + pod.Spec.Tolerations = []corev1.Toleration{ + {Key: corev1.TaintNodeUnschedulable}, + } + + decision := DecidePod(pod, DefaultOptions()) + + Expect(decision).To(Equal(Decision{Action: ActionIgnore, Reason: ReasonUnschedulableToleration})) + Expect(decision.BlocksDrain()).To(BeFalse()) + }) + + DescribeTable("should apply Kubernetes toleration matching for the unschedulable taint", + func(toleration corev1.Toleration, expectedDecision Decision) { + pod := basePod() + pod.Spec.Tolerations = []corev1.Toleration{toleration} + + decision := DecidePod(pod, DefaultOptions()) + + Expect(decision).To(Equal(expectedDecision)) + }, + Entry("matches Exists on the NoSchedule unschedulable taint", + corev1.Toleration{ + Key: corev1.TaintNodeUnschedulable, + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }, + Decision{Action: ActionIgnore, Reason: ReasonUnschedulableToleration}, + ), + Entry("does not match a different effect", + corev1.Toleration{ + Key: corev1.TaintNodeUnschedulable, + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoExecute, + }, + Decision{Action: ActionEvict, Reason: ReasonEviction}, + ), + Entry("does not match a different value with Equal", + corev1.Toleration{ + Key: corev1.TaintNodeUnschedulable, + Operator: corev1.TolerationOpEqual, + Value: "true", + Effect: corev1.TaintEffectNoSchedule, + }, + Decision{Action: ActionEvict, Reason: ReasonEviction}, + ), + ) + + It("should ignore daemonset pods by default", func() { + pod := basePod() + pod.OwnerReferences[0].Kind = daemonSetKind + + decision := DecidePod(pod, DefaultOptions()) + + Expect(decision).To(Equal(Decision{Action: ActionIgnore, Reason: ReasonDaemonSet})) + Expect(decision.BlocksDrain()).To(BeFalse()) + }) + + It("should drain daemonset pods when ignoreDaemonSets is false", func() { + pod := basePod() + pod.OwnerReferences[0].Kind = daemonSetKind + options := DefaultOptions() + options.IgnoreDaemonSets = false + + decision := DecidePod(pod, options) + + Expect(decision).To(Equal(Decision{Action: ActionEvict, Reason: ReasonEviction})) + }) + + It("should never drain kube-system pods", func() { + pod := basePod() + pod.Namespace = "kube-system" + + decision := DecidePod(pod, DefaultOptions()) + + Expect(decision).To(Equal(Decision{Action: ActionIgnore, Reason: ReasonKubeSystem})) + Expect(decision.BlocksDrain()).To(BeFalse()) + }) + + It("should ignore mirror pods", func() { + pod := basePod() + pod.Annotations = map[string]string{ + MirrorPodAnnotationKey: "mirror-id", + } + + decision := DecidePod(pod, DefaultOptions()) + + Expect(decision).To(Equal(Decision{Action: ActionIgnore, Reason: ReasonMirrorPod})) + Expect(decision.BlocksDrain()).To(BeFalse()) + }) + + It("should evict unmanaged pods by default", func() { + pod := basePod() + pod.OwnerReferences = nil + + decision := DecidePod(pod, DefaultOptions()) + + Expect(decision).To(Equal(Decision{Action: ActionEvict, Reason: ReasonEviction})) + }) + + It("should block on unmanaged pods when force is false", func() { + pod := basePod() + pod.OwnerReferences = nil + options := DefaultOptions() + options.Force = false + + decision := DecidePod(pod, options) + + Expect(decision).To(Equal(Decision{Action: ActionBlock, Reason: ReasonUnmanaged})) + Expect(decision.BlocksDrain()).To(BeTrue()) + Expect(decision.RequiresAction()).To(BeFalse()) + }) + + It("should evict emptyDir pods by default", func() { + pod := basePod() + pod.Spec.Volumes = []corev1.Volume{ + {Name: "scratch", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + } + + decision := DecidePod(pod, DefaultOptions()) + + Expect(decision).To(Equal(Decision{Action: ActionEvict, Reason: ReasonEviction})) + }) + + It("should block on emptyDir pods when deleteEmptyDirData is false", func() { + pod := basePod() + pod.Spec.Volumes = []corev1.Volume{ + {Name: "scratch", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + } + options := DefaultOptions() + options.DeleteEmptyDirData = false + + decision := DecidePod(pod, options) + + Expect(decision).To(Equal(Decision{Action: ActionBlock, Reason: ReasonEmptyDir})) + Expect(decision.BlocksDrain()).To(BeTrue()) + Expect(decision.RequiresAction()).To(BeFalse()) + }) +}) + +var _ = Describe("OptionsFromConfig", func() { + It("should return defaults for nil config", func() { + options := OptionsFromConfig(nil) + + Expect(options).To(Equal(DefaultOptions())) + }) + + It("should apply configured field overrides", func() { + options := OptionsFromConfig(&v1alpha1.DrainConfig{ + DisableEviction: boolPtr(true), + DeleteEmptyDirData: boolPtr(false), + Force: boolPtr(false), + IgnoreDaemonSets: boolPtr(false), + GracePeriod: durationPtr(2 * time.Second), + }) + + Expect(options.DisableEviction).To(BeTrue()) + Expect(options.DeleteEmptyDirData).To(BeFalse()) + Expect(options.Force).To(BeFalse()) + Expect(options.IgnoreDaemonSets).To(BeFalse()) + Expect(options.GracePeriodSeconds).To(Equal(int64Ptr(2))) + }) + + DescribeTable("should round grace period up to whole seconds", + func(gracePeriod time.Duration, expectedSeconds int64) { + options := OptionsFromConfig(&v1alpha1.DrainConfig{ + GracePeriod: durationPtr(gracePeriod), + }) + + Expect(options.GracePeriodSeconds).To(Equal(int64Ptr(expectedSeconds))) + }, + Entry("zero seconds", 0*time.Second, int64(0)), + Entry("sub-second", 500*time.Millisecond, int64(1)), + Entry("whole second", 1*time.Second, int64(1)), + Entry("partial second", 1500*time.Millisecond, int64(2)), + ) +}) + +var _ = Describe("TimedOut", func() { + start := metav1.NewTime(time.Date(2026, time.June, 5, 12, 0, 0, 0, time.UTC)) + timeout := metav1.Duration{Duration: 5 * time.Second} + zeroTimeout := metav1.Duration{} + + DescribeTable("should evaluate timeout boundaries", + func(startedAt *metav1.Time, timeout *metav1.Duration, now time.Time, expected bool) { + Expect(TimedOut(startedAt, timeout, now)).To(Equal(expected)) + }, + Entry("nil start", nil, &timeout, start.Add(6*time.Second), false), + Entry("nil timeout", &start, nil, start.Add(6*time.Second), false), + Entry("zero timeout", &start, &zeroTimeout, start.Add(6*time.Second), false), + Entry("just before deadline", &start, &timeout, start.Add(5*time.Second-time.Nanosecond), false), + Entry("exactly at deadline", &start, &timeout, start.Add(5*time.Second), true), + Entry("after deadline", &start, &timeout, start.Add(6*time.Second), true), + ) +}) + +var _ = Describe("delete options", func() { + It("should omit delete options when grace period is unset", func() { + options := DefaultOptions() + + Expect(options.DeleteOptions()).To(BeNil()) + Expect(options.EvictionDeleteOptions()).To(BeNil()) + }) + + It("should carry configured grace period into delete options", func() { + options := Options{GracePeriodSeconds: int64Ptr(7)} + + deleteOptions := options.DeleteOptions() + Expect(deleteOptions).To(HaveLen(1)) + + applied := &client.DeleteOptions{} + for _, option := range deleteOptions { + option.ApplyToDelete(applied) + } + Expect(applied.GracePeriodSeconds).To(Equal(int64Ptr(7))) + + evictionDeleteOptions := options.EvictionDeleteOptions() + Expect(evictionDeleteOptions).NotTo(BeNil()) + Expect(evictionDeleteOptions.GracePeriodSeconds).To(Equal(int64Ptr(7))) + }) +}) diff --git a/operator/internal/wrapper/mock/SkyhookNode.go b/operator/internal/wrapper/mock/SkyhookNode.go index d0bb9b9f..dea77ae0 100644 --- a/operator/internal/wrapper/mock/SkyhookNode.go +++ b/operator/internal/wrapper/mock/SkyhookNode.go @@ -27,7 +27,8 @@ import ( "github.com/NVIDIA/nodewright/operator/internal/wrapper" "github.com/go-logr/logr" mock "github.com/stretchr/testify/mock" - "k8s.io/api/core/v1" + v10 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" ) // NewMockSkyhookNode creates a new instance of MockSkyhookNode. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. @@ -134,6 +135,39 @@ func (_c *MockSkyhookNode_CleanupSCRMetadata_Call) RunAndReturn(run func()) *Moc return _c } +// ClearDrainStart provides a mock function for the type MockSkyhookNode +func (_mock *MockSkyhookNode) ClearDrainStart() { + _mock.Called() + return +} + +// MockSkyhookNode_ClearDrainStart_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ClearDrainStart' +type MockSkyhookNode_ClearDrainStart_Call struct { + *mock.Call +} + +// ClearDrainStart is a helper method to define mock.On call +func (_e *MockSkyhookNode_Expecter) ClearDrainStart() *MockSkyhookNode_ClearDrainStart_Call { + return &MockSkyhookNode_ClearDrainStart_Call{Call: _e.mock.On("ClearDrainStart")} +} + +func (_c *MockSkyhookNode_ClearDrainStart_Call) Run(run func()) *MockSkyhookNode_ClearDrainStart_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockSkyhookNode_ClearDrainStart_Call) Return() *MockSkyhookNode_ClearDrainStart_Call { + _c.Call.Return() + return _c +} + +func (_c *MockSkyhookNode_ClearDrainStart_Call) RunAndReturn(run func()) *MockSkyhookNode_ClearDrainStart_Call { + _c.Run(run) + return _c +} + // Cordon provides a mock function for the type MockSkyhookNode func (_mock *MockSkyhookNode) Cordon() { _mock.Called() @@ -167,6 +201,61 @@ func (_c *MockSkyhookNode_Cordon_Call) RunAndReturn(run func()) *MockSkyhookNode return _c } +// DrainStartedAt provides a mock function for the type MockSkyhookNode +func (_mock *MockSkyhookNode) DrainStartedAt() (*v1.Time, error) { + ret := _mock.Called() + + if len(ret) == 0 { + panic("no return value specified for DrainStartedAt") + } + + var r0 *v1.Time + var r1 error + if returnFunc, ok := ret.Get(0).(func() (*v1.Time, error)); ok { + return returnFunc() + } + if returnFunc, ok := ret.Get(0).(func() *v1.Time); ok { + r0 = returnFunc() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v1.Time) + } + } + if returnFunc, ok := ret.Get(1).(func() error); ok { + r1 = returnFunc() + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockSkyhookNode_DrainStartedAt_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DrainStartedAt' +type MockSkyhookNode_DrainStartedAt_Call struct { + *mock.Call +} + +// DrainStartedAt is a helper method to define mock.On call +func (_e *MockSkyhookNode_Expecter) DrainStartedAt() *MockSkyhookNode_DrainStartedAt_Call { + return &MockSkyhookNode_DrainStartedAt_Call{Call: _e.mock.On("DrainStartedAt")} +} + +func (_c *MockSkyhookNode_DrainStartedAt_Call) Run(run func()) *MockSkyhookNode_DrainStartedAt_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockSkyhookNode_DrainStartedAt_Call) Return(time *v1.Time, err error) *MockSkyhookNode_DrainStartedAt_Call { + _c.Call.Return(time, err) + return _c +} + +func (_c *MockSkyhookNode_DrainStartedAt_Call) RunAndReturn(run func() (*v1.Time, error)) *MockSkyhookNode_DrainStartedAt_Call { + _c.Call.Return(run) + return _c +} + // GetComplete provides a mock function for the type MockSkyhookNode func (_mock *MockSkyhookNode) GetComplete() []string { ret := _mock.Called() @@ -214,19 +303,19 @@ func (_c *MockSkyhookNode_GetComplete_Call) RunAndReturn(run func() []string) *M } // GetNode provides a mock function for the type MockSkyhookNode -func (_mock *MockSkyhookNode) GetNode() *v1.Node { +func (_mock *MockSkyhookNode) GetNode() *v10.Node { ret := _mock.Called() if len(ret) == 0 { panic("no return value specified for GetNode") } - var r0 *v1.Node - if returnFunc, ok := ret.Get(0).(func() *v1.Node); ok { + var r0 *v10.Node + if returnFunc, ok := ret.Get(0).(func() *v10.Node); ok { r0 = returnFunc() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*v1.Node) + r0 = ret.Get(0).(*v10.Node) } } return r0 @@ -249,12 +338,12 @@ func (_c *MockSkyhookNode_GetNode_Call) Run(run func()) *MockSkyhookNode_GetNode return _c } -func (_c *MockSkyhookNode_GetNode_Call) Return(node *v1.Node) *MockSkyhookNode_GetNode_Call { +func (_c *MockSkyhookNode_GetNode_Call) Return(node *v10.Node) *MockSkyhookNode_GetNode_Call { _c.Call.Return(node) return _c } -func (_c *MockSkyhookNode_GetNode_Call) RunAndReturn(run func() *v1.Node) *MockSkyhookNode_GetNode_Call { +func (_c *MockSkyhookNode_GetNode_Call) RunAndReturn(run func() *v10.Node) *MockSkyhookNode_GetNode_Call { _c.Call.Return(run) return _c } @@ -1052,6 +1141,46 @@ func (_c *MockSkyhookNode_SetVersion_Call) RunAndReturn(run func()) *MockSkyhook return _c } +// StartDrain provides a mock function for the type MockSkyhookNode +func (_mock *MockSkyhookNode) StartDrain(startedAt v1.Time) { + _mock.Called(startedAt) + return +} + +// MockSkyhookNode_StartDrain_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'StartDrain' +type MockSkyhookNode_StartDrain_Call struct { + *mock.Call +} + +// StartDrain is a helper method to define mock.On call +// - startedAt v1.Time +func (_e *MockSkyhookNode_Expecter) StartDrain(startedAt interface{}) *MockSkyhookNode_StartDrain_Call { + return &MockSkyhookNode_StartDrain_Call{Call: _e.mock.On("StartDrain", startedAt)} +} + +func (_c *MockSkyhookNode_StartDrain_Call) Run(run func(startedAt v1.Time)) *MockSkyhookNode_StartDrain_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 v1.Time + if args[0] != nil { + arg0 = args[0].(v1.Time) + } + run( + arg0, + ) + }) + return _c +} + +func (_c *MockSkyhookNode_StartDrain_Call) Return() *MockSkyhookNode_StartDrain_Call { + _c.Call.Return() + return _c +} + +func (_c *MockSkyhookNode_StartDrain_Call) RunAndReturn(run func(startedAt v1.Time)) *MockSkyhookNode_StartDrain_Call { + _c.Run(run) + return _c +} + // State provides a mock function for the type MockSkyhookNode func (_mock *MockSkyhookNode) State() (v1alpha1.NodeState, error) { ret := _mock.Called() diff --git a/operator/internal/wrapper/mock/SkyhookNodeOnly.go b/operator/internal/wrapper/mock/SkyhookNodeOnly.go index 0e08d5ed..fffcbbd1 100644 --- a/operator/internal/wrapper/mock/SkyhookNodeOnly.go +++ b/operator/internal/wrapper/mock/SkyhookNodeOnly.go @@ -26,7 +26,8 @@ import ( "github.com/NVIDIA/nodewright/operator/api/v1alpha1" "github.com/go-logr/logr" mock "github.com/stretchr/testify/mock" - "k8s.io/api/core/v1" + v10 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" ) // NewMockSkyhookNodeOnly creates a new instance of MockSkyhookNodeOnly. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. @@ -100,6 +101,39 @@ func (_c *MockSkyhookNodeOnly_Changed_Call) RunAndReturn(run func() bool) *MockS return _c } +// ClearDrainStart provides a mock function for the type MockSkyhookNodeOnly +func (_mock *MockSkyhookNodeOnly) ClearDrainStart() { + _mock.Called() + return +} + +// MockSkyhookNodeOnly_ClearDrainStart_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ClearDrainStart' +type MockSkyhookNodeOnly_ClearDrainStart_Call struct { + *mock.Call +} + +// ClearDrainStart is a helper method to define mock.On call +func (_e *MockSkyhookNodeOnly_Expecter) ClearDrainStart() *MockSkyhookNodeOnly_ClearDrainStart_Call { + return &MockSkyhookNodeOnly_ClearDrainStart_Call{Call: _e.mock.On("ClearDrainStart")} +} + +func (_c *MockSkyhookNodeOnly_ClearDrainStart_Call) Run(run func()) *MockSkyhookNodeOnly_ClearDrainStart_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockSkyhookNodeOnly_ClearDrainStart_Call) Return() *MockSkyhookNodeOnly_ClearDrainStart_Call { + _c.Call.Return() + return _c +} + +func (_c *MockSkyhookNodeOnly_ClearDrainStart_Call) RunAndReturn(run func()) *MockSkyhookNodeOnly_ClearDrainStart_Call { + _c.Run(run) + return _c +} + // Cordon provides a mock function for the type MockSkyhookNodeOnly func (_mock *MockSkyhookNodeOnly) Cordon() { _mock.Called() @@ -133,20 +167,75 @@ func (_c *MockSkyhookNodeOnly_Cordon_Call) RunAndReturn(run func()) *MockSkyhook return _c } +// DrainStartedAt provides a mock function for the type MockSkyhookNodeOnly +func (_mock *MockSkyhookNodeOnly) DrainStartedAt() (*v1.Time, error) { + ret := _mock.Called() + + if len(ret) == 0 { + panic("no return value specified for DrainStartedAt") + } + + var r0 *v1.Time + var r1 error + if returnFunc, ok := ret.Get(0).(func() (*v1.Time, error)); ok { + return returnFunc() + } + if returnFunc, ok := ret.Get(0).(func() *v1.Time); ok { + r0 = returnFunc() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v1.Time) + } + } + if returnFunc, ok := ret.Get(1).(func() error); ok { + r1 = returnFunc() + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockSkyhookNodeOnly_DrainStartedAt_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DrainStartedAt' +type MockSkyhookNodeOnly_DrainStartedAt_Call struct { + *mock.Call +} + +// DrainStartedAt is a helper method to define mock.On call +func (_e *MockSkyhookNodeOnly_Expecter) DrainStartedAt() *MockSkyhookNodeOnly_DrainStartedAt_Call { + return &MockSkyhookNodeOnly_DrainStartedAt_Call{Call: _e.mock.On("DrainStartedAt")} +} + +func (_c *MockSkyhookNodeOnly_DrainStartedAt_Call) Run(run func()) *MockSkyhookNodeOnly_DrainStartedAt_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockSkyhookNodeOnly_DrainStartedAt_Call) Return(time *v1.Time, err error) *MockSkyhookNodeOnly_DrainStartedAt_Call { + _c.Call.Return(time, err) + return _c +} + +func (_c *MockSkyhookNodeOnly_DrainStartedAt_Call) RunAndReturn(run func() (*v1.Time, error)) *MockSkyhookNodeOnly_DrainStartedAt_Call { + _c.Call.Return(run) + return _c +} + // GetNode provides a mock function for the type MockSkyhookNodeOnly -func (_mock *MockSkyhookNodeOnly) GetNode() *v1.Node { +func (_mock *MockSkyhookNodeOnly) GetNode() *v10.Node { ret := _mock.Called() if len(ret) == 0 { panic("no return value specified for GetNode") } - var r0 *v1.Node - if returnFunc, ok := ret.Get(0).(func() *v1.Node); ok { + var r0 *v10.Node + if returnFunc, ok := ret.Get(0).(func() *v10.Node); ok { r0 = returnFunc() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*v1.Node) + r0 = ret.Get(0).(*v10.Node) } } return r0 @@ -169,12 +258,12 @@ func (_c *MockSkyhookNodeOnly_GetNode_Call) Run(run func()) *MockSkyhookNodeOnly return _c } -func (_c *MockSkyhookNodeOnly_GetNode_Call) Return(node *v1.Node) *MockSkyhookNodeOnly_GetNode_Call { +func (_c *MockSkyhookNodeOnly_GetNode_Call) Return(node *v10.Node) *MockSkyhookNodeOnly_GetNode_Call { _c.Call.Return(node) return _c } -func (_c *MockSkyhookNodeOnly_GetNode_Call) RunAndReturn(run func() *v1.Node) *MockSkyhookNodeOnly_GetNode_Call { +func (_c *MockSkyhookNodeOnly_GetNode_Call) RunAndReturn(run func() *v10.Node) *MockSkyhookNodeOnly_GetNode_Call { _c.Call.Return(run) return _c } @@ -584,6 +673,46 @@ func (_c *MockSkyhookNodeOnly_SetVersion_Call) RunAndReturn(run func()) *MockSky return _c } +// StartDrain provides a mock function for the type MockSkyhookNodeOnly +func (_mock *MockSkyhookNodeOnly) StartDrain(startedAt v1.Time) { + _mock.Called(startedAt) + return +} + +// MockSkyhookNodeOnly_StartDrain_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'StartDrain' +type MockSkyhookNodeOnly_StartDrain_Call struct { + *mock.Call +} + +// StartDrain is a helper method to define mock.On call +// - startedAt v1.Time +func (_e *MockSkyhookNodeOnly_Expecter) StartDrain(startedAt interface{}) *MockSkyhookNodeOnly_StartDrain_Call { + return &MockSkyhookNodeOnly_StartDrain_Call{Call: _e.mock.On("StartDrain", startedAt)} +} + +func (_c *MockSkyhookNodeOnly_StartDrain_Call) Run(run func(startedAt v1.Time)) *MockSkyhookNodeOnly_StartDrain_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 v1.Time + if args[0] != nil { + arg0 = args[0].(v1.Time) + } + run( + arg0, + ) + }) + return _c +} + +func (_c *MockSkyhookNodeOnly_StartDrain_Call) Return() *MockSkyhookNodeOnly_StartDrain_Call { + _c.Call.Return() + return _c +} + +func (_c *MockSkyhookNodeOnly_StartDrain_Call) RunAndReturn(run func(startedAt v1.Time)) *MockSkyhookNodeOnly_StartDrain_Call { + _c.Run(run) + return _c +} + // State provides a mock function for the type MockSkyhookNodeOnly func (_mock *MockSkyhookNodeOnly) State() (v1alpha1.NodeState, error) { ret := _mock.Called() diff --git a/operator/internal/wrapper/node.go b/operator/internal/wrapper/node.go index 52cdfde3..8709feef 100644 --- a/operator/internal/wrapper/node.go +++ b/operator/internal/wrapper/node.go @@ -23,6 +23,7 @@ import ( "fmt" "sort" "strings" + "time" "github.com/NVIDIA/nodewright/operator/api/v1alpha1" "github.com/NVIDIA/nodewright/operator/internal/graph" @@ -98,6 +99,12 @@ type SkyhookNodeOnly interface { RemoveTaint(key string) // Cordon marks the node unschedulable and records the cordon in annotations for this Skyhook. Cordon() + // StartDrain records when draining started for this Skyhook on this node. + StartDrain(startedAt metav1.Time) + // DrainStartedAt returns when draining started for this Skyhook on this node. + DrainStartedAt() (*metav1.Time, error) + // ClearDrainStart removes the drain start marker for this Skyhook on this node. + ClearDrainStart() // Uncordon marks the node schedulable and removes this Skyhook's cordon annotation if present. Uncordon() // Reset clears Skyhook-related state and annotations so the node can be reconfigured from scratch. @@ -166,6 +173,10 @@ type skyhookNode struct { updated bool } +func (node *skyhookNode) drainStartAnnotationKey() string { + return fmt.Sprintf("%s/drainStart_%s", v1alpha1.METADATA_PREFIX, node.skyhookName) +} + // GetSkyhook returns the Skyhook associated with this node, or nil if only a name was set. func (node *skyhookNode) GetSkyhook() *Skyhook { return node.skyhook @@ -469,12 +480,65 @@ func (node *skyhookNode) HasSkyhookAnnotations() bool { func (node *skyhookNode) Cordon() { _, ok := node.Annotations[fmt.Sprintf("%s/cordon_%s", v1alpha1.METADATA_PREFIX, node.skyhookName)] if !node.Spec.Unschedulable || !ok { + if node.Annotations == nil { + node.Annotations = make(map[string]string) + } node.Spec.Unschedulable = true node.Annotations[fmt.Sprintf("%s/cordon_%s", v1alpha1.METADATA_PREFIX, node.skyhookName)] = "true" node.updated = true } } +// StartDrain records when draining started for this Skyhook on this node. +func (node *skyhookNode) StartDrain(startedAt metav1.Time) { + if node.Annotations == nil { + node.Annotations = make(map[string]string) + } + + key := node.drainStartAnnotationKey() + if _, ok := node.Annotations[key]; ok { + return + } + + node.Annotations[key] = startedAt.Time.Format(time.RFC3339Nano) + node.updated = true +} + +// DrainStartedAt returns when draining started for this Skyhook on this node. +func (node *skyhookNode) DrainStartedAt() (*metav1.Time, error) { + if node.Annotations == nil { + return nil, nil + } + + value, ok := node.Annotations[node.drainStartAnnotationKey()] + if !ok { + return nil, nil + } + + parsed, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + return nil, fmt.Errorf("error parsing drain start annotation: %w", err) + } + + startedAt := metav1.NewTime(parsed) + return &startedAt, nil +} + +// ClearDrainStart removes the drain start marker for this Skyhook on this node. +func (node *skyhookNode) ClearDrainStart() { + if node.Annotations == nil { + return + } + + key := node.drainStartAnnotationKey() + if _, ok := node.Annotations[key]; !ok { + return + } + + delete(node.Annotations, key) + node.updated = true +} + // Uncordon marks the node schedulable and removes this Skyhook's cordon annotation if present. func (node *skyhookNode) Uncordon() { @@ -496,8 +560,10 @@ func (node *skyhookNode) Reset() { node.skyhook.Updated = true delete(node.Annotations, fmt.Sprintf("%s/cordon_%s", v1alpha1.METADATA_PREFIX, node.skyhookName)) + delete(node.Annotations, fmt.Sprintf("%s/drainStart_%s", v1alpha1.METADATA_PREFIX, node.skyhookName)) delete(node.Annotations, fmt.Sprintf("%s/nodeState_%s", v1alpha1.METADATA_PREFIX, node.skyhookName)) delete(node.Annotations, fmt.Sprintf("%s/status_%s", v1alpha1.METADATA_PREFIX, node.skyhookName)) + delete(node.Annotations, fmt.Sprintf("%s/version_%s", v1alpha1.METADATA_PREFIX, node.skyhookName)) delete(node.Labels, fmt.Sprintf("%s/status_%s", v1alpha1.METADATA_PREFIX, node.skyhookName)) diff --git a/operator/internal/wrapper/node_test.go b/operator/internal/wrapper/node_test.go index 94843985..0119c79c 100644 --- a/operator/internal/wrapper/node_test.go +++ b/operator/internal/wrapper/node_test.go @@ -19,6 +19,8 @@ package wrapper import ( + "time" + "github.com/NVIDIA/nodewright/operator/api/v1alpha1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -178,6 +180,99 @@ var _ = Describe("SkyhookNode", func() { }) }) + Context("DrainStart", func() { + It("should record, read, and clear the drain start annotation", func() { + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + }, + } + + sn, err := NewSkyhookNode(node, &v1alpha1.Skyhook{ + ObjectMeta: metav1.ObjectMeta{Name: "my-skyhook"}, + Spec: v1alpha1.SkyhookSpec{Packages: v1alpha1.Packages{}}, + }) + Expect(err).ToNot(HaveOccurred()) + + started := metav1.NewTime(time.Date(2026, time.June, 2, 13, 14, 15, 0, time.UTC)) + sn.StartDrain(started) + + Expect(node.Annotations).To(HaveKeyWithValue("skyhook.nvidia.com/drainStart_my-skyhook", "2026-06-02T13:14:15Z")) + Expect(sn.Changed()).To(BeTrue()) + + drainStartedAt, err := sn.DrainStartedAt() + Expect(err).ToNot(HaveOccurred()) + Expect(drainStartedAt).To(Equal(&started)) + + sn.ClearDrainStart() + Expect(node.Annotations).ToNot(HaveKey("skyhook.nvidia.com/drainStart_my-skyhook")) + }) + + It("should return an error for a malformed drain start annotation", func() { + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Annotations: map[string]string{ + "skyhook.nvidia.com/drainStart_my-skyhook": "not-a-time", + }, + }, + } + + sn, err := NewSkyhookNode(node, &v1alpha1.Skyhook{ + ObjectMeta: metav1.ObjectMeta{Name: "my-skyhook"}, + Spec: v1alpha1.SkyhookSpec{Packages: v1alpha1.Packages{}}, + }) + Expect(err).ToNot(HaveOccurred()) + + drainStartedAt, err := sn.DrainStartedAt() + Expect(err).To(HaveOccurred()) + Expect(drainStartedAt).To(BeNil()) + }) + }) + + Context("Reset", func() { + It("should remove drain start and other node metadata for the skyhook", func() { + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Annotations: map[string]string{ + "skyhook.nvidia.com/cordon_my-skyhook": "true", + "skyhook.nvidia.com/drainStart_my-skyhook": "2026-06-02T13:14:15Z", + "skyhook.nvidia.com/nodeState_my-skyhook": "{}", + "skyhook.nvidia.com/status_my-skyhook": "erroring", + "skyhook.nvidia.com/version_my-skyhook": "1.0.0", + }, + Labels: map[string]string{ + "skyhook.nvidia.com/status_my-skyhook": "erroring", + }, + }, + } + skyhook := &v1alpha1.Skyhook{ + ObjectMeta: metav1.ObjectMeta{Name: "my-skyhook"}, + Spec: v1alpha1.SkyhookSpec{Packages: v1alpha1.Packages{}}, + Status: v1alpha1.SkyhookStatus{ + NodeState: map[string]v1alpha1.NodeState{"test-node": {}}, + NodeStatus: map[string]v1alpha1.Status{"test-node": v1alpha1.StatusErroring}, + }, + } + + sn, err := NewSkyhookNode(node, skyhook) + Expect(err).ToNot(HaveOccurred()) + + sn.Reset() + + Expect(node.Annotations).ToNot(HaveKey("skyhook.nvidia.com/cordon_my-skyhook")) + Expect(node.Annotations).ToNot(HaveKey("skyhook.nvidia.com/drainStart_my-skyhook")) + Expect(node.Annotations).ToNot(HaveKey("skyhook.nvidia.com/nodeState_my-skyhook")) + Expect(node.Annotations).ToNot(HaveKey("skyhook.nvidia.com/status_my-skyhook")) + Expect(node.Annotations).ToNot(HaveKey("skyhook.nvidia.com/version_my-skyhook")) + Expect(node.Labels).ToNot(HaveKey("skyhook.nvidia.com/status_my-skyhook")) + Expect(skyhook.Status.NodeState).ToNot(HaveKey("test-node")) + Expect(skyhook.Status.NodeStatus).ToNot(HaveKey("test-node")) + Expect(skyhook.Status.Status).To(Equal(v1alpha1.StatusUnknown)) + }) + }) + Context("CleanupSCRMetadata", func() { It("should remove matching keys and preserve others", func() { node := &corev1.Node{ diff --git a/scripts/gen-changelog.sh b/scripts/gen-changelog.sh index 821f66ef..d298d7b9 100755 --- a/scripts/gen-changelog.sh +++ b/scripts/gen-changelog.sh @@ -1,4 +1,21 @@ #!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 #