diff --git a/api/nvidia/v1/clusterpolicy_types.go b/api/nvidia/v1/clusterpolicy_types.go index 12f8e9dcde..361d9d53f7 100644 --- a/api/nvidia/v1/clusterpolicy_types.go +++ b/api/nvidia/v1/clusterpolicy_types.go @@ -2205,6 +2205,14 @@ func (d *DriverSpec) IsVGPULicensingEnabled() bool { return d.LicensingConfig.ConfigMapName != "" || d.LicensingConfig.SecretName != "" } +// IsAutoUpgradeEnabled returns true if auto upgrade is enabled +func (d *DriverSpec) IsAutoUpgradeEnabled() bool { + if d.UpgradePolicy == nil { + return false + } + return d.UpgradePolicy.AutoUpgrade +} + // IsEnabled returns true if device-plugin is enabled(default) through gpu-operator func (p *DevicePluginSpec) IsEnabled() bool { if p.Enabled == nil { diff --git a/cmd/gpu-operator/main.go b/cmd/gpu-operator/main.go index 9ac5df1072..a6895a7a00 100644 --- a/cmd/gpu-operator/main.go +++ b/cmd/gpu-operator/main.go @@ -216,6 +216,16 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "NVIDIADriver") os.Exit(1) } + + if err = (&controllers.NodeLabelingReconciler{ + Namespace: operatorNamespace, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Log: ctrl.Log.WithName("controllers").WithName("NodeLabeling"), + }).SetupWithManager(ctx, mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "NodeLabeling") + os.Exit(1) + } // +kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("health", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") diff --git a/controllers/clusterpolicy_controller.go b/controllers/clusterpolicy_controller.go index bb89047e7c..8623e0963c 100644 --- a/controllers/clusterpolicy_controller.go +++ b/controllers/clusterpolicy_controller.go @@ -280,9 +280,8 @@ func addWatchNewGPUNode(r *ClusterPolicyReconciler, c controller.Controller, mgr oldLabels := e.ObjectOld.GetLabels() nodeName := e.ObjectNew.GetName() - gpuCommonLabelMissing := hasGPULabels(newLabels) && !hasCommonGPULabel(newLabels) - gpuCommonLabelOutdated := !hasGPULabels(newLabels) && hasCommonGPULabel(newLabels) - migManagerLabelMissing := hasMIGCapableGPU(newLabels) && !hasMIGManagerLabel(newLabels) + // Trigger when NodeLabelingReconciler sets gpu.present=true on a new GPU node. + gpuCommonLabelAdded := !hasCommonGPULabel(oldLabels) && hasCommonGPULabel(newLabels) commonOperandsLabelChanged := hasOperandsDisabled(oldLabels) != hasOperandsDisabled(newLabels) oldGPUWorkloadConfig, _ := getWorkloadConfig(oldLabels, true) @@ -293,9 +292,7 @@ func addWatchNewGPUNode(r *ClusterPolicyReconciler, c controller.Controller, mgr newOSTreeLabel := newLabels[nfdOSTreeVersionLabelKey] osTreeLabelChanged := oldOSTreeLabel != newOSTreeLabel - needsUpdate := gpuCommonLabelMissing || - gpuCommonLabelOutdated || - migManagerLabelMissing || + needsUpdate := gpuCommonLabelAdded || commonOperandsLabelChanged || gpuWorkloadConfigLabelChanged || osTreeLabelChanged @@ -303,9 +300,7 @@ func addWatchNewGPUNode(r *ClusterPolicyReconciler, c controller.Controller, mgr if needsUpdate { r.Log.Info("Node needs an update", "name", nodeName, - "gpuCommonLabelMissing", gpuCommonLabelMissing, - "gpuCommonLabelOutdated", gpuCommonLabelOutdated, - "migManagerLabelMissing", migManagerLabelMissing, + "gpuCommonLabelAdded", gpuCommonLabelAdded, "commonOperandsLabelChanged", commonOperandsLabelChanged, "gpuWorkloadConfigLabelChanged", gpuWorkloadConfigLabelChanged, "osTreeLabelChanged", osTreeLabelChanged, diff --git a/controllers/nodelabeling_controller.go b/controllers/nodelabeling_controller.go new file mode 100644 index 0000000000..4fe4641c2d --- /dev/null +++ b/controllers/nodelabeling_controller.go @@ -0,0 +1,509 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package controllers + +import ( + "context" + "fmt" + + "github.com/NVIDIA/k8s-operator-libs/pkg/upgrade" + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + gpuv1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1" + nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1" + "github.com/NVIDIA/gpu-operator/internal/consts" + nvidiadriverutil "github.com/NVIDIA/gpu-operator/internal/nvidiadriver" +) + +const nodeLabelingControllerSingletonName = "cluster" + +// NodeLabelingReconciler applies GPU-Operator related labels and annotations to Kubernetes nodes. +// All node label write operations for the GPU Operator are centralized here. +type NodeLabelingReconciler struct { + client.Client + Scheme *runtime.Scheme + Namespace string + Log logr.Logger +} + +// nodeLabelingController holds per-reconcile state so that helper methods don't need to +// re-receive that state as arguments. +type nodeLabelingController struct { + client client.Client + namespace string + clusterPolicy *gpuv1.ClusterPolicy + logger logr.Logger +} + +// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch;update;patch + +// Reconcile applies GPU-Operator related labels and annotations to all cluster nodes. +func (r *NodeLabelingReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + r.Log.Info("Reconciling node labels") + + clusterPolicyList := &gpuv1.ClusterPolicyList{} + if err := r.List(ctx, clusterPolicyList); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to list ClusterPolicy: %w", err) + } + + // (cdesiniotis) Return early if a ClusterPolicy CR does not exist. + // This means that nodes will not get labeled unless a ClusterPolicy + // CR has been created. This may be relaxed in the future when the + // NVIDIA DRA Driver for GPUs is integrated with the GPU Operator + // and new CRDs are introduced. + if len(clusterPolicyList.Items) == 0 { + r.Log.Info("No ClusterPolicy CR exists, skipping node labeling") + return reconcile.Result{}, nil + } + clusterPolicy := &clusterPolicyList.Items[0] + + nlc := &nodeLabelingController{ + client: r.Client, + namespace: r.Namespace, + clusterPolicy: clusterPolicy, + logger: r.Log, + } + + if err := nlc.labelGPUNodes(ctx); err != nil { + return reconcile.Result{}, err + } + + if nlc.clusterPolicy.Spec.Driver.UseNvidiaDriverCRDType() { + if _, err := nvidiadriverutil.AssignOwners(ctx, r.Client); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to assign NVIDIADriver owners to nodes: %w", err) + } + if err := nlc.labelNodesWithOrphanedDriverPods(ctx); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to label nodes with orphaned NVIDIA driver pods: %w", err) + } + } + + if err := nlc.applyDriverAutoUpgradeAnnotation(ctx); err != nil { + return reconcile.Result{}, err + } + + return reconcile.Result{}, nil +} + +func (nlc *nodeLabelingController) labelGPUNodes(ctx context.Context) error { + nodeList := &corev1.NodeList{} + if err := nlc.client.List(ctx, nodeList); err != nil { + return fmt.Errorf("unable to list nodes: %w", err) + } + + for _, node := range nodeList.Items { + original := node.DeepCopy() + labels := node.GetLabels() + modified := false + + if nlc.reconcileCommonGPULabel(labels, node.Name) { + node.SetLabels(labels) + modified = true + } + + if nlc.updateGPUStateLabels(labels, node.Name) { + node.SetLabels(labels) + modified = true + } + + if modified { + if err := nlc.client.Patch(ctx, &node, client.MergeFrom(original)); err != nil { + return fmt.Errorf("unable to label node %s: %w", node.Name, err) + } + } + } + return nil +} + +// reconcileCommonGPULabel keeps nvidia.com/gpu.present in sync with NFD GPU PCI labels. +// Returns true if labels were modified. +func (nlc *nodeLabelingController) reconcileCommonGPULabel(labels map[string]string, nodeName string) bool { + if !hasCommonGPULabel(labels) && hasGPULabels(labels) { + nlc.logger.Info("Node has GPU(s), setting common GPU label", "NodeName", nodeName) + labels[commonGPULabelKey] = commonGPULabelValue + return true + } + if hasCommonGPULabel(labels) && !hasGPULabels(labels) { + nlc.logger.Info("Node no longer has GPUs, clearing GPU labels", "NodeName", nodeName) + labels[commonGPULabelKey] = "false" + return true + } + return false +} + +// updateGPUStateLabels syncs nvidia.com/gpu.deploy.* labels and sets the MIG config label when +// appropriate. If the node does not have the common GPU label, all state labels are removed. +// Returns true if labels were modified. +func (nlc *nodeLabelingController) updateGPUStateLabels(labels map[string]string, nodeName string) bool { + if !hasCommonGPULabel(labels) { + return removeAllGPUStateLabels(labels) + } + + cp := nlc.clusterPolicy + sandboxEnabled := cp != nil && cp.Spec.SandboxWorkloads.IsEnabled() + sandboxMode := "" + if cp != nil { + sandboxMode = cp.Spec.SandboxWorkloads.Mode + } + + config, err := getWorkloadConfig(labels, sandboxEnabled) + if err != nil { + nlc.logger.Info("WARNING: failed to get GPU workload config for node; using default", + "NodeName", nodeName, "SandboxEnabled", sandboxEnabled, + "Error", err, "defaultGPUWorkloadConfig", defaultGPUWorkloadConfig) + } + gpuWorkloadConfig := &gpuWorkloadConfiguration{ + config: config, + sandboxMode: sandboxMode, + node: nodeName, + log: nlc.logger, + } + modified := gpuWorkloadConfig.updateGPUStateLabels(labels) + + if cp != nil && cp.Spec.MIGManager.IsEnabled() && hasMIGCapableGPU(labels) && !hasMIGConfigLabel(labels) { + migConfigDefault := "" + if cp.Spec.MIGManager.Config != nil { + migConfigDefault = cp.Spec.MIGManager.Config.Default + } + if migConfigDefault == migConfigDisabledValue { + nlc.logger.Info("Setting MIG config label", "NodeName", nodeName, + "Label", migConfigLabelKey, "Value", migConfigDisabledValue) + labels[migConfigLabelKey] = migConfigDisabledValue + modified = true + } + } + return modified +} + +func (nlc *nodeLabelingController) setDriverAutoUpgradeAnnotation(ctx context.Context, node *corev1.Node, autoUpgradeEnabled bool) error { + annotationValue, annotationExists := node.Annotations[driverAutoUpgradeAnnotationKey] + updateRequired := false + if autoUpgradeEnabled { + updateRequired = !annotationExists || annotationValue != "true" + } else { + updateRequired = annotationExists + } + if !updateRequired { + return nil + } + + original := node.DeepCopy() + if node.Annotations == nil { + node.Annotations = map[string]string{} + } + if autoUpgradeEnabled { + node.Annotations[driverAutoUpgradeAnnotationKey] = "true" + } else { + delete(node.Annotations, driverAutoUpgradeAnnotationKey) + } + if err := nlc.client.Patch(ctx, node, client.MergeFrom(original)); err != nil { + nlc.logger.Error(err, "Failed to patch driver auto-upgrade annotation", + "node", node.Name, "enabled", autoUpgradeEnabled) + return err + } + + return nil +} + +// applyDriverAutoUpgradeAnnotation sets or clears the driver auto-upgrade annotation on GPU nodes. +func (nlc *nodeLabelingController) applyDriverAutoUpgradeAnnotation(ctx context.Context) error { + cp := nlc.clusterPolicy + + if cp.Spec.Driver.UseNvidiaDriverCRDType() && !cp.Spec.SandboxWorkloads.IsEnabled() { + return nlc.applyDriverAutoUpgradeAnnotationForNVD(ctx) + } + + autoUpgradeEnabled := cp.Spec.Driver.IsEnabled() && + cp.Spec.Driver.IsAutoUpgradeEnabled() && + !cp.Spec.SandboxWorkloads.IsEnabled() + + nodeList := &corev1.NodeList{} + if err := nlc.client.List(ctx, nodeList, client.MatchingLabels{consts.GPUPresentLabel: "true"}); err != nil { + return fmt.Errorf("unable to list nodes: %w", err) + } + + for _, node := range nodeList.Items { + err := nlc.setDriverAutoUpgradeAnnotation(ctx, &node, autoUpgradeEnabled) + if err != nil { + return fmt.Errorf("failed to set driver auto-upgrade annotation on node %q: %w", node.Name, err) + } + } + return nil +} + +func (nlc *nodeLabelingController) applyDriverAutoUpgradeAnnotationForNVD(ctx context.Context) error { + nvidiaDriverList := &nvidiav1alpha1.NVIDIADriverList{} + if err := nlc.client.List(ctx, nvidiaDriverList); err != nil { + return fmt.Errorf("failed to list NVIDIADriver instances: %w", err) + } + + for _, nvd := range nvidiaDriverList.Items { + nodeList := &corev1.NodeList{} + if err := nlc.client.List(ctx, nodeList, client.MatchingLabels{consts.NVIDIADriverOwnerLabel: nvd.Name}); err != nil { + nlc.logger.Error(err, "Failed to list nodes for NVIDIADriver", "name", nvd.Name) + return err + } + autoUpgradeEnabled := nvd.Spec.GetUpgradePolicyWithDefaults().AutoUpgrade + for _, node := range nodeList.Items { + err := nlc.setDriverAutoUpgradeAnnotation(ctx, &node, autoUpgradeEnabled) + if err != nil { + return fmt.Errorf("failed to set driver auto-upgrade annotation on node %q: %w", node.Name, err) + } + } + } + + return nil +} + +// labelNodesWithOrphanedDriverPods marks nodes that still have unowned (orphaned) ClusterPolicy +// driver pods so the upgrade controller can replace them in the normal upgrade flow. +func (nlc *nodeLabelingController) labelNodesWithOrphanedDriverPods(ctx context.Context) error { + nvidiaDrivers := &nvidiav1alpha1.NVIDIADriverList{} + if err := nlc.client.List(ctx, nvidiaDrivers); err != nil { + return fmt.Errorf("failed to list NVIDIADriver CRs: %w", err) + } + if len(nvidiaDrivers.Items) == 0 { + return nil + } + + pods := &corev1.PodList{} + if err := nlc.client.List(ctx, pods, + client.InNamespace(nlc.namespace), + client.MatchingLabels{AppComponentLabelKey: DriverAppComponentLabelValue}, + ); err != nil { + return fmt.Errorf("failed to list NVIDIA driver pods: %w", err) + } + + for _, pod := range pods.Items { + if len(pod.OwnerReferences) > 0 || pod.Status.Phase != corev1.PodRunning || pod.Spec.NodeName == "" { + continue + } + + node := &corev1.Node{} + if err := nlc.client.Get(ctx, types.NamespacedName{Name: pod.Spec.NodeName}, node); err != nil { + nlc.logger.Error(err, "failed to get node for orphaned driver pod", "pod", pod.Name, "node", pod.Spec.NodeName) + continue + } + if !nodeOwnedByNVIDIADriver(node, nvidiaDrivers.Items) { + continue + } + + upgradeStateLabel := upgrade.GetUpgradeStateLabelKey() + upgradeState := upgrade.UpgradeStateUnknown + if node.Labels != nil { + upgradeState = node.Labels[upgradeStateLabel] + } + if !isDriverUpgradeRequestAllowed(upgradeState) { + continue + } + + original := node.DeepCopy() + if node.Labels == nil { + node.Labels = map[string]string{} + } + node.Labels[upgradeStateLabel] = upgrade.UpgradeStateUpgradeRequired + if err := nlc.client.Patch(ctx, node, client.MergeFrom(original)); err != nil { + return fmt.Errorf("failed to label node %q for orphaned driver pod %q: %w", node.Name, pod.Name, err) + } + } + return nil +} + +// nodeOwnedByNVIDIADriver returns true when the node has an owner label matching a live NVIDIADriver. +func nodeOwnedByNVIDIADriver(node *corev1.Node, nvidiaDrivers []nvidiav1alpha1.NVIDIADriver) bool { + if node.Labels == nil || node.Labels[consts.NVIDIADriverOwnerLabel] == "" { + return false + } + for _, nvidiaDriver := range nvidiaDrivers { + if nvidiaDriver.HasDeletionTimestamp() { + continue + } + if node.Labels[consts.NVIDIADriverOwnerLabel] == nvidiaDriver.Name { + return true + } + } + return false +} + +// isDriverUpgradeRequestAllowed returns true when migration can request a driver upgrade +// without overwriting an active or failed upgrade state. +func isDriverUpgradeRequestAllowed(upgradeState string) bool { + return upgradeState == upgrade.UpgradeStateUnknown || upgradeState == upgrade.UpgradeStateDone +} + +// SetupWithManager registers the NodeLabelingReconciler with the controller-runtime manager. +func (r *NodeLabelingReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { + mapToSingleton := func(_ context.Context, _ client.Object) []reconcile.Request { + return []reconcile.Request{{NamespacedName: types.NamespacedName{Name: nodeLabelingControllerSingletonName}}} + } + + c, err := controller.New("node-labeling-controller", mgr, controller.Options{ + Reconciler: r, + MaxConcurrentReconciles: 1, + RateLimiter: workqueue.NewTypedItemExponentialFailureRateLimiter[reconcile.Request](minDelayCR, maxDelayCR), + }) + if err != nil { + return fmt.Errorf("error creating node-labeling controller: %w", err) + } + + clusterPolicyMapFn := func(ctx context.Context, cp *gpuv1.ClusterPolicy) []reconcile.Request { + return mapToSingleton(ctx, cp) + } + if err := c.Watch(source.Kind( + mgr.GetCache(), + &gpuv1.ClusterPolicy{}, + handler.TypedEnqueueRequestsFromMapFunc(clusterPolicyMapFn), + predicate.TypedGenerationChangedPredicate[*gpuv1.ClusterPolicy]{}, + )); err != nil { + return fmt.Errorf("error watching ClusterPolicy: %w", err) + } + + // Watch NVIDIADriver including delete events so owner labels are cleaned up promptly. + nvidiaDriverMapFn := func(ctx context.Context, nd *nvidiav1alpha1.NVIDIADriver) []reconcile.Request { + return mapToSingleton(ctx, nd) + } + if err := c.Watch(source.Kind( + mgr.GetCache(), + &nvidiav1alpha1.NVIDIADriver{}, + handler.TypedEnqueueRequestsFromMapFunc(nvidiaDriverMapFn), + predicate.TypedGenerationChangedPredicate[*nvidiav1alpha1.NVIDIADriver]{}, + )); err != nil { + return fmt.Errorf("error watching NVIDIADriver: %w", err) + } + + nodePredicate := predicate.TypedFuncs[*corev1.Node]{ + CreateFunc: func(e event.TypedCreateEvent[*corev1.Node]) bool { + labels := e.Object.GetLabels() + return hasGPULabels(labels) + }, + UpdateFunc: func(e event.TypedUpdateEvent[*corev1.Node]) bool { + newLabels := e.ObjectNew.GetLabels() + oldLabels := e.ObjectOld.GetLabels() + nodeName := e.ObjectNew.GetName() + + gpuCommonLabelMissing := hasGPULabels(newLabels) && !hasCommonGPULabel(newLabels) + gpuCommonLabelOutdated := !hasGPULabels(newLabels) && hasCommonGPULabel(newLabels) + commonOperandsLabelChanged := hasOperandsDisabled(oldLabels) != hasOperandsDisabled(newLabels) + + oldGPUWorkloadConfig, _ := getWorkloadConfig(oldLabels, true) + newGPUWorkloadConfig, _ := getWorkloadConfig(newLabels, true) + gpuWorkloadConfigLabelChanged := oldGPUWorkloadConfig != newGPUWorkloadConfig + + oldOSTreeLabel := oldLabels[nfdOSTreeVersionLabelKey] + newOSTreeLabel := newLabels[nfdOSTreeVersionLabelKey] + osTreeLabelChanged := oldOSTreeLabel != newOSTreeLabel + + nvidiaDriverOwnerLabelChanged := oldLabels[consts.NVIDIADriverOwnerLabel] != newLabels[consts.NVIDIADriverOwnerLabel] + + needsUpdate := gpuCommonLabelMissing || + gpuCommonLabelOutdated || + commonOperandsLabelChanged || + gpuWorkloadConfigLabelChanged || + osTreeLabelChanged || + nvidiaDriverOwnerLabelChanged + + // When an NVIDIADriver daemonset pod is running on the node, check if any + // label which is configured in the NVIDIADriver's node selector has changed. + nvidiaDriverNodeSelectorLabelChanged := false + if !needsUpdate && newLabels[consts.NVIDIADriverOwnerLabel] != "" { + name := newLabels[consts.NVIDIADriverOwnerLabel] + nvidiaDriver := &nvidiav1alpha1.NVIDIADriver{} + err := r.Get(ctx, types.NamespacedName{Name: name}, nvidiaDriver) + if err != nil { + r.Log.Error(err, "failed to get NVIDIADriver object that owns this node", "name", name, "node", nodeName) + return false + } + for key := range nvidiaDriver.Spec.NodeSelector { + if oldLabels[key] != newLabels[key] { + nvidiaDriverNodeSelectorLabelChanged = true + needsUpdate = true + break + } + } + } + + if needsUpdate { + r.Log.Info("Node needs an update", + "name", nodeName, + "gpuCommonLabelMissing", gpuCommonLabelMissing, + "gpuCommonLabelOutdated", gpuCommonLabelOutdated, + "commonOperandsLabelChanged", commonOperandsLabelChanged, + "gpuWorkloadConfigLabelChanged", gpuWorkloadConfigLabelChanged, + "osTreeLabelChanged", osTreeLabelChanged, + "nvidiaDriverOwnerLabelChanged", nvidiaDriverOwnerLabelChanged, + "nvidiaDriverNodeSelectorLabelChanged", nvidiaDriverNodeSelectorLabelChanged, + ) + } + return needsUpdate + }, + DeleteFunc: func(e event.TypedDeleteEvent[*corev1.Node]) bool { + return false + }, + } + nodeMapFn := func(ctx context.Context, n *corev1.Node) []reconcile.Request { + return mapToSingleton(ctx, n) + } + if err := c.Watch(source.Kind( + mgr.GetCache(), + &corev1.Node{}, + handler.TypedEnqueueRequestsFromMapFunc(nodeMapFn), + nodePredicate, + )); err != nil { + return fmt.Errorf("error watching Nodes: %w", err) + } + + // Trigger on driver pods becoming Running so orphaned pods are detected promptly. + podPredicate := predicate.TypedFuncs[*corev1.Pod]{ + CreateFunc: func(e event.TypedCreateEvent[*corev1.Pod]) bool { + return e.Object.GetLabels()[AppComponentLabelKey] == DriverAppComponentLabelValue + }, + UpdateFunc: func(e event.TypedUpdateEvent[*corev1.Pod]) bool { + if e.ObjectNew.GetLabels()[AppComponentLabelKey] != DriverAppComponentLabelValue { + return false + } + return e.ObjectOld.Status.Phase != corev1.PodRunning && + e.ObjectNew.Status.Phase == corev1.PodRunning + }, + DeleteFunc: func(e event.TypedDeleteEvent[*corev1.Pod]) bool { + return false + }, + } + podMapFn := func(ctx context.Context, p *corev1.Pod) []reconcile.Request { + return mapToSingleton(ctx, p) + } + if err := c.Watch(source.Kind( + mgr.GetCache(), + &corev1.Pod{}, + handler.TypedEnqueueRequestsFromMapFunc(podMapFn), + podPredicate, + )); err != nil { + return fmt.Errorf("error watching Pods: %w", err) + } + + return nil +} diff --git a/controllers/nodelabeling_controller_test.go b/controllers/nodelabeling_controller_test.go new file mode 100644 index 0000000000..d2253bfaae --- /dev/null +++ b/controllers/nodelabeling_controller_test.go @@ -0,0 +1,583 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package controllers + +import ( + "context" + "testing" + + "github.com/NVIDIA/k8s-operator-libs/pkg/upgrade" + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + gpuv1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1" + nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1" + "github.com/NVIDIA/gpu-operator/internal/consts" +) + +// mergeLabels merges multiple label maps into one (last write wins). +func mergeLabels(maps ...map[string]string) map[string]string { + out := make(map[string]string) + for _, m := range maps { + for k, v := range m { + out[k] = v + } + } + return out +} + +func TestReconcileCommonGPULabel(t *testing.T) { + tests := []struct { + description string + initialLabels map[string]string + expectedLabels map[string]string + }{ + { + description: "empty", + initialLabels: map[string]string{}, + expectedLabels: map[string]string{}, + }, + { + description: "GPU PCI label present, common GPU label missing", + initialLabels: map[string]string{ + "feature.node.kubernetes.io/pci-10de.present": "true", + }, + expectedLabels: map[string]string{ + "feature.node.kubernetes.io/pci-10de.present": "true", + commonGPULabelKey: "true", + }, + }, + { + description: "GPU PCI label present, common GPU label is false", + initialLabels: map[string]string{ + "feature.node.kubernetes.io/pci-10de.present": "true", + commonGPULabelKey: "false", + }, + expectedLabels: map[string]string{ + "feature.node.kubernetes.io/pci-10de.present": "true", + commonGPULabelKey: "true", + }, + }, + { + description: "GPU PCI label present, common GPU label is true", + initialLabels: map[string]string{commonGPULabelKey: "true"}, + expectedLabels: map[string]string{commonGPULabelKey: "false"}, + }, + { + description: "GPU PCI label present, common GPU label is true", + initialLabels: map[string]string{ + "feature.node.kubernetes.io/pci-10de.present": "true", + commonGPULabelKey: "true", + }, + expectedLabels: map[string]string{ + "feature.node.kubernetes.io/pci-10de.present": "true", + commonGPULabelKey: "true", + }, + }, + { + description: "GPU PCI label missing, common GPU label is false", + initialLabels: map[string]string{commonGPULabelKey: "false"}, + expectedLabels: map[string]string{commonGPULabelKey: "false"}, + }, + } + + for _, tc := range tests { + t.Run(tc.description, func(t *testing.T) { + nlc := &nodeLabelingController{ + clusterPolicy: &gpuv1.ClusterPolicy{}, + logger: logr.Discard(), + } + labels := tc.initialLabels + nlc.reconcileCommonGPULabel(labels, "test-node") + assert.Equal(t, tc.expectedLabels, labels) + }) + } +} + +func TestUpdateGPUStateLabels(t *testing.T) { + tests := []struct { + name string + clusterPolicy *gpuv1.ClusterPolicy + initialLabels map[string]string + expectedLabels map[string]string + }{ + { + name: "empty", + clusterPolicy: &gpuv1.ClusterPolicy{}, + initialLabels: map[string]string{}, + expectedLabels: map[string]string{}, + }, + { + name: "no common GPU label, all state labels removed, non-state labels preserved", + clusterPolicy: &gpuv1.ClusterPolicy{}, + initialLabels: map[string]string{ + "nvidia.com/gpu.deploy.driver": "true", + "nvidia.com/gpu.deploy.device-plugin": "true", + "foo": "bar", + }, + expectedLabels: map[string]string{ + "foo": "bar", + }, + }, + { + name: "container workload, container state labels applied", + clusterPolicy: &gpuv1.ClusterPolicy{}, + initialLabels: map[string]string{commonGPULabelKey: commonGPULabelValue}, + expectedLabels: mergeLabels( + map[string]string{commonGPULabelKey: commonGPULabelValue}, + gpuStateLabels[gpuWorkloadConfigContainer], + ), + }, + { + name: "operands disabled, all state labels removed", + clusterPolicy: &gpuv1.ClusterPolicy{}, + initialLabels: mergeLabels( + map[string]string{ + commonGPULabelKey: commonGPULabelValue, + commonOperandsLabelKey: "false", + }, + gpuStateLabels[gpuWorkloadConfigContainer], + ), + expectedLabels: map[string]string{ + commonGPULabelKey: commonGPULabelValue, + commonOperandsLabelKey: "false", + }, + }, + { + name: "sandboxWorkloads enabled, mode=kubevirt, workloadConfig=passthrough", + clusterPolicy: &gpuv1.ClusterPolicy{ + Spec: gpuv1.ClusterPolicySpec{ + SandboxWorkloads: gpuv1.SandboxWorkloadsSpec{ + Enabled: ptr.To(true), + Mode: string(gpuv1.KubeVirt), + }, + }, + }, + initialLabels: map[string]string{ + commonGPULabelKey: commonGPULabelValue, + gpuWorkloadConfigLabelKey: gpuWorkloadConfigVMPassthrough, + }, + expectedLabels: mergeLabels( + map[string]string{ + commonGPULabelKey: commonGPULabelValue, + gpuWorkloadConfigLabelKey: gpuWorkloadConfigVMPassthrough, + }, + getEffectiveStateLabels(gpuWorkloadConfigVMPassthrough, string(gpuv1.KubeVirt)), + ), + }, + { + name: "sandboxWorkloads enabled, mode=kubevirt, workloadConfig=vm-vgpu", + clusterPolicy: &gpuv1.ClusterPolicy{ + Spec: gpuv1.ClusterPolicySpec{ + SandboxWorkloads: gpuv1.SandboxWorkloadsSpec{ + Enabled: ptr.To(true), + Mode: string(gpuv1.KubeVirt), + }, + }, + }, + initialLabels: map[string]string{ + commonGPULabelKey: commonGPULabelValue, + gpuWorkloadConfigLabelKey: gpuWorkloadConfigVMVgpu, + }, + expectedLabels: mergeLabels( + map[string]string{ + commonGPULabelKey: commonGPULabelValue, + gpuWorkloadConfigLabelKey: gpuWorkloadConfigVMVgpu, + }, + getEffectiveStateLabels(gpuWorkloadConfigVMVgpu, string(gpuv1.KubeVirt)), + ), + }, + { + name: "sandboxWorkloads enabled, mode=kata, workloadConfig=passthrough", + clusterPolicy: &gpuv1.ClusterPolicy{ + Spec: gpuv1.ClusterPolicySpec{ + SandboxWorkloads: gpuv1.SandboxWorkloadsSpec{ + Enabled: ptr.To(true), + Mode: string(gpuv1.Kata), + }, + }, + }, + initialLabels: map[string]string{ + commonGPULabelKey: commonGPULabelValue, + gpuWorkloadConfigLabelKey: gpuWorkloadConfigVMPassthrough, + }, + expectedLabels: mergeLabels( + map[string]string{ + commonGPULabelKey: commonGPULabelValue, + gpuWorkloadConfigLabelKey: gpuWorkloadConfigVMPassthrough, + }, + getEffectiveStateLabels(gpuWorkloadConfigVMPassthrough, string(gpuv1.Kata)), + ), + }, + { + name: "sandboxWorkloads enabled, mode=kata, workloadConfig=vm-vgpu", + clusterPolicy: &gpuv1.ClusterPolicy{ + Spec: gpuv1.ClusterPolicySpec{ + SandboxWorkloads: gpuv1.SandboxWorkloadsSpec{ + Enabled: ptr.To(true), + Mode: string(gpuv1.Kata), + }, + }, + }, + initialLabels: map[string]string{ + commonGPULabelKey: commonGPULabelValue, + gpuWorkloadConfigLabelKey: gpuWorkloadConfigVMVgpu, + }, + expectedLabels: mergeLabels( + map[string]string{ + commonGPULabelKey: commonGPULabelValue, + gpuWorkloadConfigLabelKey: gpuWorkloadConfigVMVgpu, + }, + getEffectiveStateLabels(gpuWorkloadConfigVMVgpu, string(gpuv1.Kata)), + ), + }, + { + name: "sandboxWorkloads enabled, mode=kubevirt, workloadConfig switched from container to passthrough", + clusterPolicy: &gpuv1.ClusterPolicy{ + Spec: gpuv1.ClusterPolicySpec{ + SandboxWorkloads: gpuv1.SandboxWorkloadsSpec{ + Enabled: ptr.To(true), + Mode: string(gpuv1.KubeVirt), + }, + }, + }, + initialLabels: mergeLabels( + map[string]string{ + commonGPULabelKey: commonGPULabelValue, + gpuWorkloadConfigLabelKey: gpuWorkloadConfigVMPassthrough, + }, + getEffectiveStateLabels(gpuWorkloadConfigContainer, string(gpuv1.KubeVirt)), + ), + expectedLabels: mergeLabels( + map[string]string{ + commonGPULabelKey: commonGPULabelValue, + gpuWorkloadConfigLabelKey: gpuWorkloadConfigVMPassthrough, + }, + getEffectiveStateLabels(gpuWorkloadConfigVMPassthrough, string(gpuv1.KubeVirt)), + ), + }, + { + name: "MIG-capable node, MIG manager deploy label added and mig.config set to all-disabled", + clusterPolicy: &gpuv1.ClusterPolicy{ + Spec: gpuv1.ClusterPolicySpec{ + MIGManager: gpuv1.MIGManagerSpec{ + Enabled: ptr.To(true), + Config: &gpuv1.MIGPartedConfigSpec{Default: migConfigDisabledValue}, + }, + }, + }, + initialLabels: map[string]string{ + commonGPULabelKey: commonGPULabelValue, + migCapableLabelKey: migCapableLabelValue, + }, + expectedLabels: mergeLabels( + map[string]string{ + commonGPULabelKey: commonGPULabelValue, + migCapableLabelKey: migCapableLabelValue, + migManagerLabelKey: migManagerLabelValue, + migConfigLabelKey: migConfigDisabledValue, + }, + gpuStateLabels[gpuWorkloadConfigContainer], + ), + }, + { + name: "MIG-capable node with existing mig.config label", + clusterPolicy: &gpuv1.ClusterPolicy{ + Spec: gpuv1.ClusterPolicySpec{ + MIGManager: gpuv1.MIGManagerSpec{ + Enabled: ptr.To(true), + Config: &gpuv1.MIGPartedConfigSpec{Default: migConfigDisabledValue}, + }, + }, + }, + initialLabels: map[string]string{ + commonGPULabelKey: commonGPULabelValue, + migCapableLabelKey: migCapableLabelValue, + migConfigLabelKey: "all-1g.10gb", + }, + expectedLabels: mergeLabels( + map[string]string{ + commonGPULabelKey: commonGPULabelValue, + migCapableLabelKey: migCapableLabelValue, + migManagerLabelKey: migManagerLabelValue, + migConfigLabelKey: "all-1g.10gb", + }, + gpuStateLabels[gpuWorkloadConfigContainer], + ), + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + nlc := &nodeLabelingController{ + clusterPolicy: tc.clusterPolicy, + logger: logr.Discard(), + } + labels := mergeLabels(tc.initialLabels) + nlc.updateGPUStateLabels(labels, "test-node") + assert.Equal(t, tc.expectedLabels, labels) + }) + } +} + +func TestSetDriverAutoUpgradeAnnotation(t *testing.T) { + tests := []struct { + name string + initialAnnotations map[string]string + autoUpgradeEnabled bool + expectedAnnotations map[string]string + }{ + { + name: "autoUpgrade enabled, annotation absent → annotation set", + initialAnnotations: nil, + autoUpgradeEnabled: true, + expectedAnnotations: map[string]string{driverAutoUpgradeAnnotationKey: "true"}, + }, + { + name: "autoUpgrade enabled, annotation already true → no patch", + initialAnnotations: map[string]string{driverAutoUpgradeAnnotationKey: "true"}, + autoUpgradeEnabled: true, + expectedAnnotations: map[string]string{driverAutoUpgradeAnnotationKey: "true"}, + }, + { + name: "autoUpgrade enabled, annotation is false → annotation set to true", + initialAnnotations: map[string]string{driverAutoUpgradeAnnotationKey: "false"}, + autoUpgradeEnabled: true, + expectedAnnotations: map[string]string{driverAutoUpgradeAnnotationKey: "true"}, + }, + { + name: "autoUpgrade disabled, annotation present → annotation removed", + initialAnnotations: map[string]string{driverAutoUpgradeAnnotationKey: "true"}, + autoUpgradeEnabled: false, + expectedAnnotations: nil, + }, + { + name: "autoUpgrade disabled, annotation absent → no patch", + initialAnnotations: nil, + autoUpgradeEnabled: false, + expectedAnnotations: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Annotations: tc.initialAnnotations, + }, + } + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node).Build() + + nlc := &nodeLabelingController{ + client: fakeClient, + logger: logr.Discard(), + } + + err := nlc.setDriverAutoUpgradeAnnotation(context.Background(), node, tc.autoUpgradeEnabled) + require.NoError(t, err) + + updated := &corev1.Node{} + require.NoError(t, fakeClient.Get(context.Background(), types.NamespacedName{Name: "test-node"}, updated)) + assert.Equal(t, tc.expectedAnnotations, updated.Annotations) + }) + } +} + +func TestLabelNodesWithOrphanedDriverPods(t *testing.T) { + const namespace = "test-ns" + const driverName = "gpu-driver" + + upgradeStateLabel := upgrade.GetUpgradeStateLabelKey() + + // liveDriver returns a NVIDIADriver with no deletion timestamp. + liveDriver := func() *nvidiav1alpha1.NVIDIADriver { + return &nvidiav1alpha1.NVIDIADriver{ + ObjectMeta: metav1.ObjectMeta{Name: driverName}, + } + } + + // ownedNode returns a node that carries the NVIDIADriverOwnerLabel for driverName + // and optionally an upgrade state label. + ownedNode := func(name, upgradeState string) *corev1.Node { + labels := map[string]string{consts.NVIDIADriverOwnerLabel: driverName} + if upgradeState != "" { + labels[upgradeStateLabel] = upgradeState + } + return &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: labels}} + } + + // orphanedPod returns a Running driver pod with no owner references on the given node. + orphanedPod := func(name, nodeName string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: map[string]string{AppComponentLabelKey: DriverAppComponentLabelValue}, + }, + Spec: corev1.PodSpec{NodeName: nodeName}, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + } + } + + tests := []struct { + name string + // objects registered in the fake client + nvidiaDrivers []*nvidiav1alpha1.NVIDIADriver + nodes []*corev1.Node + pods []*corev1.Pod + // expected value of upgradeStateLabel on the named node after the call; + // empty string means the label should be absent. + expectedUpgradeState map[string]string + }{ + { + name: "no NVIDIADriver CRs → early return, node not labeled", + nvidiaDrivers: nil, + nodes: []*corev1.Node{ownedNode("node-1", "")}, + pods: []*corev1.Pod{orphanedPod("pod-1", "node-1")}, + expectedUpgradeState: map[string]string{"node-1": ""}, + }, + { + name: "orphaned pod on owned node, no upgrade state → labeled upgrade-required", + nvidiaDrivers: []*nvidiav1alpha1.NVIDIADriver{liveDriver()}, + nodes: []*corev1.Node{ownedNode("node-1", "")}, + pods: []*corev1.Pod{orphanedPod("pod-1", "node-1")}, + expectedUpgradeState: map[string]string{"node-1": upgrade.UpgradeStateUpgradeRequired}, + }, + { + name: "orphaned pod on owned node, upgrade-done state → labeled upgrade-required", + nvidiaDrivers: []*nvidiav1alpha1.NVIDIADriver{liveDriver()}, + nodes: []*corev1.Node{ownedNode("node-1", upgrade.UpgradeStateDone)}, + pods: []*corev1.Pod{orphanedPod("pod-1", "node-1")}, + expectedUpgradeState: map[string]string{"node-1": upgrade.UpgradeStateUpgradeRequired}, + }, + { + name: "orphaned pod on owned node, active upgrade state → not relabeled", + nvidiaDrivers: []*nvidiav1alpha1.NVIDIADriver{liveDriver()}, + nodes: []*corev1.Node{ownedNode("node-1", upgrade.UpgradeStatePodRestartRequired)}, + pods: []*corev1.Pod{orphanedPod("pod-1", "node-1")}, + expectedUpgradeState: map[string]string{"node-1": upgrade.UpgradeStatePodRestartRequired}, + }, + { + name: "orphaned pod on owned node, failed upgrade state → not relabeled", + nvidiaDrivers: []*nvidiav1alpha1.NVIDIADriver{liveDriver()}, + nodes: []*corev1.Node{ownedNode("node-1", upgrade.UpgradeStateFailed)}, + pods: []*corev1.Pod{orphanedPod("pod-1", "node-1")}, + expectedUpgradeState: map[string]string{"node-1": upgrade.UpgradeStateFailed}, + }, + { + name: "pod has owner references → skipped", + nvidiaDrivers: []*nvidiav1alpha1.NVIDIADriver{liveDriver()}, + nodes: []*corev1.Node{ownedNode("node-1", "")}, + pods: []*corev1.Pod{func() *corev1.Pod { + p := orphanedPod("pod-1", "node-1") + p.OwnerReferences = []metav1.OwnerReference{{Name: "daemonset-1"}} + return p + }()}, + expectedUpgradeState: map[string]string{"node-1": ""}, + }, + { + name: "pod not in Running phase → skipped", + nvidiaDrivers: []*nvidiav1alpha1.NVIDIADriver{liveDriver()}, + nodes: []*corev1.Node{ownedNode("node-1", "")}, + pods: []*corev1.Pod{func() *corev1.Pod { + p := orphanedPod("pod-1", "node-1") + p.Status.Phase = corev1.PodPending + return p + }()}, + expectedUpgradeState: map[string]string{"node-1": ""}, + }, + { + name: "pod has no NodeName → skipped", + nvidiaDrivers: []*nvidiav1alpha1.NVIDIADriver{liveDriver()}, + nodes: []*corev1.Node{ownedNode("node-1", "")}, + pods: []*corev1.Pod{func() *corev1.Pod { + p := orphanedPod("pod-1", "node-1") + p.Spec.NodeName = "" + return p + }()}, + expectedUpgradeState: map[string]string{"node-1": ""}, + }, + { + name: "node not owned by any NVIDIADriver → not labeled", + nvidiaDrivers: []*nvidiav1alpha1.NVIDIADriver{liveDriver()}, + nodes: []*corev1.Node{{ + ObjectMeta: metav1.ObjectMeta{Name: "node-1"}, // no NVIDIADriverOwnerLabel + }}, + pods: []*corev1.Pod{orphanedPod("pod-1", "node-1")}, + expectedUpgradeState: map[string]string{"node-1": ""}, + }, + { + name: "NVIDIADriver with deletion timestamp → node treated as unowned, not labeled", + nvidiaDrivers: []*nvidiav1alpha1.NVIDIADriver{{ + ObjectMeta: metav1.ObjectMeta{ + Name: driverName, + DeletionTimestamp: ptr.To(metav1.Now()), + Finalizers: []string{"test-finalizer"}, + }, + }}, + nodes: []*corev1.Node{ownedNode("node-1", "")}, + pods: []*corev1.Pod{orphanedPod("pod-1", "node-1")}, + expectedUpgradeState: map[string]string{"node-1": ""}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + require.NoError(t, nvidiav1alpha1.AddToScheme(scheme)) + + var objects []client.Object + for _, d := range tc.nvidiaDrivers { + objects = append(objects, d) + } + for _, n := range tc.nodes { + objects = append(objects, n) + } + for _, p := range tc.pods { + objects = append(objects, p) + } + + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + nlc := &nodeLabelingController{ + client: fakeClient, + namespace: namespace, + logger: logr.Discard(), + } + + require.NoError(t, nlc.labelNodesWithOrphanedDriverPods(context.Background())) + + for nodeName, expectedUpgradeState := range tc.expectedUpgradeState { + node := &corev1.Node{} + require.NoError(t, fakeClient.Get(context.Background(), types.NamespacedName{Name: nodeName}, node)) + assert.Equal(t, expectedUpgradeState, node.Labels[upgradeStateLabel], "node %q upgrade state", nodeName) + } + }) + } +} diff --git a/controllers/nvidiadriver_controller.go b/controllers/nvidiadriver_controller.go index 61a9c3866c..5dc142de3b 100644 --- a/controllers/nvidiadriver_controller.go +++ b/controllers/nvidiadriver_controller.go @@ -45,7 +45,6 @@ import ( "github.com/NVIDIA/gpu-operator/controllers/clusterinfo" "github.com/NVIDIA/gpu-operator/internal/conditions" "github.com/NVIDIA/gpu-operator/internal/consts" - nvidiadriverutil "github.com/NVIDIA/gpu-operator/internal/nvidiadriver" "github.com/NVIDIA/gpu-operator/internal/state" "github.com/NVIDIA/gpu-operator/internal/validator" ) @@ -84,8 +83,7 @@ func (r *NVIDIADriverReconciler) Reconcile(ctx context.Context, req ctrl.Request if err := r.Get(ctx, req.NamespacedName, instance); err != nil { if apierrors.IsNotFound(err) { // Request object not found, could have been deleted after reconcile request. - // Re-run owner assignment so deleting the last NVIDIADriver clears stale node owner labels. - return r.cleanupNVIDIADriverOwnerLabels(ctx) + return reconcile.Result{}, nil } wrappedErr := fmt.Errorf("error getting NVIDIADriver object: %w", err) logger.Error(err, "error getting NVIDIADriver object") @@ -97,8 +95,7 @@ func (r *NVIDIADriverReconciler) Reconcile(ctx context.Context, req ctrl.Request return reconcile.Result{}, wrappedErr } if instance.HasDeletionTimestamp() { - logger.Info("NVIDIADriver delete requested; cleaning up owner labels") - return r.cleanupNVIDIADriverOwnerLabels(ctx) + return reconcile.Result{}, nil } // Get the singleton NVIDIA ClusterPolicy object in the cluster. @@ -156,19 +153,6 @@ func (r *NVIDIADriverReconciler) Reconcile(ctx context.Context, req ctrl.Request return reconcile.Result{}, nil } - changed, err := nvidiadriverutil.AssignOwners(ctx, r.Client) - if err != nil { - logger.Error(err, "failed to assign NVIDIADriver owners to nodes") - instance.Status.State = nvidiav1alpha1.NotReady - if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, err.Error()); condErr != nil { - logger.Error(condErr, "failed to set condition") - } - return reconcile.Result{}, err - } - if changed { - return reconcile.Result{RequeueAfter: time.Second}, nil - } - if instance.Spec.UsePrecompiledDrivers() && (instance.Spec.IsGDSEnabled() || instance.Spec.IsGDRCopyEnabled()) { err := errors.New("GPUDirect Storage driver (nvidia-fs) and/or GDRCopy driver is not supported along with pre-compiled NVIDIA drivers") logger.Error(err, "unsupported driver combination detected") @@ -205,11 +189,6 @@ func (r *NVIDIADriverReconciler) Reconcile(ctx context.Context, req ctrl.Request // Sync state and update status managerStatus := r.stateManager.SyncState(ctx, instance, infoCatalog) - if err := r.labelNodesWithOrphanedDriverPods(ctx); err != nil { - logger.Error(err, "failed to label nodes with orphaned NVIDIA driver pods") - return reconcile.Result{}, err - } - // update CR status if err := r.updateCrStatus(ctx, instance, managerStatus); err != nil { return ctrl.Result{}, err @@ -328,33 +307,6 @@ func dedupeReconcileRequests(requests []reconcile.Request) []reconcile.Request { return deduped } -// cleanupNVIDIADriverOwnerLabels re-runs owner assignment after a delete event -// when ClusterPolicy is still configured for NVIDIADriver mode. This lets -// AssignOwners remove stale nvidia.com/gpu-operator.driver.owner labels when no remaining -// NVIDIADriver matches a GPU node, including deletion of the last NVIDIADriver. -func (r *NVIDIADriverReconciler) cleanupNVIDIADriverOwnerLabels(ctx context.Context) (reconcile.Result, error) { - clusterPolicyList := &gpuv1.ClusterPolicyList{} - if err := r.List(ctx, clusterPolicyList); err != nil { - wrappedErr := fmt.Errorf("error getting ClusterPolicy list: %w", err) - log.FromContext(ctx).Error(wrappedErr, "failed to cleanup NVIDIADriver owner labels") - return reconcile.Result{}, wrappedErr - } - - if len(clusterPolicyList.Items) == 0 { - return reconcile.Result{}, nil - } - - if !clusterPolicyList.Items[0].Spec.Driver.UseNvidiaDriverCRDType() { - return reconcile.Result{}, nil - } - - if _, err := nvidiadriverutil.AssignOwners(ctx, r.Client); err != nil { - log.FromContext(ctx).Error(err, "failed to cleanup NVIDIADriver owner labels") - return reconcile.Result{}, err - } - return reconcile.Result{}, nil -} - // SetupWithManager sets up the controller with the Manager. func (r *NVIDIADriverReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { // Create state manager diff --git a/controllers/nvidiadriver_controller_test.go b/controllers/nvidiadriver_controller_test.go index 2298cf5399..c13f9538ca 100644 --- a/controllers/nvidiadriver_controller_test.go +++ b/controllers/nvidiadriver_controller_test.go @@ -39,7 +39,6 @@ import ( gpuv1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1" nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1" - "github.com/NVIDIA/gpu-operator/internal/consts" "github.com/NVIDIA/gpu-operator/internal/validator" ) @@ -272,143 +271,6 @@ func TestReconcileConflictSetsNotReadyState(t *testing.T) { require.Equal(t, nvidiav1alpha1.NotReady, updater.LastErrorState) } -func TestReconcileReturnsErrorWhenOwnerAssignmentFails(t *testing.T) { - scheme := runtime.NewScheme() - require.NoError(t, nvidiav1alpha1.AddToScheme(scheme)) - require.NoError(t, gpuv1.AddToScheme(scheme)) - require.NoError(t, corev1.AddToScheme(scheme)) - - driver := &nvidiav1alpha1.NVIDIADriver{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-driver", - Namespace: "default", - }, - Spec: nvidiav1alpha1.NVIDIADriverSpec{ - NodeSelector: map[string]string{"nodepool": "a"}, - }, - } - cp := &gpuv1.ClusterPolicy{ - ObjectMeta: metav1.ObjectMeta{Name: "default"}, - Spec: gpuv1.ClusterPolicySpec{ - Driver: gpuv1.DriverSpec{ - UseNvidiaDriverCRD: ptr.To(true), - }, - }, - } - node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{ - Name: "gpu-node", - Labels: map[string]string{ - "nodepool": "a", - "nvidia.com/gpu.present": "true", - }, - }} - patchErr := errors.New("patch failed") - k8sClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cp, driver, node).Build() - updater := &FakeConditionUpdater{} - - reconciler := &NVIDIADriverReconciler{ - Client: &patchFailingClient{Client: k8sClient, patchErr: patchErr}, - Scheme: scheme, - conditionUpdater: updater, - nodeSelectorValidator: &FakeNodeSelectorValidator{}, - } - - _, err := reconciler.Reconcile(context.Background(), ctrl.Request{ - NamespacedName: types.NamespacedName{ - Name: driver.Name, - Namespace: driver.Namespace, - }, - }) - - require.Error(t, err) - require.ErrorContains(t, err, patchErr.Error()) - require.Equal(t, nvidiav1alpha1.NotReady, updater.LastErrorState) -} - -func TestReconcileCleansOwnerLabelsWhenDeletedNVIDIADriverWasLast(t *testing.T) { - scheme := runtime.NewScheme() - require.NoError(t, nvidiav1alpha1.AddToScheme(scheme)) - require.NoError(t, gpuv1.AddToScheme(scheme)) - require.NoError(t, corev1.AddToScheme(scheme)) - - cp := &gpuv1.ClusterPolicy{ - ObjectMeta: metav1.ObjectMeta{Name: "default"}, - Spec: gpuv1.ClusterPolicySpec{ - Driver: gpuv1.DriverSpec{ - UseNvidiaDriverCRD: ptr.To(true), - }, - }, - } - node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{ - Name: "gpu-node", - Labels: map[string]string{ - consts.GPUPresentLabel: "true", - consts.NVIDIADriverOwnerLabel: "demo-gold", - }, - }} - k8sClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cp, node).Build() - reconciler := &NVIDIADriverReconciler{ - Client: k8sClient, - Scheme: scheme, - } - - _, err := reconciler.Reconcile(context.Background(), ctrl.Request{ - NamespacedName: types.NamespacedName{Name: "demo-gold"}, - }) - require.NoError(t, err) - - require.NoError(t, k8sClient.Get(context.Background(), types.NamespacedName{Name: "gpu-node"}, node)) - require.NotContains(t, node.Labels, consts.NVIDIADriverOwnerLabel) -} - -func TestReconcileCleansOwnerLabelsForTerminatingNVIDIADriver(t *testing.T) { - scheme := runtime.NewScheme() - require.NoError(t, nvidiav1alpha1.AddToScheme(scheme)) - require.NoError(t, gpuv1.AddToScheme(scheme)) - require.NoError(t, corev1.AddToScheme(scheme)) - - deleteTime := metav1.Now() - driver := &nvidiav1alpha1.NVIDIADriver{ - ObjectMeta: metav1.ObjectMeta{ - Name: "demo-gold", - DeletionTimestamp: &deleteTime, - Finalizers: []string{"test-finalizer"}, - }, - Spec: nvidiav1alpha1.NVIDIADriverSpec{ - NodeSelector: map[string]string{"nodepool": "gold"}, - }, - } - cp := &gpuv1.ClusterPolicy{ - ObjectMeta: metav1.ObjectMeta{Name: "default"}, - Spec: gpuv1.ClusterPolicySpec{ - Driver: gpuv1.DriverSpec{ - UseNvidiaDriverCRD: ptr.To(true), - }, - }, - } - node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{ - Name: "gpu-node", - Labels: map[string]string{ - consts.GPUPresentLabel: "true", - consts.NVIDIADriverOwnerLabel: "demo-gold", - "nodepool": "gold", - }, - }} - k8sClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cp, driver, node).Build() - reconciler := &NVIDIADriverReconciler{ - Client: k8sClient, - Scheme: scheme, - } - - _, err := reconciler.Reconcile(context.Background(), ctrl.Request{ - NamespacedName: types.NamespacedName{Name: "demo-gold"}, - }) - require.NoError(t, err) - - require.NoError(t, k8sClient.Get(context.Background(), types.NamespacedName{Name: "gpu-node"}, node)) - require.NotContains(t, node.Labels, consts.NVIDIADriverOwnerLabel) -} - func TestEnqueueAllNVIDIADrivers(t *testing.T) { scheme := runtime.NewScheme() require.NoError(t, nvidiav1alpha1.AddToScheme(scheme)) diff --git a/controllers/nvidiadriver_migration.go b/controllers/nvidiadriver_migration.go deleted file mode 100644 index 75e25e7532..0000000000 --- a/controllers/nvidiadriver_migration.go +++ /dev/null @@ -1,109 +0,0 @@ -/** -# Copyright (c) NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -**/ - -package controllers - -import ( - "context" - "fmt" - - "github.com/NVIDIA/k8s-operator-libs/pkg/upgrade" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" - - nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1" - "github.com/NVIDIA/gpu-operator/internal/consts" -) - -// labelNodesWithOrphanedDriverPods marks NVIDIADriver-owned nodes that still have orphaned -// ClusterPolicy driver pods so the driver upgrade controller can replace those pods in -// the normal controlled upgrade flow. -func (r *NVIDIADriverReconciler) labelNodesWithOrphanedDriverPods(ctx context.Context) error { - nvidiaDrivers := &nvidiav1alpha1.NVIDIADriverList{} - if err := r.List(ctx, nvidiaDrivers); err != nil { - return fmt.Errorf("failed to list NVIDIADriver CRs: %w", err) - } - if len(nvidiaDrivers.Items) == 0 { - return nil - } - - pods := &corev1.PodList{} - if err := r.List(ctx, pods, - client.InNamespace(r.Namespace), - client.MatchingLabels{AppComponentLabelKey: DriverAppComponentLabelValue}, - ); err != nil { - return fmt.Errorf("failed to list NVIDIA driver pods: %w", err) - } - - for _, pod := range pods.Items { - if len(pod.OwnerReferences) > 0 || pod.Status.Phase != corev1.PodRunning || pod.Spec.NodeName == "" { - continue - } - - node := &corev1.Node{} - if err := r.Get(ctx, types.NamespacedName{Name: pod.Spec.NodeName}, node); err != nil { - log.FromContext(ctx).Error(err, "failed to get node for orphaned NVIDIA driver pod", "pod", pod.Name, "node", pod.Spec.NodeName) - continue - } - if !nodeOwnedByNVIDIADriver(node, nvidiaDrivers.Items) { - continue - } - - upgradeStateLabel := upgrade.GetUpgradeStateLabelKey() - upgradeState := upgrade.UpgradeStateUnknown - if node.Labels != nil { - upgradeState = node.Labels[upgradeStateLabel] - } - if !isDriverUpgradeRequestAllowed(upgradeState) { - continue - } - originalNode := node.DeepCopy() - if node.Labels == nil { - node.Labels = map[string]string{} - } - node.Labels[upgradeStateLabel] = upgrade.UpgradeStateUpgradeRequired - if err := r.Patch(ctx, node, client.MergeFrom(originalNode)); err != nil { - return fmt.Errorf("failed to label node %q with orphaned NVIDIA driver pod %q: %w", node.Name, pod.Name, err) - } - } - - return nil -} - -// isDriverUpgradeRequestAllowed returns true when migration can request a -// driver upgrade without rewinding an active or failed upgrade state. -func isDriverUpgradeRequestAllowed(upgradeState string) bool { - return upgradeState == upgrade.UpgradeStateUnknown || upgradeState == upgrade.UpgradeStateDone -} - -// nodeOwnedByNVIDIADriver returns true when the node has an owner label for a live NVIDIADriver. -func nodeOwnedByNVIDIADriver(node *corev1.Node, nvidiaDrivers []nvidiav1alpha1.NVIDIADriver) bool { - if node.Labels == nil || node.Labels[consts.NVIDIADriverOwnerLabel] == "" { - return false - } - - for _, nvidiaDriver := range nvidiaDrivers { - if nvidiaDriver.HasDeletionTimestamp() { - continue - } - if node.Labels[consts.NVIDIADriverOwnerLabel] == nvidiaDriver.Name { - return true - } - } - return false -} diff --git a/controllers/object_controls_test.go b/controllers/object_controls_test.go index 197138c3cd..1f96ff5643 100644 --- a/controllers/object_controls_test.go +++ b/controllers/object_controls_test.go @@ -388,12 +388,12 @@ func TestLabelNodesWithOrphanedDriverPodsRequestsUpgradeOnlyForOwnedAllowedState objects := []client.Object{driver, nodeWithoutUpgradeState, nodeWithDoneState, nodeWithActiveState, nodeWithFailedState, unownedMatchingNode} objects = append(objects, pods...) k8sClient := fake.NewClientBuilder().WithScheme(testScheme).WithObjects(objects...).Build() - reconciler := &NVIDIADriverReconciler{ - Client: k8sClient, - Namespace: namespace, + nlc := &nodeLabelingController{ + client: k8sClient, + namespace: namespace, } - require.NoError(t, reconciler.labelNodesWithOrphanedDriverPods(context.Background())) + require.NoError(t, nlc.labelNodesWithOrphanedDriverPods(context.Background())) require.NoError(t, k8sClient.Get(context.Background(), types.NamespacedName{Name: nodeWithoutUpgradeState.Name}, nodeWithoutUpgradeState)) require.Equal(t, upgrade.UpgradeStateUpgradeRequired, nodeWithoutUpgradeState.Labels[upgradeStateLabel]) @@ -432,17 +432,17 @@ func TestLabelNodesWithOrphanedDriverPodsReturnsPatchError(t *testing.T) { } pod := orphanedDriverPod("orphaned-driver-pod", namespace, node.Name) k8sClient := fake.NewClientBuilder().WithScheme(testScheme).WithObjects(driver, node, pod).Build() - reconciler := &NVIDIADriverReconciler{ - Client: &patchFailingClient{ + nlc := &nodeLabelingController{ + client: &patchFailingClient{ Client: k8sClient, patchErr: errors.New("patch failed"), }, - Namespace: namespace, + namespace: namespace, } - err := reconciler.labelNodesWithOrphanedDriverPods(context.Background()) + err := nlc.labelNodesWithOrphanedDriverPods(context.Background()) - require.ErrorContains(t, err, "failed to label node \"node-with-orphaned-pod\" with orphaned NVIDIA driver pod \"orphaned-driver-pod\"") + require.ErrorContains(t, err, "failed to label node \"node-with-orphaned-pod\" for orphaned driver pod \"orphaned-driver-pod\"") require.ErrorContains(t, err, "patch failed") } @@ -546,10 +546,18 @@ func setup() error { clusterPolicyController.operatorMetrics = InitOperatorMetrics() - hasNFDLabels, gpuNodeCount, err := clusterPolicyController.labelGPUNodes() - if err != nil { + // Label GPU nodes via nodeLabelingController (mirrors production behavior) + nlc := &nodeLabelingController{ + client: client, + } + if err := nlc.labelGPUNodes(ctx); err != nil { return fmt.Errorf("unable to label nodes in cluster: %v", err) } + + hasNFDLabels, gpuNodeCount, err := clusterPolicyController.discoverGPUNodes() + if err != nil { + return fmt.Errorf("unable to discover GPU nodes in cluster: %v", err) + } if gpuNodeCount == 0 { return fmt.Errorf("no gpu nodes in mock cluster") } diff --git a/controllers/state_manager.go b/controllers/state_manager.go index 8c2a943d00..b046875aaa 100644 --- a/controllers/state_manager.go +++ b/controllers/state_manager.go @@ -459,162 +459,39 @@ func (w *gpuWorkloadConfiguration) removeGPUStateLabels(labels map[string]string return modified } -func (n *ClusterPolicyController) applyDriverAutoUpgradeAnnotation() error { - // fetch all nodes - opts := []client.ListOption{} - list := &corev1.NodeList{} - err := n.client.List(n.ctx, list, opts...) - if err != nil { - return fmt.Errorf("unable to list nodes to check annotations, err %s", err.Error()) - } - for _, node := range list.Items { - node := node - labels := node.GetLabels() - if !hasCommonGPULabel(labels) { - // not a gpu node - continue - } - // set node annotation for driver auto-upgrade - updateRequired := false - value := "true" - annotationValue, annotationExists := node.Annotations[driverAutoUpgradeAnnotationKey] - if (n.singleton.Spec.Driver.IsEnabled() || n.singleton.Spec.Driver.UseNvidiaDriverCRDType()) && - n.singleton.Spec.Driver.UpgradePolicy != nil && - n.singleton.Spec.Driver.UpgradePolicy.AutoUpgrade && - !n.sandboxEnabled { - // check if we need to add the annotation - if !annotationExists { - updateRequired = true - } else if annotationValue != "true" { - updateRequired = true - } - } else { - // check if we need to remove the annotation - if annotationExists { - updateRequired = true - } - value = "null" - } - if !updateRequired { - continue - } - // update annotation - node.Annotations[driverAutoUpgradeAnnotationKey] = value - if value == "null" { - // remove annotation if value is null - delete(node.Annotations, driverAutoUpgradeAnnotationKey) - } - err := n.client.Update(n.ctx, &node) - if err != nil { - n.logger.Info("Failed to update node state annotation on a node", - "node", node.Name, - "annotationKey", driverAutoUpgradeAnnotationKey, - "annotationValue", value, "error", err) - return err - } - } - return nil -} - -// labelGPUNodes labels nodes with GPU's with NVIDIA common label -// it return clusterHasNFDLabels (bool), gpuNodesTotal (int), error -func (n *ClusterPolicyController) labelGPUNodes() (bool, int, error) { +// discoverGPUNodes reads all cluster nodes and returns whether any NFD labels are present +// and how many GPU nodes (with nvidia.com/gpu.present=true) exist. +// Node label writes are handled by NodeLabelingReconciler. +func (n *ClusterPolicyController) discoverGPUNodes() (bool, int, error) { ctx := n.ctx - // fetch all nodes - opts := []client.ListOption{} list := &corev1.NodeList{} - err := n.client.List(ctx, list, opts...) - if err != nil { - return false, 0, fmt.Errorf("unable to list nodes to check labels: %w", err) + if err := n.client.List(ctx, list); err != nil { + return false, 0, fmt.Errorf("unable to list nodes: %w", err) } + clusterHasNFDLabels := false gpuNodesTotal := 0 for _, node := range list.Items { - node := node - updateLabels := false - nodeOriginal := node.DeepCopy() - // get node labels labels := node.GetLabels() if !clusterHasNFDLabels { clusterHasNFDLabels = hasNFDLabels(labels) } - config, err := getWorkloadConfig(labels, n.sandboxEnabled) - if err != nil { - n.logger.Info("WARNING: failed to get GPU workload config for node; using default", - "NodeName", node.Name, "SandboxEnabled", n.sandboxEnabled, - "Error", err, "defaultGPUWorkloadConfig", defaultGPUWorkloadConfig) - } - n.logger.Info("GPU workload configuration", "NodeName", node.Name, "GpuWorkloadConfig", config) - mode := n.singleton.Spec.SandboxWorkloads.Mode - gpuWorkloadConfig := &gpuWorkloadConfiguration{config: config, sandboxMode: mode, node: node.Name, log: n.logger} - if !hasCommonGPULabel(labels) && hasGPULabels(labels) { - n.logger.Info("Node has GPU(s)", "NodeName", node.Name) - // label the node with common Nvidia GPU label - n.logger.Info("Setting node label", "NodeName", node.Name, "Label", commonGPULabelKey, "Value", commonGPULabelValue) - labels[commonGPULabelKey] = commonGPULabelValue - // update node labels - node.SetLabels(labels) - updateLabels = true - } else if hasCommonGPULabel(labels) && !hasGPULabels(labels) { - // previously labelled node and no longer has GPUs - // label node to reset common Nvidia GPU label - n.logger.Info("Node no longer has GPUs", "NodeName", node.Name) - n.logger.Info("Setting node label", "Label", commonGPULabelKey, "Value", "false") - labels[commonGPULabelKey] = "false" - n.logger.Info("Disabling all operands for node", "NodeName", node.Name) - removeAllGPUStateLabels(labels) - // update node labels - node.SetLabels(labels) - updateLabels = true - } - - if hasCommonGPULabel(labels) { - // If node has GPU, then add state labels as per the workload type - n.logger.Info("Checking GPU state labels on the node", "NodeName", node.Name) - if gpuWorkloadConfig.updateGPUStateLabels(labels) { - n.logger.Info("Applying correct GPU state labels to the node", "NodeName", node.Name) - node.SetLabels(labels) - updateLabels = true - } - // Disable MIG on the node explicitly where no MIG config is specified - if n.singleton.Spec.MIGManager.IsEnabled() && hasMIGCapableGPU(labels) && !hasMIGConfigLabel(labels) { - if n.singleton.Spec.MIGManager.Config != nil && n.singleton.Spec.MIGManager.Config.Default == migConfigDisabledValue { - n.logger.Info("Setting MIG config label", "NodeName", node.Name, "Label", migConfigLabelKey, "Value", migConfigDisabledValue) - labels[migConfigLabelKey] = migConfigDisabledValue - node.SetLabels(labels) - updateLabels = true - } - } - // increment GPU node count - gpuNodesTotal++ - - // add GPU node CoreOS version for OCP - if n.ocpDriverToolkit.requested { - rhcosVersion, ok := labels[nfdOSTreeVersionLabelKey] - if ok { - n.ocpDriverToolkit.rhcosVersions[rhcosVersion] = true - n.logger.V(1).Info("GPU node running RHCOS", - "nodeName", node.Name, - "RHCOS version", rhcosVersion, - ) - } else { - n.logger.Info("node doesn't have the proper NFD RHCOS version label.", - "nodeName", node.Name, - "nfdLabel", nfdOSTreeVersionLabelKey, - ) - } - } + if !hasCommonGPULabel(labels) { + continue } - - // update node with the latest labels - if updateLabels { - err = n.client.Patch(ctx, &node, client.MergeFrom(nodeOriginal)) - if err != nil { - return false, 0, fmt.Errorf("unable to label node %s for the GPU Operator deployment, err %s", - node.Name, err.Error()) + gpuNodesTotal++ + if n.ocpDriverToolkit.requested { + rhcosVersion, ok := labels[nfdOSTreeVersionLabelKey] + if ok { + n.ocpDriverToolkit.rhcosVersions[rhcosVersion] = true + n.logger.V(1).Info("GPU node running RHCOS", + "nodeName", node.Name, "RHCOS version", rhcosVersion) + } else { + n.logger.Info("node doesn't have the proper NFD RHCOS version label.", + "nodeName", node.Name, "nfdLabel", nfdOSTreeVersionLabelKey) } } - } // end node loop + } n.logger.Info("Number of nodes with GPU label", "NodeCount", gpuNodesTotal) n.operatorMetrics.gpuNodesTotal.Set(float64(gpuNodesTotal)) @@ -949,8 +826,8 @@ func (n *ClusterPolicyController) init(ctx context.Context, reconciler *ClusterP n.logger.Info("Pod Security Admission labels added to GPU Operator namespace", "namespace", n.operatorNamespace) } - // fetch all nodes and label gpu nodes - hasNFDLabels, gpuNodeCount, err := n.labelGPUNodes() + // discover GPU nodes (labels are written by NodeLabelingReconciler) + hasNFDLabels, gpuNodeCount, err := n.discoverGPUNodes() if err != nil { return err } @@ -968,11 +845,6 @@ func (n *ClusterPolicyController) init(ctx context.Context, reconciler *ClusterP n.gpuNodeOSRelease = gpuNodeOSRelease n.gpuNodeOSTag = gpuNodeOSTag } - // fetch all nodes and annotate gpu nodes - err = n.applyDriverAutoUpgradeAnnotation() - if err != nil { - return err - } // detect the container runtime on worker nodes err = n.getRuntime()