From 6228680be71f98ca4a567c78e5b4d27706c90658 Mon Sep 17 00:00:00 2001 From: Harshal Patil <12152047+harche@users.noreply.github.com> Date: Fri, 21 Nov 2025 09:54:16 -0500 Subject: [PATCH 1/3] use nvidia runtime handler for the daemonset Signed-off-by: Harshal Patil <12152047+harche@users.noreply.github.com> --- bindata/assets/instaslice-operator/daemonset.yaml | 1 + hack/manifests/gpu-cluster-policy.yaml | 3 +++ 2 files changed, 4 insertions(+) diff --git a/bindata/assets/instaslice-operator/daemonset.yaml b/bindata/assets/instaslice-operator/daemonset.yaml index 759f348ff..933f6e0fb 100644 --- a/bindata/assets/instaslice-operator/daemonset.yaml +++ b/bindata/assets/instaslice-operator/daemonset.yaml @@ -16,6 +16,7 @@ spec: spec: nodeSelector: nvidia.com/mig.capable: "true" + runtimeClassName: nvidia securityContext: runAsNonRoot: false seccompProfile: diff --git a/hack/manifests/gpu-cluster-policy.yaml b/hack/manifests/gpu-cluster-policy.yaml index 6e45f6d94..4bc783da9 100644 --- a/hack/manifests/gpu-cluster-policy.yaml +++ b/hack/manifests/gpu-cluster-policy.yaml @@ -26,6 +26,9 @@ spec: certConfig: name: "" enabled: true + env: + - name: NVIDIA_RUNTIME_SET_AS_DEFAULT + value: "true" kernelModuleConfig: name: "" licensingConfig: From a724540fa6c4c1ec92a61929d7fd3c9e2f77c9a6 Mon Sep 17 00:00:00 2001 From: Harshal Patil <12152047+harche@users.noreply.github.com> Date: Tue, 25 Nov 2025 08:57:54 -0500 Subject: [PATCH 2/3] Remove nvidia runtime from daemonset to see if tests pass Signed-off-by: Harshal Patil <12152047+harche@users.noreply.github.com> --- bindata/assets/instaslice-operator/daemonset.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/bindata/assets/instaslice-operator/daemonset.yaml b/bindata/assets/instaslice-operator/daemonset.yaml index 933f6e0fb..759f348ff 100644 --- a/bindata/assets/instaslice-operator/daemonset.yaml +++ b/bindata/assets/instaslice-operator/daemonset.yaml @@ -16,7 +16,6 @@ spec: spec: nodeSelector: nvidia.com/mig.capable: "true" - runtimeClassName: nvidia securityContext: runAsNonRoot: false seccompProfile: From f41318c0ff36086949f3666e3ae46a668c658fda Mon Sep 17 00:00:00 2001 From: Harshal Patil <12152047+harche@users.noreply.github.com> Date: Wed, 26 Nov 2025 12:19:00 -0500 Subject: [PATCH 3/3] set oci runtime handler to nvidia-legay for daemonset and workload Signed-off-by: Harshal Patil <12152047+harche@users.noreply.github.com> --- .../assets/instaslice-operator/daemonset.yaml | 1 + hack/manifests/gpu-cluster-policy.yaml | 3 --- pkg/webhook/webhook.go | 5 +++++ pkg/webhook/webhook_test.go | 16 ++++++++++++++++ 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/bindata/assets/instaslice-operator/daemonset.yaml b/bindata/assets/instaslice-operator/daemonset.yaml index 759f348ff..d4f02827d 100644 --- a/bindata/assets/instaslice-operator/daemonset.yaml +++ b/bindata/assets/instaslice-operator/daemonset.yaml @@ -16,6 +16,7 @@ spec: spec: nodeSelector: nvidia.com/mig.capable: "true" + runtimeClassName: nvidia-legacy securityContext: runAsNonRoot: false seccompProfile: diff --git a/hack/manifests/gpu-cluster-policy.yaml b/hack/manifests/gpu-cluster-policy.yaml index 4bc783da9..6e45f6d94 100644 --- a/hack/manifests/gpu-cluster-policy.yaml +++ b/hack/manifests/gpu-cluster-policy.yaml @@ -26,9 +26,6 @@ spec: certConfig: name: "" enabled: true - env: - - name: NVIDIA_RUNTIME_SET_AS_DEFAULT - value: "true" kernelModuleConfig: name: "" licensingConfig: diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go index da6b696ed..2e943e858 100644 --- a/pkg/webhook/webhook.go +++ b/pkg/webhook/webhook.go @@ -154,6 +154,11 @@ func (s *InstasliceWebhook) mutatePod(pod *corev1.Pod) ([]byte, error) { if needsScheduler { mutatedPod.Spec.SchedulerName = secondaryScheduler klog.InfoS("using secondary scheduler", "name", mutatedPod.Name) + // Set nvidia-legacy runtime for MIG workloads to avoid CDI resolution issues + // with the nvidia runtime's CDI mode + runtimeClass := "nvidia-legacy" + mutatedPod.Spec.RuntimeClassName = &runtimeClass + klog.InfoS("setting runtimeClassName for MIG workload", "name", mutatedPod.Name, "runtimeClassName", runtimeClass) } klog.InfoS("finished pod mutation", "mutatedPod", mutatedPod) diff --git a/pkg/webhook/webhook_test.go b/pkg/webhook/webhook_test.go index cb3470bbf..57fe18dc5 100644 --- a/pkg/webhook/webhook_test.go +++ b/pkg/webhook/webhook_test.go @@ -48,6 +48,10 @@ func TestMutatePodNvidiaResource(t *testing.T) { t.Fatalf("expected scheduler %s, got %s", secondaryScheduler, mutated.Spec.SchedulerName) } + if mutated.Spec.RuntimeClassName == nil || *mutated.Spec.RuntimeClassName != "nvidia-legacy" { + t.Fatalf("expected runtimeClassName nvidia-legacy") + } + limits := mutated.Spec.Containers[0].Resources.Limits if _, ok := limits[corev1.ResourceName("nvidia.com/mig-1g.5gb")]; ok { t.Fatalf("nvidia resource still present") @@ -96,6 +100,9 @@ func TestMutatePodEphemeralNvidiaResource(t *testing.T) { if mutated.Spec.SchedulerName != secondaryScheduler { t.Fatalf("expected scheduler set") } + if mutated.Spec.RuntimeClassName == nil || *mutated.Spec.RuntimeClassName != "nvidia-legacy" { + t.Fatalf("expected runtimeClassName nvidia-legacy") + } limits := mutated.Spec.EphemeralContainers[0].Resources.Limits if _, ok := limits[corev1.ResourceName("mig.das.com/1g.5gb")]; !ok { t.Fatalf("instaslice resource missing") @@ -143,6 +150,9 @@ func TestMutatePodOverrideValues(t *testing.T) { if mutated.Spec.SchedulerName != secondaryScheduler { t.Fatalf("scheduler not overridden") } + if mutated.Spec.RuntimeClassName == nil || *mutated.Spec.RuntimeClassName != "nvidia-legacy" { + t.Fatalf("expected runtimeClassName nvidia-legacy") + } envs := mutated.Spec.Containers[0].Env if len(envs) != 2 { @@ -194,6 +204,9 @@ func TestMutatePodInstaResource(t *testing.T) { if mutated.Spec.SchedulerName != secondaryScheduler { t.Fatalf("expected scheduler set") } + if mutated.Spec.RuntimeClassName == nil || *mutated.Spec.RuntimeClassName != "nvidia-legacy" { + t.Fatalf("expected runtimeClassName nvidia-legacy") + } if _, ok := mutated.Spec.Containers[0].Resources.Limits[corev1.ResourceName("mig.das.com/1g.5gb")]; !ok { t.Fatalf("instaslice resource missing") } @@ -223,6 +236,9 @@ func TestMutatePodNoResource(t *testing.T) { if mutated.Spec.SchedulerName != "" { t.Fatalf("expected scheduler not set") } + if mutated.Spec.RuntimeClassName != nil { + t.Fatalf("expected runtimeClassName not set for non-GPU pod") + } if len(mutated.Spec.Containers[0].Env) != 0 { t.Fatalf("env vars should not be added") }