From 491d214dffe5f70900f9d36eb3e47bffc7607311 Mon Sep 17 00:00:00 2001 From: Prakash Meena Date: Sun, 13 Sep 2026 13:21:43 +0530 Subject: [PATCH] cdi/generate: Add --no-ipc-sockets flag to exclude IPC sockets When running containers under sandboxed runtimes like gVisor (runsc) or Kata Containers, bind mounting host IPC domain sockets (such as /run/nvidia-persistenced/socket, /run/nvidia-fabricmanager/socket, or /tmp/nvidia-mps) causes container creation to fail with ENXIO because sandboxed gofers open existing mount points with O_CREAT. Furthermore, multi-tenant and secure container environments often prohibit sharing host IPC sockets into untrusted containers. While nvcdi already implemented FeatureDisableIPCDiscoverer, it was never exposed as a flag or environment variable in nvidia-ctk cdi generate or systemd refresh services. This commit exposes: - CLI flag: --no-ipc-sockets (aliases: --disable-ipc-sockets, --disable-ipc-discoverer) - Environment variable: NVIDIA_CTK_CDI_GENERATE_NO_IPC_SOCKETS - Configuration option documented in nvidia-cdi-refresh.env - Comprehensive unit tests covering flag validation, spec generation, and nvcdilib discoverer toggling. Fixes #2034 Signed-off-by: Prakash Meena --- cmd/nvidia-ctk/cdi/generate/generate.go | 16 ++- cmd/nvidia-ctk/cdi/generate/generate_test.go | 104 +++++++++++++++++++ deployments/systemd/nvidia-cdi-refresh.env | 5 + internal/discover/ipc.go | 2 +- pkg/nvcdi/driver-nvml_test.go | 55 ++++++++++ 5 files changed, 179 insertions(+), 3 deletions(-) create mode 100644 pkg/nvcdi/driver-nvml_test.go diff --git a/cmd/nvidia-ctk/cdi/generate/generate.go b/cmd/nvidia-ctk/cdi/generate/generate.go index 7c29e9dd9..19dc48c90 100644 --- a/cmd/nvidia-ctk/cdi/generate/generate.go +++ b/cmd/nvidia-ctk/cdi/generate/generate.go @@ -75,8 +75,9 @@ type options struct { CompatContainerRoot string } - noAllDevice bool - deviceIDs []string + noIPCSockets bool + noAllDevice bool + deviceIDs []string // the following are used for dependency injection during spec generation. nvmllib nvml.Interface @@ -243,6 +244,13 @@ func (m command) build() *cli.Command { Destination: &opts.featureFlags, Sources: cli.EnvVars("NVIDIA_CTK_CDI_GENERATE_FEATURE_FLAGS"), }, + &cli.BoolFlag{ + Name: "no-ipc-sockets", + Aliases: []string{"disable-ipc-sockets", "disable-ipc-discoverer"}, + Usage: "Do not include NVIDIA IPC sockets (nvidia-persistenced, nvidia-fabricmanager, MPS) in the generated CDI specification", + Destination: &opts.noIPCSockets, + Sources: cli.EnvVars("NVIDIA_CTK_CDI_GENERATE_NO_IPC_SOCKETS"), + }, &cli.BoolFlag{ Name: "no-all-device", Usage: "Don't generate an `all` device for the resultant spec", @@ -310,6 +318,10 @@ func (m command) validateFlags(c *cli.Command, opts *options) error { m.logger.Warningf("Disabling generation of 'all' device") opts.noAllDevice = true } + + if opts.noIPCSockets && !slices.Contains(opts.featureFlags, string(nvcdi.FeatureDisableIPCDiscoverer)) { + opts.featureFlags = append(opts.featureFlags, string(nvcdi.FeatureDisableIPCDiscoverer)) + } return nil } diff --git a/cmd/nvidia-ctk/cdi/generate/generate_test.go b/cmd/nvidia-ctk/cdi/generate/generate_test.go index a954d2ba7..99e84ae8a 100644 --- a/cmd/nvidia-ctk/cdi/generate/generate_test.go +++ b/cmd/nvidia-ctk/cdi/generate/generate_test.go @@ -32,6 +32,7 @@ import ( "github.com/NVIDIA/nvidia-container-toolkit/internal/devices" "github.com/NVIDIA/nvidia-container-toolkit/internal/test" + "github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi" ) func TestGenerateSpec(t *testing.T) { @@ -491,6 +492,109 @@ containerEdits: - nodev - rbind - rprivate +`, + }, + { + description: "no-ipc-sockets", + options: options{ + format: "yaml", + mode: "nvml", + vendor: "example.com", + class: "device", + driverRoot: driverRoot, + noIPCSockets: true, + }, + expectedOptions: options{ + format: "yaml", + mode: "nvml", + vendor: "example.com", + class: "device", + nvidiaCDIHookPath: "/usr/bin/nvidia-cdi-hook", + driverRoot: driverRoot, + noIPCSockets: true, + featureFlags: []string{string(nvcdi.FeatureDisableIPCDiscoverer)}, + }, + expectedSpec: `--- +cdiVersion: 0.5.0 +kind: example.com/device +devices: + - name: "0" + containerEdits: + deviceNodes: + - path: /dev/nvidia0 + hostPath: {{ .driverRoot }}/dev/nvidia0 + - name: all + containerEdits: + deviceNodes: + - path: /dev/nvidia0 + hostPath: {{ .driverRoot }}/dev/nvidia0 +containerEdits: + env: + - NVIDIA_CTK_LIBCUDA_DIR=/lib/x86_64-linux-gnu + - NVIDIA_VISIBLE_DEVICES=void + deviceNodes: + - path: /dev/nvidiactl + hostPath: {{ .driverRoot }}/dev/nvidiactl + hooks: + - hookName: createContainer + path: /usr/bin/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - create-symlinks + - --link + - libcuda.so.1::/lib/x86_64-linux-gnu/libcuda.so + env: + - NVIDIA_CTK_DEBUG=false + - hookName: createContainer + path: /usr/bin/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - enable-cuda-compat + - --host-driver-version=999.88.77 + env: + - NVIDIA_CTK_DEBUG=false + - hookName: createContainer + path: /usr/bin/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - update-ldcache + - --folder + - /lib/x86_64-linux-gnu + - --folder + - /lib/x86_64-linux-gnu/vdpau + env: + - NVIDIA_CTK_DEBUG=false + - hookName: createContainer + path: /usr/bin/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - disable-device-node-modification + env: + - NVIDIA_CTK_DEBUG=false + - hookName: createContainer + path: /usr/bin/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - update-application-profile + env: + - NVIDIA_CTK_DEBUG=false + mounts: + - hostPath: {{ .driverRoot }}/lib/x86_64-linux-gnu/libcuda.so.999.88.77 + containerPath: /lib/x86_64-linux-gnu/libcuda.so.999.88.77 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: {{ .driverRoot }}/lib/x86_64-linux-gnu/vdpau/libvdpau_nvidia.so.999.88.77 + containerPath: /lib/x86_64-linux-gnu/vdpau/libvdpau_nvidia.so.999.88.77 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate `, }, } diff --git a/deployments/systemd/nvidia-cdi-refresh.env b/deployments/systemd/nvidia-cdi-refresh.env index 45315914b..7f65bdc4a 100644 --- a/deployments/systemd/nvidia-cdi-refresh.env +++ b/deployments/systemd/nvidia-cdi-refresh.env @@ -27,3 +27,8 @@ # # NVIDIA_DRIVER_ROOT=/ # NVIDIA_DEV_ROOT=/ + +# To exclude NVIDIA IPC sockets (/run/nvidia-persistenced/socket, +# /run/nvidia-fabricmanager/socket, /tmp/nvidia-mps) from the generated CDI spec +# (useful for sandboxed runtimes like gVisor/runsc or Kata Containers), uncomment: +# NVIDIA_CTK_CDI_GENERATE_NO_IPC_SOCKETS=true diff --git a/internal/discover/ipc.go b/internal/discover/ipc.go index 3a5611044..6382ffccc 100644 --- a/internal/discover/ipc.go +++ b/internal/discover/ipc.go @@ -32,7 +32,7 @@ var ipcMountOptions = []string{ type ipcMounts mounts -// NewIPCDiscoverer creats a discoverer for NVIDIA IPC sockets. +// NewIPCDiscoverer creates a discoverer for NVIDIA IPC sockets. func NewIPCDiscoverer(logger logger.Interface, driverRoot string) (Discover, error) { sockets := newMounts( logger, diff --git a/pkg/nvcdi/driver-nvml_test.go b/pkg/nvcdi/driver-nvml_test.go new file mode 100644 index 000000000..f703e110b --- /dev/null +++ b/pkg/nvcdi/driver-nvml_test.go @@ -0,0 +1,55 @@ +/** +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package nvcdi + +import ( + "testing" + + testlog "github.com/sirupsen/logrus/hooks/test" + "github.com/stretchr/testify/require" + + "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/root" +) + +func TestNewIPCDiscoverer(t *testing.T) { + logger, _ := testlog.NewNullLogger() + + t.Run("default enables IPC discoverer", func(t *testing.T) { + l := &nvcdilib{ + logger: logger, + driver: root.New(root.WithDriverRoot("/")), + featureFlags: make(map[FeatureFlag]bool), + } + d, err := l.newIPCDiscoverer() + require.NoError(t, err) + require.NotNil(t, d) + }) + + t.Run("FeatureDisableIPCDiscoverer disables IPC discoverer", func(t *testing.T) { + l := &nvcdilib{ + logger: logger, + driver: root.New(root.WithDriverRoot("/")), + featureFlags: map[FeatureFlag]bool{ + FeatureDisableIPCDiscoverer: true, + }, + } + d, err := l.newIPCDiscoverer() + require.NoError(t, err) + require.Nil(t, d) + }) +}