From f11a6e7607e50c4db9d4d8f12a69bab4a6e94339 Mon Sep 17 00:00:00 2001 From: Francesco Romani Date: Thu, 30 Apr 2026 15:04:56 +0200 Subject: [PATCH] devices: optional support to check device add optional support to also check alignment of devices, discovered from the env variable the plugins inject in the container environment. Signed-off-by: Francesco Romani --- api/v0/types.go | 18 +++-- internal/cli/align.go | 10 ++- pkg/align/align.go | 48 +++++++++++- pkg/align/align_test.go | 139 ++++++++++++++++++++++++++++++++++ pkg/resources/devices.go | 63 +++++++++++++++ pkg/resources/devices_test.go | 118 +++++++++++++++++++++++++++++ pkg/resources/resources.go | 11 ++- 7 files changed, 395 insertions(+), 12 deletions(-) create mode 100644 pkg/resources/devices.go create mode 100644 pkg/resources/devices_test.go diff --git a/api/v0/types.go b/api/v0/types.go index 2a41567..2fc1781 100644 --- a/api/v0/types.go +++ b/api/v0/types.go @@ -44,17 +44,19 @@ type AlignedInfo struct { } type UnalignedInfo struct { - SMT ContainerResourcesDetails `json:"smt,omitempty"` - LLC ContainerResourcesDetails `json:"llc,omitempty"` - NUMA ContainerResourcesDetails `json:"numa,omitempty"` - Memory ContainerResourcesDetails `json:"memory,omitempty"` + SMT ContainerResourcesDetails `json:"smt,omitempty"` + LLC ContainerResourcesDetails `json:"llc,omitempty"` + NUMA ContainerResourcesDetails `json:"numa,omitempty"` + Memory ContainerResourcesDetails `json:"memory,omitempty"` + Devices ContainerResourcesDetails `json:"devices,omitempty"` } type Alignment struct { - SMT bool `json:"smt"` - LLC bool `json:"llc"` - NUMA bool `json:"numa"` - Memory bool `json:"memory"` + SMT bool `json:"smt"` + LLC bool `json:"llc"` + NUMA bool `json:"numa"` + Memory bool `json:"memory"` + Devices *bool `json:"devices,omitempty"` } type Allocation struct { diff --git a/internal/cli/align.go b/internal/cli/align.go index 7e49e12..a641d82 100644 --- a/internal/cli/align.go +++ b/internal/cli/align.go @@ -28,9 +28,13 @@ import ( "github.com/ffromani/ctrreschk/pkg/resources" ) -type AlignOptions struct{} +type AlignOptions struct { + DeviceEnvPrefixes []string +} func NewAlignCommand(env *environ.Environ, opts *Options) *cobra.Command { + alignOpts := AlignOptions{} + alignCmd := &cobra.Command{ Use: "align", Short: "show resource alignment properties", @@ -39,6 +43,9 @@ func NewAlignCommand(env *environ.Environ, opts *Options) *cobra.Command { if err != nil { return err } + if len(alignOpts.DeviceEnvPrefixes) > 0 { + container.Devices = resources.DiscoverDevicesFromEnv(env, os.Environ(), alignOpts.DeviceEnvPrefixes) + } machine, err := machine.Discover(env) if err != nil { return err @@ -57,6 +64,7 @@ func NewAlignCommand(env *environ.Environ, opts *Options) *cobra.Command { } alignCmd.PersistentFlags().StringVarP(&env.DataPath, "machinedata", "M", "", "read fake machine data from path, don't read real data from the system") + alignCmd.PersistentFlags().StringSliceVar(&alignOpts.DeviceEnvPrefixes, "device-env-prefix", nil, "env var prefixes for device PCI addresses (e.g. SRIOVNETWORK_VF_,PCIDEVICE_)") return alignCmd } diff --git a/pkg/align/align.go b/pkg/align/align.go index 1444552..25c8c74 100644 --- a/pkg/align/align.go +++ b/pkg/align/align.go @@ -45,8 +45,9 @@ func Check(env *environ.Environ, container resources.Resources, machine machine. checkLLC(env, &resp, container.CPUs.Clone(), rmap) checkNUMA(env, &resp, container.CPUs.Clone(), rmap) checkMemory(env, &resp, container.CPUs.Clone(), container.MEMs.Clone(), rmap) + checkDevices(env, &resp, container.CPUs.Clone(), container.Devices, rmap) - env.Log.V(2).Info("alignment check complete", "smt", resp.Alignment.SMT, "llc", resp.Alignment.LLC, "numa", resp.Alignment.NUMA, "memory", resp.Alignment.Memory) + env.Log.V(2).Info("alignment check complete", "smt", resp.Alignment.SMT, "llc", resp.Alignment.LLC, "numa", resp.Alignment.NUMA, "memory", resp.Alignment.Memory, "devices", resp.Alignment.Devices) return resp, nil } @@ -194,6 +195,51 @@ func checkMemory(env *environ.Environ, resp *apiv0.Allocation, cpus cpuset.CPUSe } } +func checkDevices(env *environ.Environ, resp *apiv0.Allocation, cpus cpuset.CPUSet, devices []resources.DeviceInfo, rmap rMap) { + if len(devices) == 0 { + env.Log.V(1).Info("no devices to check, skipping device alignment check") + return + } + + cpuNUMANodes := cpuset.New() + for numaID := range rmap.numa { + numaCPUs := rmap.numa.CPUSet(numaID) + if !cpus.Intersection(numaCPUs).IsEmpty() { + cpuNUMANodes = cpuNUMANodes.Union(cpuset.New(numaID)) + } + } + + aligned := true + for _, dev := range devices { + if dev.NUMANode == -1 { + env.Log.V(2).Info("device NUMA node unknown, skipping", "pciAddress", dev.PCIAddress) + continue + } + + env.Log.V(2).Info("check device alignment", "pciAddress", dev.PCIAddress, "deviceNUMA", dev.NUMANode, "cpuNUMANodes", cpuNUMANodes.String()) + + if cpuNUMANodes.Contains(dev.NUMANode) { + if resp.Aligned == nil { + resp.Aligned = apiv0.NewAlignedInfo() + } + dets := resp.Aligned.NUMA[dev.NUMANode] + dets.Devices = append(dets.Devices, dev.PCIAddress) + resp.Aligned.NUMA[dev.NUMANode] = dets + } else { + aligned = false + if resp.Unaligned == nil { + resp.Unaligned = &apiv0.UnalignedInfo{} + } + resp.Unaligned.Devices.Devices = append(resp.Unaligned.Devices.Devices, dev.PCIAddress) + if !slices.Contains(resp.Unaligned.Devices.NUMANodes, dev.NUMANode) { + resp.Unaligned.Devices.NUMANodes = append(resp.Unaligned.Devices.NUMANodes, dev.NUMANode) + } + } + } + + resp.Alignment.Devices = &aligned +} + // Reverse ID MAP (PhysicalID|LLCID|NUMAID) -> LogicalIDs type ridMap map[int][]int diff --git a/pkg/align/align_test.go b/pkg/align/align_test.go index b418770..4c47c57 100644 --- a/pkg/align/align_test.go +++ b/pkg/align/align_test.go @@ -158,6 +158,143 @@ func TestCheck(t *testing.T) { }, }, }, + { + name: "device aligned with cpu numa node", + res: resources.Resources{ + CPUs: cpuset.New(0, 16), + MEMs: cpuset.New(0), + Devices: []resources.DeviceInfo{ + {EnvVar: "SRIOVNETWORK_VF_DEV", PCIAddress: "0000:05:10.2", NUMANode: 0}, + }, + }, + expectedAlloc: apiv0.Allocation{ + Alignment: apiv0.Alignment{ + SMT: true, + LLC: true, + NUMA: true, + Memory: true, + Devices: boolPtr(true), + }, + Aligned: &apiv0.AlignedInfo{ + LLC: map[int]apiv0.ContainerResourcesDetails{ + 0: { + CPUs: []int{0, 16}, + }, + }, + NUMA: map[int]apiv0.ContainerResourcesDetails{ + 0: { + CPUs: []int{0, 16}, + Devices: []string{"0000:05:10.2"}, + }, + }, + Memory: map[int]apiv0.ContainerResourcesDetails{ + 0: { + CPUs: []int{0, 16}, + MemoryMiB: 63705, + MemoryPercent: 100.0, + }, + }, + }, + }, + }, + { + name: "device on wrong numa node", + res: resources.Resources{ + CPUs: cpuset.New(0, 16), + MEMs: cpuset.New(0), + Devices: []resources.DeviceInfo{ + {EnvVar: "SRIOVNETWORK_VF_DEV", PCIAddress: "0000:05:10.2", NUMANode: 1}, + }, + }, + expectedAlloc: apiv0.Allocation{ + Alignment: apiv0.Alignment{ + SMT: true, + LLC: true, + NUMA: true, + Memory: true, + Devices: boolPtr(false), + }, + Aligned: &apiv0.AlignedInfo{ + LLC: map[int]apiv0.ContainerResourcesDetails{ + 0: { + CPUs: []int{0, 16}, + }, + }, + NUMA: map[int]apiv0.ContainerResourcesDetails{ + 0: { + CPUs: []int{0, 16}, + }, + }, + Memory: map[int]apiv0.ContainerResourcesDetails{ + 0: { + CPUs: []int{0, 16}, + MemoryMiB: 63705, + MemoryPercent: 100.0, + }, + }, + }, + Unaligned: &apiv0.UnalignedInfo{ + Devices: apiv0.ContainerResourcesDetails{ + Devices: []string{"0000:05:10.2"}, + NUMANodes: []int{1}, + }, + }, + }, + }, + { + name: "device with unknown numa node skipped", + res: resources.Resources{ + CPUs: cpuset.New(0, 16), + Devices: []resources.DeviceInfo{ + {EnvVar: "SRIOVNETWORK_VF_DEV", PCIAddress: "0000:05:10.2", NUMANode: -1}, + }, + }, + expectedAlloc: apiv0.Allocation{ + Alignment: apiv0.Alignment{ + SMT: true, + LLC: true, + NUMA: true, + Devices: boolPtr(true), + }, + Aligned: &apiv0.AlignedInfo{ + LLC: map[int]apiv0.ContainerResourcesDetails{ + 0: { + CPUs: []int{0, 16}, + }, + }, + NUMA: map[int]apiv0.ContainerResourcesDetails{ + 0: { + CPUs: []int{0, 16}, + }, + }, + }, + }, + }, + { + name: "no devices means no device alignment field", + res: resources.Resources{ + CPUs: cpuset.New(0, 16), + }, + expectedAlloc: apiv0.Allocation{ + Alignment: apiv0.Alignment{ + SMT: true, + LLC: true, + NUMA: true, + }, + Aligned: &apiv0.AlignedInfo{ + LLC: map[int]apiv0.ContainerResourcesDetails{ + 0: { + CPUs: []int{0, 16}, + }, + }, + NUMA: map[int]apiv0.ContainerResourcesDetails{ + 0: { + CPUs: []int{0, 16}, + }, + }, + }, + }, + }, } for _, tt := range testCases { t.Run(tt.name, func(t *testing.T) { @@ -179,6 +316,8 @@ func TestCheck(t *testing.T) { } } +func boolPtr(b bool) *bool { return &b } + func toJSON(v any) string { data, err := json.Marshal(v) if err != nil { diff --git a/pkg/resources/devices.go b/pkg/resources/devices.go new file mode 100644 index 0000000..af94a2e --- /dev/null +++ b/pkg/resources/devices.go @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/ffromani/ctrreschk/pkg/environ" +) + +func DiscoverDevicesFromEnv(env *environ.Environ, osEnviron []string, prefixes []string) []DeviceInfo { + var devices []DeviceInfo + for _, entry := range osEnviron { + name, value, ok := strings.Cut(entry, "=") + if !ok || value == "" { + continue + } + if !matchesAnyPrefix(name, prefixes) { + continue + } + for _, addr := range strings.Split(value, ",") { + addr = strings.TrimSpace(addr) + if addr == "" { + continue + } + numaNode := readDeviceNUMANode(env, addr) + devices = append(devices, DeviceInfo{ + EnvVar: name, + PCIAddress: addr, + NUMANode: numaNode, + }) + env.Log.V(2).Info("discovered device", "envVar", name, "pciAddress", addr, "numaNode", numaNode) + } + } + return devices +} + +func matchesAnyPrefix(name string, prefixes []string) bool { + for _, prefix := range prefixes { + if strings.HasPrefix(name, prefix) { + return true + } + } + return false +} + +func readDeviceNUMANode(env *environ.Environ, pciAddress string) int { + numaPath := filepath.Join(env.Root.Sys, "bus", "pci", "devices", pciAddress, "numa_node") + data, err := os.ReadFile(numaPath) + if err != nil { + env.Log.V(1).Info("cannot read device NUMA node, skipping", "pciAddress", pciAddress, "error", err) + return -1 + } + node, err := strconv.Atoi(strings.TrimSpace(string(data))) + if err != nil { + env.Log.V(1).Info("cannot parse device NUMA node, skipping", "pciAddress", pciAddress, "error", err) + return -1 + } + return node +} diff --git a/pkg/resources/devices_test.go b/pkg/resources/devices_test.go new file mode 100644 index 0000000..cfd836d --- /dev/null +++ b/pkg/resources/devices_test.go @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "os" + "path/filepath" + "testing" + + "github.com/ffromani/ctrreschk/pkg/environ" +) + +func TestDiscoverDevicesFromEnv(t *testing.T) { + testCases := []struct { + name string + environ []string + prefixes []string + sysfs map[string]string // pciAddress -> numa_node content + expected []DeviceInfo + }{ + { + name: "no matching prefixes", + environ: []string{"FOO_BAR=0000:05:10.2"}, + prefixes: []string{"SRIOVNETWORK_VF_"}, + expected: nil, + }, + { + name: "single device", + environ: []string{"SRIOVNETWORK_VF_DEVICE_0000_05_10_2=0000:05:10.2"}, + prefixes: []string{"SRIOVNETWORK_VF_"}, + sysfs: map[string]string{"0000:05:10.2": "0\n"}, + expected: []DeviceInfo{ + {EnvVar: "SRIOVNETWORK_VF_DEVICE_0000_05_10_2", PCIAddress: "0000:05:10.2", NUMANode: 0}, + }, + }, + { + name: "multiple devices comma separated", + environ: []string{"PCIDEVICE_IO_NICS=0000:86:00.0,0000:87:00.0"}, + prefixes: []string{"PCIDEVICE_"}, + sysfs: map[string]string{"0000:86:00.0": "0\n", "0000:87:00.0": "1\n"}, + expected: []DeviceInfo{ + {EnvVar: "PCIDEVICE_IO_NICS", PCIAddress: "0000:86:00.0", NUMANode: 0}, + {EnvVar: "PCIDEVICE_IO_NICS", PCIAddress: "0000:87:00.0", NUMANode: 1}, + }, + }, + { + name: "missing sysfs entry returns -1", + environ: []string{"SRIOVNETWORK_VF_DEV=0000:99:00.0"}, + prefixes: []string{"SRIOVNETWORK_VF_"}, + sysfs: map[string]string{}, + expected: []DeviceInfo{ + {EnvVar: "SRIOVNETWORK_VF_DEV", PCIAddress: "0000:99:00.0", NUMANode: -1}, + }, + }, + { + name: "numa node -1 from sysfs", + environ: []string{"SRIOVNETWORK_VF_DEV=0000:05:10.2"}, + prefixes: []string{"SRIOVNETWORK_VF_"}, + sysfs: map[string]string{"0000:05:10.2": "-1\n"}, + expected: []DeviceInfo{ + {EnvVar: "SRIOVNETWORK_VF_DEV", PCIAddress: "0000:05:10.2", NUMANode: -1}, + }, + }, + { + name: "multiple prefixes", + environ: []string{"SRIOVNETWORK_VF_A=0000:05:10.2", "PCIDEVICE_B=0000:06:00.0", "OTHER=ignored"}, + prefixes: []string{"SRIOVNETWORK_VF_", "PCIDEVICE_"}, + sysfs: map[string]string{"0000:05:10.2": "0\n", "0000:06:00.0": "0\n"}, + expected: []DeviceInfo{ + {EnvVar: "SRIOVNETWORK_VF_A", PCIAddress: "0000:05:10.2", NUMANode: 0}, + {EnvVar: "PCIDEVICE_B", PCIAddress: "0000:06:00.0", NUMANode: 0}, + }, + }, + { + name: "empty value skipped", + environ: []string{"SRIOVNETWORK_VF_DEV="}, + prefixes: []string{"SRIOVNETWORK_VF_"}, + expected: nil, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + env := &environ.Environ{ + Root: environ.FS{Sys: tmpDir}, + Log: environ.DefaultLog(), + } + + for addr, content := range tt.sysfs { + devDir := filepath.Join(tmpDir, "bus", "pci", "devices", addr) + if err := os.MkdirAll(devDir, os.ModePerm); err != nil { + t.Fatalf("cannot create sysfs dir: %v", err) + } + if err := os.WriteFile(filepath.Join(devDir, "numa_node"), []byte(content), 0o644); err != nil { + t.Fatalf("cannot write numa_node: %v", err) + } + } + + got := DiscoverDevicesFromEnv(env, tt.environ, tt.prefixes) + + if len(got) != len(tt.expected) { + t.Fatalf("expected %d devices, got %d: %+v", len(tt.expected), len(got), got) + } + for i, exp := range tt.expected { + if got[i].EnvVar != exp.EnvVar { + t.Errorf("device[%d] EnvVar: expected %q, got %q", i, exp.EnvVar, got[i].EnvVar) + } + if got[i].PCIAddress != exp.PCIAddress { + t.Errorf("device[%d] PCIAddress: expected %q, got %q", i, exp.PCIAddress, got[i].PCIAddress) + } + if got[i].NUMANode != exp.NUMANode { + t.Errorf("device[%d] NUMANode: expected %d, got %d", i, exp.NUMANode, got[i].NUMANode) + } + } + }) + } +} diff --git a/pkg/resources/resources.go b/pkg/resources/resources.go index 3a659f7..add4235 100644 --- a/pkg/resources/resources.go +++ b/pkg/resources/resources.go @@ -23,9 +23,16 @@ import ( "github.com/ffromani/ctrreschk/pkg/environ" ) +type DeviceInfo struct { + EnvVar string + PCIAddress string + NUMANode int // -1 if unknown +} + type Resources struct { - CPUs cpuset.CPUSet - MEMs cpuset.CPUSet + CPUs cpuset.CPUSet + MEMs cpuset.CPUSet + Devices []DeviceInfo } func Discover(env *environ.Environ) (Resources, error) {