Skip to content

devices: optional support to check device #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ffromani merged 1 commit into from
Apr 30, 2026
Merged

devices: optional support to check device #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 10 additions & 8 deletions api/v0/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,17 +44,19 @@ type AlignedInfo struct {
}

type UnalignedInfo struct {
SMT ContainerResourcesDetails `json:"smt,omitempty"`
LLC ContainerResourcesDetails `json:"llc,omitempty"`
NUMA ContainerResourcesDetails `json:"numa,omitempty"`
Memory ContainerResourcesDetails `json:"memory,omitempty"`
SMT ContainerResourcesDetails `json:"smt,omitempty"`
LLC ContainerResourcesDetails `json:"llc,omitempty"`
NUMA ContainerResourcesDetails `json:"numa,omitempty"`
Memory ContainerResourcesDetails `json:"memory,omitempty"`
Devices ContainerResourcesDetails `json:"devices,omitempty"`
}

type Alignment struct {
SMT bool `json:"smt"`
LLC bool `json:"llc"`
NUMA bool `json:"numa"`
Memory bool `json:"memory"`
SMT bool `json:"smt"`
LLC bool `json:"llc"`
NUMA bool `json:"numa"`
Memory bool `json:"memory"`
Devices *bool `json:"devices,omitempty"`
}

type Allocation struct {
Expand Down
10 changes: 9 additions & 1 deletion internal/cli/align.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,13 @@ import (
"github.com/ffromani/ctrreschk/pkg/resources"
)

type AlignOptions struct{}
type AlignOptions struct {
DeviceEnvPrefixes []string
}

func NewAlignCommand(env *environ.Environ, opts *Options) *cobra.Command {
alignOpts := AlignOptions{}

alignCmd := &cobra.Command{
Use: "align",
Short: "show resource alignment properties",
Expand All @@ -39,6 +43,9 @@ func NewAlignCommand(env *environ.Environ, opts *Options) *cobra.Command {
if err != nil {
return err
}
if len(alignOpts.DeviceEnvPrefixes) > 0 {
container.Devices = resources.DiscoverDevicesFromEnv(env, os.Environ(), alignOpts.DeviceEnvPrefixes)
}
machine, err := machine.Discover(env)
if err != nil {
return err
Expand All @@ -57,6 +64,7 @@ func NewAlignCommand(env *environ.Environ, opts *Options) *cobra.Command {
}

alignCmd.PersistentFlags().StringVarP(&env.DataPath, "machinedata", "M", "", "read fake machine data from path, don't read real data from the system")
alignCmd.PersistentFlags().StringSliceVar(&alignOpts.DeviceEnvPrefixes, "device-env-prefix", nil, "env var prefixes for device PCI addresses (e.g. SRIOVNETWORK_VF_,PCIDEVICE_)")

return alignCmd
}
48 changes: 47 additions & 1 deletion pkg/align/align.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,9 @@ func Check(env *environ.Environ, container resources.Resources, machine machine.
checkLLC(env, &resp, container.CPUs.Clone(), rmap)
checkNUMA(env, &resp, container.CPUs.Clone(), rmap)
checkMemory(env, &resp, container.CPUs.Clone(), container.MEMs.Clone(), rmap)
checkDevices(env, &resp, container.CPUs.Clone(), container.Devices, rmap)

env.Log.V(2).Info("alignment check complete", "smt", resp.Alignment.SMT, "llc", resp.Alignment.LLC, "numa", resp.Alignment.NUMA, "memory", resp.Alignment.Memory)
env.Log.V(2).Info("alignment check complete", "smt", resp.Alignment.SMT, "llc", resp.Alignment.LLC, "numa", resp.Alignment.NUMA, "memory", resp.Alignment.Memory, "devices", resp.Alignment.Devices)

return resp, nil
}
Expand Down Expand Up @@ -194,6 +195,51 @@ func checkMemory(env *environ.Environ, resp *apiv0.Allocation, cpus cpuset.CPUSe
}
}

func checkDevices(env *environ.Environ, resp *apiv0.Allocation, cpus cpuset.CPUSet, devices []resources.DeviceInfo, rmap rMap) {
if len(devices) == 0 {
env.Log.V(1).Info("no devices to check, skipping device alignment check")
return
}

cpuNUMANodes := cpuset.New()
for numaID := range rmap.numa {
numaCPUs := rmap.numa.CPUSet(numaID)
if !cpus.Intersection(numaCPUs).IsEmpty() {
cpuNUMANodes = cpuNUMANodes.Union(cpuset.New(numaID))
}
}

aligned := true
for _, dev := range devices {
if dev.NUMANode == -1 {
env.Log.V(2).Info("device NUMA node unknown, skipping", "pciAddress", dev.PCIAddress)
continue
}

env.Log.V(2).Info("check device alignment", "pciAddress", dev.PCIAddress, "deviceNUMA", dev.NUMANode, "cpuNUMANodes", cpuNUMANodes.String())

if cpuNUMANodes.Contains(dev.NUMANode) {
if resp.Aligned == nil {
resp.Aligned = apiv0.NewAlignedInfo()
}
dets := resp.Aligned.NUMA[dev.NUMANode]
dets.Devices = append(dets.Devices, dev.PCIAddress)
resp.Aligned.NUMA[dev.NUMANode] = dets
} else {
aligned = false
if resp.Unaligned == nil {
resp.Unaligned = &apiv0.UnalignedInfo{}
}
resp.Unaligned.Devices.Devices = append(resp.Unaligned.Devices.Devices, dev.PCIAddress)
if !slices.Contains(resp.Unaligned.Devices.NUMANodes, dev.NUMANode) {
resp.Unaligned.Devices.NUMANodes = append(resp.Unaligned.Devices.NUMANodes, dev.NUMANode)
}
}
}

resp.Alignment.Devices = &aligned
}

// Reverse ID MAP (PhysicalID|LLCID|NUMAID) -> LogicalIDs
type ridMap map[int][]int

Expand Down
139 changes: 139 additions & 0 deletions pkg/align/align_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,143 @@ func TestCheck(t *testing.T) {
},
},
},
{
name: "device aligned with cpu numa node",
res: resources.Resources{
CPUs: cpuset.New(0, 16),
MEMs: cpuset.New(0),
Devices: []resources.DeviceInfo{
{EnvVar: "SRIOVNETWORK_VF_DEV", PCIAddress: "0000:05:10.2", NUMANode: 0},
},
},
expectedAlloc: apiv0.Allocation{
Alignment: apiv0.Alignment{
SMT: true,
LLC: true,
NUMA: true,
Memory: true,
Devices: boolPtr(true),
},
Aligned: &apiv0.AlignedInfo{
LLC: map[int]apiv0.ContainerResourcesDetails{
0: {
CPUs: []int{0, 16},
},
},
NUMA: map[int]apiv0.ContainerResourcesDetails{
0: {
CPUs: []int{0, 16},
Devices: []string{"0000:05:10.2"},
},
},
Memory: map[int]apiv0.ContainerResourcesDetails{
0: {
CPUs: []int{0, 16},
MemoryMiB: 63705,
MemoryPercent: 100.0,
},
},
},
},
},
{
name: "device on wrong numa node",
res: resources.Resources{
CPUs: cpuset.New(0, 16),
MEMs: cpuset.New(0),
Devices: []resources.DeviceInfo{
{EnvVar: "SRIOVNETWORK_VF_DEV", PCIAddress: "0000:05:10.2", NUMANode: 1},
},
},
expectedAlloc: apiv0.Allocation{
Alignment: apiv0.Alignment{
SMT: true,
LLC: true,
NUMA: true,
Memory: true,
Devices: boolPtr(false),
},
Aligned: &apiv0.AlignedInfo{
LLC: map[int]apiv0.ContainerResourcesDetails{
0: {
CPUs: []int{0, 16},
},
},
NUMA: map[int]apiv0.ContainerResourcesDetails{
0: {
CPUs: []int{0, 16},
},
},
Memory: map[int]apiv0.ContainerResourcesDetails{
0: {
CPUs: []int{0, 16},
MemoryMiB: 63705,
MemoryPercent: 100.0,
},
},
},
Unaligned: &apiv0.UnalignedInfo{
Devices: apiv0.ContainerResourcesDetails{
Devices: []string{"0000:05:10.2"},
NUMANodes: []int{1},
},
},
},
},
{
name: "device with unknown numa node skipped",
res: resources.Resources{
CPUs: cpuset.New(0, 16),
Devices: []resources.DeviceInfo{
{EnvVar: "SRIOVNETWORK_VF_DEV", PCIAddress: "0000:05:10.2", NUMANode: -1},
},
},
expectedAlloc: apiv0.Allocation{
Alignment: apiv0.Alignment{
SMT: true,
LLC: true,
NUMA: true,
Devices: boolPtr(true),
},
Aligned: &apiv0.AlignedInfo{
LLC: map[int]apiv0.ContainerResourcesDetails{
0: {
CPUs: []int{0, 16},
},
},
NUMA: map[int]apiv0.ContainerResourcesDetails{
0: {
CPUs: []int{0, 16},
},
},
},
},
},
{
name: "no devices means no device alignment field",
res: resources.Resources{
CPUs: cpuset.New(0, 16),
},
expectedAlloc: apiv0.Allocation{
Alignment: apiv0.Alignment{
SMT: true,
LLC: true,
NUMA: true,
},
Aligned: &apiv0.AlignedInfo{
LLC: map[int]apiv0.ContainerResourcesDetails{
0: {
CPUs: []int{0, 16},
},
},
NUMA: map[int]apiv0.ContainerResourcesDetails{
0: {
CPUs: []int{0, 16},
},
},
},
},
},
}
for _, tt := range testCases {
t.Run(tt.name, func(t *testing.T) {
Expand All @@ -179,6 +316,8 @@ func TestCheck(t *testing.T) {
}
}

func boolPtr(b bool) *bool { return &b }

func toJSON(v any) string {
data, err := json.Marshal(v)
if err != nil {
Expand Down
63 changes: 63 additions & 0 deletions pkg/resources/devices.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
// SPDX-License-Identifier: Apache-2.0

package resources

import (
"os"
"path/filepath"
"strconv"
"strings"

"github.com/ffromani/ctrreschk/pkg/environ"
)

func DiscoverDevicesFromEnv(env *environ.Environ, osEnviron []string, prefixes []string) []DeviceInfo {
var devices []DeviceInfo
for _, entry := range osEnviron {
name, value, ok := strings.Cut(entry, "=")
if !ok || value == "" {
continue
}
if !matchesAnyPrefix(name, prefixes) {
continue
}
for _, addr := range strings.Split(value, ",") {
addr = strings.TrimSpace(addr)
if addr == "" {
continue
}
numaNode := readDeviceNUMANode(env, addr)
devices = append(devices, DeviceInfo{
EnvVar: name,
PCIAddress: addr,
NUMANode: numaNode,
})
env.Log.V(2).Info("discovered device", "envVar", name, "pciAddress", addr, "numaNode", numaNode)
}
}
return devices
}

func matchesAnyPrefix(name string, prefixes []string) bool {
for _, prefix := range prefixes {
if strings.HasPrefix(name, prefix) {
return true
}
}
return false
}

func readDeviceNUMANode(env *environ.Environ, pciAddress string) int {
numaPath := filepath.Join(env.Root.Sys, "bus", "pci", "devices", pciAddress, "numa_node")
data, err := os.ReadFile(numaPath)
if err != nil {
env.Log.V(1).Info("cannot read device NUMA node, skipping", "pciAddress", pciAddress, "error", err)
return -1
}
node, err := strconv.Atoi(strings.TrimSpace(string(data)))
if err != nil {
env.Log.V(1).Info("cannot parse device NUMA node, skipping", "pciAddress", pciAddress, "error", err)
return -1
}
return node
}
Loading
Loading