cmd: add alignmem command

api/v0/types.go

@@ -62,3 +62,15 @@ type Allocation struct {
 	Aligned   *AlignedInfo   `json:"aligned,omitempty"`
 	Unaligned *UnalignedInfo `json:"unaligned,omitempty"`
 }
+
+type NUMAMapsNodeInfo struct {
+	Pages   int64 `json:"pages"`
+	SizeKiB int64 `json:"sizeKiB"`
+}
+
+type NUMAMapsInfo struct {
+	Nodes       map[int]NUMAMapsNodeInfo `json:"nodes,omitempty"`
+	LocalPages  int64                    `json:"localPages"`
+	RemotePages int64                    `json:"remotePages"`
+	Local       bool                     `json:"local"`
+}

internal/cli/alignmem.go

@@ -0,0 +1,96 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package cli
+
+import (
+	"encoding/json"
+	"os"
+
+	"github.com/spf13/cobra"
+
+	"k8s.io/utils/cpuset"
+
+	apiv0 "github.com/ffromani/ctrreschk/api/v0"
+	"github.com/ffromani/ctrreschk/pkg/cgroups"
+	"github.com/ffromani/ctrreschk/pkg/environ"
+	"github.com/ffromani/ctrreschk/pkg/machine"
+	"github.com/ffromani/ctrreschk/pkg/numamaps"
+)
+
+func NewAlignMemCommand(env *environ.Environ, opts *Options) *cobra.Command {
+	alignMemCmd := &cobra.Command{
+		Use:   "alignmem",
+		Short: "verify actual memory NUMA placement via numa_maps",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cpus, err := cgroups.Cpuset(env)
+			if err != nil {
+				return err
+			}
+
+			mach, err := machine.Discover(env)
+			if err != nil {
+				return err
+			}
+
+			nm, err := numamaps.Read(env)
+			if err != nil {
+				return err
+			}
+
+			cpuNUMANodes := cpuNUMANodesFromTopology(cpus, mach)
+			env.Log.V(2).Info("alignmem", "cpuNUMANodes", cpuNUMANodes.String())
+
+			result := buildNUMAMapsInfo(nm, cpuNUMANodes)
+
+			err = json.NewEncoder(os.Stdout).Encode(result)
+			if err != nil {
+				return err
+			}
+			return MainLoop(opts)
+		},
+		Args: cobra.NoArgs,
+	}
+
+	alignMemCmd.PersistentFlags().StringVarP(&env.DataPath, "machinedata", "M", "", "read fake machine data from path, don't read real data from the system")
+
+	return alignMemCmd
+}
+
+func cpuNUMANodesFromTopology(cpus cpuset.CPUSet, mach machine.Machine) cpuset.CPUSet {
+	result := cpuset.New()
+	for _, node := range mach.Topology.Nodes {
+		for _, core := range node.Cores {
+			coreCPUs := cpuset.New(core.LogicalProcessors...)
+			if !cpus.Intersection(coreCPUs).IsEmpty() {
+				result = result.Union(cpuset.New(node.ID))
+				break
+			}
+		}
+	}
+	return result
+}
+
+func buildNUMAMapsInfo(nm numamaps.NumaMaps, cpuNUMANodes cpuset.CPUSet) apiv0.NUMAMapsInfo {
+	pagesByNode := nm.TotalPagesByNode()
+	bytesByNode := nm.TotalBytesByNode()
+
+	info := apiv0.NUMAMapsInfo{
+		Nodes: make(map[int]apiv0.NUMAMapsNodeInfo),
+	}
+
+	for nodeID, pages := range pagesByNode {
+		info.Nodes[nodeID] = apiv0.NUMAMapsNodeInfo{
+			Pages:   pages,
+			SizeKiB: bytesByNode[nodeID] / 1024,
+		}
+		if cpuNUMANodes.Contains(nodeID) {
+			info.LocalPages += pages
+		} else {
+			info.RemotePages += pages
+		}
+	}
+
+	info.Local = info.RemotePages == 0 && info.LocalPages > 0
+
+	return info
+}

internal/cli/root.go

@@ -64,6 +64,7 @@ func NewRootCommand(env *environ.Environ, extraCmds ...NewCommandFunc) *cobra.Co
 
 	root.AddCommand(
 		NewAlignCommand(env, &opts),
+		NewAlignMemCommand(env, &opts),
 		NewInfoCommand(env, &opts),
 		NewK8SCommand(env, &opts),
 		NewPauseCommand(env, &opts),

pkg/environ/environ.go

@@ -25,12 +25,14 @@ import (
 )
 
 type FS struct {
-	Sys string
+	Sys  string
+	Proc string
 }
 
 func DefaultFS() FS {
 	return FS{
-		Sys: "/sys",
+		Sys:  "/sys",
+		Proc: "/proc",
 	}
 }
 

pkg/numamaps/numamaps.go

@@ -0,0 +1,143 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package numamaps
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/ffromani/ctrreschk/pkg/environ"
+)
+
+const (
+	ProcSelfPath = "self"
+	NumaMapsFile = "numa_maps"
+)
+
+var numaPageRe = regexp.MustCompile(`^N(\d+)=(\d+)$`)
+
+type VMA struct {
+	Address        uint64
+	Policy         string
+	FilePath       string
+	NUMAPages      map[int]int64
+	KernPageSizeKB int64
+}
+
+type NumaMaps struct {
+	VMAs []VMA
+}
+
+func NumaMapsPath(env *environ.Environ) string {
+	return filepath.Join(env.Root.Proc, ProcSelfPath, NumaMapsFile)
+}
+
+func Read(env *environ.Environ) (NumaMaps, error) {
+	path := NumaMapsPath(env)
+	env.Log.V(2).Info("reading numa_maps", "path", path)
+
+	f, err := os.Open(path)
+	if err != nil {
+		return NumaMaps{}, err
+	}
+	defer f.Close()
+
+	return parse(f)
+}
+
+func parse(f *os.File) (NumaMaps, error) {
+	var result NumaMaps
+	scanner := bufio.NewScanner(f)
+
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" {
+			continue
+		}
+		vma, err := parseLine(line)
+		if err != nil {
+			return NumaMaps{}, fmt.Errorf("parsing line %q: %w", line, err)
+		}
+		result.VMAs = append(result.VMAs, vma)
+	}
+	if err := scanner.Err(); err != nil {
+		return NumaMaps{}, err
+	}
+	return result, nil
+}
+
+func parseLine(line string) (VMA, error) {
+	fields := strings.Fields(line)
+	if len(fields) < 2 {
+		return VMA{}, fmt.Errorf("too few fields")
+	}
+
+	addr, err := strconv.ParseUint(fields[0], 16, 64)
+	if err != nil {
+		return VMA{}, fmt.Errorf("invalid address %q: %w", fields[0], err)
+	}
+
+	vma := VMA{
+		Address:        addr,
+		Policy:         fields[1],
+		NUMAPages:      make(map[int]int64),
+		KernPageSizeKB: 4, // default
+	}
+
+	for _, field := range fields[2:] {
+		if m := numaPageRe.FindStringSubmatch(field); m != nil {
+			nodeID, _ := strconv.Atoi(m[1])
+			pages, _ := strconv.ParseInt(m[2], 10, 64)
+			vma.NUMAPages[nodeID] = pages
+		} else if rest, ok := strings.CutPrefix(field, "file="); ok {
+			vma.FilePath = rest
+		} else if rest, ok := strings.CutPrefix(field, "kernelpagesize_kB="); ok {
+			if v, err := strconv.ParseInt(rest, 10, 64); err == nil {
+				vma.KernPageSizeKB = v
+			}
+		}
+	}
+
+	return vma, nil
+}
+
+func (nm NumaMaps) NUMANodes() []int {
+	seen := make(map[int]struct{})
+	for _, vma := range nm.VMAs {
+		for nodeID := range vma.NUMAPages {
+			seen[nodeID] = struct{}{}
+		}
+	}
+	nodes := make([]int, 0, len(seen))
+	for nodeID := range seen {
+		nodes = append(nodes, nodeID)
+	}
+	sort.Ints(nodes)
+	return nodes
+}
+
+func (nm NumaMaps) TotalPagesByNode() map[int]int64 {
+	totals := make(map[int]int64)
+	for _, vma := range nm.VMAs {
+		for nodeID, pages := range vma.NUMAPages {
+			totals[nodeID] += pages
+		}
+	}
+	return totals
+}
+
+func (nm NumaMaps) TotalBytesByNode() map[int]int64 {
+	totals := make(map[int]int64)
+	for _, vma := range nm.VMAs {
+		for nodeID, pages := range vma.NUMAPages {
+			totals[nodeID] += pages * vma.KernPageSizeKB * 1024
+		}
+	}
+	return totals
+}