Fix reported CPU usage on Podman

environment/docker/stats.go

@@ -5,15 +5,23 @@ import (
 	"io"
 	"math"
 	"strings"
+	"sync"
 	"time"
 
 	"emperror.dev/errors"
+	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/goccy/go-json"
 
 	"github.com/pelican/wings/environment"
 )
 
+var runtimeDetection struct {
+	sync.Mutex
+	detected bool
+	isPodman bool
+}
+
 // Uptime returns the current uptime of the container in milliseconds. If the
 // container is not currently running this will return 0.
 func (e *Environment) Uptime(ctx context.Context) (int64, error) {
@@ -52,6 +60,12 @@ func (e *Environment) pollResources(ctx context.Context) error {
 		e.log().WithField("error", err).Warn("failed to calculate container uptime")
 	}
 
+	isPodman, err := e.isPodman(ctx)
+	if err != nil {
+		e.log().WithField("error", err).Warn("failed to detect container runtime, using wall time for CPU calculation")
+		isPodman = true
+	}
+
 	dec := json.NewDecoder(stats.Body)
 	for {
 		select {
@@ -82,7 +96,7 @@ func (e *Environment) pollResources(ctx context.Context) error {
 				Uptime:      uptime,
 				Memory:      calculateDockerMemory(v.MemoryStats),
 				MemoryLimit: v.MemoryStats.Limit,
-				CpuAbsolute: calculateDockerAbsoluteCpu(v.PreCPUStats, v.CPUStats),
+				CpuAbsolute: calculateDockerAbsoluteCpu(v, isPodman),
 				Network:     environment.NetworkStats{},
 			}
 
@@ -130,28 +144,66 @@ func calculateDockerMemory(stats container.MemoryStats) uint64 {
 // Calculates the absolute CPU usage used by the server process on the system, not constrained
 // by the defined CPU limits on the container.
 //
-// @see https://github.com/docker/cli/blob/aa097cf1aa19099da70930460250797c8920b709/cli/command/container/stats_helpers.go#L166
-func calculateDockerAbsoluteCpu(pStats container.CPUStats, stats container.CPUStats) float64 {
-	// Calculate the change in CPU usage between the current and previous reading.
-	cpuDelta := float64(stats.CPUUsage.TotalUsage) - float64(pStats.CPUUsage.TotalUsage)
+// Podman's Docker-compatible API does not provide Docker-equivalent values for SystemUsage, so
+// its CPU usage must instead be compared to the elapsed time between samples.
+func calculateDockerAbsoluteCpu(stats container.StatsResponse, useWallTime bool) float64 {
+	current := stats.CPUStats.CPUUsage.TotalUsage
+	previous := stats.PreCPUStats.CPUUsage.TotalUsage
+	if current <= previous {
+		return 0
+	}
 
-	// Calculate the change for the entire system's CPU usage between current and previous reading.
-	systemDelta := float64(stats.SystemUsage) - float64(pStats.SystemUsage)
+	cpuDelta := float64(current - previous)
+	if useWallTime {
+		if stats.PreRead.IsZero() || !stats.Read.After(stats.PreRead) {
+			return 0
+		}
 
-	// Calculate the total number of CPU cores being used.
-	cpus := float64(stats.OnlineCPUs)
-	if cpus == 0.0 {
-		cpus = float64(len(stats.CPUUsage.PercpuUsage))
+		timeDelta := float64(stats.Read.Sub(stats.PreRead).Nanoseconds())
+		return math.Round((cpuDelta/timeDelta)*100*1000) / 1000
 	}
 
-	percent := 0.0
-	if systemDelta > 0.0 && cpuDelta > 0.0 {
-		percent = (cpuDelta / systemDelta) * 100.0
+	currentSystem := stats.CPUStats.SystemUsage
+	previousSystem := stats.PreCPUStats.SystemUsage
+	if currentSystem <= previousSystem {
+		return 0
+	}
 
-		if cpus > 0 {
-			percent *= cpus
-		}
+	cpus := float64(stats.CPUStats.OnlineCPUs)
+	if cpus == 0 {
+		cpus = float64(len(stats.CPUStats.CPUUsage.PercpuUsage))
+	}
+
+	percent := (cpuDelta / float64(currentSystem-previousSystem)) * 100
+	if cpus > 0 {
+		percent *= cpus
 	}
 
 	return math.Round(percent*1000) / 1000
 }
+
+func (e *Environment) isPodman(ctx context.Context) (bool, error) {
+	runtimeDetection.Lock()
+	defer runtimeDetection.Unlock()
+	if runtimeDetection.detected {
+		return runtimeDetection.isPodman, nil
+	}
+
+	version, err := e.client.ServerVersion(ctx)
+	if err != nil {
+		return false, err
+	}
+
+	runtimeDetection.detected = true
+	runtimeDetection.isPodman = isPodmanVersion(version)
+	return runtimeDetection.isPodman, nil
+}
+
+func isPodmanVersion(version types.Version) bool {
+	for _, component := range version.Components {
+		if strings.EqualFold(component.Name, "Podman Engine") {
+			return true
+		}
+	}
+	return false
+}

environment/docker/stats_test.go

@@ -0,0 +1,110 @@
+package docker
+
+import (
+	"testing"
+	"time"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCalculateDockerAbsoluteCpu(t *testing.T) {
+	base := time.Date(2026, time.July, 13, 12, 0, 0, 0, time.UTC)
+
+	tests := []struct {
+		name        string
+		stats       container.StatsResponse
+		useWallTime bool
+		expected    float64
+	}{
+		{
+			name: "Docker system usage",
+			stats: func() container.StatsResponse {
+				stats := cpuStatsResponse(base, 2*time.Second, 5_000_000_000, 6_000_000_000)
+				stats.PreCPUStats.SystemUsage = 10_000_000_000
+				stats.CPUStats.SystemUsage = 74_000_000_000
+				stats.CPUStats.OnlineCPUs = 64
+				return stats
+			}(),
+			expected: 100,
+		},
+		{
+			name:        "multiple cores with wall time",
+			stats:       cpuStatsResponse(base, time.Second, 5_000_000_000, 7_500_000_000),
+			useWallTime: true,
+			expected:    250,
+		},
+		{
+			name:        "rounds wall time to three decimal places",
+			stats:       cpuStatsResponse(base, 3*time.Second, 5_000_000_000, 6_000_000_000),
+			useWallTime: true,
+			expected:    33.333,
+		},
+		{
+			name: "Podman system usage",
+			stats: func() container.StatsResponse {
+				stats := cpuStatsResponse(base, time.Second, 5_000_000_000, 6_000_000_000)
+				stats.PreCPUStats.SystemUsage = 1_000_000_000
+				stats.CPUStats.SystemUsage = 2_000_000_000
+				stats.CPUStats.OnlineCPUs = 64
+				return stats
+			}(),
+			useWallTime: true,
+			expected:    100,
+		},
+		{
+			name:        "no CPU usage",
+			stats:       cpuStatsResponse(base, time.Second, 5_000_000_000, 5_000_000_000),
+			useWallTime: true,
+			expected:    0,
+		},
+		{
+			name:        "CPU counter reset",
+			stats:       cpuStatsResponse(base, time.Second, 5_000_000_000, 1_000_000_000),
+			useWallTime: true,
+			expected:    0,
+		},
+		{
+			name:        "missing previous timestamp",
+			stats:       cpuStatsResponse(time.Time{}, time.Second, 5_000_000_000, 6_000_000_000),
+			useWallTime: true,
+			expected:    0,
+		},
+		{
+			name:        "non-increasing timestamp",
+			stats:       cpuStatsResponse(base, 0, 5_000_000_000, 6_000_000_000),
+			useWallTime: true,
+			expected:    0,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			require.Equal(t, tt.expected, calculateDockerAbsoluteCpu(tt.stats, tt.useWallTime))
+		})
+	}
+}
+
+func TestIsPodmanVersion(t *testing.T) {
+	version := types.Version{
+		Components: []types.ComponentVersion{{Name: "Podman Engine"}},
+	}
+	require.True(t, isPodmanVersion(version))
+
+	version.Components[0].Name = "Engine"
+	require.False(t, isPodmanVersion(version))
+}
+
+func cpuStatsResponse(preRead time.Time, elapsed time.Duration, previous, current uint64) container.StatsResponse {
+	return container.StatsResponse{
+		Read:    preRead.Add(elapsed),
+		PreRead: preRead,
+		CPUStats: container.CPUStats{
+			CPUUsage: container.CPUUsage{TotalUsage: current},
+		},
+		PreCPUStats: container.CPUStats{
+			CPUUsage: container.CPUUsage{TotalUsage: previous},
+		},
+	}
+}