diff --git a/src/runtime/export_test.go b/src/runtime/export_test.go index 15365bff38f326..c33bd1539a0462 100644 --- a/src/runtime/export_test.go +++ b/src/runtime/export_test.go @@ -1404,25 +1404,26 @@ func GCTestPointerClass(p unsafe.Pointer) string { const Raceenabled = raceenabled const ( - GCBackgroundUtilization = gcBackgroundUtilization - GCGoalUtilization = gcGoalUtilization DefaultHeapMinimum = defaultHeapMinimum MemoryLimitHeapGoalHeadroomPercent = memoryLimitHeapGoalHeadroomPercent MemoryLimitMinHeapGoalHeadroom = memoryLimitMinHeapGoalHeadroom ) +var GCBackgroundUtilization = gcController.gcRatio +var GCGoalUtilization = gcGoalUtilization + type GCController struct { gcControllerState } -func NewGCController(gcPercent int, memoryLimit int64) *GCController { +func NewGCController(gcPercent int, memoryLimit int64, gcRatio float64) *GCController { // Force the controller to escape. We're going to // do 64-bit atomics on it, and if it gets stack-allocated // on a 32-bit architecture, it may get allocated unaligned // space. g := Escape(new(GCController)) g.gcControllerState.test = true // Mark it as a test copy. - g.init(int32(gcPercent), memoryLimit) + g.init(int32(gcPercent), memoryLimit, gcRatio) return g } diff --git a/src/runtime/gcratio_bench_test.go b/src/runtime/gcratio_bench_test.go new file mode 100644 index 00000000000000..a99f2ecb98225e --- /dev/null +++ b/src/runtime/gcratio_bench_test.go @@ -0,0 +1,104 @@ +// Copyright 2026 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime_test + +import ( + "runtime/debug" + "testing" +) + +// gcRatioNode is a heap-allocated object with a pointer-heavy layout: a +// slice of pointers plus a slice of words. A large live set of these +// objects, combined with a low GOGC, keeps the concurrent GC marking in +// almost continuous operation, so throughput is dominated by how many +// background mark workers the pacer dedicates (GOGCRATIO * GOMAXPROCS). +// See runtime/mgcpacer.go readGOGCRATIO. +type gcRatioNode struct { + ptrs []*gcRatioNode + data []uint64 + sum uint64 +} + +func newGCRatioNode(ptrs, data int) *gcRatioNode { + return &gcRatioNode{ + ptrs: make([]*gcRatioNode, ptrs), + data: make([]uint64, data), + } +} + +// linkGCRatioNodes connects every node to a few of its successors so that +// GC marking has to follow real pointer chains rather than only scanning +// the pool roots. +func linkGCRatioNodes(pool []*gcRatioNode) { + for i, n := range pool { + for j := range n.ptrs { + n.ptrs[j] = pool[(i+j+1)%len(pool)] + } + } +} + +// runGCBackground measures an allocation-heavy workload over a large +// pointer-heavy live set in which objects are continuously replaced, so +// GC runs frequently. Whether marking is the bottleneck depends on GOGC: +// at the default 100 most of each cycle goes to the mutator and the +// number of background mark workers barely matters, while at low GOGC +// marking dominates and throughput is decided by how many workers the +// pacer dedicates (GOGCRATIO * GOMAXPROCS). Three wrappers below sample +// three GC-pressure regimes so the benchmark captures both the regular +// case (no regression expected) and the GC-bound case (large effect). +// +// Sub-benchmarks vary the live-set size and pointer density. Run with: +// +// GOGCRATIO=25 GOMAXPROCS=N go test -run '^$' -bench BenchmarkGCBackground +func runGCBackground(b *testing.B, gogc int) { + debug.SetGCPercent(gogc) + + for _, tc := range []struct { + name string + live int + ptrs int + data int + }{ + {"gcbound", 32768, 32, 64}, + {"gcbound-large", 65536, 32, 64}, + {"gcbound-dense", 32768, 64, 32}, + } { + b.Run(tc.name, func(b *testing.B) { + pool := make([]*gcRatioNode, tc.live) + for i := range pool { + pool[i] = newGCRatioNode(tc.ptrs, tc.data) + } + linkGCRatioNodes(pool) + idx := 0 + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + pool[idx] = newGCRatioNode(tc.ptrs, tc.data) + idx = (idx + 1) % tc.live + } + }) + } +} + +// BenchmarkGCBackgroundDefault runs at GOGC=100, the regime of most real +// workloads: marking is far from the bottleneck, so GOGCRATIO should have +// a negligible effect. Guards against regressions in the common case. +func BenchmarkGCBackgroundDefault(b *testing.B) { + runGCBackground(b, 100) +} + +// BenchmarkGCBackgroundModerate runs at GOGC=20, an intermediate GC +// pressure where marking starts to consume a visible share of the CPU. +func BenchmarkGCBackgroundModerate(b *testing.B) { + runGCBackground(b, 20) +} + +// BenchmarkGCBackgroundDense runs at GOGC=5, where marking is the +// bottleneck and the number of background mark workers (governed by +// GOGCRATIO) directly decides how much of the available CPU goes to +// marking. Throughput is expected to vary strongly with the setting. +func BenchmarkGCBackgroundDense(b *testing.B) { + runGCBackground(b, 5) +} diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index a3bed4b3eb62f9..23a8972bd6e396 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -186,7 +186,7 @@ func gcinit() { // Initialize GC pacer state. // Use the environment variable GOGC for the initial gcPercent value. // Use the environment variable GOMEMLIMIT for the initial memoryLimit value. - gcController.init(readGOGC(), readGOMEMLIMIT()) + gcController.init(readGOGC(), readGOMEMLIMIT(), readGOGCRATIO()) // Set up the cleanup block ptr mask. for i := range cleanupBlockPtrMask { @@ -280,12 +280,12 @@ const ( // gcMarkWorkerFractionalMode indicates that a P is currently // running the "fractional" mark worker. The fractional worker - // is necessary when GOMAXPROCS*gcBackgroundUtilization is not + // is necessary when GOMAXPROCS*gcController.gcRatio is not // an integer and using only dedicated workers would result in - // utilization too far from the target of gcBackgroundUtilization. + // utilization too far from the target of gcController.gcRatio. // The fractional worker should run until it is preempted and // will be scheduled to pick up the fractional part of - // GOMAXPROCS*gcBackgroundUtilization. + // GOMAXPROCS*gcController.gcRatio. gcMarkWorkerFractionalMode // gcMarkWorkerIdleMode indicates that a P is running the mark diff --git a/src/runtime/mgclimit.go b/src/runtime/mgclimit.go index 80aeb71cad6185..8c42b2c56b3b33 100644 --- a/src/runtime/mgclimit.go +++ b/src/runtime/mgclimit.go @@ -228,7 +228,7 @@ func (l *gcCPULimiterState) updateLocked(now int64) { // Compute total GC time. windowGCTime := assistTime if l.gcEnabled { - windowGCTime += int64(float64(windowTotalTime) * gcBackgroundUtilization) + windowGCTime += int64(float64(windowTotalTime) * gcController.gcRatio) } // Subtract out all idle time from the total time. Do this after computing diff --git a/src/runtime/mgclimit_test.go b/src/runtime/mgclimit_test.go index 124da03ef1d9f3..8d3eaa6beeac8b 100644 --- a/src/runtime/mgclimit_test.go +++ b/src/runtime/mgclimit_test.go @@ -76,13 +76,13 @@ func TestGCCPULimiter(t *testing.T) { // Test passing time without assists during a GC. Specifically, just enough to drain the bucket to // exactly procs nanoseconds (easier to get to because of rounding). // - // The window we need to drain the bucket is 1/(1-2*gcBackgroundUtilization) times the current fill: + // The window we need to drain the bucket is 1/(1-2*gcController.gcRatio) times the current fill: // - // fill + (window * procs * gcBackgroundUtilization - window * procs * (1-gcBackgroundUtilization)) = n - // fill = n - (window * procs * gcBackgroundUtilization - window * procs * (1-gcBackgroundUtilization)) - // fill = n + window * procs * ((1-gcBackgroundUtilization) - gcBackgroundUtilization) - // fill = n + window * procs * (1-2*gcBackgroundUtilization) - // window = (fill - n) / (procs * (1-2*gcBackgroundUtilization))) + // fill + (window * procs * gcController.gcRatio - window * procs * (1-gcController.gcRatio)) = n + // fill = n - (window * procs * gcController.gcRatio - window * procs * (1-gcController.gcRatio)) + // fill = n + window * procs * ((1-gcController.gcRatio) - gcController.gcRatio) + // fill = n + window * procs * (1-2*gcController.gcRatio) + // window = (fill - n) / (procs * (1-2*gcController.gcRatio))) // // And here we want n=procs: factor := (1 / (1 - 2*GCBackgroundUtilization)) diff --git a/src/runtime/mgcpacer.go b/src/runtime/mgcpacer.go index 388cce83cd2388..e77762f33d8781 100644 --- a/src/runtime/mgcpacer.go +++ b/src/runtime/mgcpacer.go @@ -14,30 +14,6 @@ import ( ) const ( - // gcGoalUtilization is the goal CPU utilization for - // marking as a fraction of GOMAXPROCS. - // - // Increasing the goal utilization will shorten GC cycles as the GC - // has more resources behind it, lessening costs from the write barrier, - // but comes at the cost of increasing mutator latency. - gcGoalUtilization = gcBackgroundUtilization - - // gcBackgroundUtilization is the fixed CPU utilization for background - // marking. It must be <= gcGoalUtilization. The difference between - // gcGoalUtilization and gcBackgroundUtilization will be made up by - // mark assists. The scheduler will aim to use within 50% of this - // goal. - // - // As a general rule, there's little reason to set gcBackgroundUtilization - // < gcGoalUtilization. One reason might be in mostly idle applications, - // where goroutines are unlikely to assist at all, so the actual - // utilization will be lower than the goal. But this is moot point - // because the idle mark workers already soak up idle CPU resources. - // These two values are still kept separate however because they are - // distinct conceptually, and in previous iterations of the pacer the - // distinction was more important. - gcBackgroundUtilization = 0.25 - // gcCreditSlack is the amount of scan work credit that can // accumulate locally before updating gcController.heapScanWork and, // optionally, gcController.bgScanCredit. Lower values give a more @@ -75,6 +51,14 @@ const ( memoryLimitHeapGoalHeadroomPercent = 3 ) +// gcGoalUtilization is the goal CPU utilization for +// marking as a fraction of GOMAXPROCS. +// +// Increasing the goal utilization will shorten GC cycles as the GC +// has more resources behind it, lessening costs from the write barrier, +// but comes at the cost of increasing mutator latency. +var gcGoalUtilization = gcController.gcRatio + // gcController implements the GC pacing controller that determines // when to trigger concurrent garbage collection and how much marking // work to do in mutator assists and background marking. @@ -90,6 +74,11 @@ const ( var gcController gcControllerState type gcControllerState struct { + // gcController.gcRatio be optional, value equals gcratio/100.0. + // Initialized from GOGCRATIO, which in the range of (1, 99). + // Default GOGCRATIO is 25. + gcRatio float64 + // Initialized from GOGC. GOGC=off means no GC. gcPercent atomic.Int32 @@ -368,11 +357,12 @@ type gcControllerState struct { _ cpu.CacheLinePad } -func (c *gcControllerState) init(gcPercent int32, memoryLimit int64) { +func (c *gcControllerState) init(gcPercent int32, memoryLimit int64, gcRatio float64) { c.heapMinimum = defaultHeapMinimum c.triggered = ^uint64(0) c.setGCPercent(gcPercent) c.setMemoryLimit(memoryLimit) + c.setGOGCRatio(gcRatio) c.commit(true) // No sweep phase in the first GC cycle. // N.B. Don't bother calling traceHeapGoal. Tracing is never enabled at // initialization time. @@ -400,13 +390,13 @@ func (c *gcControllerState) startCycle(markStartTime int64, procs int, trigger g // dedicated workers so that the utilization is closest to // 25%. For small GOMAXPROCS, this would introduce too much // error, so we add fractional workers in that case. - totalUtilizationGoal := float64(procs) * gcBackgroundUtilization + totalUtilizationGoal := float64(procs) * gcController.gcRatio dedicatedMarkWorkersNeeded := int64(totalUtilizationGoal + 0.5) utilError := float64(dedicatedMarkWorkersNeeded)/totalUtilizationGoal - 1 const maxUtilError = 0.3 if utilError < -maxUtilError || utilError > maxUtilError { // Rounding put us more than 30% off our goal. With - // gcBackgroundUtilization of 25%, this happens for + // gcController.gcRatio of 25%, this happens for // GOMAXPROCS<=3 or GOMAXPROCS=6. Enable fractional // workers to compensate. if float64(dedicatedMarkWorkersNeeded) > totalUtilizationGoal { @@ -606,7 +596,7 @@ func (c *gcControllerState) endCycle(now int64, procs int, userForced bool) { assistDuration := now - c.markStartTime // Assume background mark hit its utilization goal. - utilization := gcBackgroundUtilization + utilization := gcController.gcRatio // Add assist utilization; avoid divide by zero. if assistDuration > 0 { utilization += float64(c.assistTime.Load()) / float64(assistDuration*int64(procs)) @@ -1426,6 +1416,39 @@ func readGOMEMLIMIT() int64 { return n } +func (c *gcControllerState) setGOGCRatio(in float64) float64 { + if !c.test { + assertWorldStoppedOrLockHeld(&mheap_.lock) + } + + out := c.gcRatio + c.gcRatio = in + + return out +} + +func readGOGCRATIO() float64 { + p := gogetenv("GOGCRATIO") + if p == "" { + return 0.25 + } + n, ok := parseByteCount(p) + if !ok { + print("GOGCRATIO=", p, "\n") + throw("malformed GOGCRATIO; get the wrong value") + } + + if n < 1 { + n = 1 + } else if n > 99 { + n = 99 + } + + out := float64(n) / 100.0 + + return out +} + // addIdleMarkWorker attempts to add a new idle mark worker. // // If this returns true, the caller must become an idle mark worker unless diff --git a/src/runtime/mgcpacer_test.go b/src/runtime/mgcpacer_test.go index 4b9cbf558909d4..3158f717579297 100644 --- a/src/runtime/mgcpacer_test.go +++ b/src/runtime/mgcpacer_test.go @@ -24,6 +24,7 @@ func TestGcPacer(t *testing.T) { name: "Steady", gcPercent: 100, memoryLimit: math.MaxInt64, + gcRatio: 0.25, globalsBytes: 32 << 10, nCores: 8, allocRate: constant(33.0), @@ -49,6 +50,7 @@ func TestGcPacer(t *testing.T) { name: "SteadyBigStacks", gcPercent: 100, memoryLimit: math.MaxInt64, + gcRatio: 0.25, globalsBytes: 32 << 10, nCores: 8, allocRate: constant(132.0), @@ -77,6 +79,7 @@ func TestGcPacer(t *testing.T) { name: "SteadyBigGlobals", gcPercent: 100, memoryLimit: math.MaxInt64, + gcRatio: 0.25, globalsBytes: 128 << 20, nCores: 8, allocRate: constant(132.0), @@ -105,6 +108,7 @@ func TestGcPacer(t *testing.T) { name: "StepAlloc", gcPercent: 100, memoryLimit: math.MaxInt64, + gcRatio: 0.25, globalsBytes: 32 << 10, nCores: 8, allocRate: constant(33.0).sum(ramp(66.0, 1).delay(50)), @@ -128,6 +132,7 @@ func TestGcPacer(t *testing.T) { name: "HeavyStepAlloc", gcPercent: 100, memoryLimit: math.MaxInt64, + gcRatio: 0.25, globalsBytes: 32 << 10, nCores: 8, allocRate: constant(33).sum(ramp(330, 1).delay(50)), @@ -151,6 +156,7 @@ func TestGcPacer(t *testing.T) { name: "StepScannableFrac", gcPercent: 100, memoryLimit: math.MaxInt64, + gcRatio: 0.25, globalsBytes: 32 << 10, nCores: 8, allocRate: constant(128.0), @@ -176,6 +182,7 @@ func TestGcPacer(t *testing.T) { name: "HighGOGC", gcPercent: 1500, memoryLimit: math.MaxInt64, + gcRatio: 0.25, globalsBytes: 32 << 10, nCores: 8, allocRate: random(7, 0x53).offset(165), @@ -217,6 +224,7 @@ func TestGcPacer(t *testing.T) { name: "OscAlloc", gcPercent: 100, memoryLimit: math.MaxInt64, + gcRatio: 0.25, globalsBytes: 32 << 10, nCores: 8, allocRate: oscillate(13, 0, 8).offset(67), @@ -241,6 +249,7 @@ func TestGcPacer(t *testing.T) { name: "JitterAlloc", gcPercent: 100, memoryLimit: math.MaxInt64, + gcRatio: 0.25, globalsBytes: 32 << 10, nCores: 8, allocRate: random(13, 0xf).offset(132), @@ -266,6 +275,7 @@ func TestGcPacer(t *testing.T) { name: "HeavyJitterAlloc", gcPercent: 100, memoryLimit: math.MaxInt64, + gcRatio: 0.25, globalsBytes: 32 << 10, nCores: 8, allocRate: random(33.0, 0x0).offset(330), @@ -295,6 +305,7 @@ func TestGcPacer(t *testing.T) { name: "SmallHeapSlowAlloc", gcPercent: 100, memoryLimit: math.MaxInt64, + gcRatio: 0.25, globalsBytes: 32 << 10, nCores: 8, allocRate: constant(1.0), @@ -332,6 +343,7 @@ func TestGcPacer(t *testing.T) { name: "MediumHeapSlowAlloc", gcPercent: 100, memoryLimit: math.MaxInt64, + gcRatio: 0.25, globalsBytes: 32 << 10, nCores: 8, allocRate: constant(1.0), @@ -369,6 +381,7 @@ func TestGcPacer(t *testing.T) { name: "LargeHeapSlowAlloc", gcPercent: 100, memoryLimit: math.MaxInt64, + gcRatio: 0.25, globalsBytes: 32 << 10, nCores: 8, allocRate: constant(1.0), @@ -407,6 +420,7 @@ func TestGcPacer(t *testing.T) { name: "SteadyMemoryLimit", gcPercent: 100, memoryLimit: 512 << 20, + gcRatio: 0.25, globalsBytes: 32 << 10, nCores: 8, allocRate: constant(33.0), @@ -436,6 +450,7 @@ func TestGcPacer(t *testing.T) { name: "SteadyMemoryLimitNoGCPercent", gcPercent: -1, memoryLimit: 512 << 20, + gcRatio: 0.25, globalsBytes: 32 << 10, nCores: 8, allocRate: constant(33.0), @@ -465,6 +480,7 @@ func TestGcPacer(t *testing.T) { name: "ExceedMemoryLimit", gcPercent: 100, memoryLimit: 512 << 20, + gcRatio: 0.25, globalsBytes: 32 << 10, nCores: 8, allocRate: constant(33.0), @@ -499,6 +515,7 @@ func TestGcPacer(t *testing.T) { name: "ExceedMemoryLimitNoGCPercent", gcPercent: -1, memoryLimit: 512 << 20, + gcRatio: 0.25, globalsBytes: 32 << 10, nCores: 8, allocRate: constant(33.0), @@ -538,6 +555,7 @@ func TestGcPacer(t *testing.T) { name: "MaintainMemoryLimit", gcPercent: 100, memoryLimit: 512 << 20, + gcRatio: 0.25, globalsBytes: 32 << 10, nCores: 8, allocRate: constant(33.0), @@ -571,6 +589,7 @@ func TestGcPacer(t *testing.T) { name: "MaintainMemoryLimitNoGCPercent", gcPercent: -1, memoryLimit: 512 << 20, + gcRatio: 0.25, globalsBytes: 32 << 10, nCores: 8, allocRate: constant(33.0), @@ -606,7 +625,7 @@ func TestGcPacer(t *testing.T) { t.Run(e.name, func(t *testing.T) { t.Parallel() - c := NewGCController(e.gcPercent, e.memoryLimit) + c := NewGCController(e.gcPercent, e.memoryLimit, e.gcRatio) var bytesAllocatedBlackLast int64 results := make([]gcCycleResult, 0, e.length) for i := 0; i < e.length; i++ { @@ -761,6 +780,7 @@ type gcExecTest struct { gcPercent int memoryLimit int64 + gcRatio float64 globalsBytes uint64 nCores int @@ -1033,7 +1053,7 @@ func applyMemoryLimitHeapGoalHeadroom(goal uint64) uint64 { func TestIdleMarkWorkerCount(t *testing.T) { const workers = 10 - c := NewGCController(100, math.MaxInt64) + c := NewGCController(100, math.MaxInt64, 0.25) c.SetMaxIdleMarkWorkers(workers) for i := 0; i < workers; i++ { if !c.NeedIdleMarkWorker() {