Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions src/runtime/export_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1404,25 +1404,26 @@ func GCTestPointerClass(p unsafe.Pointer) string {
const Raceenabled = raceenabled

const (
GCBackgroundUtilization = gcBackgroundUtilization
GCGoalUtilization = gcGoalUtilization
DefaultHeapMinimum = defaultHeapMinimum
MemoryLimitHeapGoalHeadroomPercent = memoryLimitHeapGoalHeadroomPercent
MemoryLimitMinHeapGoalHeadroom = memoryLimitMinHeapGoalHeadroom
)

var GCBackgroundUtilization = gcController.gcRatio
var GCGoalUtilization = gcGoalUtilization

type GCController struct {
gcControllerState
}

func NewGCController(gcPercent int, memoryLimit int64) *GCController {
func NewGCController(gcPercent int, memoryLimit int64, gcRatio float64) *GCController {
// Force the controller to escape. We're going to
// do 64-bit atomics on it, and if it gets stack-allocated
// on a 32-bit architecture, it may get allocated unaligned
// space.
g := Escape(new(GCController))
g.gcControllerState.test = true // Mark it as a test copy.
g.init(int32(gcPercent), memoryLimit)
g.init(int32(gcPercent), memoryLimit, gcRatio)
return g
}

Expand Down
104 changes: 104 additions & 0 deletions src/runtime/gcratio_bench_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
// Copyright 2026 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime_test

import (
"runtime/debug"
"testing"
)

// gcRatioNode is a heap-allocated object with a pointer-heavy layout: a
// slice of pointers plus a slice of words. A large live set of these
// objects, combined with a low GOGC, keeps the concurrent GC marking in
// almost continuous operation, so throughput is dominated by how many
// background mark workers the pacer dedicates (GOGCRATIO * GOMAXPROCS).
// See runtime/mgcpacer.go readGOGCRATIO.
type gcRatioNode struct {
ptrs []*gcRatioNode
data []uint64
sum uint64
}

func newGCRatioNode(ptrs, data int) *gcRatioNode {
return &gcRatioNode{
ptrs: make([]*gcRatioNode, ptrs),
data: make([]uint64, data),
}
}

// linkGCRatioNodes connects every node to a few of its successors so that
// GC marking has to follow real pointer chains rather than only scanning
// the pool roots.
func linkGCRatioNodes(pool []*gcRatioNode) {
for i, n := range pool {
for j := range n.ptrs {
n.ptrs[j] = pool[(i+j+1)%len(pool)]
}
}
}

// runGCBackground measures an allocation-heavy workload over a large
// pointer-heavy live set in which objects are continuously replaced, so
// GC runs frequently. Whether marking is the bottleneck depends on GOGC:
// at the default 100 most of each cycle goes to the mutator and the
// number of background mark workers barely matters, while at low GOGC
// marking dominates and throughput is decided by how many workers the
// pacer dedicates (GOGCRATIO * GOMAXPROCS). Three wrappers below sample
// three GC-pressure regimes so the benchmark captures both the regular
// case (no regression expected) and the GC-bound case (large effect).
//
// Sub-benchmarks vary the live-set size and pointer density. Run with:
//
// GOGCRATIO=25 GOMAXPROCS=N go test -run '^$' -bench BenchmarkGCBackground
func runGCBackground(b *testing.B, gogc int) {
debug.SetGCPercent(gogc)

for _, tc := range []struct {
name string
live int
ptrs int
data int
}{
{"gcbound", 32768, 32, 64},
{"gcbound-large", 65536, 32, 64},
{"gcbound-dense", 32768, 64, 32},
} {
b.Run(tc.name, func(b *testing.B) {
pool := make([]*gcRatioNode, tc.live)
for i := range pool {
pool[i] = newGCRatioNode(tc.ptrs, tc.data)
}
linkGCRatioNodes(pool)
idx := 0
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
pool[idx] = newGCRatioNode(tc.ptrs, tc.data)
idx = (idx + 1) % tc.live
}
})
}
}

// BenchmarkGCBackgroundDefault runs at GOGC=100, the regime of most real
// workloads: marking is far from the bottleneck, so GOGCRATIO should have
// a negligible effect. Guards against regressions in the common case.
func BenchmarkGCBackgroundDefault(b *testing.B) {
runGCBackground(b, 100)
}

// BenchmarkGCBackgroundModerate runs at GOGC=20, an intermediate GC
// pressure where marking starts to consume a visible share of the CPU.
func BenchmarkGCBackgroundModerate(b *testing.B) {
runGCBackground(b, 20)
}

// BenchmarkGCBackgroundDense runs at GOGC=5, where marking is the
// bottleneck and the number of background mark workers (governed by
// GOGCRATIO) directly decides how much of the available CPU goes to
// marking. Throughput is expected to vary strongly with the setting.
func BenchmarkGCBackgroundDense(b *testing.B) {
runGCBackground(b, 5)
}
8 changes: 4 additions & 4 deletions src/runtime/mgc.go
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ func gcinit() {
// Initialize GC pacer state.
// Use the environment variable GOGC for the initial gcPercent value.
// Use the environment variable GOMEMLIMIT for the initial memoryLimit value.
gcController.init(readGOGC(), readGOMEMLIMIT())
gcController.init(readGOGC(), readGOMEMLIMIT(), readGOGCRATIO())

// Set up the cleanup block ptr mask.
for i := range cleanupBlockPtrMask {
Expand Down Expand Up @@ -280,12 +280,12 @@ const (

// gcMarkWorkerFractionalMode indicates that a P is currently
// running the "fractional" mark worker. The fractional worker
// is necessary when GOMAXPROCS*gcBackgroundUtilization is not
// is necessary when GOMAXPROCS*gcController.gcRatio is not
// an integer and using only dedicated workers would result in
// utilization too far from the target of gcBackgroundUtilization.
// utilization too far from the target of gcController.gcRatio.
// The fractional worker should run until it is preempted and
// will be scheduled to pick up the fractional part of
// GOMAXPROCS*gcBackgroundUtilization.
// GOMAXPROCS*gcController.gcRatio.
gcMarkWorkerFractionalMode

// gcMarkWorkerIdleMode indicates that a P is running the mark
Expand Down
2 changes: 1 addition & 1 deletion src/runtime/mgclimit.go
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ func (l *gcCPULimiterState) updateLocked(now int64) {
// Compute total GC time.
windowGCTime := assistTime
if l.gcEnabled {
windowGCTime += int64(float64(windowTotalTime) * gcBackgroundUtilization)
windowGCTime += int64(float64(windowTotalTime) * gcController.gcRatio)
}

// Subtract out all idle time from the total time. Do this after computing
Expand Down
12 changes: 6 additions & 6 deletions src/runtime/mgclimit_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,13 +76,13 @@ func TestGCCPULimiter(t *testing.T) {
// Test passing time without assists during a GC. Specifically, just enough to drain the bucket to
// exactly procs nanoseconds (easier to get to because of rounding).
//
// The window we need to drain the bucket is 1/(1-2*gcBackgroundUtilization) times the current fill:
// The window we need to drain the bucket is 1/(1-2*gcController.gcRatio) times the current fill:
//
// fill + (window * procs * gcBackgroundUtilization - window * procs * (1-gcBackgroundUtilization)) = n
// fill = n - (window * procs * gcBackgroundUtilization - window * procs * (1-gcBackgroundUtilization))
// fill = n + window * procs * ((1-gcBackgroundUtilization) - gcBackgroundUtilization)
// fill = n + window * procs * (1-2*gcBackgroundUtilization)
// window = (fill - n) / (procs * (1-2*gcBackgroundUtilization)))
// fill + (window * procs * gcController.gcRatio - window * procs * (1-gcController.gcRatio)) = n
// fill = n - (window * procs * gcController.gcRatio - window * procs * (1-gcController.gcRatio))
// fill = n + window * procs * ((1-gcController.gcRatio) - gcController.gcRatio)
// fill = n + window * procs * (1-2*gcController.gcRatio)
// window = (fill - n) / (procs * (1-2*gcController.gcRatio)))
//
// And here we want n=procs:
factor := (1 / (1 - 2*GCBackgroundUtilization))
Expand Down
79 changes: 51 additions & 28 deletions src/runtime/mgcpacer.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,30 +14,6 @@ import (
)

const (
// gcGoalUtilization is the goal CPU utilization for
// marking as a fraction of GOMAXPROCS.
//
// Increasing the goal utilization will shorten GC cycles as the GC
// has more resources behind it, lessening costs from the write barrier,
// but comes at the cost of increasing mutator latency.
gcGoalUtilization = gcBackgroundUtilization

// gcBackgroundUtilization is the fixed CPU utilization for background
// marking. It must be <= gcGoalUtilization. The difference between
// gcGoalUtilization and gcBackgroundUtilization will be made up by
// mark assists. The scheduler will aim to use within 50% of this
// goal.
//
// As a general rule, there's little reason to set gcBackgroundUtilization
// < gcGoalUtilization. One reason might be in mostly idle applications,
// where goroutines are unlikely to assist at all, so the actual
// utilization will be lower than the goal. But this is moot point
// because the idle mark workers already soak up idle CPU resources.
// These two values are still kept separate however because they are
// distinct conceptually, and in previous iterations of the pacer the
// distinction was more important.
gcBackgroundUtilization = 0.25

// gcCreditSlack is the amount of scan work credit that can
// accumulate locally before updating gcController.heapScanWork and,
// optionally, gcController.bgScanCredit. Lower values give a more
Expand Down Expand Up @@ -75,6 +51,14 @@ const (
memoryLimitHeapGoalHeadroomPercent = 3
)

// gcGoalUtilization is the goal CPU utilization for
// marking as a fraction of GOMAXPROCS.
//
// Increasing the goal utilization will shorten GC cycles as the GC
// has more resources behind it, lessening costs from the write barrier,
// but comes at the cost of increasing mutator latency.
var gcGoalUtilization = gcController.gcRatio

// gcController implements the GC pacing controller that determines
// when to trigger concurrent garbage collection and how much marking
// work to do in mutator assists and background marking.
Expand All @@ -90,6 +74,11 @@ const (
var gcController gcControllerState

type gcControllerState struct {
// gcController.gcRatio be optional, value equals gcratio/100.0.
// Initialized from GOGCRATIO, which in the range of (1, 99).
// Default GOGCRATIO is 25.
gcRatio float64

// Initialized from GOGC. GOGC=off means no GC.
gcPercent atomic.Int32

Expand Down Expand Up @@ -368,11 +357,12 @@ type gcControllerState struct {
_ cpu.CacheLinePad
}

func (c *gcControllerState) init(gcPercent int32, memoryLimit int64) {
func (c *gcControllerState) init(gcPercent int32, memoryLimit int64, gcRatio float64) {
c.heapMinimum = defaultHeapMinimum
c.triggered = ^uint64(0)
c.setGCPercent(gcPercent)
c.setMemoryLimit(memoryLimit)
c.setGOGCRatio(gcRatio)
c.commit(true) // No sweep phase in the first GC cycle.
// N.B. Don't bother calling traceHeapGoal. Tracing is never enabled at
// initialization time.
Expand Down Expand Up @@ -400,13 +390,13 @@ func (c *gcControllerState) startCycle(markStartTime int64, procs int, trigger g
// dedicated workers so that the utilization is closest to
// 25%. For small GOMAXPROCS, this would introduce too much
// error, so we add fractional workers in that case.
totalUtilizationGoal := float64(procs) * gcBackgroundUtilization
totalUtilizationGoal := float64(procs) * gcController.gcRatio
dedicatedMarkWorkersNeeded := int64(totalUtilizationGoal + 0.5)
utilError := float64(dedicatedMarkWorkersNeeded)/totalUtilizationGoal - 1
const maxUtilError = 0.3
if utilError < -maxUtilError || utilError > maxUtilError {
// Rounding put us more than 30% off our goal. With
// gcBackgroundUtilization of 25%, this happens for
// gcController.gcRatio of 25%, this happens for
// GOMAXPROCS<=3 or GOMAXPROCS=6. Enable fractional
// workers to compensate.
if float64(dedicatedMarkWorkersNeeded) > totalUtilizationGoal {
Expand Down Expand Up @@ -606,7 +596,7 @@ func (c *gcControllerState) endCycle(now int64, procs int, userForced bool) {
assistDuration := now - c.markStartTime

// Assume background mark hit its utilization goal.
utilization := gcBackgroundUtilization
utilization := gcController.gcRatio
// Add assist utilization; avoid divide by zero.
if assistDuration > 0 {
utilization += float64(c.assistTime.Load()) / float64(assistDuration*int64(procs))
Expand Down Expand Up @@ -1426,6 +1416,39 @@ func readGOMEMLIMIT() int64 {
return n
}

func (c *gcControllerState) setGOGCRatio(in float64) float64 {
if !c.test {
assertWorldStoppedOrLockHeld(&mheap_.lock)
}

out := c.gcRatio
c.gcRatio = in

return out
}

func readGOGCRATIO() float64 {
p := gogetenv("GOGCRATIO")
if p == "" {
return 0.25
}
n, ok := parseByteCount(p)
if !ok {
print("GOGCRATIO=", p, "\n")
throw("malformed GOGCRATIO; get the wrong value")
}

if n < 1 {
n = 1
} else if n > 99 {
n = 99
}

out := float64(n) / 100.0

return out
}

// addIdleMarkWorker attempts to add a new idle mark worker.
//
// If this returns true, the caller must become an idle mark worker unless
Expand Down
Loading
Loading