Skip to content

fix: remove project label from projectstorage metrics to reduce cardinality #558

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zachsmith1 merged 1 commit into from
Apr 2, 2026
Merged

fix: remove project label from projectstorage metrics to reduce cardinality #558

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 17 additions & 19 deletions internal/apiserver/storage/project/mux.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,10 @@ var (
childCreations = k8smetrics.NewCounterVec(
&k8smetrics.CounterOpts{
Name: "projectstorage_child_creations_total",
Help: "Per-project child storage creations",
Help: "Child storage creations by resource type",
StabilityLevel: k8smetrics.ALPHA,
},
[]string{"project", "resource_group", "resource_kind"},
[]string{"resource_group", "resource_kind"},
)

firstReady = k8smetrics.NewHistogramVec(
Expand All @@ -48,7 +48,7 @@ var (
Buckets: []float64{0.02, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10},
StabilityLevel: k8smetrics.ALPHA,
},
[]string{"project", "resource_group", "resource_kind"},
[]string{"resource_group", "resource_kind"},
)

reinitErrors = k8smetrics.NewCounterVec(
Expand All @@ -57,7 +57,7 @@ var (
Help: "Ops that hit 'storage is (re)initializing'",
StabilityLevel: k8smetrics.ALPHA,
},
[]string{"project", "resource_group", "resource_kind", "verb"},
[]string{"resource_group", "resource_kind", "verb"},
)
)

Expand All @@ -69,13 +69,13 @@ func isReinitErr(err error) bool {
return err != nil && strings.Contains(err.Error(), "storage is (re)initializing")
}

func incrReinit(project, group, kind, verb string) {
reinitErrors.WithLabelValues(project, group, kind, verb).Inc()
func incrReinit(group, kind, verb string) {
reinitErrors.WithLabelValues(group, kind, verb).Inc()
}

func recordFirstReady(c *child, project, group, kind string) {
func recordFirstReady(c *child, group, kind string) {
c.readyOnce.Do(func() {
firstReady.WithLabelValues(project, group, kind).
firstReady.WithLabelValues(group, kind).
Observe(time.Since(c.created).Seconds())
})
}
Expand Down Expand Up @@ -107,21 +107,20 @@ type decoratorArgs struct {

// instrumentedStorage wraps a storage.Interface to emit metrics once per child
type instrumentedStorage struct {
inner storage.Interface
child *child
project string
inner storage.Interface
child *child

// normalized labels
group string // API group ("" => "core" when you query; we keep "" here)
kind string // resource plural
}

func (i *instrumentedStorage) markSuccess() {
recordFirstReady(i.child, i.project, i.group, i.kind)
recordFirstReady(i.child, i.group, i.kind)
}
func (i *instrumentedStorage) markReinit(verb string, err error) error {
if isReinitErr(err) {
incrReinit(i.project, i.group, i.kind, verb)
incrReinit(i.group, i.kind, verb)
}
return err
}
Expand Down Expand Up @@ -239,16 +238,15 @@ func (m *projectMux) childForProject(project string) (storage.Interface, error)
// Wrap the child once with instrumentation.
c := &child{s: s, destroy: destroy, created: time.Now()}
wrapped := &instrumentedStorage{
inner: s,
child: c,
project: project,
group: m.args.resourceGroup,
kind: m.args.resourceKind,
inner: s,
child: c,
group: m.args.resourceGroup,
kind: m.args.resourceKind,
}
c.s = wrapped

m.children[project] = c
childCreations.WithLabelValues(project, m.args.resourceGroup, m.args.resourceKind).Inc()
childCreations.WithLabelValues(m.args.resourceGroup, m.args.resourceKind).Inc()

// Bootstrap system namespace synchronously to prevent resource creation failures
if project != "" && m.loopbackConfig != nil {
Expand Down
Loading