From 280e058b0b4a2a21254ec8526652eef9d21c40e3 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Mon, 22 Jun 2026 19:14:03 -0400 Subject: [PATCH 1/5] Wire per-component cardinality limits from Config per Cardinality Limits RFC Co-Authored-By: Claude Sonnet 4.6 --- .../trace/api/config/GeneralConfig.java | 1 + .../trace/common/metrics/AggregateEntry.java | 49 ++++++++---- .../trace/common/metrics/AggregateTable.java | 8 +- .../trace/common/metrics/PeerTagSchema.java | 6 +- .../main/java/datadog/trace/api/Config.java | 11 +++ metadata/supported-configurations.json | 80 +++++++++++++++++++ 6 files changed, 135 insertions(+), 20 deletions(-) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/GeneralConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/GeneralConfig.java index c9963516f1a..79d3861d8ab 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/GeneralConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/GeneralConfig.java @@ -79,6 +79,7 @@ public final class GeneralConfig { public static final String TRACER_METRICS_MAX_PENDING = "trace.tracer.metrics.max.pending"; public static final String TRACER_METRICS_IGNORED_RESOURCES = "trace.tracer.metrics.ignored.resources"; + public static final String AZURE_APP_SERVICES = "azure.app.services"; public static final String INTERNAL_EXIT_ON_FAILURE = "trace.internal.exit.on.failure"; diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java index 29ce3d0bea4..7f93fad66a9 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java @@ -1,6 +1,7 @@ package datadog.trace.common.metrics; import datadog.metrics.api.Histogram; +import datadog.trace.api.Config; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.util.Hashtable; import datadog.trace.util.LongHashingUtils; @@ -28,36 +29,54 @@ final class AggregateEntry extends Hashtable.Entry { private static final UTF8BytesString[] EMPTY_TAGS = new UTF8BytesString[0]; - // Sentinel substitution is disabled until per-component config is wired in a follow-up PR. - // Tests that need sentinel mode should pass useBlockedSentinel=true explicitly. - static final boolean LIMITS_ENABLED = false; - - // Per-field cardinality handlers. Limits live on MetricCardinalityLimits -- see that class for - // per-field rationale. + // Per-field cardinality handlers. Limits are tunable via DD_TRACE_STATS_{field}_CARDINALITY_LIMIT + // (e.g. DD_TRACE_STATS_RESOURCE_CARDINALITY_LIMIT). Defaults live on MetricCardinalityLimits. + // Frozen at first class-load from Config. static final PropertyCardinalityHandler RESOURCE_HANDLER = - new PropertyCardinalityHandler("resource", MetricCardinalityLimits.RESOURCE, LIMITS_ENABLED); + new PropertyCardinalityHandler( + "resource", + Config.get().getTraceStatsCardinalityLimit("resource", MetricCardinalityLimits.RESOURCE)); static final PropertyCardinalityHandler SERVICE_HANDLER = - new PropertyCardinalityHandler("service", MetricCardinalityLimits.SERVICE, LIMITS_ENABLED); + new PropertyCardinalityHandler( + "service", + Config.get().getTraceStatsCardinalityLimit("service", MetricCardinalityLimits.SERVICE)); static final PropertyCardinalityHandler OPERATION_HANDLER = new PropertyCardinalityHandler( - "operation", MetricCardinalityLimits.OPERATION, LIMITS_ENABLED); + "operation", + Config.get() + .getTraceStatsCardinalityLimit("operation", MetricCardinalityLimits.OPERATION)); static final PropertyCardinalityHandler SERVICE_SOURCE_HANDLER = new PropertyCardinalityHandler( - "service_source", MetricCardinalityLimits.SERVICE_SOURCE, LIMITS_ENABLED); + "service_source", + Config.get() + .getTraceStatsCardinalityLimit( + "service_source", MetricCardinalityLimits.SERVICE_SOURCE)); static final PropertyCardinalityHandler TYPE_HANDLER = - new PropertyCardinalityHandler("type", MetricCardinalityLimits.TYPE, LIMITS_ENABLED); + new PropertyCardinalityHandler( + "type", + Config.get().getTraceStatsCardinalityLimit("type", MetricCardinalityLimits.TYPE)); static final PropertyCardinalityHandler SPAN_KIND_HANDLER = new PropertyCardinalityHandler( - "span_kind", MetricCardinalityLimits.SPAN_KIND, LIMITS_ENABLED); + "span_kind", + Config.get() + .getTraceStatsCardinalityLimit("span_kind", MetricCardinalityLimits.SPAN_KIND)); static final PropertyCardinalityHandler HTTP_METHOD_HANDLER = new PropertyCardinalityHandler( - "http_method", MetricCardinalityLimits.HTTP_METHOD, LIMITS_ENABLED); + "http_method", + Config.get() + .getTraceStatsCardinalityLimit("http_method", MetricCardinalityLimits.HTTP_METHOD)); static final PropertyCardinalityHandler HTTP_ENDPOINT_HANDLER = new PropertyCardinalityHandler( - "http_endpoint", MetricCardinalityLimits.HTTP_ENDPOINT, LIMITS_ENABLED); + "http_endpoint", + Config.get() + .getTraceStatsCardinalityLimit( + "http_endpoint", MetricCardinalityLimits.HTTP_ENDPOINT)); static final PropertyCardinalityHandler GRPC_STATUS_CODE_HANDLER = new PropertyCardinalityHandler( - "grpc_status_code", MetricCardinalityLimits.GRPC_STATUS_CODE, LIMITS_ENABLED); + "grpc_status_code", + Config.get() + .getTraceStatsCardinalityLimit( + "grpc_status_code", MetricCardinalityLimits.GRPC_STATUS_CODE)); // Single authoritative list used by resetCardinalityHandlers(). populateFrom() and hashOf() keep // named access for readability and to avoid per-span iteration overhead; this array ensures the diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java index fcd6c5fcdb4..6010fb18a5c 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java @@ -83,10 +83,10 @@ AggregateEntry findOrInsert(SpanSnapshot snapshot) { * {@code onStatsAggregateDropped}) rather than evicting an established one. Cap is sized to the * steady-state working set, so eviction is rare in the common case. * - *

How often this fires depends on {@link AggregateEntry#LIMITS_ENABLED}. With limits enabled, - * over-cap values for a given field collapse into a shared {@code blocked_by_tracer} bucket, so - * the table itself rarely reaches {@code maxAggregates}. With limits disabled (the default), - * over-cap values flow to distinct buckets and {@code maxAggregates} becomes the load-bearing + *

With per-field cardinality limits enabled, over-cap values for a given field collapse into a + * shared {@code tracer_blocked_value} bucket, so the table itself rarely reaches {@code + * maxAggregates}. Without per-field limits, over-cap values flow to distinct buckets and {@code + * maxAggregates} becomes the load-bearing * backstop -- the cursor-resumed scan was added specifically for this regime. */ private boolean evictOneStale() { diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/PeerTagSchema.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/PeerTagSchema.java index eca5c6c6a1a..fe8b255c7ca 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/PeerTagSchema.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/PeerTagSchema.java @@ -3,6 +3,7 @@ import static datadog.trace.api.DDTags.BASE_SERVICE; import datadog.communication.ddagent.DDAgentFeaturesDiscovery; +import datadog.trace.api.Config; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import java.util.Set; @@ -74,7 +75,10 @@ static PeerTagSchema of(Set names, String state) { for (int i = 0; i < names.length; i++) { this.handlers[i] = new TagCardinalityHandler( - names[i], MetricCardinalityLimits.PEER_TAG_VALUE, AggregateEntry.LIMITS_ENABLED); + names[i], + Config.get() + .getTraceStatsCardinalityLimit( + "peer_tag", MetricCardinalityLimits.PEER_TAG_VALUE)); } } diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index b10bda7c683..74e77dd8e65 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -3863,6 +3863,17 @@ public int getTraceStatsInterval() { return traceStatsInterval; } + /** + * Returns the per-cycle cardinality limit for the named stats field, following the RFC naming + * pattern {@code DD_TRACE_STATS_{tagName}_CARDINALITY_LIMIT} (e.g. {@code + * DD_TRACE_STATS_RESOURCE_CARDINALITY_LIMIT}). The caller supplies the default from {@code + * MetricCardinalityLimits} so per-field rationale stays co-located with the defaults. + */ + public int getTraceStatsCardinalityLimit(String tagName, int defaultLimit) { + return configProvider.getInteger( + "trace.stats." + tagName + ".cardinality.limit", defaultLimit); + } + public boolean isLogsInjectionEnabled() { return logsInjectionEnabled; } diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index ae081138b71..b09c46916f8 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -10641,6 +10641,86 @@ "aliases": [] } ], + "DD_TRACE_STATS_RESOURCE_CARDINALITY_LIMIT": [ + { + "version": "A", + "type": "integer", + "default": "256", + "aliases": [] + } + ], + "DD_TRACE_STATS_SERVICE_CARDINALITY_LIMIT": [ + { + "version": "A", + "type": "integer", + "default": "32", + "aliases": [] + } + ], + "DD_TRACE_STATS_OPERATION_CARDINALITY_LIMIT": [ + { + "version": "A", + "type": "integer", + "default": "64", + "aliases": [] + } + ], + "DD_TRACE_STATS_SERVICE_SOURCE_CARDINALITY_LIMIT": [ + { + "version": "A", + "type": "integer", + "default": "16", + "aliases": [] + } + ], + "DD_TRACE_STATS_TYPE_CARDINALITY_LIMIT": [ + { + "version": "A", + "type": "integer", + "default": "16", + "aliases": [] + } + ], + "DD_TRACE_STATS_SPAN_KIND_CARDINALITY_LIMIT": [ + { + "version": "A", + "type": "integer", + "default": "8", + "aliases": [] + } + ], + "DD_TRACE_STATS_HTTP_METHOD_CARDINALITY_LIMIT": [ + { + "version": "A", + "type": "integer", + "default": "16", + "aliases": [] + } + ], + "DD_TRACE_STATS_HTTP_ENDPOINT_CARDINALITY_LIMIT": [ + { + "version": "A", + "type": "integer", + "default": "64", + "aliases": [] + } + ], + "DD_TRACE_STATS_GRPC_STATUS_CODE_CARDINALITY_LIMIT": [ + { + "version": "A", + "type": "integer", + "default": "24", + "aliases": [] + } + ], + "DD_TRACE_STATS_PEER_TAG_CARDINALITY_LIMIT": [ + { + "version": "A", + "type": "integer", + "default": "512", + "aliases": [] + } + ], "DD_TRACE_STATUS404DECORATOR_ENABLED": [ { "version": "A", From 78a7f0daf88c7b2f81df8fd5d0962cc566a04805 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Wed, 24 Jun 2026 13:16:32 -0400 Subject: [PATCH 2/5] Raise RESOURCE cardinality limit to 1024 to avoid premature collapse Co-Authored-By: Claude Sonnet 4.6 --- .../trace/common/metrics/MetricCardinalityLimits.java | 5 +++-- metadata/supported-configurations.json | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricCardinalityLimits.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricCardinalityLimits.java index 58dcd3a70a1..1917f6add57 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricCardinalityLimits.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricCardinalityLimits.java @@ -14,9 +14,10 @@ private MetricCardinalityLimits() {} /** * Distinct {@code resource.name} values per cycle. Highest-cardinality field by far: DB-query - * obfuscations, HTTP route templates, custom resources. Typical service: 30-200 unique. + * obfuscations, HTTP route templates, custom resources. Typical service: 30-200 unique; 1024 + * leaves headroom for high-cardinality SQL/HTTP workloads without risking premature collapse. */ - static final int RESOURCE = 256; + static final int RESOURCE = 1024; /** * Distinct {@code service.name} values per cycle. Local service plus downstream peer-service diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index b09c46916f8..afe9bf20a78 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -10645,7 +10645,7 @@ { "version": "A", "type": "integer", - "default": "256", + "default": "1024", "aliases": [] } ], From 5b550ddfbbeb4e156774968c02327c885fdaca75 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 30 Jun 2026 20:54:22 -0400 Subject: [PATCH 3/5] Apply spotless formatting Co-Authored-By: Claude Sonnet 4.6 --- .../java/datadog/trace/common/metrics/AggregateEntry.java | 3 +-- .../java/datadog/trace/common/metrics/AggregateTable.java | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java index 7f93fad66a9..ff1cea55768 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java @@ -53,8 +53,7 @@ final class AggregateEntry extends Hashtable.Entry { "service_source", MetricCardinalityLimits.SERVICE_SOURCE)); static final PropertyCardinalityHandler TYPE_HANDLER = new PropertyCardinalityHandler( - "type", - Config.get().getTraceStatsCardinalityLimit("type", MetricCardinalityLimits.TYPE)); + "type", Config.get().getTraceStatsCardinalityLimit("type", MetricCardinalityLimits.TYPE)); static final PropertyCardinalityHandler SPAN_KIND_HANDLER = new PropertyCardinalityHandler( "span_kind", diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java index 6010fb18a5c..6f2072f6495 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java @@ -86,8 +86,8 @@ AggregateEntry findOrInsert(SpanSnapshot snapshot) { *

With per-field cardinality limits enabled, over-cap values for a given field collapse into a * shared {@code tracer_blocked_value} bucket, so the table itself rarely reaches {@code * maxAggregates}. Without per-field limits, over-cap values flow to distinct buckets and {@code - * maxAggregates} becomes the load-bearing - * backstop -- the cursor-resumed scan was added specifically for this regime. + * maxAggregates} becomes the load-bearing backstop -- the cursor-resumed scan was added + * specifically for this regime. */ private boolean evictOneStale() { // Two passes -- [cursor, length) then [0, cursor) -- using the half-open-range iterator. The From c50dd1dbf36c7a6cf81a77420a5b834d19167642 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Wed, 1 Jul 2026 13:08:25 -0400 Subject: [PATCH 4/5] Apply spotless formatting to Config.java Co-Authored-By: Claude Sonnet 4.6 --- internal-api/src/main/java/datadog/trace/api/Config.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 74e77dd8e65..f0f73e70dba 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -3870,8 +3870,7 @@ public int getTraceStatsInterval() { * MetricCardinalityLimits} so per-field rationale stays co-located with the defaults. */ public int getTraceStatsCardinalityLimit(String tagName, int defaultLimit) { - return configProvider.getInteger( - "trace.stats." + tagName + ".cardinality.limit", defaultLimit); + return configProvider.getInteger("trace.stats." + tagName + ".cardinality.limit", defaultLimit); } public boolean isLogsInjectionEnabled() { From e558d99cb64d3f4d035207944ecd88ba87cb8d46 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Wed, 1 Jul 2026 13:15:10 -0400 Subject: [PATCH 5/5] Fix supported-configurations.json: use int not integer for cardinality limits Co-Authored-By: Claude Sonnet 4.6 --- metadata/supported-configurations.json | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index afe9bf20a78..c373c484727 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -10644,7 +10644,7 @@ "DD_TRACE_STATS_RESOURCE_CARDINALITY_LIMIT": [ { "version": "A", - "type": "integer", + "type": "int", "default": "1024", "aliases": [] } @@ -10652,7 +10652,7 @@ "DD_TRACE_STATS_SERVICE_CARDINALITY_LIMIT": [ { "version": "A", - "type": "integer", + "type": "int", "default": "32", "aliases": [] } @@ -10660,7 +10660,7 @@ "DD_TRACE_STATS_OPERATION_CARDINALITY_LIMIT": [ { "version": "A", - "type": "integer", + "type": "int", "default": "64", "aliases": [] } @@ -10668,7 +10668,7 @@ "DD_TRACE_STATS_SERVICE_SOURCE_CARDINALITY_LIMIT": [ { "version": "A", - "type": "integer", + "type": "int", "default": "16", "aliases": [] } @@ -10676,7 +10676,7 @@ "DD_TRACE_STATS_TYPE_CARDINALITY_LIMIT": [ { "version": "A", - "type": "integer", + "type": "int", "default": "16", "aliases": [] } @@ -10684,7 +10684,7 @@ "DD_TRACE_STATS_SPAN_KIND_CARDINALITY_LIMIT": [ { "version": "A", - "type": "integer", + "type": "int", "default": "8", "aliases": [] } @@ -10692,7 +10692,7 @@ "DD_TRACE_STATS_HTTP_METHOD_CARDINALITY_LIMIT": [ { "version": "A", - "type": "integer", + "type": "int", "default": "16", "aliases": [] } @@ -10700,7 +10700,7 @@ "DD_TRACE_STATS_HTTP_ENDPOINT_CARDINALITY_LIMIT": [ { "version": "A", - "type": "integer", + "type": "int", "default": "64", "aliases": [] } @@ -10708,7 +10708,7 @@ "DD_TRACE_STATS_GRPC_STATUS_CODE_CARDINALITY_LIMIT": [ { "version": "A", - "type": "integer", + "type": "int", "default": "24", "aliases": [] } @@ -10716,7 +10716,7 @@ "DD_TRACE_STATS_PEER_TAG_CARDINALITY_LIMIT": [ { "version": "A", - "type": "integer", + "type": "int", "default": "512", "aliases": [] }