From 571938b6c764b76a1316e0bf98b008d13bd820b6 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:47:46 -0400 Subject: [PATCH 01/26] refactor(evp): centralize EVP proxy routing --- .../communication/BackendApiFactory.java | 54 +++-- .../java/datadog/communication/EvpProxy.java | 15 ++ .../datadog/communication/EvpProxyApi.java | 8 +- .../ddagent/DDAgentFeaturesDiscovery.java | 8 +- .../DDAgentFeaturesDiscoveryTest.groovy | 6 + .../communication/BackendApiFactoryTest.java | 191 ++++++++++++++++++ .../common/writer/ddintake/DDEvpProxyApi.java | 4 +- .../writer/ddintake/LLMObsSpanMapper.java | 3 +- .../writer/ddintake/DDEvpProxyApiTest.groovy | 5 +- 9 files changed, 270 insertions(+), 24 deletions(-) create mode 100644 communication/src/main/java/datadog/communication/EvpProxy.java create mode 100644 communication/src/test/java/datadog/communication/BackendApiFactoryTest.java diff --git a/communication/src/main/java/datadog/communication/BackendApiFactory.java b/communication/src/main/java/datadog/communication/BackendApiFactory.java index 3ce78b88c22..ba5d68554ba 100644 --- a/communication/src/main/java/datadog/communication/BackendApiFactory.java +++ b/communication/src/main/java/datadog/communication/BackendApiFactory.java @@ -24,6 +24,11 @@ public BackendApiFactory(Config config, SharedCommunicationObjects sharedCommuni } public @Nullable BackendApi createBackendApi(Intake intake) { + return createBackendApi(intake, null, true); + } + + public @Nullable BackendApi createBackendApi( + Intake intake, @Nullable String preferredEvpProxyEndpoint, boolean responseCompression) { HttpRetryPolicy.Factory retryPolicyFactory = new HttpRetryPolicy.Factory(5, 100, 2.0, true); if (intake.isAgentlessEnabled(config)) { @@ -46,23 +51,40 @@ public BackendApiFactory(Config config, SharedCommunicationObjects sharedCommuni DDAgentFeaturesDiscovery featuresDiscovery = sharedCommunicationObjects.featuresDiscovery(config); featuresDiscovery.discoverIfOutdated(); - if (featuresDiscovery.supportsEvpProxy()) { - String traceId = config.getIdGenerationStrategy().generateTraceId().toString(); - String evpProxyEndpoint = featuresDiscovery.getEvpProxyEndpoint(); - HttpUrl evpProxyUrl = sharedCommunicationObjects.agentUrl.resolve(evpProxyEndpoint); - String subdomain = intake.getUrlPrefix(); - return new EvpProxyApi( - traceId, - evpProxyUrl, - subdomain, - retryPolicyFactory, - sharedCommunicationObjects.agentHttpClient, - true); + String evpProxyEndpoint; + if (preferredEvpProxyEndpoint != null) { + if (!featuresDiscovery.supportsEvpProxyEndpoint(preferredEvpProxyEndpoint)) { + log.warn( + "Cannot create backend API client for {} since agent does not support requested EVP" + + " proxy endpoint {}", + intake, + preferredEvpProxyEndpoint); + return null; + } + evpProxyEndpoint = preferredEvpProxyEndpoint; + } else if (featuresDiscovery.supportsEvpProxy()) { + evpProxyEndpoint = featuresDiscovery.getEvpProxyEndpoint(); + } else { + log.warn( + "Cannot create backend API client since agentless mode is disabled, " + + "and agent does not support EVP proxy"); + return null; } - log.warn( - "Cannot create backend API client since agentless mode is disabled, " - + "and agent does not support EVP proxy"); - return null; + String traceId = config.getIdGenerationStrategy().generateTraceId().toString(); + log.debug( + "Creating EVP proxy client for {} using endpoint {} with responseCompression={}", + intake, + evpProxyEndpoint, + responseCompression); + HttpUrl evpProxyUrl = sharedCommunicationObjects.agentUrl.resolve(evpProxyEndpoint); + String subdomain = intake.getUrlPrefix(); + return new EvpProxyApi( + traceId, + evpProxyUrl, + subdomain, + retryPolicyFactory, + sharedCommunicationObjects.agentHttpClient, + responseCompression); } } diff --git a/communication/src/main/java/datadog/communication/EvpProxy.java b/communication/src/main/java/datadog/communication/EvpProxy.java new file mode 100644 index 00000000000..c2453bccb25 --- /dev/null +++ b/communication/src/main/java/datadog/communication/EvpProxy.java @@ -0,0 +1,15 @@ +package datadog.communication; + +/** Shared EVP proxy constants. */ +public final class EvpProxy { + + public static final String SUBDOMAIN_HEADER = "X-Datadog-EVP-Subdomain"; + + /** + * Default SDK-side target for uncompressed EVP request bodies. Writers may split batches at or + * below this size to keep Agent proxy requests comfortably bounded. + */ + public static final int PAYLOAD_SIZE_LIMIT_BYTES = 5 * 1024 * 1024; + + private EvpProxy() {} +} diff --git a/communication/src/main/java/datadog/communication/EvpProxyApi.java b/communication/src/main/java/datadog/communication/EvpProxyApi.java index 83037ab9663..b00838c68f3 100644 --- a/communication/src/main/java/datadog/communication/EvpProxyApi.java +++ b/communication/src/main/java/datadog/communication/EvpProxyApi.java @@ -20,7 +20,6 @@ public class EvpProxyApi implements BackendApi { private static final Logger log = LoggerFactory.getLogger(EvpProxyApi.class); private static final String API_VERSION = "v2"; - private static final String X_DATADOG_EVP_SUBDOMAIN_HEADER = "X-Datadog-EVP-Subdomain"; private static final String X_DATADOG_TRACE_ID_HEADER = "x-datadog-trace-id"; private static final String X_DATADOG_PARENT_ID_HEADER = "x-datadog-parent-id"; private static final String ACCEPT_ENCODING_HEADER = "Accept-Encoding"; @@ -62,7 +61,7 @@ public T post( Request.Builder requestBuilder = new Request.Builder() .url(url) - .addHeader(X_DATADOG_EVP_SUBDOMAIN_HEADER, subdomain) + .addHeader(EvpProxy.SUBDOMAIN_HEADER, subdomain) .addHeader(X_DATADOG_TRACE_ID_HEADER, traceId) .addHeader(X_DATADOG_PARENT_ID_HEADER, traceId); @@ -79,6 +78,11 @@ public T post( } final Request request = requestBuilder.post(requestBody).build(); + log.debug( + "Posting EVP request to {} with responseCompression={} requestCompression={}", + url, + responseCompression, + requestCompression); try (okhttp3.Response response = OkHttpUtils.sendWithRetries(httpClient, retryPolicyFactory, request)) { diff --git a/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java b/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java index 7cef3e02924..ba263a7f6e6 100644 --- a/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java +++ b/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java @@ -98,6 +98,7 @@ private static class State { String debuggerSnapshotEndpoint; String debuggerDiagnosticsEndpoint; String evpProxyEndpoint; + Set evpProxyEndpoints = emptySet(); String version; String telemetryProxyEndpoint; Set peerTags = emptySet(); @@ -140,7 +141,7 @@ protected long getFeaturesDiscoveryMinDelayMillis() { private synchronized void discoverIfOutdated(final long maxElapsedMs) { final long now = System.currentTimeMillis(); final long elapsed = now - discoveryState.lastTimeDiscovered; - if (elapsed > maxElapsedMs) { + if (elapsed >= maxElapsedMs) { final State newState = new State(); doDiscovery(newState); newState.lastTimeDiscovered = now; @@ -288,6 +289,7 @@ private boolean processInfoResponse(State newState, String response) { break; } } + newState.evpProxyEndpoints = unmodifiableSet(endpoints); for (String endpoint : telemetryProxyEndpoints) { if (containsEndpoint(endpoints, endpoint)) { @@ -422,6 +424,10 @@ public String getEvpProxyEndpoint() { return discoveryState.evpProxyEndpoint; } + public boolean supportsEvpProxyEndpoint(String endpoint) { + return containsEndpoint(discoveryState.evpProxyEndpoints, endpoint); + } + public HttpUrl buildUrl(String endpoint) { return agentBaseUrl.resolve(endpoint); } diff --git a/communication/src/test/groovy/datadog/communication/ddagent/DDAgentFeaturesDiscoveryTest.groovy b/communication/src/test/groovy/datadog/communication/ddagent/DDAgentFeaturesDiscoveryTest.groovy index 8bcda6eb811..aab3c240d92 100644 --- a/communication/src/test/groovy/datadog/communication/ddagent/DDAgentFeaturesDiscoveryTest.groovy +++ b/communication/src/test/groovy/datadog/communication/ddagent/DDAgentFeaturesDiscoveryTest.groovy @@ -6,6 +6,8 @@ import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V05_ENDPOIN import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V06_METRICS_ENDPOINT import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V07_CONFIG_ENDPOINT import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V1_ENDPOINT +import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V2_EVP_PROXY_ENDPOINT +import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V4_EVP_PROXY_ENDPOINT import static datadog.communication.http.OkHttpUtils.DATADOG_CONTAINER_ID import static datadog.communication.http.OkHttpUtils.DATADOG_CONTAINER_TAGS_HASH import static datadog.trace.api.ProtocolVersion.V0_4 @@ -75,6 +77,8 @@ class DDAgentFeaturesDiscoveryTest extends DDSpecification { features.supportsEvpProxy() features.supportsContentEncodingHeadersWithEvpProxy() features.getEvpProxyEndpoint() == "evp_proxy/v4/" + features.supportsEvpProxyEndpoint(V2_EVP_PROXY_ENDPOINT) + features.supportsEvpProxyEndpoint(V4_EVP_PROXY_ENDPOINT) features.getVersion() == "0.99.0" !features.supportsLongRunning() !features.supportsTelemetryProxy() @@ -489,6 +493,8 @@ class DDAgentFeaturesDiscoveryTest extends DDSpecification { 1 * client.newCall(_) >> { Request request -> infoResponse(request, INFO_WITH_OLD_EVP_PROXY) } features.supportsEvpProxy() features.getEvpProxyEndpoint() == "evp_proxy/v2/" // v3 is advertised, but the tracer should ignore it + features.supportsEvpProxyEndpoint(V2_EVP_PROXY_ENDPOINT) + !features.supportsEvpProxyEndpoint(V4_EVP_PROXY_ENDPOINT) !features.supportsContentEncodingHeadersWithEvpProxy() features.supportsDebugger() features.getDebuggerSnapshotEndpoint() == "debugger/v1/diagnostics" diff --git a/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java b/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java new file mode 100644 index 00000000000..c287790b4ae --- /dev/null +++ b/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java @@ -0,0 +1,191 @@ +package datadog.communication; + +import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V2_EVP_PROXY_ENDPOINT; +import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V4_EVP_PROXY_ENDPOINT; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import datadog.communication.ddagent.DDAgentFeaturesDiscovery; +import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.metrics.api.Monitoring; +import datadog.trace.api.Config; +import datadog.trace.api.ProtocolVersion; +import datadog.trace.api.intake.Intake; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import okhttp3.HttpUrl; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.RequestBody; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; +import org.junit.jupiter.api.Test; + +class BackendApiFactoryTest { + + private static final MediaType JSON = MediaType.parse("application/json"); + + @Test + void preferredEvpProxyEndpointMustBeAdvertisedByAgent() { + final FakeFeaturesDiscovery discovery = + new FakeFeaturesDiscovery( + V4_EVP_PROXY_ENDPOINT, Collections.singleton(V4_EVP_PROXY_ENDPOINT)); + final BackendApiFactory factory = + new BackendApiFactory(Config.get(), sharedCommunicationObjects(discovery, null)); + + assertNull(factory.createBackendApi(Intake.EVENT_PLATFORM, V2_EVP_PROXY_ENDPOINT, false)); + } + + @Test + void preferredEvpProxyEndpointUsesRequestedRouteWhenAdvertised() throws Exception { + final MockWebServer agent = new MockWebServer(); + agent.enqueue(new MockResponse().setResponseCode(200).setBody("{}")); + agent.start(); + try { + final FakeFeaturesDiscovery discovery = + new FakeFeaturesDiscovery( + V4_EVP_PROXY_ENDPOINT, + new HashSet<>(Arrays.asList(V4_EVP_PROXY_ENDPOINT, V2_EVP_PROXY_ENDPOINT))); + final BackendApiFactory factory = + new BackendApiFactory( + Config.get(), sharedCommunicationObjects(discovery, agent.url("/"))); + final BackendApi api = + factory.createBackendApi(Intake.EVENT_PLATFORM, V2_EVP_PROXY_ENDPOINT, false); + + assertNotNull(api); + api.post( + "flagevaluation", + RequestBody.create(JSON, "{}".getBytes(StandardCharsets.UTF_8)), + stream -> null, + null, + false); + + final RecordedRequest request = agent.takeRequest(); + assertEquals("/evp_proxy/v2/api/v2/flagevaluation", request.getPath()); + } finally { + agent.shutdown(); + } + } + + @Test + void preferredEvpProxyEndpointDoesNotRequireLegacyDefaultRoute() throws Exception { + final String futureEndpoint = "evp_proxy/v9/"; + final MockWebServer agent = new MockWebServer(); + agent.enqueue(new MockResponse().setResponseCode(200).setBody("{}")); + agent.start(); + try { + final FakeFeaturesDiscovery discovery = + new FakeFeaturesDiscovery(null, Collections.singleton(futureEndpoint)); + final BackendApiFactory factory = + new BackendApiFactory( + Config.get(), sharedCommunicationObjects(discovery, agent.url("/"))); + final BackendApi api = + factory.createBackendApi(Intake.EVENT_PLATFORM, futureEndpoint, false); + + assertNotNull(api); + api.post( + "flagevaluation", + RequestBody.create(JSON, "{}".getBytes(StandardCharsets.UTF_8)), + stream -> null, + null, + false); + + final RecordedRequest request = agent.takeRequest(); + assertEquals("/evp_proxy/v9/api/v2/flagevaluation", request.getPath()); + } finally { + agent.shutdown(); + } + } + + @Test + void defaultEvpProxyEndpointSupportsDisabledResponseCompression() throws Exception { + final MockWebServer agent = new MockWebServer(); + agent.enqueue(new MockResponse().setResponseCode(200).setBody("{}")); + agent.start(); + try { + final FakeFeaturesDiscovery discovery = + new FakeFeaturesDiscovery( + V4_EVP_PROXY_ENDPOINT, + new HashSet<>(Arrays.asList(V4_EVP_PROXY_ENDPOINT, V2_EVP_PROXY_ENDPOINT))); + final BackendApiFactory factory = + new BackendApiFactory( + Config.get(), sharedCommunicationObjects(discovery, agent.url("/"))); + final BackendApi api = factory.createBackendApi(Intake.EVENT_PLATFORM, null, false); + + assertNotNull(api); + api.post( + "flagevaluation", + RequestBody.create(JSON, "{}".getBytes(StandardCharsets.UTF_8)), + stream -> null, + null, + false); + + final RecordedRequest request = agent.takeRequest(); + assertEquals("/evp_proxy/v4/api/v2/flagevaluation", request.getPath()); + } finally { + agent.shutdown(); + } + } + + private static SharedCommunicationObjects sharedCommunicationObjects( + final DDAgentFeaturesDiscovery discovery, final HttpUrl agentUrl) { + final TestSharedCommunicationObjects sco = new TestSharedCommunicationObjects(discovery); + sco.agentUrl = agentUrl != null ? agentUrl : HttpUrl.get("http://localhost:8126/"); + sco.agentHttpClient = new OkHttpClient(); + return sco; + } + + private static final class TestSharedCommunicationObjects extends SharedCommunicationObjects { + private final DDAgentFeaturesDiscovery discovery; + + private TestSharedCommunicationObjects(final DDAgentFeaturesDiscovery discovery) { + this.discovery = discovery; + } + + @Override + public DDAgentFeaturesDiscovery featuresDiscovery(final Config config) { + return discovery; + } + } + + private static final class FakeFeaturesDiscovery extends DDAgentFeaturesDiscovery { + private final String evpProxyEndpoint; + private final Set evpProxyEndpoints; + + private FakeFeaturesDiscovery( + final String evpProxyEndpoint, final Set evpProxyEndpoints) { + super( + new OkHttpClient(), + Monitoring.DISABLED, + HttpUrl.get("http://localhost:8126/"), + ProtocolVersion.V0_5, + true, + false); + this.evpProxyEndpoint = evpProxyEndpoint; + this.evpProxyEndpoints = evpProxyEndpoints; + } + + @Override + public void discoverIfOutdated() {} + + @Override + public String getEvpProxyEndpoint() { + return evpProxyEndpoint; + } + + @Override + public boolean supportsEvpProxyEndpoint(final String endpoint) { + return evpProxyEndpoints.contains(endpoint) || evpProxyEndpoints.contains("/" + endpoint); + } + + @Override + public boolean supportsEvpProxy() { + return evpProxyEndpoint != null; + } + } +} diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddintake/DDEvpProxyApi.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddintake/DDEvpProxyApi.java index e911a980c31..9ec4eca5f4e 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddintake/DDEvpProxyApi.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddintake/DDEvpProxyApi.java @@ -3,6 +3,7 @@ import static datadog.trace.common.writer.DDIntakeWriter.DEFAULT_INTAKE_TIMEOUT; import static datadog.trace.common.writer.DDIntakeWriter.DEFAULT_INTAKE_VERSION; +import datadog.communication.EvpProxy; import datadog.communication.http.HttpRetryPolicy; import datadog.communication.http.OkHttpUtils; import datadog.trace.api.civisibility.InstrumentationBridge; @@ -26,7 +27,6 @@ public class DDEvpProxyApi extends RemoteApi { private static final Logger log = LoggerFactory.getLogger(DDEvpProxyApi.class); - private static final String DD_EVP_SUBDOMAIN_HEADER = "X-Datadog-EVP-Subdomain"; private static final String CONTENT_ENCODING_HEADER = "Content-Encoding"; private static final String GZIP_CONTENT_TYPE = "gzip"; @@ -131,7 +131,7 @@ public Response sendSerializedTraces(Payload payload) { Request.Builder builder = new Request.Builder() .url(proxiedApiUrl) - .addHeader(DD_EVP_SUBDOMAIN_HEADER, subdomain) + .addHeader(EvpProxy.SUBDOMAIN_HEADER, subdomain) .tag(OkHttpUtils.CustomListener.class, telemetryListener); if (isCompressionEnabled()) { diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index 7849052b9d3..1f14502cc27 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -2,6 +2,7 @@ import static datadog.communication.http.OkHttpUtils.gzippedMsgpackRequestBodyOf; +import datadog.communication.EvpProxy; import datadog.communication.serialization.GrowableBuffer; import datadog.communication.serialization.Writable; import datadog.communication.serialization.msgpack.MsgPackWriter; @@ -99,7 +100,7 @@ public class LLMObsSpanMapper implements RemoteMapper { private int spansWritten; public LLMObsSpanMapper() { - this(5 << 20); + this(EvpProxy.PAYLOAD_SIZE_LIMIT_BYTES); } private LLMObsSpanMapper(int size) { diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDEvpProxyApiTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDEvpProxyApiTest.groovy index fd797695d99..52ad90a9514 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDEvpProxyApiTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDEvpProxyApiTest.groovy @@ -2,6 +2,7 @@ package datadog.trace.common.writer.ddintake import com.fasterxml.jackson.core.type.TypeReference import com.fasterxml.jackson.databind.ObjectMapper +import datadog.communication.EvpProxy import datadog.communication.serialization.ByteBufferConsumer import datadog.communication.serialization.FlushingBuffer import datadog.communication.serialization.msgpack.MsgPackWriter @@ -64,7 +65,7 @@ class DDEvpProxyApiTest extends DDCoreSpecification { clientResponse.status().present clientResponse.status().asInt == 200 agentEvpProxy.getLastRequest().path == path - agentEvpProxy.getLastRequest().getHeader(DDEvpProxyApi.DD_EVP_SUBDOMAIN_HEADER) == intakeSubdomain + agentEvpProxy.getLastRequest().getHeader(EvpProxy.SUBDOMAIN_HEADER) == intakeSubdomain cleanup: agentEvpProxy.close() @@ -100,7 +101,7 @@ class DDEvpProxyApiTest extends DDCoreSpecification { clientResponse.status().present clientResponse.status().asInt == 200 agentEvpProxy.getLastRequest().path == path - agentEvpProxy.getLastRequest().getHeader(DDEvpProxyApi.DD_EVP_SUBDOMAIN_HEADER) == intakeSubdomain + agentEvpProxy.getLastRequest().getHeader(EvpProxy.SUBDOMAIN_HEADER) == intakeSubdomain cleanup: agentEvpProxy.close() From 4949ed7ee4b3c67b386a6b27c2fc214d905565cd Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:47:55 -0400 Subject: [PATCH 02/26] refactor(feature-flagging): share EVP publishing --- .../featureflag/ExposureWriterImpl.java | 33 ++------ .../featureflag/FeatureFlagEvpContext.java | 22 ++++++ .../featureflag/FeatureFlagEvpPublisher.java | 79 +++++++++++++++++++ .../FeatureFlagEvpPublisherTest.java | 72 +++++++++++++++++ 4 files changed, 179 insertions(+), 27 deletions(-) create mode 100644 products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpContext.java create mode 100644 products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpPublisher.java create mode 100644 products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagEvpPublisherTest.java diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java index eddcd520f27..1333fcead13 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java @@ -4,25 +4,19 @@ import static datadog.trace.util.AgentThreadFactory.newAgentThread; import static java.util.concurrent.TimeUnit.SECONDS; -import com.squareup.moshi.JsonAdapter; -import com.squareup.moshi.Moshi; import datadog.common.queue.MessagePassingBlockingQueue; import datadog.common.queue.Queues; -import datadog.communication.BackendApi; import datadog.communication.BackendApiFactory; import datadog.communication.ddagent.SharedCommunicationObjects; import datadog.trace.api.Config; import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.api.featureflag.exposure.ExposureEvent; import datadog.trace.api.featureflag.exposure.ExposuresRequest; -import datadog.trace.api.intake.Intake; import datadog.trace.api.internal.VisibleForTesting; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; -import okhttp3.RequestBody; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,6 +26,7 @@ public class ExposureWriterImpl implements ExposureWriter { private static final int DEFAULT_CAPACITY = 1 << 16; // 65536 elements private static final int DEFAULT_FLUSH_INTERVAL_IN_SECONDS = 1; private static final int FLUSH_THRESHOLD = 100; + private static final String EXPOSURES_ROUTE = "exposures"; private final MessagePassingBlockingQueue queue; private final Thread serializerThread; @@ -47,21 +42,13 @@ public ExposureWriterImpl(final SharedCommunicationObjects sco, final Config con final SharedCommunicationObjects sco, final Config config) { this.queue = Queues.mpscBlockingConsumerArrayQueue(capacity); - final Map context = new HashMap<>(4); - context.put("service", config.getServiceName() == null ? "unknown" : config.getServiceName()); - if (config.getEnv() != null) { - context.put("env", config.getEnv()); - } - if (config.getVersion() != null) { - context.put("version", config.getVersion()); - } final ExposureSerializingHandler serializer = new ExposureSerializingHandler( new BackendApiFactory(config, sco), queue, flushInterval, timeUnit, - context, + FeatureFlagEvpContext.from(config), this::close); this.serializerThread = newAgentThread(FEATURE_FLAG_EXPOSURE_PROCESSOR, serializer); } @@ -100,10 +87,7 @@ private static class ExposureSerializingHandler implements Runnable { private final long ticksRequiredToFlush; private long lastTicks; - private final JsonAdapter jsonAdapter; - private final BackendApiFactory backendApiFactory; - private BackendApi evp; - + private final FeatureFlagEvpPublisher evpPublisher; private final Map context; private final ExposureCache cache; @@ -119,8 +103,7 @@ public ExposureSerializingHandler( final Runnable errorCallback) { this.queue = queue; this.cache = new LRUExposureCache(queue.capacity()); - this.jsonAdapter = new Moshi.Builder().build().adapter(ExposuresRequest.class); - this.backendApiFactory = backendApiFactory; + this.evpPublisher = new FeatureFlagEvpPublisher<>(backendApiFactory, ExposuresRequest.class); this.context = context; this.lastTicks = System.nanoTime(); @@ -133,8 +116,7 @@ public ExposureSerializingHandler( @Override public void run() { - evp = backendApiFactory.createBackendApi(Intake.EVENT_PLATFORM); - if (evp == null) { + if (!evpPublisher.start()) { errorCallback.run(); throw new IllegalArgumentException("EVP Proxy not available"); } @@ -180,10 +162,7 @@ protected void flushIfNecessary() { if (shouldFlush()) { try { final ExposuresRequest exposures = new ExposuresRequest(this.context, this.buffer); - final String reqBod = jsonAdapter.toJson(exposures); - final RequestBody requestBody = - RequestBody.create(okhttp3.MediaType.parse("application/json"), reqBod); - evp.post("exposures", requestBody, stream -> null, null, false); + evpPublisher.post(EXPOSURES_ROUTE, exposures); this.buffer.clear(); } catch (Exception e) { LOGGER.error("Could not submit exposures", e); diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpContext.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpContext.java new file mode 100644 index 00000000000..c964efa6c7f --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpContext.java @@ -0,0 +1,22 @@ +package com.datadog.featureflag; + +import datadog.trace.api.Config; +import java.util.HashMap; +import java.util.Map; + +final class FeatureFlagEvpContext { + + private FeatureFlagEvpContext() {} + + static Map from(final Config config) { + final Map context = new HashMap<>(4); + context.put("service", config.getServiceName() == null ? "unknown" : config.getServiceName()); + if (config.getEnv() != null) { + context.put("env", config.getEnv()); + } + if (config.getVersion() != null) { + context.put("version", config.getVersion()); + } + return context; + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpPublisher.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpPublisher.java new file mode 100644 index 00000000000..f0694d17aa4 --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpPublisher.java @@ -0,0 +1,79 @@ +package com.datadog.featureflag; + +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.Moshi; +import datadog.communication.BackendApi; +import datadog.communication.BackendApiFactory; +import datadog.trace.api.intake.Intake; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import okhttp3.MediaType; +import okhttp3.RequestBody; + +final class FeatureFlagEvpPublisher { + + private static final MediaType JSON = MediaType.parse("application/json"); + + private final BackendApiFactory backendApiFactory; + private final String preferredEvpProxyEndpoint; + private final boolean responseCompression; + private final JsonAdapter jsonAdapter; + private BackendApi evp; + + FeatureFlagEvpPublisher(final BackendApiFactory backendApiFactory, final Class requestType) { + this(backendApiFactory, requestType, null, true); + } + + FeatureFlagEvpPublisher( + final BackendApiFactory backendApiFactory, + final Class requestType, + final boolean responseCompression) { + this(backendApiFactory, requestType, null, responseCompression); + } + + FeatureFlagEvpPublisher( + final BackendApiFactory backendApiFactory, + final Class requestType, + final String preferredEvpProxyEndpoint, + final boolean responseCompression) { + this.backendApiFactory = backendApiFactory; + this.preferredEvpProxyEndpoint = preferredEvpProxyEndpoint; + this.responseCompression = responseCompression; + this.jsonAdapter = new Moshi.Builder().build().adapter(requestType); + } + + boolean start() { + if (evp == null) { + evp = + preferredEvpProxyEndpoint == null && responseCompression + ? backendApiFactory.createBackendApi(Intake.EVENT_PLATFORM) + : backendApiFactory.createBackendApi( + Intake.EVENT_PLATFORM, preferredEvpProxyEndpoint, responseCompression); + } + return evp != null; + } + + void post(final String route, final T request) throws IOException { + post(route, serialize(request)); + } + + byte[] serialize(final T request) { + return utf8Bytes(jsonAdapter.toJson(request)); + } + + void post(final String route, final byte[] json) throws IOException { + if (!start()) { + throw new IllegalStateException("EVP Proxy not available"); + } + final RequestBody requestBody = RequestBody.create(JSON, json); + evp.post(route, requestBody, stream -> null, null, false); + } + + static byte[] utf8Bytes(final String json) { + try { + return json.getBytes("UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new AssertionError("UTF-8 must be available", e); + } + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagEvpPublisherTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagEvpPublisherTest.java new file mode 100644 index 00000000000..b5044f43c7a --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagEvpPublisherTest.java @@ -0,0 +1,72 @@ +package com.datadog.featureflag; + +import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V4_EVP_PROXY_ENDPOINT; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import datadog.communication.BackendApi; +import datadog.communication.BackendApiFactory; +import datadog.trace.api.intake.Intake; +import okhttp3.RequestBody; +import org.junit.jupiter.api.Test; + +class FeatureFlagEvpPublisherTest { + + @Test + void defaultPublisherUsesDefaultBackendApiFactoryPath() { + final BackendApi backendApi = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(Intake.EVENT_PLATFORM)).thenReturn(backendApi); + + final FeatureFlagEvpPublisher publisher = + new FeatureFlagEvpPublisher<>(factory, TestRequest.class); + + publisher.start(); + + verify(factory).createBackendApi(Intake.EVENT_PLATFORM); + verifyNoMoreInteractions(factory); + } + + @Test + void responseCompressionCanBeDisabledWithoutPinnedEndpoint() { + final BackendApi backendApi = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(Intake.EVENT_PLATFORM, null, false)).thenReturn(backendApi); + + final FeatureFlagEvpPublisher publisher = + new FeatureFlagEvpPublisher<>(factory, TestRequest.class, false); + + publisher.start(); + + verify(factory).createBackendApi(Intake.EVENT_PLATFORM, null, false); + } + + @Test + void preferredEndpointAndRequestCompressionAreForwardedToBackendApi() throws Exception { + final BackendApi backendApi = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(Intake.EVENT_PLATFORM, V4_EVP_PROXY_ENDPOINT, false)) + .thenReturn(backendApi); + final FeatureFlagEvpPublisher publisher = + new FeatureFlagEvpPublisher<>(factory, TestRequest.class, V4_EVP_PROXY_ENDPOINT, false); + + publisher.post("flagevaluation", new TestRequest("value")); + + verify(factory).createBackendApi(Intake.EVENT_PLATFORM, V4_EVP_PROXY_ENDPOINT, false); + verify(backendApi) + .post(eq("flagevaluation"), any(RequestBody.class), any(), isNull(), eq(false)); + } + + static class TestRequest { + public final String value; + + TestRequest(final String value) { + this.value = value; + } + } +} From 4aa06476b70d15ecf5298ccc5f16f6e1ce229cbd Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:47:56 -0400 Subject: [PATCH 03/26] feat(openfeature): add flagevaluation contract --- .../api/config/FeatureFlaggingConfig.java | 8 ++ metadata/supported-configurations.json | 8 ++ .../featureflag/FeatureFlaggingGateway.java | 29 +++++ .../flagevaluation/FlagEvalEvent.java | 108 ++++++++++++++++++ .../flagevaluation/FlagEvaluationWriter.java | 27 +++++ 5 files changed, 180 insertions(+) create mode 100644 products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEvent.java create mode 100644 products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvaluationWriter.java diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java index 28151f88864..5f567908254 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java @@ -3,4 +3,12 @@ public class FeatureFlaggingConfig { public static final String FLAGGING_PROVIDER_ENABLED = "experimental.flagging.provider.enabled"; + + /** + * Killswitch for the EVP {@code flagevaluation} emission path. Default: enabled. Disabling it + * turns off EVP flag-evaluation counts while leaving the OTel {@code feature_flag.evaluations} + * metric path untouched. Maps to {@code DD_FLAGGING_EVALUATION_COUNTS_ENABLED}. + */ + public static final String FLAGGING_EVALUATION_COUNTS_ENABLED = + "flagging.evaluation.counts.enabled"; } diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index ae081138b71..7206f291599 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -1489,6 +1489,14 @@ "aliases": [] } ], + "DD_FLAGGING_EVALUATION_COUNTS_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "true", + "aliases": [] + } + ], "DD_FORCE_CLEAR_TEXT_HTTP_FOR_INTAKE_CLIENT": [ { "version": "A", diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java index b9d73ffa7ab..a0856fc0ab3 100644 --- a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java +++ b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java @@ -1,6 +1,7 @@ package datadog.trace.api.featureflag; import datadog.trace.api.featureflag.exposure.ExposureEvent; +import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; @@ -19,6 +20,15 @@ public interface ExposureListener extends Consumer {} private static final AtomicReference CURRENT_CONFIG = new AtomicReference<>(); + /** + * The active EVP flagevaluation writer. Registered by {@code FlagEvaluationWriterImpl.start()} + * when the killswitch {@code DD_FLAGGING_EVALUATION_COUNTS_ENABLED} is on (default). Read by + * {@code FlagEvalLoggingHook} to route evaluations into the two-tier aggregator. {@code null} + * when the EVP path is disabled. + */ + private static final AtomicReference FLAG_EVAL_WRITER = + new AtomicReference<>(); + private FeatureFlaggingGateway() {} public static void addConfigListener(final ConfigListener listener) { @@ -49,4 +59,23 @@ public static void removeExposureListener(final ExposureListener listener) { public static void dispatch(final ExposureEvent event) { EXPOSURE_LISTENERS.forEach(listener -> listener.accept(event)); } + + /** + * Registers the active EVP flagevaluation writer. Called by {@code + * FlagEvaluationWriterImpl.start()} when the feature is enabled. Replaces any previously + * registered writer. + * + * @param writer the writer to register, or {@code null} to deregister + */ + public static void setFlagEvalWriter(final FlagEvaluationWriter writer) { + FLAG_EVAL_WRITER.set(writer); + } + + /** + * Returns the active EVP flagevaluation writer, or {@code null} when disabled (killswitch off or + * not yet started). + */ + public static FlagEvaluationWriter getFlagEvalWriter() { + return FLAG_EVAL_WRITER.get(); + } } diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEvent.java b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEvent.java new file mode 100644 index 00000000000..83397e5dccd --- /dev/null +++ b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEvent.java @@ -0,0 +1,108 @@ +package datadog.trace.api.featureflag.flagevaluation; + +import java.util.Collections; +import java.util.Map; +import java.util.function.Supplier; + +/** + * Lightweight data record capturing a single flag evaluation for EVP flagevaluation emission. + * + *

This is the currency passed from the {@code FlagEvalLoggingHook} (feature-flagging-api) to the + * {@code FlagEvaluationWriter} (feature-flagging-lib) via a non-blocking bounded queue. + * + *

Scalar fields are captured at hook-fire time on the evaluation thread. Context attributes can + * be supplied lazily so recursive flattening happens on the writer thread, not on the evaluation + * path. No aggregation happens here. + */ +public final class FlagEvalEvent { + + /** The feature flag key. Never null. */ + public final String flagKey; + + /** + * The OpenFeature variant key selected for the evaluation. {@code null} means the default value + * was returned (runtime default). + */ + public final String variant; + + /** The allocation key from flag metadata ("allocationKey"). May be null. */ + public final String allocationKey; + + /** The targeting key from the evaluation context. May be null. */ + public final String targetingKey; + + /** + * The evaluation error message when the evaluation failed, else {@code null}. Sourced from the + * OpenFeature evaluation details (error message, falling back to the error code). + */ + public final String errorMessage; + + /** + * Evaluation timestamp in milliseconds since epoch. Stamped at eval-entry time from flag metadata + * key {@code "dd.eval.timestamp_ms"}, or falls back to hook-fire time when absent. This ensures + * first/last_evaluation reflect evaluation time, not hook-fire time. + */ + public final long evalTimeMs; + + /** + * Flattened evaluation context attributes. Used for the full-tier canonical context key. May be + * empty but never null. + */ + public final Map attrs; + + private final Supplier> attrsSupplier; + + public FlagEvalEvent( + final String flagKey, + final String variant, + final String allocationKey, + final String targetingKey, + final long evalTimeMs, + final Map attrs) { + this(flagKey, variant, allocationKey, targetingKey, null, evalTimeMs, attrs); + } + + public FlagEvalEvent( + final String flagKey, + final String variant, + final String allocationKey, + final String targetingKey, + final String errorMessage, + final long evalTimeMs, + final Map attrs) { + this.flagKey = flagKey; + this.variant = variant; + this.allocationKey = allocationKey; + this.targetingKey = targetingKey; + this.errorMessage = errorMessage; + this.evalTimeMs = evalTimeMs; + this.attrs = attrs != null ? attrs : Collections.emptyMap(); + this.attrsSupplier = null; + } + + public FlagEvalEvent( + final String flagKey, + final String variant, + final String allocationKey, + final String targetingKey, + final String errorMessage, + final long evalTimeMs, + final Supplier> attrsSupplier) { + this.flagKey = flagKey; + this.variant = variant; + this.allocationKey = allocationKey; + this.targetingKey = targetingKey; + this.errorMessage = errorMessage; + this.evalTimeMs = evalTimeMs; + this.attrs = Collections.emptyMap(); + this.attrsSupplier = attrsSupplier; + } + + public Map contextAttributes() { + if (attrsSupplier == null) { + return attrs; + } + final Map supplied = attrsSupplier.get(); + return supplied != null ? supplied : Collections.emptyMap(); + } +} diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvaluationWriter.java b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvaluationWriter.java new file mode 100644 index 00000000000..9bfe40d11ca --- /dev/null +++ b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvaluationWriter.java @@ -0,0 +1,27 @@ +package datadog.trace.api.featureflag.flagevaluation; + +/** + * Defines an EVP flagevaluation writer responsible for aggregating flag evaluation events and + * flushing them to the EVP proxy. + * + *

Implementations must use a background thread (serializing handler) for aggregation and + * transport. The {@link #enqueue(FlagEvalEvent)} method must be non-blocking and callable from the + * OpenFeature hook thread without backpressure. + */ +public interface FlagEvaluationWriter extends AutoCloseable { + + /** + * Non-blocking enqueue of a flag evaluation event. May silently drop the event if the internal + * bounded queue is full (best-effort, observable via drop counter). + * + * @param event the flag evaluation event captured at hook-fire time + */ + void enqueue(FlagEvalEvent event); + + /** Starts the background serializing thread. Must be called once after construction. */ + void start(); + + /** Stops the background thread and releases resources. */ + @Override + void close(); +} From ac126fb44a2a143612b3bda30ed605c42faef8f8 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:47:56 -0400 Subject: [PATCH 04/26] refactor(openfeature): name metrics hook explicitly --- ...{FlagEvalHook.java => FlagEvalMetricsHook.java} | 4 ++-- .../datadog/trace/api/openfeature/Provider.java | 12 ++++++------ ...lHookTest.java => FlagEvalMetricsHookTest.java} | 14 +++++++------- .../trace/api/openfeature/ProviderTest.java | 4 ++-- 4 files changed, 17 insertions(+), 17 deletions(-) rename products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/{FlagEvalHook.java => FlagEvalMetricsHook.java} (91%) rename products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/{FlagEvalHookTest.java => FlagEvalMetricsHookTest.java} (89%) diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalHook.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalMetricsHook.java similarity index 91% rename from products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalHook.java rename to products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalMetricsHook.java index 1132602a53f..a3e0dbfc83c 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalHook.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalMetricsHook.java @@ -7,11 +7,11 @@ import dev.openfeature.sdk.ImmutableMetadata; import java.util.Map; -class FlagEvalHook implements Hook { +class FlagEvalMetricsHook implements Hook { private final FlagEvalMetrics metrics; - FlagEvalHook(FlagEvalMetrics metrics) { + FlagEvalMetricsHook(FlagEvalMetrics metrics) { this.metrics = metrics; } diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java index c5e8898fa4a..670d4d3642d 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java @@ -34,7 +34,7 @@ public class Provider extends EventProvider implements Metadata { private final AtomicReference initializationState = new AtomicReference<>(InitializationState.NOT_STARTED); private final FlagEvalMetrics flagEvalMetrics; - private final FlagEvalHook flagEvalHook; + private final FlagEvalMetricsHook flagEvalMetricsHook; public Provider() { this(DEFAULT_OPTIONS, null); @@ -48,16 +48,16 @@ public Provider(final Options options) { this.options = options; this.evaluator = evaluator; FlagEvalMetrics metrics = null; - FlagEvalHook hook = null; + FlagEvalMetricsHook hook = null; try { metrics = new FlagEvalMetrics(); - hook = new FlagEvalHook(metrics); + hook = new FlagEvalMetricsHook(metrics); } catch (LinkageError | Exception e) { // This outer catch fires when the metrics helper itself can't load (OTel API absent). log.warn("Evaluation metrics unavailable — OTel API classes not on classpath", e); } this.flagEvalMetrics = metrics; - this.flagEvalHook = hook; + this.flagEvalMetricsHook = hook; } @Override @@ -167,10 +167,10 @@ private Evaluator buildEvaluator() throws Exception { @Override public List getProviderHooks() { - if (flagEvalHook == null) { + if (flagEvalMetricsHook == null) { return Collections.emptyList(); } - return Collections.singletonList(flagEvalHook); + return Collections.singletonList(flagEvalMetricsHook); } @Override diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalHookTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalMetricsHookTest.java similarity index 89% rename from products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalHookTest.java rename to products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalMetricsHookTest.java index 8ed17d91cbb..322a06d2de7 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalHookTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalMetricsHookTest.java @@ -13,12 +13,12 @@ import java.util.Collections; import org.junit.jupiter.api.Test; -class FlagEvalHookTest { +class FlagEvalMetricsHookTest { @Test void finallyAfterRecordsBasicEvaluation() { FlagEvalMetrics metrics = mock(FlagEvalMetrics.class); - FlagEvalHook hook = new FlagEvalHook(metrics); + FlagEvalMetricsHook hook = new FlagEvalMetricsHook(metrics); FlagEvaluationDetails details = FlagEvaluationDetails.builder() @@ -44,7 +44,7 @@ void finallyAfterRecordsBasicEvaluation() { @Test void finallyAfterRecordsErrorEvaluation() { FlagEvalMetrics metrics = mock(FlagEvalMetrics.class); - FlagEvalHook hook = new FlagEvalHook(metrics); + FlagEvalMetricsHook hook = new FlagEvalMetricsHook(metrics); FlagEvaluationDetails details = FlagEvaluationDetails.builder() @@ -68,7 +68,7 @@ void finallyAfterRecordsErrorEvaluation() { @Test void finallyAfterHandlesNullFlagMetadata() { FlagEvalMetrics metrics = mock(FlagEvalMetrics.class); - FlagEvalHook hook = new FlagEvalHook(metrics); + FlagEvalMetricsHook hook = new FlagEvalMetricsHook(metrics); FlagEvaluationDetails details = FlagEvaluationDetails.builder() @@ -87,7 +87,7 @@ void finallyAfterHandlesNullFlagMetadata() { @Test void finallyAfterHandlesNullVariantAndReason() { FlagEvalMetrics metrics = mock(FlagEvalMetrics.class); - FlagEvalHook hook = new FlagEvalHook(metrics); + FlagEvalMetricsHook hook = new FlagEvalMetricsHook(metrics); FlagEvaluationDetails details = FlagEvaluationDetails.builder().flagKey("my-flag").value("default").build(); @@ -100,7 +100,7 @@ void finallyAfterHandlesNullVariantAndReason() { @Test void finallyAfterNeverThrows() { FlagEvalMetrics metrics = mock(FlagEvalMetrics.class); - FlagEvalHook hook = new FlagEvalHook(metrics); + FlagEvalMetricsHook hook = new FlagEvalMetricsHook(metrics); // Should not throw even with completely null inputs hook.finallyAfter(null, null, null); @@ -110,7 +110,7 @@ void finallyAfterNeverThrows() { @Test void finallyAfterIsNoOpWhenMetricsIsNull() { - FlagEvalHook hook = new FlagEvalHook(null); + FlagEvalMetricsHook hook = new FlagEvalMetricsHook(null); FlagEvaluationDetails details = FlagEvaluationDetails.builder() diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java index 27d4dd5d2b5..06bbe7ef8d2 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java @@ -326,12 +326,12 @@ protected Class loadEvaluatorClass() throws ClassNotFoundException { } @Test - public void testGetProviderHooksReturnsFlagEvalHook() { + public void testGetProviderHooksReturnsFlagEvalMetricsHook() { Provider provider = new Provider(new Options().initTimeout(10, MILLISECONDS), mock(Evaluator.class)); List hooks = provider.getProviderHooks(); assertThat(hooks.size(), equalTo(1)); - assertThat(hooks.get(0) instanceof FlagEvalHook, equalTo(true)); + assertThat(hooks.get(0) instanceof FlagEvalMetricsHook, equalTo(true)); } @Test From 7a43658da6bbf774948e73bb37af0506184fa5f0 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:47:57 -0400 Subject: [PATCH 05/26] feat(openfeature): add flagevaluation logging hook --- .../trace/api/openfeature/DDEvaluator.java | 6 +- .../api/openfeature/FlagEvalLoggingHook.java | 125 ++++++++++++++++++ 2 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalLoggingHook.java diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java index 91c0aafdc7a..762ffe5997c 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java @@ -387,11 +387,15 @@ private static ProviderEvaluation resolveVariant( + e.getMessage()); } + // Stamp eval-time at the resolution point so first/last_evaluation reflect evaluation time, + // not hook-fire time. Passed to the hook via provider metadata "dd.eval.timestamp_ms". + final long evalTimestampMs = System.currentTimeMillis(); final ImmutableMetadata.ImmutableMetadataBuilder metadataBuilder = ImmutableMetadata.builder() .addString("flagKey", flag.key) .addString("variationType", flag.variationType.name()) - .addString("allocationKey", allocation.key); + .addString("allocationKey", allocation.key) + .addLong("dd.eval.timestamp_ms", evalTimestampMs); final ProviderEvaluation result = ProviderEvaluation.builder() .value(mappedValue) diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalLoggingHook.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalLoggingHook.java new file mode 100644 index 00000000000..b6a08548fc0 --- /dev/null +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalLoggingHook.java @@ -0,0 +1,125 @@ +package datadog.trace.api.openfeature; + +import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; +import dev.openfeature.sdk.FlagEvaluationDetails; +import dev.openfeature.sdk.Hook; +import dev.openfeature.sdk.HookContext; +import dev.openfeature.sdk.ImmutableMetadata; +import java.util.Collections; +import java.util.Map; +import java.util.function.Supplier; + +/** + * OpenFeature {@code Hook} that captures flag evaluation events for EVP {@code flagevaluation} + * emission. + * + *

Contract: {@code finallyAfter} does ONLY cheap scalar extraction + a non-blocking offer to the + * writer's bounded queue. No inline aggregation on the hook thread. + * + *

This hook is registered alongside the existing OTel {@link FlagEvalMetricsHook} - it does NOT + * replace it (the existing OTel metrics hook is left unchanged). + * + *

The writer is resolved lazily from {@link FeatureFlaggingGateway#getFlagEvalWriter()} on each + * call, so the hook is always safe to register - if the writer is absent (killswitch off or not yet + * started) it is a no-op. + */ +class FlagEvalLoggingHook implements Hook { + + /** + * Singleton instance: always registered when the provider is created; harmless when writer=null + * (killswitch off or not yet started). + */ + static final FlagEvalLoggingHook INSTANCE = new FlagEvalLoggingHook<>(); + + /** + * Writer resolver. Production instances resolve through {@link FeatureFlaggingGateway}; tests can + * inject a direct writer or a resolver that simulates old-bootstrap linkage failures. + */ + private final Supplier writerSupplier; + + /** Production constructor - resolves writer from gateway. */ + FlagEvalLoggingHook() { + this(FeatureFlaggingGateway::getFlagEvalWriter); + } + + /** Test-only constructor - injects a writer directly, bypassing the gateway. */ + FlagEvalLoggingHook(final FlagEvaluationWriter writer) { + this(() -> writer); + } + + /** Test-only constructor - injects a writer resolver directly, bypassing the gateway. */ + FlagEvalLoggingHook(final Supplier writerSupplier) { + this.writerSupplier = writerSupplier; + } + + /** + * Cheap capture + non-blocking enqueue only. Runs at the {@code finally} stage so it covers + * success, error, and default-value paths. + */ + @Override + public void finallyAfter( + final HookContext ctx, + final FlagEvaluationDetails details, + final Map hints) { + try { + final FlagEvaluationWriter w = writerSupplier.get(); + if (w == null || details == null) { + return; + } + + // Cheap scalar extraction - no JSON, no map lookups beyond metadata.asMap() + final String flagKey = details.getFlagKey(); + final ImmutableMetadata metadata = details.getFlagMetadata(); + + // allocationKey: "allocationKey" (camelCase) - consistent with FlagEvalMetricsHook.java + final String allocationKey = metadata != null ? metadata.getString("allocationKey") : null; + + // eval-time: from flag metadata "dd.eval.timestamp_ms" (Long), fallback to hook-fire time. + // ImmutableMetadata.getLong available since sdk 1.4+. + final Long evalTimeObj = metadata != null ? metadata.getLong("dd.eval.timestamp_ms") : null; + final long evalTimeMs = evalTimeObj != null ? evalTimeObj : System.currentTimeMillis(); + + // variant: the OpenFeature variant key (same source as the OTel FlagEvalMetricsHook), NOT the + // evaluated value. A null variant means no variant was selected (runtime default). + final String variant = details.getVariant(); + + // error message: prefer the human-readable message; fall back to the error code name when + // the message is empty (some providers populate only the code). null on success. + String errorMessage = details.getErrorMessage(); + if ((errorMessage == null || errorMessage.isEmpty()) && details.getErrorCode() != null) { + errorMessage = details.getErrorCode().name(); + } + if (errorMessage != null && errorMessage.isEmpty()) { + errorMessage = null; + } + + // targetingKey from evaluation context + final String targetingKey = + ctx != null && ctx.getCtx() != null ? ctx.getCtx().getTargetingKey() : null; + + w.enqueue( + new FlagEvalEvent( + flagKey, + variant, + allocationKey, + targetingKey, + errorMessage, + evalTimeMs, + () -> extractAttrs(ctx))); + } catch (LinkageError | Exception e) { + // Never let EVP recording break flag evaluation + } + } + + /** Extracts converted, flattened attributes from the evaluation context. */ + private Map extractAttrs(final HookContext ctx) { + if (ctx == null || ctx.getCtx() == null) { + return Collections.emptyMap(); + } + final Map attrs = DDEvaluator.flattenContext(ctx.getCtx()); + attrs.remove("targetingKey"); + return attrs; + } +} From 4ec77cae03b4a8aa4cda903ecaf860676630d359 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:47:57 -0400 Subject: [PATCH 06/26] test(openfeature): cover flagevaluation logging hook --- .../openfeature/FlagEvalLoggingHookTest.java | 384 ++++++++++++++++++ 1 file changed, 384 insertions(+) create mode 100644 products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalLoggingHookTest.java diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalLoggingHookTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalLoggingHookTest.java new file mode 100644 index 00000000000..61643e6bb61 --- /dev/null +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalLoggingHookTest.java @@ -0,0 +1,384 @@ +package datadog.trace.api.openfeature; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; +import dev.openfeature.sdk.ErrorCode; +import dev.openfeature.sdk.FlagEvaluationDetails; +import dev.openfeature.sdk.FlagValueType; +import dev.openfeature.sdk.HookContext; +import dev.openfeature.sdk.ImmutableMetadata; +import dev.openfeature.sdk.MutableContext; +import dev.openfeature.sdk.Reason; +import dev.openfeature.sdk.Value; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link FlagEvalLoggingHook}: cheap capture, non-blocking enqueue, eval-time + * metadata, absent-variant detection, and killswitch-via-writer-null behaviour. + */ +class FlagEvalLoggingHookTest { + + // ---- helpers ---- + + /** + * Creates a writer that captures the enqueued event for assertion. Uses an anonymous class since + * FlagEvaluationWriter has multiple abstract methods. + */ + private FlagEvaluationWriter capturingWriter(final AtomicReference ref) { + return new FlagEvaluationWriter() { + @Override + public void enqueue(final FlagEvalEvent event) { + ref.set(event); + } + + @Override + public void start() {} + + @Override + public void close() {} + }; + } + + private static FlagEvalLoggingHook hookWithWriter(final FlagEvaluationWriter writer) { + return new FlagEvalLoggingHook<>(writer); + } + + private static FlagEvaluationDetails details( + final String flagKey, + final Object value, + final String variant, + final String reason, + final ImmutableMetadata metadata) { + final FlagEvaluationDetails.FlagEvaluationDetailsBuilder builder = + FlagEvaluationDetails.builder().flagKey(flagKey).value(value).reason(reason); + if (variant != null) { + builder.variant(variant); + } + if (metadata != null) { + builder.flagMetadata(metadata); + } + return builder.build(); + } + + private static HookContext hookCtxWithTargetingKey( + final String flagKey, final String targetingKey) { + final MutableContext ctx = new MutableContext(targetingKey); + return HookContext.builder() + .flagKey(flagKey) + .type(FlagValueType.STRING) + .defaultValue("default") + .ctx(ctx) + .build(); + } + + // ---- test: hook calls writer.enqueue once with flagKey, variant, allocationKey ---- + + @Test + void finallyAfterEnqueuesEventWithAllBasicFields() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final FlagEvaluationDetails det = + details( + "my-flag", + "on-value", + "on", + Reason.TARGETING_MATCH.name(), + ImmutableMetadata.builder().addString("allocationKey", "alloc-1").build()); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get(), "writer.enqueue must be called once"); + final FlagEvalEvent e = captured.get(); + assertEquals("my-flag", e.flagKey); + assertEquals("on", e.variant, "variant must be the OpenFeature variant key"); + assertEquals("alloc-1", e.allocationKey); + } + + // ---- variant comes from details.getVariant(), NOT details.getValue() ---- + + @Test + void variantIsTheVariantKeyNotTheEvaluatedValue() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + // value and variant DIFFER, so a value-vs-variant mistake is detectable. + final FlagEvaluationDetails det = + details( + "g1-flag", + "the-evaluated-value", // value + "the-variant-key", // variant + Reason.TARGETING_MATCH.name(), + null); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertEquals( + "the-variant-key", + captured.get().variant, + "variant must be sourced from details.getVariant(), not details.getValue()"); + } + + // ---- test: evalTimeMs from metadata "dd.eval.timestamp_ms" ---- + + @Test + void evalTimeMsComesFromMetadataWhenPresent() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final long expectedTimestamp = 1_700_000_000_000L; + final FlagEvaluationDetails det = + details( + "ts-flag", + "v", + "v", + Reason.SPLIT.name(), + ImmutableMetadata.builder() + .addString("allocationKey", "a") + .addLong("dd.eval.timestamp_ms", expectedTimestamp) + .build()); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertEquals( + expectedTimestamp, + captured.get().evalTimeMs, + "evalTimeMs must come from dd.eval.timestamp_ms metadata when present"); + } + + // ---- test: evalTimeMs falls back to System.currentTimeMillis() when absent ---- + + @Test + void evalTimeMsFallsBackToCurrentTimeWhenMetadataAbsent() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final long before = System.currentTimeMillis(); + final FlagEvaluationDetails det = + details("ts-flag", "v", "v", Reason.SPLIT.name(), null); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + final long after = System.currentTimeMillis(); + assertNotNull(captured.get()); + final long ts = captured.get().evalTimeMs; + assertTrue( + ts >= before && ts <= after, + "evalTimeMs must fall back to hook-fire time when metadata absent. got: " + ts); + } + + // ---- test: absent variant -> variant is null -> runtime default ---- + + @Test + void absentVariantProducesNullVariant() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + // A runtime default returns the default value but no variant. + final FlagEvaluationDetails det = + details("def-flag", "default-value", null, Reason.DEFAULT.name(), null); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertNull(captured.get().variant, "Absent variant must stay null (runtime default)"); + } + + // ---- test: error message captured from details (error object support) ---- + + @Test + void errorMessageCapturedFromDetails() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final FlagEvaluationDetails det = + FlagEvaluationDetails.builder() + .flagKey("err-flag") + .value("default") + .reason(Reason.ERROR.name()) + .errorCode(ErrorCode.TYPE_MISMATCH) + .errorMessage("value does not match declared type") + .build(); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertEquals( + "value does not match declared type", + captured.get().errorMessage, + "errorMessage must be captured from the evaluation details"); + } + + // ---- test: error code used as fallback message when error message is empty ---- + + @Test + void errorCodeUsedAsFallbackWhenMessageEmpty() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final FlagEvaluationDetails det = + FlagEvaluationDetails.builder() + .flagKey("err-flag") + .value("default") + .reason(Reason.ERROR.name()) + .errorCode(ErrorCode.FLAG_NOT_FOUND) + .build(); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertEquals( + "FLAG_NOT_FOUND", + captured.get().errorMessage, + "error code name must be used when no error message is present"); + } + + // ---- test: success path has no error message ---- + + @Test + void successPathHasNullErrorMessage() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final FlagEvaluationDetails det = + details("ok-flag", "v", "v", Reason.TARGETING_MATCH.name(), null); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertNull(captured.get().errorMessage, "success path must have no error message"); + } + + // ---- test: hook does NO aggregation on the hook thread ---- + + @Test + void finallyAfterOnlyCallsEnqueueNoOtherWriterMethods() { + final FlagEvaluationWriter writer = mock(FlagEvaluationWriter.class); + final FlagEvalLoggingHook hook = hookWithWriter(writer); + + final FlagEvaluationDetails det = + details("flag", "v", "v", Reason.TARGETING_MATCH.name(), null); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + // Exactly one enqueue call, no start/close/aggregate + verify(writer, times(1)).enqueue(any(FlagEvalEvent.class)); + verify(writer, never()).close(); + verify(writer, never()).start(); + } + + // ---- test: writer=null -> no-op (killswitch off / not yet started) ---- + + @Test + void writerNullIsNoOp() { + final FlagEvalLoggingHook hook = hookWithWriter(null); + final FlagEvaluationDetails det = + details("flag", "v", "v", Reason.TARGETING_MATCH.name(), null); + + // Must not throw; nothing is enqueued + hook.finallyAfter(null, det, Collections.emptyMap()); + } + + @Test + void writerLookupLinkageErrorIsNoOp() { + final FlagEvalLoggingHook hook = + new FlagEvalLoggingHook<>( + () -> { + throw new NoSuchMethodError("old bootstrap"); + }); + final FlagEvaluationDetails det = + details("flag", "v", "v", Reason.TARGETING_MATCH.name(), null); + + assertDoesNotThrow(() -> hook.finallyAfter(null, det, Collections.emptyMap())); + } + + // ---- test: details=null -> no-op ---- + + @Test + void detailsNullIsNoOp() { + final FlagEvaluationWriter writer = mock(FlagEvaluationWriter.class); + final FlagEvalLoggingHook hook = hookWithWriter(writer); + + // Should not throw + hook.finallyAfter(null, null, Collections.emptyMap()); + + verifyNoInteractions(writer); + } + + // ---- test: targetingKey extracted from evaluation context ---- + + @Test + void targetingKeyExtractedFromContext() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final FlagEvaluationDetails det = + details("ctx-flag", "v", "v", Reason.SPLIT.name(), null); + + final HookContext hookCtx = hookCtxWithTargetingKey("ctx-flag", "user-42"); + + hook.finallyAfter(hookCtx, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertEquals( + "user-42", + captured.get().targetingKey, + "targetingKey must be extracted from the evaluation context"); + } + + @Test + void contextAttributesAreFlattenedAndConvertedAfterEnqueue() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final Map profile = new HashMap<>(); + profile.put("tier", "gold"); + final Map attributes = new HashMap<>(); + attributes.put("score", 42); + attributes.put("profile", profile); + final MutableContext context = + new MutableContext(Value.objectToValue(attributes).asStructure().asMap()); + context.setTargetingKey("user-42"); + + final HookContext hookCtx = + HookContext.builder() + .flagKey("ctx-flag") + .type(FlagValueType.STRING) + .defaultValue("default") + .ctx(context) + .build(); + final FlagEvaluationDetails det = + details("ctx-flag", "v", "v", Reason.TARGETING_MATCH.name(), null); + + hook.finallyAfter(hookCtx, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertTrue(captured.get().attrs.isEmpty(), "hook must not flatten context before enqueue"); + final Map attrs = captured.get().contextAttributes(); + assertEquals(42, attrs.get("score")); + assertEquals("gold", attrs.get("profile.tier")); + assertFalse(attrs.containsKey("targetingKey")); + assertTrue( + attrs.values().stream().noneMatch(Value.class::isInstance), + "context attrs must contain converted scalar values, not OpenFeature Value wrappers"); + } +} From 6fa7ade3e47b905b2aef4ff995f41205d6f0727a Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 2 Jul 2026 00:07:51 -0400 Subject: [PATCH 07/26] fix(openfeature): snapshot flagevaluation context --- .../trace/api/openfeature/DDEvaluator.java | 102 +++++++++++++++--- .../api/openfeature/FlagEvalLoggingHook.java | 23 ++-- .../api/openfeature/DDEvaluatorTest.java | 10 ++ .../openfeature/FlagEvalLoggingHookTest.java | 45 ++++++++ 4 files changed, 161 insertions(+), 19 deletions(-) diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java index 762ffe5997c..0a137cc5669 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java @@ -19,6 +19,7 @@ import dev.openfeature.sdk.ErrorCode; import dev.openfeature.sdk.EvaluationContext; import dev.openfeature.sdk.ImmutableMetadata; +import dev.openfeature.sdk.ImmutableStructure; import dev.openfeature.sdk.ProviderEvaluation; import dev.openfeature.sdk.Reason; import dev.openfeature.sdk.Structure; @@ -27,12 +28,16 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.Deque; import java.util.HashMap; import java.util.HashSet; +import java.util.IdentityHashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.CountDownLatch; @@ -504,38 +509,109 @@ private static String allocationKey(final ProviderEvaluation resolution) } static AbstractMap flattenContext(final EvaluationContext context) { - final Set keys = context.keySet(); + return flattenValues(snapshotValues(context)); + } + + static Map snapshotValues(final EvaluationContext context) { + final HashMap values = new HashMap<>(); + final Set seenContainers = Collections.newSetFromMap(new IdentityHashMap<>()); + for (final String key : context.keySet()) { + values.put(key, snapshotValue(context.getValue(key), seenContainers)); + } + return values; + } + + private static Value snapshotValue(final Value value, final Set seenContainers) { + if (value == null) { + return null; + } else if (value.isNull()) { + return new Value(); + } else if (value.isBoolean()) { + return new Value(value.asBoolean()); + } else if (value.isNumber()) { + final Object number = value.asObject(); + return number instanceof Integer + ? new Value((Integer) number) + : new Value(((Number) number).doubleValue()); + } else if (value.isString()) { + return new Value(value.asString()); + } else if (value.isInstant()) { + return new Value(value.asInstant()); + } else if (value.isList()) { + final List list = value.asList(); + if (!seenContainers.add(list)) { + return new Value(); + } + final List snapshot = new ArrayList<>(list.size()); + for (final Value item : list) { + snapshot.add(snapshotValue(item, seenContainers)); + } + seenContainers.remove(list); + return new Value(Collections.unmodifiableList(snapshot)); + } else if (value.isStructure()) { + final Structure structure = value.asStructure(); + if (!seenContainers.add(structure)) { + return new Value(); + } + final Map snapshot = new HashMap<>(); + for (final String key : structure.keySet()) { + snapshot.put(key, snapshotValue(structure.getValue(key), seenContainers)); + } + seenContainers.remove(structure); + return new Value(new ImmutableStructure(snapshot)); + } + throw new IllegalArgumentException("Unsupported OpenFeature value type: " + value); + } + + static AbstractMap flattenValues(final Map values) { final HashMap result = new HashMap<>(); - final Set seen = new HashSet<>(); - for (final String key : keys) { + final Set seenContainers = Collections.newSetFromMap(new IdentityHashMap<>()); + for (final Map.Entry root : values.entrySet()) { final Deque deque = new LinkedList<>(); - deque.push(new FlattenEntry(key, context.getValue(key))); + deque.push(new FlattenEntry(root.getKey(), root.getValue())); while (!deque.isEmpty()) { final FlattenEntry entry = deque.pop(); final Value value = entry.value; - if (value == null || seen.add(value)) { - if (value == null) { - result.put(entry.key, null); - } else if (value.isList()) { - final List list = value.asList(); + if (value == null) { + result.put(entry.key, null); + } else if (value.isList()) { + final List list = value.asList(); + if (seenContainers.add(list)) { for (int i = 0; i < list.size(); i++) { deque.push(new FlattenEntry(entry.key + "[" + i + "]", list.get(i))); } - } else if (value.isStructure()) { - final Structure structure = value.asStructure(); + } + } else if (value.isStructure()) { + final Structure structure = value.asStructure(); + if (seenContainers.add(structure)) { for (final String property : structure.keySet()) { deque.push( new FlattenEntry(entry.key + "." + property, structure.getValue(property))); } - } else { - result.put(entry.key, context.convertValue(value)); } + } else { + result.put(entry.key, convertValue(value)); } } } return result; } + private static Object convertValue(final Value value) { + if (value == null || value.isNull()) { + return null; + } else if (value.isBoolean()) { + return value.asBoolean(); + } else if (value.isNumber()) { + return value.asObject(); + } else if (value.isString()) { + return value.asString(); + } else if (value.isInstant()) { + return value.asInstant(); + } + throw new IllegalArgumentException("Unsupported OpenFeature value type: " + value); + } + @FunctionalInterface private interface NumberComparator { boolean compare(double a, double b); diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalLoggingHook.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalLoggingHook.java index b6a08548fc0..c394a154e01 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalLoggingHook.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalLoggingHook.java @@ -3,10 +3,12 @@ import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; +import dev.openfeature.sdk.EvaluationContext; import dev.openfeature.sdk.FlagEvaluationDetails; import dev.openfeature.sdk.Hook; import dev.openfeature.sdk.HookContext; import dev.openfeature.sdk.ImmutableMetadata; +import dev.openfeature.sdk.Value; import java.util.Collections; import java.util.Map; import java.util.function.Supplier; @@ -98,6 +100,7 @@ public void finallyAfter( // targetingKey from evaluation context final String targetingKey = ctx != null && ctx.getCtx() != null ? ctx.getCtx().getTargetingKey() : null; + final Map attrs = snapshotAttrs(ctx); w.enqueue( new FlagEvalEvent( @@ -107,19 +110,27 @@ public void finallyAfter( targetingKey, errorMessage, evalTimeMs, - () -> extractAttrs(ctx))); + () -> extractAttrs(attrs))); } catch (LinkageError | Exception e) { // Never let EVP recording break flag evaluation } } - /** Extracts converted, flattened attributes from the evaluation context. */ - private Map extractAttrs(final HookContext ctx) { + private Map snapshotAttrs(final HookContext ctx) { if (ctx == null || ctx.getCtx() == null) { return Collections.emptyMap(); } - final Map attrs = DDEvaluator.flattenContext(ctx.getCtx()); - attrs.remove("targetingKey"); - return attrs; + final EvaluationContext context = ctx.getCtx(); + final Map attrs = DDEvaluator.snapshotValues(context); + attrs.remove(EvaluationContext.TARGETING_KEY); + return attrs.isEmpty() ? Collections.emptyMap() : attrs; + } + + /** Extracts converted, flattened attributes from the evaluation context. */ + private Map extractAttrs(final Map attrs) { + if (attrs.isEmpty()) { + return Collections.emptyMap(); + } + return DDEvaluator.flattenValues(attrs); } } diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java index bb86c409bad..eee6661f024 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java @@ -15,6 +15,7 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasEntry; @@ -233,6 +234,10 @@ private static Arguments[] flatteningTestCases() { Arguments.of( mapOf("map", mapOf("key1", 1, "key2", 2, "key3", mapOf("key4", 4))), mapOf("map.key1", 1, "map.key2", 2, "map.key3.key4", 4))); + arguments.add( + Arguments.of( + mapOf("plan", "gold", "cohort", "gold"), + mapOf("plan", "gold", "cohort", "gold"))); return arguments.toArray(new Arguments[0]); } @@ -561,6 +566,11 @@ public void testEvaluate(final TestCase testCase) { if (expectedAllocation != null) { assertThat(details.getFlagMetadata().getString("allocationKey"), equalTo(expectedAllocation)); } + if (expected.errorCode == null && expected.variant != null) { + final Long evalTimestampMs = details.getFlagMetadata().getLong("dd.eval.timestamp_ms"); + assertThat(evalTimestampMs, notNullValue()); + assertThat(evalTimestampMs > 0, equalTo(true)); + } if (shouldDispatchExposure(expected)) { verify(exposureListener, times(1)).accept(exposureEventCaptor.capture()); final ExposureEvent capturedEvent = exposureEventCaptor.getValue(); diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalLoggingHookTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalLoggingHookTest.java index 61643e6bb61..9fa50b03fdc 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalLoggingHookTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalLoggingHookTest.java @@ -21,10 +21,13 @@ import dev.openfeature.sdk.HookContext; import dev.openfeature.sdk.ImmutableMetadata; import dev.openfeature.sdk.MutableContext; +import dev.openfeature.sdk.MutableStructure; import dev.openfeature.sdk.Reason; import dev.openfeature.sdk.Value; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; @@ -381,4 +384,46 @@ void contextAttributesAreFlattenedAndConvertedAfterEnqueue() { attrs.values().stream().noneMatch(Value.class::isInstance), "context attrs must contain converted scalar values, not OpenFeature Value wrappers"); } + + @Test + void contextAttributesUseEnqueueTimeSnapshot() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final MutableContext context = new MutableContext("user-42"); + context.add("region", "us-east-1"); + final MutableStructure profile = new MutableStructure(); + profile.add("tier", "gold"); + context.add("profile", profile); + final List cohorts = new ArrayList<>(); + cohorts.add(Value.objectToValue("beta")); + context.add("cohorts", cohorts); + + final HookContext hookCtx = + HookContext.builder() + .flagKey("ctx-flag") + .type(FlagValueType.STRING) + .defaultValue("default") + .ctx(context) + .build(); + final FlagEvaluationDetails det = + details("ctx-flag", "v", "v", Reason.TARGETING_MATCH.name(), null); + + hook.finallyAfter(hookCtx, det, Collections.emptyMap()); + context.add("region", "eu-west-1"); + context.add("late", "ignored"); + profile.add("tier", "platinum"); + profile.add("late", "ignored"); + cohorts.set(0, Value.objectToValue("ga")); + cohorts.add(Value.objectToValue("late")); + + assertNotNull(captured.get()); + final Map attrs = captured.get().contextAttributes(); + assertEquals("us-east-1", attrs.get("region")); + assertEquals("gold", attrs.get("profile.tier")); + assertEquals("beta", attrs.get("cohorts[0]")); + assertFalse(attrs.containsKey("late")); + assertFalse(attrs.containsKey("profile.late")); + assertFalse(attrs.containsKey("cohorts[1]")); + } } From 986a9af6be6f86e240b54b7c905e24fa99bf56f9 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:47:58 -0400 Subject: [PATCH 08/26] feat(openfeature): register flagevaluation hook --- .../trace/api/openfeature/Provider.java | 22 ++++- .../trace/api/openfeature/ProviderTest.java | 81 ++++++++++++++++++- 2 files changed, 97 insertions(+), 6 deletions(-) diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java index 670d4d3642d..e01a85f6fc5 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java @@ -16,6 +16,7 @@ import dev.openfeature.sdk.exceptions.OpenFeatureError; import dev.openfeature.sdk.exceptions.ProviderNotReadyError; import java.lang.reflect.Constructor; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; @@ -167,10 +168,25 @@ private Evaluator buildEvaluator() throws Exception { @Override public List getProviderHooks() { - if (flagEvalMetricsHook == null) { - return Collections.emptyList(); + final List hooks = new ArrayList<>(2); + if (flagEvalMetricsHook != null) { + hooks.add(flagEvalMetricsHook); } - return Collections.singletonList(flagEvalMetricsHook); + // EVP flagevaluation hook: always registered; no-op when writer is absent (killswitch off). + // Writer is resolved lazily from FeatureFlaggingGateway.getFlagEvalWriter() on each call. + try { + final Hook flagEvalLoggingHook = buildFlagEvalLoggingHook(); + if (flagEvalLoggingHook != null) { + hooks.add(flagEvalLoggingHook); + } + } catch (LinkageError | Exception e) { + // Keep older bootstrap/API combinations working: EVP recording is best-effort. + } + return Collections.unmodifiableList(hooks); + } + + Hook buildFlagEvalLoggingHook() { + return FlagEvalLoggingHook.INSTANCE; } @Override diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java index 06bbe7ef8d2..3fd4417d6f1 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java @@ -16,6 +16,8 @@ import static org.mockito.Mockito.when; import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; import datadog.trace.api.openfeature.Provider.Options; import dev.openfeature.sdk.Client; @@ -25,6 +27,8 @@ import dev.openfeature.sdk.Features; import dev.openfeature.sdk.FlagEvaluationDetails; import dev.openfeature.sdk.Hook; +import dev.openfeature.sdk.ImmutableMetadata; +import dev.openfeature.sdk.MutableContext; import dev.openfeature.sdk.OpenFeatureAPI; import dev.openfeature.sdk.ProviderEvaluation; import dev.openfeature.sdk.ProviderEvent; @@ -67,6 +71,7 @@ public void tearDown() { executor.shutdownNow(); OpenFeatureAPI.getInstance().shutdown(); FeatureFlaggingGateway.dispatch((ServerConfiguration) null); + FeatureFlaggingGateway.setFlagEvalWriter(null); } @Test @@ -330,10 +335,66 @@ public void testGetProviderHooksReturnsFlagEvalMetricsHook() { Provider provider = new Provider(new Options().initTimeout(10, MILLISECONDS), mock(Evaluator.class)); List hooks = provider.getProviderHooks(); + // Two hooks: OTel FlagEvalMetricsHook (index 0) + FlagEvalLoggingHook (index 1) + assertThat(hooks.size(), equalTo(2)); + assertThat(hooks.get(0) instanceof FlagEvalMetricsHook, equalTo(true)); + assertThat(hooks.get(1) instanceof FlagEvalLoggingHook, equalTo(true)); + } + + @Test + public void testGetProviderHooksSkipsFlagEvalLoggingHookOnLinkageFailure() { + Provider provider = + new Provider(new Options().initTimeout(10, MILLISECONDS), mock(Evaluator.class)) { + @Override + Hook buildFlagEvalLoggingHook() { + throw new NoClassDefFoundError("old bootstrap"); + } + }; + + List hooks = provider.getProviderHooks(); + assertThat(hooks.size(), equalTo(1)); assertThat(hooks.get(0) instanceof FlagEvalMetricsHook, equalTo(true)); } + @Test + public void testClientEvaluationRoutesThroughFlagEvalLoggingHook() throws Exception { + FeatureFlaggingGateway.dispatch(mock(ServerConfiguration.class)); + final AtomicReference captured = new AtomicReference<>(); + FeatureFlaggingGateway.setFlagEvalWriter(capturingWriter(captured)); + final Evaluator evaluator = mock(Evaluator.class); + when(evaluator.initialize(eq(10L), eq(SECONDS), any())).thenReturn(true); + when(evaluator.hasConfiguration()).thenReturn(true); + when(evaluator.evaluate(eq(String.class), eq("logged-flag"), eq("default"), any())) + .thenReturn( + ProviderEvaluation.builder() + .value("value") + .reason("STATIC") + .variant("variant-1") + .flagMetadata( + ImmutableMetadata.builder() + .addString("allocationKey", "allocation-1") + .addLong("dd.eval.timestamp_ms", 1_700_000_000_000L) + .build()) + .build()); + final OpenFeatureAPI api = OpenFeatureAPI.getInstance(); + api.setProviderAndWait(new Provider(new Options().initTimeout(10, SECONDS), evaluator)); + final MutableContext context = new MutableContext("user-1"); + context.add("region", "us-east-1"); + + final FlagEvaluationDetails details = + api.getClient().getStringDetails("logged-flag", "default", context); + + assertThat(details.getValue(), equalTo("value")); + final FlagEvalEvent event = captured.get(); + assertThat(event.flagKey, equalTo("logged-flag")); + assertThat(event.variant, equalTo("variant-1")); + assertThat(event.allocationKey, equalTo("allocation-1")); + assertThat(event.targetingKey, equalTo("user-1")); + assertThat(event.evalTimeMs, equalTo(1_700_000_000_000L)); + assertThat(event.contextAttributes().get("region"), equalTo("us-east-1")); + } + @Test public void testShutdownCleansUpMetrics() throws Exception { Evaluator evaluator = mock(Evaluator.class); @@ -343,9 +404,8 @@ public void testShutdownCleansUpMetrics() throws Exception { provider.initialize(null); provider.shutdown(); verify(evaluator).shutdown(); - // After shutdown, getProviderHooks still returns a list (hook is still present but metrics is - // shut down) - assertThat(provider.getProviderHooks().size(), equalTo(1)); + // After shutdown, getProviderHooks still returns a list with both OTel + logging hooks + assertThat(provider.getProviderHooks().size(), equalTo(2)); } public interface EvaluateMethod { @@ -394,4 +454,19 @@ private static String initializationState(final Provider provider) throws Except final AtomicReference state = (AtomicReference) stateField.get(provider); return state.get().toString(); } + + private static FlagEvaluationWriter capturingWriter(final AtomicReference ref) { + return new FlagEvaluationWriter() { + @Override + public void enqueue(final FlagEvalEvent event) { + ref.set(event); + } + + @Override + public void start() {} + + @Override + public void close() {} + }; + } } From f1c37aba169c8e82a2eb6da9381b6db7439d3341 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:48:08 -0400 Subject: [PATCH 09/26] feat(feature-flagging): canonicalize flagevaluation context --- .../featureflag/FlagEvaluationAggregator.java | 194 ++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationAggregator.java diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationAggregator.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationAggregator.java new file mode 100644 index 00000000000..89b8104fa70 --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationAggregator.java @@ -0,0 +1,194 @@ +package com.datadog.featureflag; + +import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; + +final class FlagEvaluationAggregator { + + static final int MAX_CONTEXT_FIELDS = 256; + static final int MAX_FIELD_LENGTH = 256; + + private static final byte CTX_TAG_STRING = 's'; + private static final byte CTX_TAG_BOOL = 'b'; + private static final byte CTX_TAG_INT = 'i'; + private static final byte CTX_TAG_LONG = 'l'; + private static final byte CTX_TAG_FLOAT = 'f'; + private static final byte CTX_TAG_DOUBLE = 'd'; + private static final byte CTX_TAG_OTHER = 'o'; + + private FlagEvaluationAggregator() {} + + static Map pruneContext(final Map attrs) { + if (attrs == null || attrs.isEmpty()) { + return java.util.Collections.emptyMap(); + } + final TreeMap out = new TreeMap<>(); + final TreeMap sorted = new TreeMap<>(attrs); + int count = 0; + for (final Map.Entry entry : sorted.entrySet()) { + if (count >= MAX_CONTEXT_FIELDS) { + break; + } + final Object v = entry.getValue(); + if (v instanceof String && ((String) v).length() > MAX_FIELD_LENGTH) { + continue; + } + out.put(entry.getKey(), v); + count++; + } + return out; + } + + static String canonicalContextKey(final Map prunedAttrs) { + if (prunedAttrs == null || prunedAttrs.isEmpty()) { + return ""; + } + final Map sorted = + (prunedAttrs instanceof TreeMap) ? prunedAttrs : new TreeMap<>(prunedAttrs); + final StringBuilder sb = new StringBuilder(); + for (final Map.Entry entry : sorted.entrySet()) { + appendLengthDelimited(sb, entry.getKey()); + appendContextValue(sb, entry.getValue()); + } + return sb.toString(); + } + + private static void appendLengthDelimited(final StringBuilder sb, final String s) { + sb.append(String.format("%08x", (long) s.length())); + sb.append(s); + } + + private static void appendContextValue(final StringBuilder sb, final Object v) { + if (v instanceof Boolean) { + sb.append((char) CTX_TAG_BOOL); + appendLengthDelimited(sb, v.toString()); + } else if (v instanceof Long) { + sb.append((char) CTX_TAG_LONG); + appendLengthDelimited(sb, v.toString()); + } else if (v instanceof Integer) { + sb.append((char) CTX_TAG_INT); + appendLengthDelimited(sb, v.toString()); + } else if (v instanceof Float) { + sb.append((char) CTX_TAG_FLOAT); + appendLengthDelimited(sb, v.toString()); + } else if (v instanceof Double) { + sb.append((char) CTX_TAG_DOUBLE); + appendLengthDelimited(sb, v.toString()); + } else if (v instanceof String) { + sb.append((char) CTX_TAG_STRING); + appendLengthDelimited(sb, (String) v); + } else { + sb.append((char) CTX_TAG_OTHER); + appendLengthDelimited(sb, v == null ? "" : v.toString()); + } + } + + static class EvalBucket { + long count; + long firstEvalMs; + long lastEvalMs; + boolean runtimeDefaultUsed; + String flagKey; + String variant; + String allocationKey; + String targetingKey; + String errorMessage; + Map prunedAttrs; + + EvalBucket( + final String flagKey, + final String variant, + final String allocationKey, + final String targetingKey, + final String errorMessage, + final long evalTimeMs, + final boolean runtimeDefaultUsed, + final Map prunedAttrs) { + this.flagKey = flagKey; + this.variant = variant; + this.allocationKey = allocationKey; + this.targetingKey = targetingKey; + this.errorMessage = errorMessage; + this.firstEvalMs = evalTimeMs; + this.lastEvalMs = evalTimeMs; + this.count = 1; + this.runtimeDefaultUsed = runtimeDefaultUsed; + this.prunedAttrs = prunedAttrs; + } + + int prunedContextFieldCount() { + return prunedAttrs == null ? 0 : prunedAttrs.size(); + } + + void merge(final long evalTimeMs, final boolean isDefault) { + count++; + if (evalTimeMs < firstEvalMs) { + firstEvalMs = evalTimeMs; + } + if (evalTimeMs > lastEvalMs) { + lastEvalMs = evalTimeMs; + } + if (isDefault) { + runtimeDefaultUsed = true; + } + } + } + + static final class FullKey { + private final String flagKey; + private final String variant; + private final String allocationKey; + private final boolean runtimeDefaultUsed; + private final String errorMessage; + private final String targetingKey; + private final String contextKey; + + FullKey( + final String flagKey, + final String variant, + final String allocationKey, + final boolean runtimeDefaultUsed, + final String errorMessage, + final String targetingKey, + final String contextKey) { + this.flagKey = flagKey; + this.variant = variant; + this.allocationKey = allocationKey; + this.runtimeDefaultUsed = runtimeDefaultUsed; + this.errorMessage = errorMessage; + this.targetingKey = targetingKey; + this.contextKey = contextKey; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof FullKey)) { + return false; + } + final FullKey fullKey = (FullKey) o; + return runtimeDefaultUsed == fullKey.runtimeDefaultUsed + && Objects.equals(flagKey, fullKey.flagKey) + && Objects.equals(variant, fullKey.variant) + && Objects.equals(allocationKey, fullKey.allocationKey) + && Objects.equals(errorMessage, fullKey.errorMessage) + && Objects.equals(targetingKey, fullKey.targetingKey) + && Objects.equals(contextKey, fullKey.contextKey); + } + + @Override + public int hashCode() { + return Objects.hash( + flagKey, + variant, + allocationKey, + runtimeDefaultUsed, + errorMessage, + targetingKey, + contextKey); + } + } +} From 115a407b76a5f57c48a23ae603fe65c20b98de05 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:48:08 -0400 Subject: [PATCH 10/26] feat(feature-flagging): aggregate flagevaluation rows --- .../featureflag/FlagEvaluationAggregator.java | 235 +++++++++++++++++- 1 file changed, 234 insertions(+), 1 deletion(-) diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationAggregator.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationAggregator.java index 89b8104fa70..eb15eb884a5 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationAggregator.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationAggregator.java @@ -1,11 +1,29 @@ package com.datadog.featureflag; +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicLong; final class FlagEvaluationAggregator { + static final int EVAL_SCALE_TARGET_FLAGS = 2_500; + static final int EVAL_SCALE_FULL_BUCKETS_PER_FLAG = 50; + static final int EVAL_SCALE_USERS_PER_FLAG = 1_000; + static final int EVAL_SCALE_PER_FLAG_HEADROOM_MULTIPLIER = 10; + static final int EVAL_SCALE_DEGRADED_BUCKETS_PER_FLAG = 10; + static final int EVAL_SCALE_FULL_BUCKET_TARGET = + EVAL_SCALE_TARGET_FLAGS * EVAL_SCALE_FULL_BUCKETS_PER_FLAG; + static final int EVAL_SCALE_PER_FLAG_BUCKET_TARGET = + EVAL_SCALE_PER_FLAG_HEADROOM_MULTIPLIER * EVAL_SCALE_USERS_PER_FLAG; + static final int EVAL_SCALE_DEGRADED_BUCKET_TARGET = + EVAL_SCALE_TARGET_FLAGS * EVAL_SCALE_DEGRADED_BUCKETS_PER_FLAG; + static final int GLOBAL_CAP = 131_072; + static final int PER_FLAG_CAP = EVAL_SCALE_PER_FLAG_BUCKET_TARGET; + static final int DEGRADED_CAP = 32_768; + static final int MAX_CONTEXT_FIELDS = 256; static final int MAX_FIELD_LENGTH = 256; @@ -17,7 +35,165 @@ final class FlagEvaluationAggregator { private static final byte CTX_TAG_DOUBLE = 'd'; private static final byte CTX_TAG_OTHER = 'o'; - private FlagEvaluationAggregator() {} + final Map fullTier = new HashMap<>(); + final Map degradedTier = new HashMap<>(); + final Map perFlagCount = new HashMap<>(); + final AtomicLong droppedDegradedOverflow = new AtomicLong(0); + int globalFullCount = 0; + + void aggregate(final FlagEvalEvent event) { + final boolean isDefault = event.variant == null; + final Map prunedAttrs = pruneContext(event.contextAttributes()); + final String ctxKey = canonicalContextKey(prunedAttrs); + final FullKey fullKey = buildFullKey(event, ctxKey); + + EvalBucket bucket = fullTier.get(fullKey); + if (bucket != null) { + bucket.merge(event.evalTimeMs, isDefault); + return; + } + + final int flagCount = perFlagCount.getOrDefault(event.flagKey, 0); + if (globalFullCount < GLOBAL_CAP && flagCount < PER_FLAG_CAP) { + fullTier.put( + fullKey, + new EvalBucket( + event.flagKey, + event.variant, + event.allocationKey, + event.targetingKey, + event.errorMessage, + event.evalTimeMs, + isDefault, + prunedAttrs)); + globalFullCount++; + perFlagCount.put(event.flagKey, flagCount + 1); + return; + } + + final DegradedKey degradedKey = buildDegradedKey(event); + bucket = degradedTier.get(degradedKey); + if (bucket != null) { + bucket.merge(event.evalTimeMs, isDefault); + return; + } + + if (degradedTier.size() < DEGRADED_CAP) { + degradedTier.put( + degradedKey, + new EvalBucket( + event.flagKey, + event.variant, + event.allocationKey, + null, + event.errorMessage, + event.evalTimeMs, + isDefault, + null)); + return; + } + + droppedDegradedOverflow.incrementAndGet(); + } + + boolean isEmpty() { + return fullTier.isEmpty() && degradedTier.isEmpty(); + } + + int fullTierSize() { + return fullTier.size(); + } + + long degradedEvaluationCount() { + long count = 0; + for (final EvalBucket bucket : degradedTier.values()) { + count += bucket.count; + } + return count; + } + + int bucketCount() { + return fullTier.size() + degradedTier.size(); + } + + Iterable fullBuckets() { + return fullTier.values(); + } + + Iterable degradedBuckets() { + return degradedTier.values(); + } + + void clear() { + fullTier.clear(); + degradedTier.clear(); + perFlagCount.clear(); + globalFullCount = 0; + } + + AggregatedState snapshot() { + return new AggregatedState( + new HashMap<>(fullTier), new HashMap<>(degradedTier), droppedDegradedOverflow.get()); + } + + void simulateFullTierAtCap() { + for (int i = globalFullCount; i < GLOBAL_CAP; i++) { + final String key = "synthetic-full-" + i; + fullTier.put( + new FullKey(key, "on", "alloc", false, null, null, ""), + new EvalBucket(key, "on", "alloc", null, null, 1L, false, null)); + globalFullCount++; + perFlagCount.merge(key, 1, Integer::sum); + } + } + + void simulateDegradedTierAtCap() { + for (int i = degradedTier.size(); i < DEGRADED_CAP; i++) { + final String key = "synthetic-dg-" + i; + degradedTier.put( + new DegradedKey(key, "on", "alloc", false, null), + new EvalBucket(key, "on", "alloc", null, null, 1L, false, null)); + } + } + + void addDegradedBucketForTest( + final String flagKey, + final String variant, + final String allocationKey, + final String errorMessage, + final long evalTimeMs) { + degradedTier.put( + new DegradedKey(flagKey, variant, allocationKey, variant == null, errorMessage), + new EvalBucket( + flagKey, + variant, + allocationKey, + null, + errorMessage, + evalTimeMs, + variant == null, + null)); + } + + private static FullKey buildFullKey(final FlagEvalEvent event, final String ctxKey) { + return new FullKey( + event.flagKey, + event.variant, + event.allocationKey, + event.variant == null, + event.errorMessage, + event.targetingKey, + ctxKey); + } + + private static DegradedKey buildDegradedKey(final FlagEvalEvent event) { + return new DegradedKey( + event.flagKey, + event.variant, + event.allocationKey, + event.variant == null, + event.errorMessage); + } static Map pruneContext(final Map attrs) { if (attrs == null || attrs.isEmpty()) { @@ -191,4 +367,61 @@ public int hashCode() { contextKey); } } + + static final class DegradedKey { + private final String flagKey; + private final String variant; + private final String allocationKey; + private final boolean runtimeDefaultUsed; + private final String errorMessage; + + DegradedKey( + final String flagKey, + final String variant, + final String allocationKey, + final boolean runtimeDefaultUsed, + final String errorMessage) { + this.flagKey = flagKey; + this.variant = variant; + this.allocationKey = allocationKey; + this.runtimeDefaultUsed = runtimeDefaultUsed; + this.errorMessage = errorMessage; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof DegradedKey)) { + return false; + } + final DegradedKey that = (DegradedKey) o; + return runtimeDefaultUsed == that.runtimeDefaultUsed + && Objects.equals(flagKey, that.flagKey) + && Objects.equals(variant, that.variant) + && Objects.equals(allocationKey, that.allocationKey) + && Objects.equals(errorMessage, that.errorMessage); + } + + @Override + public int hashCode() { + return Objects.hash(flagKey, variant, allocationKey, runtimeDefaultUsed, errorMessage); + } + } + + static class AggregatedState { + final Map fullTier; + final Map degradedTier; + final long droppedDegradedOverflow; + + AggregatedState( + final Map fullTier, + final Map degradedTier, + final long droppedDegradedOverflow) { + this.fullTier = fullTier; + this.degradedTier = degradedTier; + this.droppedDegradedOverflow = droppedDegradedOverflow; + } + } } From cc3b21c0c2c03aeaec09b82d0a12fb6b7e550e97 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:48:09 -0400 Subject: [PATCH 11/26] test(feature-flagging): cover flagevaluation aggregation --- .../FlagEvaluationAggregatorTest.java | 183 ++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationAggregatorTest.java diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationAggregatorTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationAggregatorTest.java new file mode 100644 index 00000000000..0adfcf35d3e --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationAggregatorTest.java @@ -0,0 +1,183 @@ +package com.datadog.featureflag; + +import static java.util.Collections.emptyMap; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class FlagEvaluationAggregatorTest { + + @Test + void identicalEventsAggregateIntoOneBucketWithCount2() { + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + + aggregator.aggregate(event("flag-a", "on", "alloc1", "user-1", 1000L, emptyMap())); + aggregator.aggregate(event("flag-a", "on", "alloc1", "user-1", 2000L, emptyMap())); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + assertEquals(1, state.fullTier.size()); + final FlagEvaluationAggregator.EvalBucket bucket = state.fullTier.values().iterator().next(); + assertEquals(2, bucket.count); + assertEquals(1000L, bucket.firstEvalMs); + assertEquals(2000L, bucket.lastEvalMs); + assertTrue(bucket.firstEvalMs <= bucket.lastEvalMs); + } + + @Test + void differentValueTypesProduceDifferentBuckets() { + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + final Map attrsInt = new HashMap<>(); + attrsInt.put("score", 1); + final Map attrsStr = new HashMap<>(); + attrsStr.put("score", "1"); + + aggregator.aggregate(event("flag-b", "on", "alloc1", "user-1", 1000L, attrsInt)); + aggregator.aggregate(event("flag-b", "on", "alloc1", "user-1", 1000L, attrsStr)); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + assertEquals(2, state.fullTier.size()); + } + + @Test + void nulCharactersInKeyFieldsDoNotCollide() { + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + + aggregator.aggregate(event("a\0b", "c", "alloc1", "user-1", 1000L, emptyMap())); + aggregator.aggregate(event("a", "b\0c", "alloc1", "user-1", 1000L, emptyMap())); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + assertEquals(2, state.fullTier.size()); + } + + @Test + void globalCapOverflowRoutesToDegradedTier() { + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + + aggregator.simulateFullTierAtCap(); + aggregator.aggregate(simpleEvent("extra-flag", "on")); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + assertTrue(state.degradedTier.size() > 0); + assertEquals(0, state.droppedDegradedOverflow); + } + + @Test + void degradedCapOverflowIncrementsDroppedCounter() { + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + + aggregator.simulateFullTierAtCap(); + aggregator.simulateDegradedTierAtCap(); + aggregator.aggregate(simpleEvent("drop-flag", "on")); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + assertTrue(state.droppedDegradedOverflow > 0); + } + + @Test + void absentVariantSetsRuntimeDefaultUsed() { + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + + aggregator.aggregate(event("flag-c", null, "alloc1", "user-1", 1000L, emptyMap())); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + assertEquals(1, state.fullTier.size()); + final FlagEvaluationAggregator.EvalBucket bucket = state.fullTier.values().iterator().next(); + assertTrue(bucket.runtimeDefaultUsed); + } + + @Test + void contextExceeding256FieldsIsPrunedToStoredPrunedAttrs() { + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + final Map hugeAttrs = new HashMap<>(); + for (int i = 0; i < 300; i++) { + hugeAttrs.put("key" + i, "v" + i); + } + + aggregator.aggregate(event("flag-d", "on", "alloc1", "user-1", 1000L, hugeAttrs)); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + final FlagEvaluationAggregator.EvalBucket bucket = state.fullTier.values().iterator().next(); + assertEquals(256, bucket.prunedContextFieldCount()); + assertEquals(256, bucket.prunedAttrs.size()); + } + + @Test + void pruningIsDeterministicSortBeforeCut() { + final Map attrs = new HashMap<>(); + for (int i = 0; i < 300; i++) { + attrs.put(String.format("k%03d", i), "v" + i); + } + + final Map p1 = FlagEvaluationAggregator.pruneContext(attrs); + final Map p2 = FlagEvaluationAggregator.pruneContext(new HashMap<>(attrs)); + + assertEquals(256, p1.size()); + assertEquals(p1.keySet(), p2.keySet()); + assertTrue(p1.containsKey("k000")); + assertTrue(p1.containsKey("k255")); + assertFalse(p1.containsKey("k256")); + assertFalse(p1.containsKey("k299")); + } + + @Test + void contextValueExceeding256CharsIsSkippedFromPrunedAttrs() { + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + final Map attrs = new HashMap<>(); + attrs.put("long-val", repeat('x', 300)); + attrs.put("short-val", "ok"); + + aggregator.aggregate(event("flag-e", "on", "alloc1", "user-1", 1000L, attrs)); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + final FlagEvaluationAggregator.EvalBucket bucket = state.fullTier.values().iterator().next(); + assertFalse(bucket.prunedAttrs.containsKey("long-val")); + assertTrue(bucket.prunedAttrs.containsKey("short-val")); + } + + @Test + void capSizingUsesNamedScaleConstants() { + assertEquals(125_000, FlagEvaluationAggregator.EVAL_SCALE_FULL_BUCKET_TARGET); + assertEquals(10_000, FlagEvaluationAggregator.EVAL_SCALE_PER_FLAG_BUCKET_TARGET); + assertEquals(25_000, FlagEvaluationAggregator.EVAL_SCALE_DEGRADED_BUCKET_TARGET); + assertEquals(131_072, FlagEvaluationAggregator.GLOBAL_CAP); + assertEquals(10_000, FlagEvaluationAggregator.PER_FLAG_CAP); + assertEquals(32_768, FlagEvaluationAggregator.DEGRADED_CAP); + } + + @Test + void flagEvalEventDoesNotCarryReason() { + final boolean hasReasonField = + Arrays.stream( + datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent.class + .getDeclaredFields()) + .anyMatch(field -> field.getName().equals("reason")); + + assertFalse(hasReasonField); + } + + private static FlagEvalEvent event( + final String flagKey, + final String variant, + final String allocationKey, + final String targetingKey, + final long evalTimeMs, + final Map attrs) { + return new FlagEvalEvent(flagKey, variant, allocationKey, targetingKey, evalTimeMs, attrs); + } + + private static FlagEvalEvent simpleEvent(final String flagKey, final String variant) { + return event(flagKey, variant, "alloc1", "user-1", 1000L, emptyMap()); + } + + private static String repeat(final char c, final int count) { + final char[] chars = new char[count]; + Arrays.fill(chars, c); + return new String(chars); + } +} From 31d7cb51169829fbebf79b6721db1c8efd97d85b Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:48:09 -0400 Subject: [PATCH 12/26] feat(feature-flagging): encode flagevaluation payloads --- .../featureflag/FlagEvaluationPayloads.java | 270 ++++++++++++++++++ 1 file changed, 270 insertions(+) create mode 100644 products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationPayloads.java diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationPayloads.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationPayloads.java new file mode 100644 index 00000000000..c2b3cbfb517 --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationPayloads.java @@ -0,0 +1,270 @@ +package com.datadog.featureflag; + +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.Moshi; +import com.squareup.moshi.Types; +import java.io.ByteArrayOutputStream; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +final class FlagEvaluationPayloads { + + private static final byte[] PAYLOAD_SUFFIX = FeatureFlagEvpPublisher.utf8Bytes("]}"); + private static final byte[] JSON_COMMA = FeatureFlagEvpPublisher.utf8Bytes(","); + private static final JsonAdapter EVENT_JSON_ADAPTER; + private static final JsonAdapter> CONTEXT_JSON_ADAPTER; + + static { + final Moshi moshi = new Moshi.Builder().build(); + EVENT_JSON_ADAPTER = moshi.adapter(FlagEvaluationEvent.class); + final Type contextType = Types.newParameterizedType(Map.class, String.class, String.class); + CONTEXT_JSON_ADAPTER = moshi.adapter(contextType); + } + + private FlagEvaluationPayloads() {} + + static class FlagEvaluationsRequest { + public final Map context; + public final List flagEvaluations; + + FlagEvaluationsRequest( + final Map context, final List flagEvaluations) { + this.context = context; + this.flagEvaluations = flagEvaluations; + } + } + + static EncodedPayloads buildPayloads( + final List events, + final Map context, + final int payloadSizeLimitBytes) { + final byte[] prefix = payloadPrefix(context); + EncodedPayloadBuilder current = new EncodedPayloadBuilder(prefix); + final List payloads = new ArrayList<>(); + long droppedPayloadLimit = 0; + long degradedPayloadLimit = 0; + + for (final FlagEvaluationEvent event : events) { + byte[] eventBytes = encodeEvent(event); + + if (!current.canAdd(eventBytes, payloadSizeLimitBytes) && !current.isEmpty()) { + payloads.add(current.toByteArray()); + current = new EncodedPayloadBuilder(prefix); + } + + if (current.canAdd(eventBytes, payloadSizeLimitBytes)) { + current.add(eventBytes); + continue; + } + + final FlagEvaluationEvent degraded = event.withoutTargetingKeyAndContext(); + if (degraded != null) { + eventBytes = encodeEvent(degraded); + if (!current.canAdd(eventBytes, payloadSizeLimitBytes) && !current.isEmpty()) { + payloads.add(current.toByteArray()); + current = new EncodedPayloadBuilder(prefix); + } + if (current.canAdd(eventBytes, payloadSizeLimitBytes)) { + current.add(eventBytes); + degradedPayloadLimit += event.evaluation_count; + continue; + } + } + + droppedPayloadLimit += event.evaluation_count; + } + + if (!current.isEmpty()) { + payloads.add(current.toByteArray()); + } + return new EncodedPayloads(payloads, droppedPayloadLimit, degradedPayloadLimit); + } + + private static byte[] payloadPrefix(final Map context) { + return FeatureFlagEvpPublisher.utf8Bytes( + "{\"context\":" + CONTEXT_JSON_ADAPTER.toJson(context) + ",\"flagEvaluations\":["); + } + + private static byte[] encodeEvent(final FlagEvaluationEvent event) { + return FeatureFlagEvpPublisher.utf8Bytes(EVENT_JSON_ADAPTER.toJson(event)); + } + + static final class EncodedPayloads { + final List bodies; + final long droppedPayloadLimit; + final long degradedPayloadLimit; + + private EncodedPayloads( + final List bodies, + final long droppedPayloadLimit, + final long degradedPayloadLimit) { + this.bodies = bodies; + this.droppedPayloadLimit = droppedPayloadLimit; + this.degradedPayloadLimit = degradedPayloadLimit; + } + } + + private static final class EncodedPayloadBuilder { + private final byte[] prefix; + private final List events = new ArrayList<>(); + private int eventBytes; + + private EncodedPayloadBuilder(final byte[] prefix) { + this.prefix = prefix; + } + + private boolean isEmpty() { + return events.isEmpty(); + } + + private boolean canAdd(final byte[] event, final int payloadSizeLimitBytes) { + return sizeWith(event) <= payloadSizeLimitBytes; + } + + private int sizeWith(final byte[] event) { + return prefix.length + PAYLOAD_SUFFIX.length + eventBytes + event.length + events.size(); + } + + private void add(final byte[] event) { + events.add(event); + eventBytes += event.length; + } + + private byte[] toByteArray() { + final int size = prefix.length + PAYLOAD_SUFFIX.length + eventBytes; + final ByteArrayOutputStream out = new ByteArrayOutputStream(size + events.size()); + out.write(prefix, 0, prefix.length); + for (int i = 0; i < events.size(); i++) { + if (i > 0) { + out.write(JSON_COMMA, 0, JSON_COMMA.length); + } + final byte[] event = events.get(i); + out.write(event, 0, event.length); + } + out.write(PAYLOAD_SUFFIX, 0, PAYLOAD_SUFFIX.length); + return out.toByteArray(); + } + } + + static class FlagEvaluationEvent { + public final long timestamp; + public final FlagKeyObject flag; + public final long first_evaluation; + public final long last_evaluation; + public final long evaluation_count; + public final KeyObject variant; + public final KeyObject allocation; + public final String targeting_key; + public final Boolean runtime_default_used; + public final EventContext context; + public final ErrorObject error; + + FlagEvaluationEvent( + final long timestamp, + final String flagKey, + final long firstEvalMs, + final long lastEvalMs, + final long count, + final String variant, + final String allocation, + final String targetingKey, + final boolean runtimeDefaultUsed, + final String errorMessage, + final Map evaluationAttrs) { + this.timestamp = timestamp; + this.flag = new FlagKeyObject(flagKey); + this.first_evaluation = firstEvalMs; + this.last_evaluation = lastEvalMs; + this.evaluation_count = count; + this.variant = (variant != null && !variant.isEmpty()) ? new KeyObject(variant) : null; + this.allocation = + (allocation != null && !allocation.isEmpty()) ? new KeyObject(allocation) : null; + this.targeting_key = targetingKey; + this.runtime_default_used = runtimeDefaultUsed ? Boolean.TRUE : null; + this.context = + (evaluationAttrs != null && !evaluationAttrs.isEmpty()) + ? new EventContext(evaluationAttrs) + : null; + this.error = + (errorMessage != null && !errorMessage.isEmpty()) ? new ErrorObject(errorMessage) : null; + } + + static FlagEvaluationEvent fromBucket( + final FlagEvaluationAggregator.EvalBucket bucket, + final boolean isFullTier, + final long flushTimeMs) { + return new FlagEvaluationEvent( + flushTimeMs, + bucket.flagKey, + bucket.firstEvalMs, + bucket.lastEvalMs, + bucket.count, + bucket.variant, + bucket.allocationKey, + isFullTier ? bucket.targetingKey : null, + bucket.runtimeDefaultUsed, + bucket.errorMessage, + isFullTier ? bucket.prunedAttrs : null); + } + + FlagEvaluationEvent withoutTargetingKeyAndContext() { + if (targeting_key == null && context == null) { + return null; + } + return new FlagEvaluationEvent( + timestamp, + flag.key, + first_evaluation, + last_evaluation, + evaluation_count, + keyOf(variant), + keyOf(allocation), + null, + Boolean.TRUE.equals(runtime_default_used), + messageOf(error), + null); + } + } + + private static String keyOf(final KeyObject object) { + return object == null ? null : object.key; + } + + private static String messageOf(final ErrorObject object) { + return object == null ? null : object.message; + } + + static class KeyObject { + public final String key; + + KeyObject(final String key) { + this.key = key; + } + } + + static class FlagKeyObject { + public final String key; + + FlagKeyObject(final String key) { + this.key = key; + } + } + + static class ErrorObject { + public final String message; + + ErrorObject(final String message) { + this.message = message; + } + } + + static class EventContext { + public final Map evaluation; + + EventContext(final Map evaluation) { + this.evaluation = evaluation; + } + } +} From e43f93acd0506c0b22dfef39467776d0e5a835fb Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:48:10 -0400 Subject: [PATCH 13/26] test(feature-flagging): cover flagevaluation payload encoding --- .../FlagEvaluationPayloadsTest.java | 251 ++++++++++++++++++ 1 file changed, 251 insertions(+) create mode 100644 products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java new file mode 100644 index 00000000000..3ba2dc8fccf --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java @@ -0,0 +1,251 @@ +package com.datadog.featureflag; + +import static java.util.Collections.emptyMap; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.Moshi; +import com.squareup.moshi.Types; +import java.lang.reflect.Type; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class FlagEvaluationPayloadsTest { + + private static final long EVAL_MS = 1_760_000_000_000L; + private static final Map CONTEXT = context(); + private static final JsonAdapter> JSON_MAP; + + static { + final Moshi moshi = new Moshi.Builder().build(); + final Type type = Types.newParameterizedType(Map.class, String.class, Object.class); + JSON_MAP = moshi.adapter(type); + } + + @Test + void fullTierPayloadUsesWorkerWireShape() throws Exception { + final Map attrs = new HashMap<>(); + attrs.put("region", "us-east-1"); + + final Map json = + firstPayload( + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList( + event("my-flag", "on", "alloc-x", "user-1", 1, attrs)), + CONTEXT, + 1_000_000)); + + final Map ev = firstEvent(json); + assertObjectWithKey(ev.get("variant"), "on"); + assertObjectWithKey(ev.get("allocation"), "alloc-x"); + assertObjectWithKey(ev.get("flag"), "my-flag"); + final Map ctx = (Map) ev.get("context"); + assertNotNull(ctx); + final Map evalAttrs = (Map) ctx.get("evaluation"); + assertNotNull(evalAttrs); + assertEquals("us-east-1", evalAttrs.get("region")); + assertFalse(ev.containsKey("reason")); + } + + @Test + void eventFromFullBucketUsesFlushTimeAndEvaluationBounds() throws Exception { + final FlagEvaluationAggregator.EvalBucket bucket = + new FlagEvaluationAggregator.EvalBucket( + "ts-flag", "on", "alloc1", "user-1", null, EVAL_MS, false, emptyMap()); + bucket.merge(EVAL_MS + 10, false); + final long flushTimeMs = EVAL_MS + 5_000; + + final Map json = + firstPayload( + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList( + FlagEvaluationPayloads.FlagEvaluationEvent.fromBucket( + bucket, true, flushTimeMs)), + CONTEXT, + 1_000_000)); + + final Map ev = firstEvent(json); + assertEquals((double) flushTimeMs, ((Number) ev.get("timestamp")).doubleValue()); + assertEquals((double) EVAL_MS, ((Number) ev.get("first_evaluation")).doubleValue()); + assertEquals((double) (EVAL_MS + 10), ((Number) ev.get("last_evaluation")).doubleValue()); + assertEquals(2.0, ((Number) ev.get("evaluation_count")).doubleValue()); + } + + @Test + void degradedTierEventOmitsTargetingKeyAndContext() throws Exception { + final FlagEvaluationAggregator.EvalBucket bucket = + new FlagEvaluationAggregator.EvalBucket( + "dg-flag", "on", "alloc1", null, null, EVAL_MS, false, null); + + final Map json = + firstPayload( + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList( + FlagEvaluationPayloads.FlagEvaluationEvent.fromBucket(bucket, false, EVAL_MS)), + CONTEXT, + 1_000_000)); + + final Map ev = firstEvent(json); + assertNull(ev.get("targeting_key")); + assertNull(ev.get("context")); + } + + @Test + void splitPayloadsByEncodedSize() throws Exception { + final Map attrs = new HashMap<>(); + attrs.put("payload", repeat('x', 180)); + final java.util.ArrayList events = + new java.util.ArrayList<>(); + for (int i = 0; i < 4; i++) { + events.add(event("split-flag-" + i, "on", "alloc1", "user-" + i, 1, attrs)); + } + + final FlagEvaluationPayloads.EncodedPayloads payloads = + FlagEvaluationPayloads.buildPayloads(events, CONTEXT, 1_100); + + assertTrue(payloads.bodies.size() > 1); + int eventCount = 0; + for (final byte[] body : payloads.bodies) { + assertTrue(body.length <= 1_100); + eventCount += eventCount(parse(body)); + } + assertEquals(4, eventCount); + assertEquals(0, payloads.droppedPayloadLimit); + assertEquals(0, payloads.degradedPayloadLimit); + } + + @Test + void oversizedFullPayloadRowIsDegradedBeforeDrop() throws Exception { + final Map attrs = new HashMap<>(); + for (int i = 0; i < 4; i++) { + attrs.put("payload-" + i, repeat('x', 200)); + } + + final FlagEvaluationPayloads.EncodedPayloads payloads = + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList( + event("oversized-full", "on", "alloc1", "user-1", 2, attrs)), + CONTEXT, + 512); + + assertEquals(1, payloads.bodies.size()); + assertTrue(payloads.bodies.get(0).length <= 512); + assertEquals(0, payloads.droppedPayloadLimit); + assertEquals(2, payloads.degradedPayloadLimit); + final Map ev = firstEvent(parse(payloads.bodies.get(0))); + assertEquals(2.0, ((Number) ev.get("evaluation_count")).doubleValue()); + assertNull(ev.get("targeting_key")); + assertNull(ev.get("context")); + } + + @Test + void oversizedDegradedPayloadRowIsDropped() { + final FlagEvaluationPayloads.EncodedPayloads payloads = + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList( + event(repeat('f', 512), "on", "alloc1", null, 2, emptyMap())), + CONTEXT, + 128); + + assertTrue(payloads.bodies.isEmpty()); + assertEquals(2, payloads.droppedPayloadLimit); + assertEquals(0, payloads.degradedPayloadLimit); + } + + @Test + void errorPayloadSerializesErrorObject() throws Exception { + final Map json = + firstPayload( + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList( + new FlagEvaluationPayloads.FlagEvaluationEvent( + EVAL_MS, + "err-flag", + EVAL_MS, + EVAL_MS, + 1, + null, + null, + null, + true, + "type mismatch", + null)), + CONTEXT, + 1_000_000)); + + final Map ev = firstEvent(json); + final Map error = (Map) ev.get("error"); + assertNotNull(error); + assertEquals("type mismatch", error.get("message")); + assertEquals(Boolean.TRUE, ev.get("runtime_default_used")); + } + + private static FlagEvaluationPayloads.FlagEvaluationEvent event( + final String flagKey, + final String variant, + final String allocation, + final String targetingKey, + final long count, + final Map attrs) { + return new FlagEvaluationPayloads.FlagEvaluationEvent( + EVAL_MS, + flagKey, + EVAL_MS, + EVAL_MS, + count, + variant, + allocation, + targetingKey, + false, + null, + attrs); + } + + private static Map firstPayload( + final FlagEvaluationPayloads.EncodedPayloads payloads) throws Exception { + assertEquals(1, payloads.bodies.size()); + return parse(payloads.bodies.get(0)); + } + + private static Map parse(final byte[] body) throws Exception { + return JSON_MAP.fromJson(new String(body, java.nio.charset.StandardCharsets.UTF_8)); + } + + @SuppressWarnings("unchecked") + private static Map firstEvent(final Map batch) { + final List events = (List) batch.get("flagEvaluations"); + assertNotNull(events); + assertFalse(events.isEmpty()); + return (Map) events.get(0); + } + + @SuppressWarnings("unchecked") + private static int eventCount(final Map batch) { + final List events = (List) batch.get("flagEvaluations"); + assertNotNull(events); + return events.size(); + } + + private static void assertObjectWithKey(final Object object, final String expectedKey) { + assertTrue(object instanceof Map); + assertEquals(expectedKey, ((Map) object).get("key")); + } + + private static String repeat(final char c, final int count) { + final char[] chars = new char[count]; + java.util.Arrays.fill(chars, c); + return new String(chars); + } + + private static Map context() { + final Map context = new HashMap<>(); + context.put("service", "test-service"); + return context; + } +} From 17d7785420fc08c081558b55a4161d179606bdb7 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:48:10 -0400 Subject: [PATCH 14/26] feat(telemetry): support tagged core metric counts --- .../api/telemetry/CoreMetricCollector.java | 8 ++++++++ .../telemetry/CoreMetricCollectorTest.groovy | 20 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/internal-api/src/main/java/datadog/trace/api/telemetry/CoreMetricCollector.java b/internal-api/src/main/java/datadog/trace/api/telemetry/CoreMetricCollector.java index d33fcf1529d..d09dc2f2ac0 100644 --- a/internal-api/src/main/java/datadog/trace/api/telemetry/CoreMetricCollector.java +++ b/internal-api/src/main/java/datadog/trace/api/telemetry/CoreMetricCollector.java @@ -29,6 +29,14 @@ private CoreMetricCollector() { this.metricsQueue = new ArrayBlockingQueue<>(RAW_QUEUE_SIZE); } + public void count(String metricName, long value, String tag) { + if (value <= 0) { + return; + } + this.metricsQueue.offer( + new CoreMetric(METRIC_NAMESPACE, true, metricName, "count", value, tag)); + } + @Override public void prepareMetrics() { // Collect span metrics diff --git a/internal-api/src/test/groovy/datadog/trace/api/telemetry/CoreMetricCollectorTest.groovy b/internal-api/src/test/groovy/datadog/trace/api/telemetry/CoreMetricCollectorTest.groovy index 205da2bd0f5..5d0b920bfb9 100644 --- a/internal-api/src/test/groovy/datadog/trace/api/telemetry/CoreMetricCollectorTest.groovy +++ b/internal-api/src/test/groovy/datadog/trace/api/telemetry/CoreMetricCollectorTest.groovy @@ -53,4 +53,24 @@ class CoreMetricCollectorTest extends DDSpecification { collector.prepareMetrics() collector.drain().size() == limit } + + def "direct count core metric"() { + setup: + def collector = CoreMetricCollector.getInstance() + collector.drain() + + when: + collector.count('flagevaluation.rows.dropped', 3, 'reason:queue_overflow') + def metrics = collector.drain() + + then: + metrics.size() == 1 + + def metric = metrics[0] + metric.type == 'count' + metric.value == 3 + metric.namespace == 'tracers' + metric.metricName == 'flagevaluation.rows.dropped' + metric.tags == ['reason:queue_overflow'] + } } From 5f88cadeb295fb2448905f735e93da6c6c1c4837 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:49:05 -0400 Subject: [PATCH 15/26] feat(feature-flagging): run flagevaluation writer lifecycle --- .../trace/util/AgentThreadFactory.java | 3 +- .../featureflag/FlagEvaluationWriterImpl.java | 294 ++++++++++++++++++ 2 files changed, 296 insertions(+), 1 deletion(-) create mode 100644 products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java diff --git a/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java b/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java index 752adb8899d..3269602a84c 100644 --- a/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java +++ b/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java @@ -66,7 +66,8 @@ public enum AgentThread { LLMOBS_EVALS_PROCESSOR("dd-llmobs-evals-processor"), - FEATURE_FLAG_EXPOSURE_PROCESSOR("dd-ffe-exposure-processor"); + FEATURE_FLAG_EXPOSURE_PROCESSOR("dd-ffe-exposure-processor"), + FEATURE_FLAG_EVALUATION_PROCESSOR("dd-ffe-evaluation-processor"); public final String threadName; diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java new file mode 100644 index 00000000000..68068d1ea4c --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java @@ -0,0 +1,294 @@ +package com.datadog.featureflag; + +import static datadog.trace.util.AgentThreadFactory.AgentThread.FEATURE_FLAG_EVALUATION_PROCESSOR; +import static datadog.trace.util.AgentThreadFactory.newAgentThread; +import static java.util.concurrent.TimeUnit.SECONDS; + +import datadog.common.queue.MessagePassingBlockingQueue; +import datadog.common.queue.Queues; +import datadog.communication.BackendApiFactory; +import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.trace.api.Config; +import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; +import datadog.trace.api.telemetry.CoreMetricCollector; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * EVP {@code flagevaluation} writer for Java. + * + *

Owns the bounded hook hand-off queue, background aggregation loop, and writer lifecycle. The + * EVP payload/posting path is layered on top of this queue in the next review step. + */ +public class FlagEvaluationWriterImpl implements FlagEvaluationWriter { + + private static final Logger LOGGER = LoggerFactory.getLogger(FlagEvaluationWriterImpl.class); + + static final int DEFAULT_CAPACITY = 1 << 16; + static final int FLUSH_INTERVAL_SECONDS = 10; + static final int EVAL_SCALE_FULL_BUCKET_TARGET = + FlagEvaluationAggregator.EVAL_SCALE_FULL_BUCKET_TARGET; + static final int EVAL_SCALE_PER_FLAG_BUCKET_TARGET = + FlagEvaluationAggregator.EVAL_SCALE_PER_FLAG_BUCKET_TARGET; + static final int EVAL_SCALE_DEGRADED_BUCKET_TARGET = + FlagEvaluationAggregator.EVAL_SCALE_DEGRADED_BUCKET_TARGET; + static final int GLOBAL_CAP = FlagEvaluationAggregator.GLOBAL_CAP; + static final int PER_FLAG_CAP = FlagEvaluationAggregator.PER_FLAG_CAP; + static final int DEGRADED_CAP = FlagEvaluationAggregator.DEGRADED_CAP; + static final String FLAG_EVALUATION_DROPPED_METRIC = "flagevaluation.rows.dropped"; + static final String FLAG_EVALUATION_DEGRADED_METRIC = "flagevaluation.rows.degraded"; + static final String DROP_REASON_QUEUE_OVERFLOW = "queue_overflow"; + static final String DROP_REASON_DEGRADED_CAP = "degraded_cap"; + static final String DROP_REASON_CLOSED = "closed"; + static final String DROP_REASON_CONTEXT_ERROR = "context_error"; + static final String DEGRADED_REASON_CARDINALITY_CAP = "cardinality_cap"; + private static final CoreMetricCollector CORE_METRICS = CoreMetricCollector.getInstance(); + + static final int MAX_CONTEXT_FIELDS = FlagEvaluationAggregator.MAX_CONTEXT_FIELDS; + static final int MAX_FIELD_LENGTH = FlagEvaluationAggregator.MAX_FIELD_LENGTH; + + private final MessagePassingBlockingQueue queue; + private final FlagEvaluationSerializingHandler serializer; + private final Thread serializerThread; + private final Object lifecycleLock = new Object(); + private final AtomicBoolean closed = new AtomicBoolean(false); + + private final AtomicLong droppedQueueOverflow = new AtomicLong(0); + + public FlagEvaluationWriterImpl(final SharedCommunicationObjects sco, final Config config) { + this(DEFAULT_CAPACITY, FLUSH_INTERVAL_SECONDS, SECONDS, sco, config); + } + + FlagEvaluationWriterImpl( + final int capacity, + final long flushInterval, + final TimeUnit timeUnit, + final SharedCommunicationObjects sco, + final Config config) { + this(capacity, flushInterval, timeUnit, new BackendApiFactory(config, sco), config); + } + + FlagEvaluationWriterImpl( + final int capacity, + final long flushInterval, + final TimeUnit timeUnit, + final BackendApiFactory backendApiFactory, + final Config config) { + this.queue = Queues.mpscBlockingConsumerArrayQueue(capacity); + this.serializer = + new FlagEvaluationSerializingHandler(queue, flushInterval, timeUnit, droppedQueueOverflow); + this.serializerThread = newAgentThread(FEATURE_FLAG_EVALUATION_PROCESSOR, serializer); + } + + @Override + public void start() { + synchronized (lifecycleLock) { + if (closed.get()) { + return; + } + FeatureFlaggingGateway.setFlagEvalWriter(this); + this.serializerThread.start(); + } + } + + void startForTest() { + synchronized (lifecycleLock) { + if (!closed.get()) { + this.serializerThread.start(); + } + } + } + + int aggregatorFullTierSizeForTest() { + return serializer.aggregator.fullTierSize(); + } + + @Override + public void close() { + synchronized (lifecycleLock) { + if (!closed.compareAndSet(false, true)) { + return; + } + FeatureFlaggingGateway.setFlagEvalWriter(null); + if (!this.serializerThread.isAlive()) { + return; + } + serializer.requestShutdown(); + this.serializerThread.interrupt(); + } + if (Thread.currentThread() == this.serializerThread) { + return; + } + try { + this.serializerThread.join(TimeUnit.SECONDS.toMillis(5)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + @Override + public void enqueue(final FlagEvalEvent event) { + if (event == null) { + return; + } + synchronized (lifecycleLock) { + if (closed.get()) { + countMetric(FLAG_EVALUATION_DROPPED_METRIC, 1, DROP_REASON_CLOSED); + return; + } + if (!queue.offer(event)) { + droppedQueueOverflow.incrementAndGet(); + } + } + } + + long droppedQueueOverflow() { + return droppedQueueOverflow.get(); + } + + FlagEvalEvent pollQueuedEventForTest() { + return queue.poll(); + } + + void flushForTest() { + serializer.flush(); + } + + static Map pruneContext(final Map attrs) { + return FlagEvaluationAggregator.pruneContext(attrs); + } + + static String canonicalContextKey(final Map prunedAttrs) { + return FlagEvaluationAggregator.canonicalContextKey(prunedAttrs); + } + + private static void countMetric(final String metricName, final long value, final String reason) { + if (value <= 0) { + return; + } + CORE_METRICS.count(metricName, value, reason == null ? null : "reason:" + reason); + } + + static class FlagEvaluationSerializingHandler implements Runnable { + private final MessagePassingBlockingQueue queue; + private final long ticksRequiredToFlush; + + @SuppressFBWarnings( + value = "AT_NONATOMIC_64BIT_PRIMITIVE", + justification = "the field is confined to the single serializer thread") + private long lastTicks; + + private final AtomicLong droppedQueueOverflow; + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + private final AtomicBoolean shutdownRequested = new AtomicBoolean(false); + + FlagEvaluationSerializingHandler( + final MessagePassingBlockingQueue queue, + final long flushInterval, + final TimeUnit timeUnit, + final AtomicLong droppedQueueOverflow) { + this.queue = queue; + this.droppedQueueOverflow = droppedQueueOverflow; + this.lastTicks = System.nanoTime(); + this.ticksRequiredToFlush = timeUnit.toNanos(flushInterval); + LOGGER.debug("starting flag evaluation serializer"); + } + + void requestShutdown() { + shutdownRequested.set(true); + } + + @Override + public void run() { + try { + runDutyCycle(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + drainAndFlush(); + } + LOGGER.debug("flag evaluation processor worker exited."); + } + + private void runDutyCycle() throws InterruptedException { + final Thread thread = Thread.currentThread(); + while (!thread.isInterrupted() && !shutdownRequested.get()) { + final FlagEvalEvent event = queue.poll(100, TimeUnit.MILLISECONDS); + if (event != null) { + aggregateEvent(event); + } + flushIfNecessary(); + } + } + + void drainAndFlush() { + FlagEvalEvent event; + while ((event = queue.poll()) != null) { + aggregateEvent(event); + } + flush(); + } + + void aggregateEvent(final FlagEvalEvent event) { + try { + aggregator.aggregate(event); + } catch (LinkageError | RuntimeException e) { + countMetric(FLAG_EVALUATION_DROPPED_METRIC, 1, DROP_REASON_CONTEXT_ERROR); + LOGGER.debug("Could not aggregate flag evaluation event", e); + } + } + + void flushIfNecessary() { + if (aggregator.isEmpty() && droppedQueueOverflow.get() == 0) { + return; + } + if (shouldFlush()) { + flush(); + } + } + + void flush() { + final long qDrops = droppedQueueOverflow.getAndSet(0); + countMetric(FLAG_EVALUATION_DROPPED_METRIC, qDrops, DROP_REASON_QUEUE_OVERFLOW); + if (qDrops > 0) { + LOGGER.warn( + "flag evaluation queue full - dropped {} evaluation(s) under backpressure" + + " (best-effort telemetry)", + qDrops); + } + final long dgDrops = aggregator.droppedDegradedOverflow.getAndSet(0); + countMetric(FLAG_EVALUATION_DROPPED_METRIC, dgDrops, DROP_REASON_DEGRADED_CAP); + if (dgDrops > 0) { + LOGGER.warn( + "degraded aggregation tier full - dropped {} evaluation(s); raise degraded cap" + + " (best-effort telemetry)", + dgDrops); + } + if (!aggregator.isEmpty()) { + countMetric( + FLAG_EVALUATION_DEGRADED_METRIC, + aggregator.degradedEvaluationCount(), + DEGRADED_REASON_CARDINALITY_CAP); + aggregator.clear(); + } + lastTicks = System.nanoTime(); + } + + private boolean shouldFlush() { + final long nanoTime = System.nanoTime(); + final long ticks = nanoTime - lastTicks; + if (ticks > ticksRequiredToFlush) { + lastTicks = nanoTime; + return true; + } + return false; + } + } +} From 597e97d10b2fe5ff0b851466147f72cf8650aa03 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:49:14 -0400 Subject: [PATCH 16/26] feat(feature-flagging): post flagevaluation payloads --- .../featureflag/FlagEvaluationWriterImpl.java | 313 +++++++++++++++++- 1 file changed, 296 insertions(+), 17 deletions(-) diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java index 68068d1ea4c..18b04d5c99a 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java @@ -7,6 +7,7 @@ import datadog.common.queue.MessagePassingBlockingQueue; import datadog.common.queue.Queues; import datadog.communication.BackendApiFactory; +import datadog.communication.EvpProxy; import datadog.communication.ddagent.SharedCommunicationObjects; import datadog.trace.api.Config; import datadog.trace.api.featureflag.FeatureFlaggingGateway; @@ -14,7 +15,10 @@ import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; import datadog.trace.api.telemetry.CoreMetricCollector; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.util.ArrayList; +import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -24,14 +28,38 @@ /** * EVP {@code flagevaluation} writer for Java. * - *

Owns the bounded hook hand-off queue, background aggregation loop, and writer lifecycle. The - * EVP payload/posting path is layered on top of this queue in the next review step. + *

Uses the same EVP publisher path as {@link ExposureWriterImpl}, with two-tier aggregation + * replacing the single-exposure buffer. Routes to the Agent-advertised EVP proxy endpoint for + * {@code /api/v2/flagevaluation}. + * + *

Two-tier aggregation contract: + * + *

    + *
  • Two-tier aggregation: full -> degraded -> drop-counted. + *
  • Full key: (flagKey, variant, allocationKey, runtimeDefault, errorMessage, targetingKey, + * canonical-context-key). + *
  • Degraded key: (flagKey, variant, allocationKey, runtimeDefault, errorMessage) - no + * targetingKey/context. + *
  • Canonical context key: sorted entries, type-tagged length-delimited encoding - NOT a hash + * (collision-safe, comparable string identity). + *
  • Context pruning: deterministic (sort before cut), <=256 fields, string values <=256 chars; + * the pruned attributes are what gets aggregated and serialized. + *
  • Caps: globalCap=131072, perFlagCap=10000, degradedCap=32768. + *
  • Eval-time: min/max of firstEvalMs/lastEvalMs across events in the same bucket. + *
  • Runtime default: absent variant means {@code runtimeDefaultUsed=true}. + *
  • Flush interval: 10 seconds. + *
  • Queue: bounded MessagePassingBlockingQueue (capacity 2^16), non-blocking offer; on overflow + * the event is dropped and the {@code droppedQueueOverflow} counter is incremented and + * surfaced on flush. + *
  • Shutdown: {@link #close()} drains the queue and performs a final flush before the worker + * thread exits. + *
*/ public class FlagEvaluationWriterImpl implements FlagEvaluationWriter { private static final Logger LOGGER = LoggerFactory.getLogger(FlagEvaluationWriterImpl.class); - static final int DEFAULT_CAPACITY = 1 << 16; + static final int DEFAULT_CAPACITY = 1 << 16; // 65536 elements static final int FLUSH_INTERVAL_SECONDS = 10; static final int EVAL_SCALE_FULL_BUCKET_TARGET = FlagEvaluationAggregator.EVAL_SCALE_FULL_BUCKET_TARGET; @@ -42,15 +70,21 @@ public class FlagEvaluationWriterImpl implements FlagEvaluationWriter { static final int GLOBAL_CAP = FlagEvaluationAggregator.GLOBAL_CAP; static final int PER_FLAG_CAP = FlagEvaluationAggregator.PER_FLAG_CAP; static final int DEGRADED_CAP = FlagEvaluationAggregator.DEGRADED_CAP; + static final int FLAG_EVALUATION_PAYLOAD_SIZE_LIMIT_BYTES = EvpProxy.PAYLOAD_SIZE_LIMIT_BYTES; static final String FLAG_EVALUATION_DROPPED_METRIC = "flagevaluation.rows.dropped"; static final String FLAG_EVALUATION_DEGRADED_METRIC = "flagevaluation.rows.degraded"; + static final String FLAG_EVALUATION_SPLITS_METRIC = "flagevaluation.payload.splits"; static final String DROP_REASON_QUEUE_OVERFLOW = "queue_overflow"; - static final String DROP_REASON_DEGRADED_CAP = "degraded_cap"; static final String DROP_REASON_CLOSED = "closed"; static final String DROP_REASON_CONTEXT_ERROR = "context_error"; + static final String DROP_REASON_DEGRADED_CAP = "degraded_cap"; + static final String DROP_REASON_PAYLOAD_LIMIT = "payload_limit"; static final String DEGRADED_REASON_CARDINALITY_CAP = "cardinality_cap"; + static final String DEGRADED_REASON_PAYLOAD_LIMIT = "payload_limit"; + private static final String FLAG_EVALUATION_ROUTE = "flagevaluation"; private static final CoreMetricCollector CORE_METRICS = CoreMetricCollector.getInstance(); + // Context pruning limits: max fields and max string value length static final int MAX_CONTEXT_FIELDS = FlagEvaluationAggregator.MAX_CONTEXT_FIELDS; static final int MAX_FIELD_LENGTH = FlagEvaluationAggregator.MAX_FIELD_LENGTH; @@ -60,6 +94,17 @@ public class FlagEvaluationWriterImpl implements FlagEvaluationWriter { private final Object lifecycleLock = new Object(); private final AtomicBoolean closed = new AtomicBoolean(false); + private static void countMetric(final String metricName, final long value, final String reason) { + if (value <= 0) { + return; + } + CORE_METRICS.count(metricName, value, reason == null ? null : "reason:" + reason); + } + + /** + * Observable counter for events dropped because the bounded hand-off queue was full when the hook + * tried to enqueue (backpressure). Incremented on the hook thread, surfaced on flush. + */ private final AtomicLong droppedQueueOverflow = new AtomicLong(0); public FlagEvaluationWriterImpl(final SharedCommunicationObjects sco, final Config config) { @@ -75,6 +120,7 @@ public FlagEvaluationWriterImpl(final SharedCommunicationObjects sco, final Conf this(capacity, flushInterval, timeUnit, new BackendApiFactory(config, sco), config); } + /** Package-private constructor allowing a {@link BackendApiFactory} to be injected for tests. */ FlagEvaluationWriterImpl( final int capacity, final long flushInterval, @@ -83,7 +129,14 @@ public FlagEvaluationWriterImpl(final SharedCommunicationObjects sco, final Conf final Config config) { this.queue = Queues.mpscBlockingConsumerArrayQueue(capacity); this.serializer = - new FlagEvaluationSerializingHandler(queue, flushInterval, timeUnit, droppedQueueOverflow); + new FlagEvaluationSerializingHandler( + backendApiFactory, + queue, + flushInterval, + timeUnit, + FeatureFlagEvpContext.from(config), + droppedQueueOverflow, + this::close); this.serializerThread = newAgentThread(FEATURE_FLAG_EVALUATION_PROCESSOR, serializer); } @@ -93,19 +146,23 @@ public void start() { if (closed.get()) { return; } + // Register with the gateway so FlagEvalLoggingHook can route evaluations to this writer FeatureFlaggingGateway.setFlagEvalWriter(this); this.serializerThread.start(); } } + /** Test seam: starts the worker thread WITHOUT registering with the global gateway. */ void startForTest() { synchronized (lifecycleLock) { - if (!closed.get()) { - this.serializerThread.start(); + if (closed.get()) { + return; } + this.serializerThread.start(); } } + /** Test seam: current full-tier bucket count in the worker's aggregator. */ int aggregatorFullTierSizeForTest() { return serializer.aggregator.fullTierSize(); } @@ -116,10 +173,12 @@ public void close() { if (!closed.compareAndSet(false, true)) { return; } + // Deregister from the gateway so no new events are enqueued. FeatureFlaggingGateway.setFlagEvalWriter(null); if (!this.serializerThread.isAlive()) { return; } + // Ask the worker to drain the queue and final-flush, then interrupt to wake it from poll(). serializer.requestShutdown(); this.serializerThread.interrupt(); } @@ -127,6 +186,7 @@ public void close() { return; } try { + // Bounded wait for the worker's final flush so queued events are not lost on shutdown. this.serializerThread.join(TimeUnit.SECONDS.toMillis(5)); } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -143,38 +203,58 @@ public void enqueue(final FlagEvalEvent event) { countMetric(FLAG_EVALUATION_DROPPED_METRIC, 1, DROP_REASON_CLOSED); return; } + // Non-blocking offer. Count overflow so loss is observable rather than silent; the count is + // surfaced on the next flush. if (!queue.offer(event)) { droppedQueueOverflow.incrementAndGet(); } } } + /** Returns the count of events dropped due to queue-overflow backpressure (observable). */ long droppedQueueOverflow() { return droppedQueueOverflow.get(); } + /** Test seam: returns one queued event without starting the worker. */ FlagEvalEvent pollQueuedEventForTest() { return queue.poll(); } + /** Test seam: flushes serializer state without starting the worker. */ void flushForTest() { serializer.flush(); } + // ---- Deterministic context pruning ---- + + /** + * Produces the deterministically-pruned context map used for both the canonical key and the + * serialized payload. Keys are sorted FIRST, then the first {@link #MAX_CONTEXT_FIELDS} + * non-oversized values are kept, so two logically-identical contexts always prune to the same + * subset (and therefore the same canonical key). Oversized string values (>{@link + * #MAX_FIELD_LENGTH} chars) are skipped. Returns an empty map for null/empty input. + */ static Map pruneContext(final Map attrs) { return FlagEvaluationAggregator.pruneContext(attrs); } + // ---- Canonical context key ---- + // Sorted entries, type-tagged length-delimited encoding. NOT a hash (collision-safe string key). + // Implementation mirrors dd-trace-go/openfeature/flagevaluation.go canonicalContextKey(). + + /** + * Builds the canonical context key from the already-pruned context map for the full-tier bucket + * identity. Expects a pruned, sorted map (e.g. produced by {@link #pruneContext(Map)}); iterating + * a {@link TreeMap} yields keys in sorted order, so the encoding is deterministic. + * + *

Returns an empty string for null/empty context. + */ static String canonicalContextKey(final Map prunedAttrs) { return FlagEvaluationAggregator.canonicalContextKey(prunedAttrs); } - private static void countMetric(final String metricName, final long value, final String reason) { - if (value <= 0) { - return; - } - CORE_METRICS.count(metricName, value, reason == null ? null : "reason:" + reason); - } + // ---- Serializing handler (background thread logic) ---- static class FlagEvaluationSerializingHandler implements Runnable { private final MessagePassingBlockingQueue queue; @@ -185,34 +265,80 @@ static class FlagEvaluationSerializingHandler implements Runnable { justification = "the field is confined to the single serializer thread") private long lastTicks; + private final FeatureFlagEvpPublisher + evpPublisher; + final Map context; private final AtomicLong droppedQueueOverflow; + private final Runnable errorCallback; + private final int payloadSizeLimitBytes; final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + + // Shutdown coordination: set by close(), drives a final drain+flush before the worker exits. private final AtomicBoolean shutdownRequested = new AtomicBoolean(false); + private final CountDownLatch finalFlushDone = new CountDownLatch(1); + + FlagEvaluationSerializingHandler( + final BackendApiFactory backendApiFactory, + final MessagePassingBlockingQueue queue, + final long flushInterval, + final TimeUnit timeUnit, + final Map context, + final AtomicLong droppedQueueOverflow, + final Runnable errorCallback) { + this( + backendApiFactory, + queue, + flushInterval, + timeUnit, + context, + droppedQueueOverflow, + errorCallback, + FLAG_EVALUATION_PAYLOAD_SIZE_LIMIT_BYTES); + } FlagEvaluationSerializingHandler( + final BackendApiFactory backendApiFactory, final MessagePassingBlockingQueue queue, final long flushInterval, final TimeUnit timeUnit, - final AtomicLong droppedQueueOverflow) { + final Map context, + final AtomicLong droppedQueueOverflow, + final Runnable errorCallback, + final int payloadSizeLimitBytes) { this.queue = queue; + this.evpPublisher = + new FeatureFlagEvpPublisher<>( + backendApiFactory, FlagEvaluationPayloads.FlagEvaluationsRequest.class, false); + this.context = context; this.droppedQueueOverflow = droppedQueueOverflow; + this.payloadSizeLimitBytes = payloadSizeLimitBytes; this.lastTicks = System.nanoTime(); this.ticksRequiredToFlush = timeUnit.toNanos(flushInterval); + this.errorCallback = errorCallback; LOGGER.debug("starting flag evaluation serializer"); } + /** Signals the worker to drain the queue and perform a final flush before exiting. */ void requestShutdown() { shutdownRequested.set(true); } @Override public void run() { + if (!evpPublisher.start()) { + finalFlushDone.countDown(); + errorCallback.run(); + throw new IllegalArgumentException("EVP Proxy not available"); + } try { runDutyCycle(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } finally { + // On exit (interrupt or shutdown request), drain everything still buffered and flush it so + // queued events are not lost on shutdown. drainAndFlush(); + finalFlushDone.countDown(); } LOGGER.debug("flag evaluation processor worker exited."); } @@ -228,6 +354,7 @@ private void runDutyCycle() throws InterruptedException { } } + /** Drains all remaining queued events and performs a final flush. Used on shutdown. */ void drainAndFlush() { FlagEvalEvent event; while ((event = queue.poll()) != null) { @@ -236,6 +363,9 @@ void drainAndFlush() { flush(); } + // ---- Aggregation logic ---- + + /** Routes an event into the full tier or degraded tier, or drops and counts on overflow. */ void aggregateEvent(final FlagEvalEvent event) { try { aggregator.aggregate(event); @@ -245,6 +375,8 @@ void aggregateEvent(final FlagEvalEvent event) { } } + // ---- Flush logic ---- + void flushIfNecessary() { if (aggregator.isEmpty() && droppedQueueOverflow.get() == 0) { return; @@ -255,6 +387,8 @@ void flushIfNecessary() { } void flush() { + // Surface backpressure (queue-overflow) drops as an observable warning even when there is + // nothing else to flush. final long qDrops = droppedQueueOverflow.getAndSet(0); countMetric(FLAG_EVALUATION_DROPPED_METRIC, qDrops, DROP_REASON_QUEUE_OVERFLOW); if (qDrops > 0) { @@ -271,14 +405,68 @@ void flush() { + " (best-effort telemetry)", dgDrops); } - if (!aggregator.isEmpty()) { + + if (aggregator.isEmpty()) { + return; + } + boolean aggregatesWereEncoded = false; + try { countMetric( FLAG_EVALUATION_DEGRADED_METRIC, aggregator.degradedEvaluationCount(), DEGRADED_REASON_CARDINALITY_CAP); - aggregator.clear(); + final List events = buildEventList(); + if (events.isEmpty()) { + return; + } + final FlagEvaluationPayloads.EncodedPayloads payloads = + FlagEvaluationPayloads.buildPayloads(events, context, payloadSizeLimitBytes); + aggregatesWereEncoded = true; + countMetric( + FLAG_EVALUATION_DROPPED_METRIC, + payloads.droppedPayloadLimit, + DROP_REASON_PAYLOAD_LIMIT); + countMetric( + FLAG_EVALUATION_DEGRADED_METRIC, + payloads.degradedPayloadLimit, + DEGRADED_REASON_PAYLOAD_LIMIT); + if (payloads.bodies.size() > 1) { + countMetric(FLAG_EVALUATION_SPLITS_METRIC, payloads.bodies.size() - 1, null); + } + if (payloads.droppedPayloadLimit > 0) { + LOGGER.warn( + "flag evaluation payload too large - dropped {} evaluation(s)" + + " (best-effort telemetry)", + payloads.droppedPayloadLimit); + } + for (final byte[] payload : payloads.bodies) { + evpPublisher.post(FLAG_EVALUATION_ROUTE, payload); + } + } catch (Exception e) { + LOGGER.error("Could not submit flag evaluations", e); + } finally { + if (aggregatesWereEncoded) { + // Once payload bytes are built, this writer is best-effort: clear encoded aggregates even + // if one split post fails so a later flush cannot duplicate already-sent rows. + aggregator.clear(); + lastTicks = System.nanoTime(); + } } - lastTicks = System.nanoTime(); + } + + private List buildEventList() { + final long flushTimeMs = System.currentTimeMillis(); + final List events = + new ArrayList<>(aggregator.bucketCount()); + for (final FlagEvaluationAggregator.EvalBucket bucket : aggregator.fullBuckets()) { + events.add( + FlagEvaluationPayloads.FlagEvaluationEvent.fromBucket(bucket, true, flushTimeMs)); + } + for (final FlagEvaluationAggregator.EvalBucket bucket : aggregator.degradedBuckets()) { + events.add( + FlagEvaluationPayloads.FlagEvaluationEvent.fromBucket(bucket, false, flushTimeMs)); + } + return events; } private boolean shouldFlush() { @@ -291,4 +479,95 @@ private boolean shouldFlush() { return false; } } + + // ---- Test-seam inner class (package-private) ---- + + /** + * Test-accessible handler that exposes {@link #drainAndAggregate()} and {@link #flush()} without + * starting a real background thread. + */ + static class SerializingHandlerForTest extends FlagEvaluationSerializingHandler { + + SerializingHandlerForTest(final BackendApiFactory factory, final Map context) { + this(factory, context, FLAG_EVALUATION_PAYLOAD_SIZE_LIMIT_BYTES); + } + + SerializingHandlerForTest( + final BackendApiFactory factory, + final Map context, + final int payloadSizeLimitBytes) { + super( + factory, + Queues.mpscBlockingConsumerArrayQueue(DEFAULT_CAPACITY), + Long.MAX_VALUE, // effectively never auto-flush + TimeUnit.NANOSECONDS, + context, + new AtomicLong(0), + () -> {}, + payloadSizeLimitBytes); + } + + private final List staged = new ArrayList<>(); + + /** Adds an event to the staged list (simulates hook enqueue). */ + void add(final FlagEvalEvent event) { + staged.add(event); + } + + /** Aggregates all staged events and returns the current aggregation state. */ + FlagEvaluationAggregator.AggregatedState drainAndAggregate() { + for (final FlagEvalEvent e : staged) { + aggregateEvent(e); + } + staged.clear(); + return aggregator.snapshot(); + } + + /** Simulates filling the full tier to GLOBAL_CAP by injecting synthetic distinct buckets. */ + void simulateFullTierAtCap() { + aggregator.simulateFullTierAtCap(); + } + + /** + * Simulates filling the degraded tier to DEGRADED_CAP by injecting synthetic distinct buckets. + */ + void simulateDegradedTierAtCap() { + aggregator.simulateDegradedTierAtCap(); + } + + void addDroppedDegradedOverflowForTest(final long count) { + aggregator.droppedDegradedOverflow.addAndGet(count); + } + + void addDegradedBucketForTest( + final String flagKey, + final String variant, + final String allocationKey, + final String errorMessage, + final long evalTimeMs) { + aggregator.addDegradedBucketForTest( + flagKey, variant, allocationKey, errorMessage, evalTimeMs); + } + + void clearAggregationForTest() { + aggregator.clear(); + } + + int fullTierSizeForTest() { + return aggregator.fullTierSize(); + } + } + + /** Factory method for test use - creates a SerializingHandlerForTest. */ + static SerializingHandlerForTest createHandlerForTest( + final BackendApiFactory factory, final Map context) { + return new SerializingHandlerForTest(factory, context); + } + + static SerializingHandlerForTest createHandlerForTest( + final BackendApiFactory factory, + final Map context, + final int payloadSizeLimitBytes) { + return new SerializingHandlerForTest(factory, context, payloadSizeLimitBytes); + } } From 58df9361656a1d171c0312a506f57202034296ca Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:49:22 -0400 Subject: [PATCH 17/26] test(feature-flagging): add flagevaluation test support --- .../FlagEvaluationTestSupport.java | 212 ++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java new file mode 100644 index 00000000000..11e5b4f407f --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java @@ -0,0 +1,212 @@ +package com.datadog.featureflag; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.Moshi; +import com.squareup.moshi.Types; +import datadog.communication.BackendApi; +import datadog.communication.BackendApiFactory; +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import datadog.trace.api.telemetry.CoreMetricCollector; +import datadog.trace.api.telemetry.MetricCollector; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import okhttp3.RequestBody; +import okio.Buffer; + +final class FlagEvaluationTestSupport { + + static final long REALISTIC_EVAL_MS = 1_760_000_000_000L; + static final JsonAdapter> JSON_MAP; + + static { + final Moshi moshi = new Moshi.Builder().build(); + final Type type = Types.newParameterizedType(Map.class, String.class, Object.class); + JSON_MAP = moshi.adapter(type); + } + + private FlagEvaluationTestSupport() {} + + static void clearCoreMetrics() { + CoreMetricCollector.getInstance().drain(); + } + + static FlagEvalEvent event( + final String flagKey, + final String variant, + final String allocationKey, + final String targetingKey, + final long evalTimeMs, + final Map attrs) { + return new FlagEvalEvent(flagKey, variant, allocationKey, targetingKey, evalTimeMs, attrs); + } + + static FlagEvalEvent errorEvent( + final String flagKey, final String errorMessage, final long evalTimeMs) { + return new FlagEvalEvent( + flagKey, null, null, null, errorMessage, evalTimeMs, java.util.Collections.emptyMap()); + } + + static FlagEvalEvent simpleEvent(final String flagKey, final String variant) { + return event(flagKey, variant, "alloc1", "user-1", 1000L, java.util.Collections.emptyMap()); + } + + static String repeat(final char c, final int count) { + final char[] chars = new char[count]; + Arrays.fill(chars, c); + return new String(chars); + } + + static TestWriterSetup buildTestWriter(final BackendApi mockEvp) { + return buildTestWriter( + mockEvp, FlagEvaluationWriterImpl.FLAG_EVALUATION_PAYLOAD_SIZE_LIMIT_BYTES); + } + + static TestWriterSetup buildTestWriter( + final BackendApi mockEvp, final int payloadSizeLimitBytes) { + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + when(factory.createBackendApi(any(), any(), eq(false))).thenReturn(mockEvp); + + final Map context = new HashMap<>(); + context.put("service", "test-service"); + + final FlagEvaluationWriterImpl.SerializingHandlerForTest handler = + FlagEvaluationWriterImpl.createHandlerForTest(factory, context, payloadSizeLimitBytes); + + return new TestWriterSetup(handler, mockEvp, factory); + } + + static CapturedJson flushAndCapture(final TestWriterSetup setup) throws Exception { + final List captured = flushAndCaptureAll(setup); + assertEquals(1, captured.size(), "Expected exactly one posted payload"); + return captured.get(0); + } + + static List flushAndCaptureAll(final TestWriterSetup setup) throws Exception { + final List captured = new ArrayList<>(); + when(setup.mockEvp.post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false))) + .thenAnswer( + inv -> { + captured.add(inv.getArgument(1)); + return null; + }); + setup.handler.drainAndAggregate(); + setup.handler.flush(); + final List json = new ArrayList<>(); + for (final RequestBody body : captured) { + json.add(readJson(body)); + } + return json; + } + + static CapturedJson readJson(final RequestBody body) throws Exception { + assertNotNull(body, "RequestBody must have been posted"); + final Buffer buf = new Buffer(); + body.writeTo(buf); + final String raw = buf.readUtf8(); + return new CapturedJson(raw, JSON_MAP.fromJson(raw)); + } + + static Map flushAndCaptureJson(final TestWriterSetup setup) throws Exception { + return flushAndCapture(setup).parsed; + } + + static long metricSum( + final Collection metrics, + final String metricName, + final String tag) { + long sum = 0; + for (final MetricCollector.Metric metric : metrics) { + if (!metricName.equals(metric.metricName)) { + continue; + } + if (tag == null) { + if (!metric.tags.isEmpty()) { + continue; + } + } else if (!metric.tags.contains(tag)) { + continue; + } + sum += metric.value.longValue(); + } + return sum; + } + + static datadog.trace.api.Config cfg() { + final datadog.trace.api.Config config = mock(datadog.trace.api.Config.class); + when(config.getServiceName()).thenReturn("test-service"); + return config; + } + + static void assertObjectWithKey(final Object o, final String expectedKey, final String msg) { + assertTrue(o instanceof Map, msg + " (must be a JSON object, not a bare string)"); + assertEquals(expectedKey, ((Map) o).get("key"), msg); + } + + @SuppressWarnings("unchecked") + static Map firstEvent(final Map batch) { + final List events = (List) batch.get("flagEvaluations"); + assertNotNull(events, "flagEvaluations array must be present"); + assertFalse(events.isEmpty(), "flagEvaluations must not be empty"); + return (Map) events.get(0); + } + + @SuppressWarnings("unchecked") + static int eventCount(final Map batch) { + final List events = (List) batch.get("flagEvaluations"); + assertNotNull(events, "flagEvaluations array must be present"); + return events.size(); + } + + @SuppressWarnings("unchecked") + static Map eventForFlag(final Map batch, final String flagKey) { + final List events = (List) batch.get("flagEvaluations"); + for (final Object o : events) { + final Map ev = (Map) o; + final Map flag = (Map) ev.get("flag"); + if (flag != null && flagKey.equals(flag.get("key"))) { + return ev; + } + } + return null; + } + + static class TestWriterSetup { + final FlagEvaluationWriterImpl.SerializingHandlerForTest handler; + final BackendApi mockEvp; + final BackendApiFactory factory; + + TestWriterSetup( + final FlagEvaluationWriterImpl.SerializingHandlerForTest handler, + final BackendApi mockEvp, + final BackendApiFactory factory) { + this.handler = handler; + this.mockEvp = mockEvp; + this.factory = factory; + } + } + + static class CapturedJson { + final String raw; + final Map parsed; + + CapturedJson(final String raw, final Map parsed) { + this.raw = raw; + this.parsed = parsed; + } + } +} From acc6e21940a1f1378dc2f6d298649e1a5d615da1 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:49:22 -0400 Subject: [PATCH 18/26] test(feature-flagging): cover flagevaluation writer --- .../FlagEvaluationWriterImplTest.java | 304 ++++++++++++++++++ 1 file changed, 304 insertions(+) create mode 100644 products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java new file mode 100644 index 00000000000..bec4ed9eaee --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java @@ -0,0 +1,304 @@ +package com.datadog.featureflag; + +import static com.datadog.featureflag.FlagEvaluationTestSupport.JSON_MAP; +import static com.datadog.featureflag.FlagEvaluationTestSupport.buildTestWriter; +import static com.datadog.featureflag.FlagEvaluationTestSupport.cfg; +import static com.datadog.featureflag.FlagEvaluationTestSupport.clearCoreMetrics; +import static com.datadog.featureflag.FlagEvaluationTestSupport.event; +import static com.datadog.featureflag.FlagEvaluationTestSupport.eventForFlag; +import static com.datadog.featureflag.FlagEvaluationTestSupport.metricSum; +import static com.datadog.featureflag.FlagEvaluationTestSupport.repeat; +import static com.datadog.featureflag.FlagEvaluationTestSupport.simpleEvent; +import static java.util.Collections.emptyMap; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import datadog.communication.BackendApi; +import datadog.communication.BackendApiFactory; +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import datadog.trace.api.intake.Intake; +import datadog.trace.api.telemetry.CoreMetricCollector; +import datadog.trace.api.telemetry.MetricCollector; +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import okhttp3.RequestBody; +import okio.Buffer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class FlagEvaluationWriterImplTest { + + @BeforeEach + void clearCoreMetricsBefore() { + clearCoreMetrics(); + } + + @AfterEach + void clearCoreMetricsAfter() { + clearCoreMetrics(); + } + + @Test + void degradedCapOverflowTelemetryIsEmittedOnFlush() { + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp); + + setup.handler.addDroppedDegradedOverflowForTest(3); + setup.handler.flush(); + + final Collection metrics = + CoreMetricCollector.getInstance().drain(); + assertEquals( + 3, + metricSum( + metrics, + FlagEvaluationWriterImpl.FLAG_EVALUATION_DROPPED_METRIC, + "reason:" + FlagEvaluationWriterImpl.DROP_REASON_DEGRADED_CAP)); + } + + @Test + void queueOverflowIncrementsObservableDropCounter() { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + when(factory.createBackendApi(any(), any(), eq(false))).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(2, 10L, TimeUnit.SECONDS, factory, cfg()); + + for (int i = 0; i < 100; i++) { + writer.enqueue(simpleEvent("of-flag", "on")); + } + + assertTrue(writer.droppedQueueOverflow() > 0); + final long queueDrops = writer.droppedQueueOverflow(); + writer.flushForTest(); + final Collection metrics = + CoreMetricCollector.getInstance().drain(); + assertEquals( + queueDrops, + metricSum( + metrics, + FlagEvaluationWriterImpl.FLAG_EVALUATION_DROPPED_METRIC, + "reason:" + FlagEvaluationWriterImpl.DROP_REASON_QUEUE_OVERFLOW)); + } + + @Test + void enqueueAfterCloseIsDroppedAndCounted() { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + when(factory.createBackendApi(any(), any(), eq(false))).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + + writer.close(); + writer.enqueue(simpleEvent("closed-flag", "on")); + + final Collection metrics = + CoreMetricCollector.getInstance().drain(); + assertEquals( + 1, + metricSum( + metrics, + FlagEvaluationWriterImpl.FLAG_EVALUATION_DROPPED_METRIC, + "reason:" + FlagEvaluationWriterImpl.DROP_REASON_CLOSED)); + assertNull(writer.pollQueuedEventForTest()); + } + + @Test + void enqueueDoesNotAggregateOnTheCallingThread() { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + when(factory.createBackendApi(any(), any(), eq(false))).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + + writer.enqueue(simpleEvent("g2-flag", "on")); + writer.enqueue(simpleEvent("g2-flag", "on")); + + assertEquals(0, writer.aggregatorFullTierSizeForTest()); + assertEquals(0, writer.droppedQueueOverflow()); + } + + @Test + void enqueueDoesNotResolveContextBeforeBuffering() { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + when(factory.createBackendApi(any(), any(), eq(false))).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + final AtomicInteger resolutions = new AtomicInteger(); + final Map attrs = new HashMap<>(); + attrs.put("tier", "gold"); + + writer.enqueue( + new FlagEvalEvent( + "lazy-flag", + "on", + "alloc1", + "user-1", + null, + 1000L, + () -> { + resolutions.incrementAndGet(); + return attrs; + })); + + final FlagEvalEvent queued = writer.pollQueuedEventForTest(); + assertNotNull(queued); + assertTrue(queued.attrs.isEmpty()); + assertEquals(0, resolutions.get()); + assertEquals("gold", queued.contextAttributes().get("tier")); + assertEquals(1, resolutions.get()); + } + + @Test + void contextMaterializationFailureDropsSingleEvent() { + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp); + setup.handler.add( + new FlagEvalEvent( + "bad-context", + "on", + "alloc1", + "user-1", + null, + 1000L, + () -> { + throw new IllegalArgumentException("bad context"); + })); + + setup.handler.drainAndAggregate(); + + final Collection metrics = + CoreMetricCollector.getInstance().drain(); + assertEquals( + 1, + metricSum( + metrics, + FlagEvaluationWriterImpl.FLAG_EVALUATION_DROPPED_METRIC, + "reason:" + FlagEvaluationWriterImpl.DROP_REASON_CONTEXT_ERROR)); + assertEquals(0, setup.handler.fullTierSizeForTest()); + } + + @Test + void closeDrainsAndFinalFlushesQueuedEvents() throws Exception { + final java.util.concurrent.CountDownLatch posted = new java.util.concurrent.CountDownLatch(1); + final RequestBody[] captured = {null}; + final BackendApi mockEvp = mock(BackendApi.class); + when(mockEvp.post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false))) + .thenAnswer( + inv -> { + captured[0] = inv.getArgument(1); + posted.countDown(); + return null; + }); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + when(factory.createBackendApi(any(), any(), eq(false))).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl( + 64, TimeUnit.DAYS.toSeconds(1), TimeUnit.SECONDS, factory, cfg()); + + writer.startForTest(); + writer.enqueue(simpleEvent("shutdown-flag", "on")); + writer.close(); + + assertTrue(posted.await(5, TimeUnit.SECONDS)); + assertNotNull(captured[0]); + final Buffer buf = new Buffer(); + captured[0].writeTo(buf); + final Map json = JSON_MAP.fromJson(buf.readUtf8()); + assertNotNull(eventForFlag(json, "shutdown-flag")); + } + + @Test + void continuousTrafficFlushesWithoutWaitingForIdle() throws Exception { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + when(factory.createBackendApi(any(), any(), eq(false))).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(1 << 12, 1, TimeUnit.MILLISECONDS, factory, cfg()); + + writer.startForTest(); + boolean posted = false; + try { + final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (System.nanoTime() < deadline) { + writer.enqueue(simpleEvent("busy-flag", "on")); + try { + verify(mockEvp, atLeastOnce()) + .post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false)); + posted = true; + break; + } catch (AssertionError ignored) { + // Keep the worker busy until the deadline. + } + } + } finally { + writer.close(); + } + + assertTrue(posted); + } + + @Test + void flushPostsToFlagevaluationEndpoint() throws Exception { + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp); + + setup.handler.add(event("flag-f", "on", "alloc1", "user-1", 1000L, emptyMap())); + setup.handler.drainAndAggregate(); + setup.handler.flush(); + + verify(setup.factory).createBackendApi(eq(Intake.EVENT_PLATFORM), isNull(), eq(false)); + verify(mockEvp).post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false)); + } + + @Test + void splitPostFailureDoesNotRetryAlreadySentPayloads() throws Exception { + final int limit = 1_100; + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp, limit); + final AtomicInteger posts = new AtomicInteger(); + doAnswer( + invocation -> { + if (posts.incrementAndGet() == 2) { + throw new IOException("boom"); + } + return null; + }) + .when(mockEvp) + .post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false)); + + for (int i = 0; i < 4; i++) { + final Map attrs = new HashMap<>(); + attrs.put("payload", repeat('x', 180)); + setup.handler.add(event("split-failure-" + i, "on", "alloc1", "user-" + i, 1000L, attrs)); + } + + setup.handler.drainAndAggregate(); + setup.handler.flush(); + assertEquals(2, posts.get()); + setup.handler.flush(); + assertEquals(2, posts.get()); + } +} From 96df04f1be66c6ebd237752e43c8f37b09ec8212 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:49:29 -0400 Subject: [PATCH 19/26] feat(feature-flagging): wire flagevaluation writer lifecycle --- .../featureflag/FeatureFlaggingSystem.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index 02689767bad..c5be9b67f82 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -2,6 +2,8 @@ import datadog.communication.ddagent.SharedCommunicationObjects; import datadog.trace.api.Config; +import datadog.trace.api.config.FeatureFlaggingConfig; +import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -11,6 +13,7 @@ public class FeatureFlaggingSystem { private static volatile RemoteConfigService CONFIG_SERVICE; private static volatile ExposureWriter EXPOSURE_WRITER; + private static volatile FlagEvaluationWriter FLAG_EVAL_WRITER; private FeatureFlaggingSystem() {} @@ -27,10 +30,29 @@ public static void start(final SharedCommunicationObjects sco) { EXPOSURE_WRITER = new ExposureWriterImpl(sco, config); EXPOSURE_WRITER.init(); + final boolean evalCountsEnabled = + config + .configProvider() + .getBoolean(FeatureFlaggingConfig.FLAGGING_EVALUATION_COUNTS_ENABLED, true); + if (evalCountsEnabled) { + final FlagEvaluationWriterImpl evalWriter = new FlagEvaluationWriterImpl(sco, config); + evalWriter.start(); + FLAG_EVAL_WRITER = evalWriter; + LOGGER.debug("Flag evaluation EVP writer started"); + } else { + LOGGER.debug( + "Flag evaluation EVP writer disabled ({}=false)", + FeatureFlaggingConfig.FLAGGING_EVALUATION_COUNTS_ENABLED); + } + LOGGER.debug("Feature Flagging system started"); } public static void stop() { + if (FLAG_EVAL_WRITER != null) { + FLAG_EVAL_WRITER.close(); + FLAG_EVAL_WRITER = null; + } if (EXPOSURE_WRITER != null) { EXPOSURE_WRITER.close(); EXPOSURE_WRITER = null; From edb264e7d8419dcaa161b4b8a077e61cbcc7ad29 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:49:30 -0400 Subject: [PATCH 20/26] test(feature-flagging): cover flagevaluation writer lifecycle --- .../FeatureFlaggingSystemTest.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java index 970c2cf16e3..82fae6ecdb0 100644 --- a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java +++ b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java @@ -1,6 +1,8 @@ package com.datadog.featureflag; import static datadog.trace.api.config.RemoteConfigConfig.REMOTE_CONFIGURATION_ENABLED; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -15,6 +17,8 @@ import datadog.remoteconfig.ConfigurationPoller; import datadog.remoteconfig.Product; import datadog.trace.api.Config; +import datadog.trace.api.config.FeatureFlaggingConfig; +import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.junit.utils.config.WithConfig; import okhttp3.HttpUrl; import okhttp3.OkHttpClient; @@ -23,6 +27,7 @@ class FeatureFlaggingSystemTest { @Test + @WithConfig(key = FeatureFlaggingConfig.FLAGGING_EVALUATION_COUNTS_ENABLED, value = "true") void testFeatureFlagSystemInitialization() { ConfigurationPoller poller = mock(ConfigurationPoller.class); DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); @@ -39,14 +44,37 @@ void testFeatureFlagSystemInitialization() { verify(poller).addCapabilities(Capabilities.CAPABILITY_FFE_FLAG_CONFIGURATION_RULES); verify(poller).addListener(eq(Product.FFE_FLAGS), any(ConfigurationDeserializer.class), any()); verify(poller).start(); + assertNotNull(FeatureFlaggingGateway.getFlagEvalWriter()); FeatureFlaggingSystem.stop(); + assertNull(FeatureFlaggingGateway.getFlagEvalWriter()); verify(poller).removeCapabilities(Capabilities.CAPABILITY_FFE_FLAG_CONFIGURATION_RULES); verify(poller).removeListeners(Product.FFE_FLAGS); verify(poller).stop(); } + @Test + @WithConfig(key = FeatureFlaggingConfig.FLAGGING_EVALUATION_COUNTS_ENABLED, value = "false") + void testFlagEvaluationWriterCanBeDisabled() { + ConfigurationPoller poller = mock(ConfigurationPoller.class); + DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); + SharedCommunicationObjects sharedCommunicationObjects = mock(SharedCommunicationObjects.class); + when(discovery.supportsEvpProxy()).thenReturn(true); + when(discovery.getEvpProxyEndpoint()).thenReturn("/evp_proxy/"); + when(sharedCommunicationObjects.configurationPoller(any(Config.class))).thenReturn(poller); + when(sharedCommunicationObjects.featuresDiscovery(any(Config.class))).thenReturn(discovery); + sharedCommunicationObjects.agentUrl = HttpUrl.get("http://localhost"); + sharedCommunicationObjects.agentHttpClient = new OkHttpClient.Builder().build(); + + try { + FeatureFlaggingSystem.start(sharedCommunicationObjects); + assertNull(FeatureFlaggingGateway.getFlagEvalWriter()); + } finally { + FeatureFlaggingSystem.stop(); + } + } + @Test @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "false") void testThatRemoteConfigIsRequired() { From 90ed6cb556c56be50657236faa366b7e0907d4e6 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:49:30 -0400 Subject: [PATCH 21/26] perf(feature-flagging): benchmark flagevaluation hot path --- .../feature-flagging-lib/build.gradle.kts | 12 ++ .../FlagEvaluationHotPathBenchmark.java | 143 ++++++++++++++++++ 2 files changed, 155 insertions(+) create mode 100644 products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java diff --git a/products/feature-flagging/feature-flagging-lib/build.gradle.kts b/products/feature-flagging/feature-flagging-lib/build.gradle.kts index 3291e239d40..73337d4676a 100644 --- a/products/feature-flagging/feature-flagging-lib/build.gradle.kts +++ b/products/feature-flagging/feature-flagging-lib/build.gradle.kts @@ -1,6 +1,7 @@ plugins { `java-library` id("dd-trace-java.version-file") + id("me.champeau.jmh") } apply(from = "$rootDir/gradle/java.gradle") @@ -28,3 +29,14 @@ dependencies { testImplementation(project(":utils:test-utils")) testImplementation(project(":dd-java-agent:testing")) } + +jmh { + jmhVersion = libs.versions.jmh.get() + duplicateClassesStrategy = DuplicatesStrategy.EXCLUDE + if (project.hasProperty("jmhIncludes")) { + includes = listOf(project.property("jmhIncludes").toString()) + } + if (project.hasProperty("jmhProf")) { + profilers = listOf(project.property("jmhProf").toString()) + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java b/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java new file mode 100644 index 00000000000..bd70b70d211 --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java @@ -0,0 +1,143 @@ +package com.datadog.featureflag; + +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; + +import datadog.communication.BackendApiFactory; +import datadog.trace.api.Config; +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import java.util.HashMap; +import java.util.Map; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Hot-path benchmark for EVP {@code flagevaluation} recording. + * + *

The OpenFeature {@code finally} hook runs synchronously on the caller's evaluation thread. The + * evaluation thread should only pay for scalar capture, lazy context supplier capture, and the + * non-blocking writer enqueue. Recursive context flattening, pruning, canonical-key construction, + * and aggregation are characterized separately as worker-thread cost. + * + *

Run: {@code ./gradlew :products:feature-flagging:feature-flagging-lib:jmh -PjmhIncludes=FlagEvaluationHotPathBenchmark}. + */ +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 2, timeUnit = SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = SECONDS) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(NANOSECONDS) +@Fork(value = 1) +public class FlagEvaluationHotPathBenchmark { + + @Param({ + "typical/100flags_50users_10fields", + "stress/10flags_1000users_250fields", + "scale/2500flags_500users_20fields" + }) + public String profile; + + private Map attrs; + private String[] flagKeys; + private String[] targetingKeys; + private int cursor; + private FlagEvaluationWriterImpl writer; + private FlagEvaluationWriterImpl.SerializingHandlerForTest handler; + + @Setup(Level.Iteration) + public void setUp() { + final Profile p = Profile.fromName(profile); + + attrs = new HashMap<>(); + for (int i = 0; i < p.numFields; i++) { + attrs.put("field" + i, "value"); + } + flagKeys = keys("bench-flag-", p.numFlags); + targetingKeys = keys("bench-user-", p.numUsers); + cursor = 0; + + final Config config = Config.get(); + final BackendApiFactory factory = new BackendApiFactory(config, null); + final Map ddContext = new HashMap<>(); + ddContext.put("service", "bench-service"); + handler = FlagEvaluationWriterImpl.createHandlerForTest(factory, ddContext); + + // Capacity large enough that the benchmark never overflows within a measurement window. + writer = new FlagEvaluationWriterImpl(1 << 20, Long.MAX_VALUE, NANOSECONDS, factory, config); + } + + /** Evaluation-thread cost: lazy capture snapshot plus non-blocking enqueue. */ + @Benchmark + public void evalThreadCapture(final Blackhole blackhole) { + final FlagEvalEvent event = nextLazyEvent(); + writer.enqueue(event); + blackhole.consume(writer.pollQueuedEventForTest()); + blackhole.consume(event); + } + + /** Worker-thread cost: materialize context, prune, canonicalize, and aggregate. */ + @Benchmark + public void workerAggregate(final Blackhole blackhole) { + final FlagEvalEvent event = nextLazyEvent(); + handler.aggregateEvent(event); + if ((cursor % 10_000) == 0) { + handler.clearAggregationForTest(); + } + blackhole.consume(handler.fullTierSizeForTest()); + } + + private FlagEvalEvent nextLazyEvent() { + final int i = cursor++; + return new FlagEvalEvent( + flagKeys[Math.floorMod(i, flagKeys.length)], + "variant-" + Math.floorMod(i, 4), + "alloc-" + Math.floorMod(i, flagKeys.length), + targetingKeys[Math.floorMod(i, targetingKeys.length)], + null, + 1_700_000_000_000L + i, + () -> attrs); + } + + private static String[] keys(final String prefix, final int count) { + final String[] out = new String[count]; + for (int i = 0; i < count; i++) { + out[i] = prefix + i; + } + return out; + } + + private static final class Profile { + private final int numFlags; + private final int numUsers; + private final int numFields; + + private Profile(final int numFlags, final int numUsers, final int numFields) { + this.numFlags = numFlags; + this.numUsers = numUsers; + this.numFields = numFields; + } + + private static Profile fromName(final String name) { + if ("typical/100flags_50users_10fields".equals(name)) { + return new Profile(100, 50, 10); + } + if ("stress/10flags_1000users_250fields".equals(name)) { + return new Profile(10, 1_000, 250); + } + if ("scale/2500flags_500users_20fields".equals(name)) { + return new Profile(2_500, 500, 20); + } + throw new IllegalArgumentException("unknown benchmark profile: " + name); + } + } +} From c73cafdb46fc8fff39cb83f5f61f8665b24b6a4f Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 2 Jul 2026 10:28:16 -0400 Subject: [PATCH 22/26] chore: apply spotless formatting --- .../test/java/datadog/communication/BackendApiFactoryTest.java | 3 +-- .../java/datadog/trace/api/openfeature/DDEvaluatorTest.java | 3 +-- .../datadog/featureflag/FlagEvaluationHotPathBenchmark.java | 3 ++- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java b/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java index c287790b4ae..38a75ddceb6 100644 --- a/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java +++ b/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java @@ -84,8 +84,7 @@ void preferredEvpProxyEndpointDoesNotRequireLegacyDefaultRoute() throws Exceptio final BackendApiFactory factory = new BackendApiFactory( Config.get(), sharedCommunicationObjects(discovery, agent.url("/"))); - final BackendApi api = - factory.createBackendApi(Intake.EVENT_PLATFORM, futureEndpoint, false); + final BackendApi api = factory.createBackendApi(Intake.EVENT_PLATFORM, futureEndpoint, false); assertNotNull(api); api.post( diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java index eee6661f024..e81b33bf4c5 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java @@ -236,8 +236,7 @@ private static Arguments[] flatteningTestCases() { mapOf("map.key1", 1, "map.key2", 2, "map.key3.key4", 4))); arguments.add( Arguments.of( - mapOf("plan", "gold", "cohort", "gold"), - mapOf("plan", "gold", "cohort", "gold"))); + mapOf("plan", "gold", "cohort", "gold"), mapOf("plan", "gold", "cohort", "gold"))); return arguments.toArray(new Arguments[0]); } diff --git a/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java b/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java index bd70b70d211..f8bbc729f59 100644 --- a/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java +++ b/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java @@ -30,7 +30,8 @@ * non-blocking writer enqueue. Recursive context flattening, pruning, canonical-key construction, * and aggregation are characterized separately as worker-thread cost. * - *

Run: {@code ./gradlew :products:feature-flagging:feature-flagging-lib:jmh -PjmhIncludes=FlagEvaluationHotPathBenchmark}. + *

Run: {@code ./gradlew :products:feature-flagging:feature-flagging-lib:jmh + * -PjmhIncludes=FlagEvaluationHotPathBenchmark}. */ @State(Scope.Benchmark) @Warmup(iterations = 3, time = 2, timeUnit = SECONDS) From 5b1456b3fbe467dd6b74ba919c85720e7f763921 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 2 Jul 2026 11:36:17 -0400 Subject: [PATCH 23/26] test(feature-flagging): cover flag evaluation jacoco paths --- .../FeatureFlagEvpPublisherTest.java | 14 ++ .../FlagEvaluationAggregatorTest.java | 133 +++++++++++++ .../FlagEvaluationPayloadsTest.java | 123 ++++++++++++ .../FlagEvaluationTestSupport.java | 13 +- .../FlagEvaluationWriterImplTest.java | 184 ++++++++++++++++++ 5 files changed, 465 insertions(+), 2 deletions(-) diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagEvpPublisherTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagEvpPublisherTest.java index b5044f43c7a..ad66a148fc7 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagEvpPublisherTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagEvpPublisherTest.java @@ -1,6 +1,8 @@ package com.datadog.featureflag; import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V4_EVP_PROXY_ENDPOINT; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isNull; @@ -62,6 +64,18 @@ void preferredEndpointAndRequestCompressionAreForwardedToBackendApi() throws Exc .post(eq("flagevaluation"), any(RequestBody.class), any(), isNull(), eq(false)); } + @Test + void postThrowsWhenEvpBackendApiCannotBeCreated() { + final BackendApiFactory factory = mock(BackendApiFactory.class); + final FeatureFlagEvpPublisher publisher = + new FeatureFlagEvpPublisher<>(factory, TestRequest.class); + + assertFalse(publisher.start()); + assertThrows( + IllegalStateException.class, + () -> publisher.post("flagevaluation", FeatureFlagEvpPublisher.utf8Bytes("{}"))); + } + static class TestRequest { public final String value; diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationAggregatorTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationAggregatorTest.java index 0adfcf35d3e..57bc7251c1e 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationAggregatorTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationAggregatorTest.java @@ -3,6 +3,7 @@ import static java.util.Collections.emptyMap; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; @@ -79,6 +80,24 @@ void degradedCapOverflowIncrementsDroppedCounter() { assertTrue(state.droppedDegradedOverflow > 0); } + @Test + void perFlagCapOverflowRoutesToDegradedTierAndMergesSameDegradedKey() { + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + aggregator.perFlagCount.put("hot-flag", FlagEvaluationAggregator.PER_FLAG_CAP); + + aggregator.aggregate(event("hot-flag", "on", "alloc1", "user-1", 1000L, emptyMap())); + aggregator.aggregate(event("hot-flag", "on", "alloc1", "user-2", 2000L, emptyMap())); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + assertEquals(0, state.fullTier.size()); + assertEquals(1, state.degradedTier.size()); + final FlagEvaluationAggregator.EvalBucket bucket = + state.degradedTier.values().iterator().next(); + assertEquals(2, bucket.count); + assertEquals(1000L, bucket.firstEvalMs); + assertEquals(2000L, bucket.lastEvalMs); + } + @Test void absentVariantSetsRuntimeDefaultUsed() { final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); @@ -125,6 +144,34 @@ void pruningIsDeterministicSortBeforeCut() { assertFalse(p1.containsKey("k299")); } + @Test + void emptyContextInputsProduceEmptyPrunedMapAndCanonicalKey() { + assertEquals(emptyMap(), FlagEvaluationAggregator.pruneContext(null)); + assertEquals(emptyMap(), FlagEvaluationAggregator.pruneContext(emptyMap())); + assertEquals("", FlagEvaluationAggregator.canonicalContextKey(null)); + assertEquals("", FlagEvaluationAggregator.canonicalContextKey(emptyMap())); + } + + @Test + void canonicalContextKeyEncodesSupportedValueTypes() { + final Map attrs = new HashMap<>(); + attrs.put("bool", true); + attrs.put("double", 1.5d); + attrs.put("float", 1.25f); + attrs.put("int", 1); + attrs.put("long", 2L); + attrs.put("null", null); + attrs.put("object", new StringBuilder("other")); + attrs.put("string", "value"); + + final String key = FlagEvaluationAggregator.canonicalContextKey(attrs); + + assertEquals(key, FlagEvaluationAggregator.canonicalContextKey(new HashMap<>(attrs))); + assertTrue(key.contains("bool")); + assertTrue(key.contains("string")); + assertTrue(key.contains("other")); + } + @Test void contextValueExceeding256CharsIsSkippedFromPrunedAttrs() { final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); @@ -161,6 +208,64 @@ void flagEvalEventDoesNotCarryReason() { assertFalse(hasReasonField); } + @Test + void evalBucketTracksBoundsDefaultStateAndNullContextFieldCount() { + final FlagEvaluationAggregator.EvalBucket bucket = + new FlagEvaluationAggregator.EvalBucket( + "bucket-flag", "on", "alloc1", "user-1", null, 1000L, false, null); + + assertEquals(0, bucket.prunedContextFieldCount()); + + bucket.merge(900L, true); + bucket.merge(1100L, false); + bucket.merge(1000L, false); + + assertEquals(4, bucket.count); + assertEquals(900L, bucket.firstEvalMs); + assertEquals(1100L, bucket.lastEvalMs); + assertTrue(bucket.runtimeDefaultUsed); + } + + @Test + void fullKeyEqualityUsesEveryDimension() { + final FlagEvaluationAggregator.FullKey base = + fullKey("flag", "on", "alloc", false, "error", "user", "ctx"); + final FlagEvaluationAggregator.FullKey same = + fullKey("flag", "on", "alloc", false, "error", "user", "ctx"); + + assertEquals(base, base); + assertEquals(base, same); + assertEquals(base.hashCode(), same.hashCode()); + assertNotEquals(base, null); + assertNotEquals(base, "not-a-key"); + assertNotEquals(base, fullKey("other", "on", "alloc", false, "error", "user", "ctx")); + assertNotEquals(base, fullKey("flag", "off", "alloc", false, "error", "user", "ctx")); + assertNotEquals(base, fullKey("flag", "on", "other", false, "error", "user", "ctx")); + assertNotEquals(base, fullKey("flag", "on", "alloc", true, "error", "user", "ctx")); + assertNotEquals(base, fullKey("flag", "on", "alloc", false, "other", "user", "ctx")); + assertNotEquals(base, fullKey("flag", "on", "alloc", false, "error", "other", "ctx")); + assertNotEquals(base, fullKey("flag", "on", "alloc", false, "error", "user", "other")); + } + + @Test + void degradedKeyEqualityUsesEveryDimension() { + final FlagEvaluationAggregator.DegradedKey base = + degradedKey("flag", "on", "alloc", false, "error"); + final FlagEvaluationAggregator.DegradedKey same = + degradedKey("flag", "on", "alloc", false, "error"); + + assertEquals(base, base); + assertEquals(base, same); + assertEquals(base.hashCode(), same.hashCode()); + assertNotEquals(base, null); + assertNotEquals(base, "not-a-key"); + assertNotEquals(base, degradedKey("other", "on", "alloc", false, "error")); + assertNotEquals(base, degradedKey("flag", "off", "alloc", false, "error")); + assertNotEquals(base, degradedKey("flag", "on", "other", false, "error")); + assertNotEquals(base, degradedKey("flag", "on", "alloc", true, "error")); + assertNotEquals(base, degradedKey("flag", "on", "alloc", false, "other")); + } + private static FlagEvalEvent event( final String flagKey, final String variant, @@ -175,6 +280,34 @@ private static FlagEvalEvent simpleEvent(final String flagKey, final String vari return event(flagKey, variant, "alloc1", "user-1", 1000L, emptyMap()); } + private static FlagEvaluationAggregator.FullKey fullKey( + final String flagKey, + final String variant, + final String allocationKey, + final boolean runtimeDefaultUsed, + final String errorMessage, + final String targetingKey, + final String contextKey) { + return new FlagEvaluationAggregator.FullKey( + flagKey, + variant, + allocationKey, + runtimeDefaultUsed, + errorMessage, + targetingKey, + contextKey); + } + + private static FlagEvaluationAggregator.DegradedKey degradedKey( + final String flagKey, + final String variant, + final String allocationKey, + final boolean runtimeDefaultUsed, + final String errorMessage) { + return new FlagEvaluationAggregator.DegradedKey( + flagKey, variant, allocationKey, runtimeDefaultUsed, errorMessage); + } + private static String repeat(final char c, final int count) { final char[] chars = new char[count]; Arrays.fill(chars, c); diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java index 3ba2dc8fccf..e3c2d120b00 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java @@ -11,6 +11,7 @@ import com.squareup.moshi.Moshi; import com.squareup.moshi.Types; import java.lang.reflect.Type; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -144,6 +145,45 @@ void oversizedFullPayloadRowIsDegradedBeforeDrop() throws Exception { assertNull(ev.get("context")); } + @Test + void oversizedFullPayloadRowStartsNewPayloadWhenDegradedRowFitsByItself() throws Exception { + final FlagEvaluationPayloads.FlagEvaluationEvent first = + event("first-flag", "on", "alloc1", "user-1", 1, emptyMap()); + final Map attrs = new HashMap<>(); + attrs.put("payload", repeat('x', 400)); + final FlagEvaluationPayloads.FlagEvaluationEvent second = + event("second-flag", "on", "alloc1", "user-2", 3, attrs); + final FlagEvaluationPayloads.FlagEvaluationEvent degradedSecond = + second.withoutTargetingKeyAndContext(); + assertNotNull(degradedSecond); + + final int firstPayloadSize = + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList(first), CONTEXT, 1_000_000) + .bodies + .get(0) + .length; + final int degradedPayloadSize = + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList(degradedSecond), CONTEXT, 1_000_000) + .bodies + .get(0) + .length; + final int limit = Math.max(firstPayloadSize, degradedPayloadSize); + + final FlagEvaluationPayloads.EncodedPayloads payloads = + FlagEvaluationPayloads.buildPayloads(Arrays.asList(first, second), CONTEXT, limit); + + assertEquals(2, payloads.bodies.size()); + assertEquals(0, payloads.droppedPayloadLimit); + assertEquals(3, payloads.degradedPayloadLimit); + assertEquals(1, eventCount(parse(payloads.bodies.get(0)))); + final Map ev = firstEvent(parse(payloads.bodies.get(1))); + assertObjectWithKey(ev.get("flag"), "second-flag"); + assertNull(ev.get("targeting_key")); + assertNull(ev.get("context")); + } + @Test void oversizedDegradedPayloadRowIsDropped() { final FlagEvaluationPayloads.EncodedPayloads payloads = @@ -158,6 +198,20 @@ void oversizedDegradedPayloadRowIsDropped() { assertEquals(0, payloads.degradedPayloadLimit); } + @Test + void oversizedFullPayloadRowIsDroppedWhenDegradedRowStillExceedsLimit() { + final FlagEvaluationPayloads.EncodedPayloads payloads = + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList( + event(repeat('f', 512), "on", "alloc1", "user-1", 2, emptyMap())), + CONTEXT, + 128); + + assertTrue(payloads.bodies.isEmpty()); + assertEquals(2, payloads.droppedPayloadLimit); + assertEquals(0, payloads.degradedPayloadLimit); + } + @Test void errorPayloadSerializesErrorObject() throws Exception { final Map json = @@ -186,6 +240,75 @@ void errorPayloadSerializesErrorObject() throws Exception { assertEquals(Boolean.TRUE, ev.get("runtime_default_used")); } + @Test + void emptyEventListProducesNoPayloads() { + final FlagEvaluationPayloads.EncodedPayloads payloads = + FlagEvaluationPayloads.buildPayloads(java.util.Collections.emptyList(), CONTEXT, 1_000_000); + + assertTrue(payloads.bodies.isEmpty()); + assertEquals(0, payloads.droppedPayloadLimit); + assertEquals(0, payloads.degradedPayloadLimit); + } + + @Test + void requestDtoStoresContextAndEvents() { + final List events = + java.util.Collections.singletonList( + event("dto-flag", "on", "alloc1", "user-1", 1, emptyMap())); + + final FlagEvaluationPayloads.FlagEvaluationsRequest request = + new FlagEvaluationPayloads.FlagEvaluationsRequest(CONTEXT, events); + + assertEquals(CONTEXT, request.context); + assertEquals(events, request.flagEvaluations); + } + + @Test + void degradingEventWithMissingOptionalFieldsKeepsOptionalObjectsAbsent() { + final FlagEvaluationPayloads.FlagEvaluationEvent degraded = + new FlagEvaluationPayloads.FlagEvaluationEvent( + EVAL_MS, + "default-flag", + EVAL_MS, + EVAL_MS, + 1, + null, + null, + "user-1", + true, + null, + null) + .withoutTargetingKeyAndContext(); + + assertNotNull(degraded); + assertNull(degraded.variant); + assertNull(degraded.allocation); + assertNull(degraded.targeting_key); + assertNull(degraded.context); + assertNull(degraded.error); + assertEquals(Boolean.TRUE, degraded.runtime_default_used); + } + + @Test + void emptyOptionalStringsAreTreatedAsAbsentWhenContextIsPresent() { + final Map attrs = new HashMap<>(); + attrs.put("tier", "gold"); + final FlagEvaluationPayloads.FlagEvaluationEvent event = + new FlagEvaluationPayloads.FlagEvaluationEvent( + EVAL_MS, "empty-optionals", EVAL_MS, EVAL_MS, 1, "", "", null, false, "", attrs); + + assertNull(event.variant); + assertNull(event.allocation); + assertNull(event.error); + assertNotNull(event.context); + + final FlagEvaluationPayloads.FlagEvaluationEvent degraded = + event.withoutTargetingKeyAndContext(); + assertNotNull(degraded); + assertNull(degraded.targeting_key); + assertNull(degraded.context); + } + private static FlagEvaluationPayloads.FlagEvaluationEvent event( final String flagKey, final String variant, diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java index 11e5b4f407f..2ec8629dbfc 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java @@ -71,8 +71,17 @@ static String repeat(final char c, final int count) { } static TestWriterSetup buildTestWriter(final BackendApi mockEvp) { - return buildTestWriter( - mockEvp, FlagEvaluationWriterImpl.FLAG_EVALUATION_PAYLOAD_SIZE_LIMIT_BYTES); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + when(factory.createBackendApi(any(), any(), eq(false))).thenReturn(mockEvp); + + final Map context = new HashMap<>(); + context.put("service", "test-service"); + + final FlagEvaluationWriterImpl.SerializingHandlerForTest handler = + FlagEvaluationWriterImpl.createHandlerForTest(factory, context); + + return new TestWriterSetup(handler, mockEvp, factory); } static TestWriterSetup buildTestWriter( diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java index bec4ed9eaee..6737e5553d6 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java @@ -6,6 +6,7 @@ import static com.datadog.featureflag.FlagEvaluationTestSupport.clearCoreMetrics; import static com.datadog.featureflag.FlagEvaluationTestSupport.event; import static com.datadog.featureflag.FlagEvaluationTestSupport.eventForFlag; +import static com.datadog.featureflag.FlagEvaluationTestSupport.flushAndCaptureJson; import static com.datadog.featureflag.FlagEvaluationTestSupport.metricSum; import static com.datadog.featureflag.FlagEvaluationTestSupport.repeat; import static com.datadog.featureflag.FlagEvaluationTestSupport.simpleEvent; @@ -13,6 +14,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -23,8 +25,12 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import datadog.common.queue.MessagePassingBlockingQueue; +import datadog.common.queue.Queues; import datadog.communication.BackendApi; import datadog.communication.BackendApiFactory; +import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; import datadog.trace.api.intake.Intake; import datadog.trace.api.telemetry.CoreMetricCollector; @@ -35,6 +41,7 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import okhttp3.RequestBody; import okio.Buffer; import org.junit.jupiter.api.AfterEach; @@ -51,6 +58,7 @@ void clearCoreMetricsBefore() { @AfterEach void clearCoreMetricsAfter() { clearCoreMetrics(); + FeatureFlaggingGateway.setFlagEvalWriter(null); } @Test @@ -71,6 +79,45 @@ void degradedCapOverflowTelemetryIsEmittedOnFlush() { "reason:" + FlagEvaluationWriterImpl.DROP_REASON_DEGRADED_CAP)); } + @Test + void publicConstructorAndContextHelpersDelegateToSharedImplementations() { + final datadog.trace.api.Config config = cfg(); + when(config.getEnv()).thenReturn("prod"); + when(config.getVersion()).thenReturn("1.2.3"); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(new SharedCommunicationObjects(true), config); + final Map attrs = new HashMap<>(); + attrs.put("b", "2"); + attrs.put("a", "1"); + + final Map pruned = FlagEvaluationWriterImpl.pruneContext(attrs); + + assertEquals(2, pruned.size()); + assertEquals( + FlagEvaluationAggregator.canonicalContextKey(pruned), + FlagEvaluationWriterImpl.canonicalContextKey(pruned)); + writer.close(); + } + + @Test + void startRegistersWriterAndCloseDeregistersIt() { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + when(factory.createBackendApi(any(), any(), eq(false))).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + + writer.start(); + assertEquals(writer, FeatureFlaggingGateway.getFlagEvalWriter()); + + writer.close(); + writer.close(); + writer.start(); + + assertNull(FeatureFlaggingGateway.getFlagEvalWriter()); + } + @Test void queueOverflowIncrementsObservableDropCounter() { final BackendApi mockEvp = mock(BackendApi.class); @@ -120,6 +167,21 @@ void enqueueAfterCloseIsDroppedAndCounted() { assertNull(writer.pollQueuedEventForTest()); } + @Test + void enqueueIgnoresNullEvent() { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + when(factory.createBackendApi(any(), any(), eq(false))).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + + writer.enqueue(null); + + assertNull(writer.pollQueuedEventForTest()); + assertEquals(0, writer.droppedQueueOverflow()); + } + @Test void enqueueDoesNotAggregateOnTheCallingThread() { final BackendApi mockEvp = mock(BackendApi.class); @@ -198,6 +260,122 @@ void contextMaterializationFailureDropsSingleEvent() { assertEquals(0, setup.handler.fullTierSizeForTest()); } + @Test + void handlerRunFailsFastWhenEvpProxyIsUnavailable() { + final BackendApiFactory factory = mock(BackendApiFactory.class); + final FlagEvaluationWriterImpl.SerializingHandlerForTest handler = + FlagEvaluationWriterImpl.createHandlerForTest(factory, context()); + + assertThrows(IllegalArgumentException.class, handler::run); + } + + @Test + void flushIfNecessarySkipsEmptyStateAndWaitsForInterval() { + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp); + + setup.handler.flushIfNecessary(); + setup.handler.add(simpleEvent("pending-flag", "on")); + setup.handler.drainAndAggregate(); + setup.handler.flushIfNecessary(); + + assertEquals(1, setup.handler.fullTierSizeForTest()); + } + + @Test + void flushIfNecessaryDoesNotReturnEarlyWhenOnlyQueueDropsArePending() { + final AtomicLong queueDrops = new AtomicLong(1); + final FlagEvaluationWriterImpl.FlagEvaluationSerializingHandler handler = + new FlagEvaluationWriterImpl.FlagEvaluationSerializingHandler( + mock(BackendApiFactory.class), + Queues.mpscBlockingConsumerArrayQueue(16), + Long.MAX_VALUE, + TimeUnit.NANOSECONDS, + context(), + queueDrops, + () -> {}); + + handler.flushIfNecessary(); + + assertEquals(1, queueDrops.get()); + } + + @Test + @SuppressWarnings("unchecked") + void workerHandlesEmptyPolls() throws Exception { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + when(factory.createBackendApi(any(), any(), eq(false))).thenReturn(mockEvp); + final MessagePassingBlockingQueue queue = + mock(MessagePassingBlockingQueue.class); + when(queue.poll(100, TimeUnit.MILLISECONDS)) + .thenAnswer( + invocation -> { + Thread.currentThread().interrupt(); + return null; + }); + final FlagEvaluationWriterImpl.FlagEvaluationSerializingHandler handler = + new FlagEvaluationWriterImpl.FlagEvaluationSerializingHandler( + factory, + queue, + Long.MAX_VALUE, + TimeUnit.NANOSECONDS, + context(), + new AtomicLong(0), + () -> {}); + + handler.run(); + + assertTrue(Thread.interrupted()); + } + + @Test + void degradedBucketsAreSerializedWithoutTargetingKeyOrContext() throws Exception { + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp); + setup.handler.addDegradedBucketForTest("degraded-flag", "on", "alloc1", null, 1000L); + + final Map json = flushAndCaptureJson(setup); + + final Map ev = eventForFlag(json, "degraded-flag"); + assertNotNull(ev); + assertNull(ev.get("targeting_key")); + assertNull(ev.get("context")); + } + + @Test + void testHandlerCanSimulateAndClearDegradedTierAtCap() { + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp); + + setup.handler.simulateDegradedTierAtCap(); + setup.handler.clearAggregationForTest(); + setup.handler.add(simpleEvent("after-clear", "on")); + setup.handler.drainAndAggregate(); + + assertEquals(1, setup.handler.fullTierSizeForTest()); + } + + @Test + void payloadLimitDropsAreCountedOnFlush() { + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp, 128); + setup.handler.add(event(repeat('f', 512), "on", "alloc1", "user-1", 1000L, emptyMap())); + + setup.handler.drainAndAggregate(); + setup.handler.flush(); + + final Collection metrics = + CoreMetricCollector.getInstance().drain(); + assertEquals( + 1, + metricSum( + metrics, + FlagEvaluationWriterImpl.FLAG_EVALUATION_DROPPED_METRIC, + "reason:" + FlagEvaluationWriterImpl.DROP_REASON_PAYLOAD_LIMIT)); + } + @Test void closeDrainsAndFinalFlushesQueuedEvents() throws Exception { final java.util.concurrent.CountDownLatch posted = new java.util.concurrent.CountDownLatch(1); @@ -301,4 +479,10 @@ void splitPostFailureDoesNotRetryAlreadySentPayloads() throws Exception { setup.handler.flush(); assertEquals(2, posts.get()); } + + private static Map context() { + final Map context = new HashMap<>(); + context.put("service", "test-service"); + return context; + } } From 6e1f81e9be0770e7f549d016120fc6c665b41016 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 2 Jul 2026 12:54:39 -0400 Subject: [PATCH 24/26] fix(feature-flagging): gate flag evaluation enqueue during shutdown --- .../java/datadog/trace/bootstrap/Agent.java | 19 ++++++ .../featureflag/FeatureFlaggingSystem.java | 9 +++ .../FeatureFlaggingSystemTest.java | 33 ++++++++- .../api/openfeature/FlagEvalLoggingHook.java | 9 ++- .../openfeature/FlagEvalLoggingHookTest.java | 33 ++++++++- .../trace/api/openfeature/ProviderTest.java | 1 + .../featureflag/FeatureFlaggingGateway.java | 16 +++++ .../FeatureFlaggingGatewayTest.java | 2 + .../featureflag/FlagEvaluationWriterImpl.java | 21 +++++- .../FlagEvaluationWriterImplTest.java | 68 +++++++++++++++++++ 10 files changed, 203 insertions(+), 8 deletions(-) diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index aae283da55c..68b30bf9f07 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -521,6 +521,9 @@ public static void shutdown(final boolean sync) { if (profilingEnabled) { shutdownProfilingAgent(sync); } + if (featureFlaggingEnabled) { + shutdownFeatureFlagging(); + } if (telemetryEnabled) { stopTelemetry(); } @@ -1211,6 +1214,22 @@ private static void shutdownLogsIntake() { } } + private static void shutdownFeatureFlagging() { + if (AGENT_CLASSLOADER == null) { + // It wasn't started, so no need to shut it down + return; + } + try { + Thread.currentThread().setContextClassLoader(AGENT_CLASSLOADER); + final Class ffSysClass = + AGENT_CLASSLOADER.loadClass("com.datadog.featureflag.FeatureFlaggingSystem"); + final Method shutdownMethod = ffSysClass.getMethod("shutdown"); + shutdownMethod.invoke(null); + } catch (final Throwable ex) { + log.error("Throwable thrown while shutting down Feature Flagging", ex); + } + } + private static void startTelemetry(Instrumentation inst, Class scoClass, Object sco) { StaticEventLogger.begin("Telemetry"); diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index c5be9b67f82..91c03afa7ca 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -3,6 +3,7 @@ import datadog.communication.ddagent.SharedCommunicationObjects; import datadog.trace.api.Config; import datadog.trace.api.config.FeatureFlaggingConfig; +import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,12 +35,14 @@ public static void start(final SharedCommunicationObjects sco) { config .configProvider() .getBoolean(FeatureFlaggingConfig.FLAGGING_EVALUATION_COUNTS_ENABLED, true); + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(evalCountsEnabled); if (evalCountsEnabled) { final FlagEvaluationWriterImpl evalWriter = new FlagEvaluationWriterImpl(sco, config); evalWriter.start(); FLAG_EVAL_WRITER = evalWriter; LOGGER.debug("Flag evaluation EVP writer started"); } else { + FeatureFlaggingGateway.setFlagEvalWriter(null); LOGGER.debug( "Flag evaluation EVP writer disabled ({}=false)", FeatureFlaggingConfig.FLAGGING_EVALUATION_COUNTS_ENABLED); @@ -49,6 +52,8 @@ public static void start(final SharedCommunicationObjects sco) { } public static void stop() { + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(false); + FeatureFlaggingGateway.setFlagEvalWriter(null); if (FLAG_EVAL_WRITER != null) { FLAG_EVAL_WRITER.close(); FLAG_EVAL_WRITER = null; @@ -63,4 +68,8 @@ public static void stop() { } LOGGER.debug("Feature Flagging system stopped"); } + + public static void shutdown() { + stop(); + } } diff --git a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java index 82fae6ecdb0..ad18710c0dd 100644 --- a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java +++ b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java @@ -1,9 +1,11 @@ package com.datadog.featureflag; import static datadog.trace.api.config.RemoteConfigConfig.REMOTE_CONFIGURATION_ENABLED; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; @@ -19,13 +21,21 @@ import datadog.trace.api.Config; import datadog.trace.api.config.FeatureFlaggingConfig; import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; import datadog.trace.junit.utils.config.WithConfig; import okhttp3.HttpUrl; import okhttp3.OkHttpClient; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; class FeatureFlaggingSystemTest { + @AfterEach + void resetFlagEvaluationGateway() { + FeatureFlaggingGateway.setFlagEvalWriter(null); + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); + } + @Test @WithConfig(key = FeatureFlaggingConfig.FLAGGING_EVALUATION_COUNTS_ENABLED, value = "true") void testFeatureFlagSystemInitialization() { @@ -38,15 +48,18 @@ void testFeatureFlagSystemInitialization() { when(sharedCommunicationObjects.featuresDiscovery(any(Config.class))).thenReturn(discovery); sharedCommunicationObjects.agentUrl = HttpUrl.get("http://localhost"); sharedCommunicationObjects.agentHttpClient = new OkHttpClient.Builder().build(); + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(false); FeatureFlaggingSystem.start(sharedCommunicationObjects); verify(poller).addCapabilities(Capabilities.CAPABILITY_FFE_FLAG_CONFIGURATION_RULES); verify(poller).addListener(eq(Product.FFE_FLAGS), any(ConfigurationDeserializer.class), any()); verify(poller).start(); + assertTrue(FeatureFlaggingGateway.isFlagEvaluationEnqueueEnabled()); assertNotNull(FeatureFlaggingGateway.getFlagEvalWriter()); - FeatureFlaggingSystem.stop(); + FeatureFlaggingSystem.shutdown(); + assertFalse(FeatureFlaggingGateway.isFlagEvaluationEnqueueEnabled()); assertNull(FeatureFlaggingGateway.getFlagEvalWriter()); verify(poller).removeCapabilities(Capabilities.CAPABILITY_FFE_FLAG_CONFIGURATION_RULES); @@ -66,15 +79,29 @@ void testFlagEvaluationWriterCanBeDisabled() { when(sharedCommunicationObjects.featuresDiscovery(any(Config.class))).thenReturn(discovery); sharedCommunicationObjects.agentUrl = HttpUrl.get("http://localhost"); sharedCommunicationObjects.agentHttpClient = new OkHttpClient.Builder().build(); + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); + FeatureFlaggingGateway.setFlagEvalWriter(mock(FlagEvaluationWriter.class)); try { FeatureFlaggingSystem.start(sharedCommunicationObjects); + assertFalse(FeatureFlaggingGateway.isFlagEvaluationEnqueueEnabled()); assertNull(FeatureFlaggingGateway.getFlagEvalWriter()); } finally { - FeatureFlaggingSystem.stop(); + FeatureFlaggingSystem.shutdown(); } } + @Test + void testFeatureFlagSystemShutdownClearsGatewayState() { + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); + FeatureFlaggingGateway.setFlagEvalWriter(mock(FlagEvaluationWriter.class)); + + FeatureFlaggingSystem.shutdown(); + + assertFalse(FeatureFlaggingGateway.isFlagEvaluationEnqueueEnabled()); + assertNull(FeatureFlaggingGateway.getFlagEvalWriter()); + } + @Test @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "false") void testThatRemoteConfigIsRequired() { @@ -85,7 +112,7 @@ void testThatRemoteConfigIsRequired() { IllegalStateException.class, () -> FeatureFlaggingSystem.start(sharedCommunicationObjects)); } finally { - FeatureFlaggingSystem.stop(); + FeatureFlaggingSystem.shutdown(); } } } diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalLoggingHook.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalLoggingHook.java index c394a154e01..5a3c852a207 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalLoggingHook.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalLoggingHook.java @@ -66,8 +66,15 @@ public void finallyAfter( final FlagEvaluationDetails details, final Map hints) { try { + if (details == null) { + return; + } + if (!FeatureFlaggingGateway.isFlagEvaluationEnqueueEnabled()) { + return; + } + final FlagEvaluationWriter w = writerSupplier.get(); - if (w == null || details == null) { + if (w == null) { return; } diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalLoggingHookTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalLoggingHookTest.java index 9fa50b03fdc..b498faf42bc 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalLoggingHookTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalLoggingHookTest.java @@ -13,6 +13,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; +import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; import dev.openfeature.sdk.ErrorCode; @@ -30,6 +31,8 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; /** @@ -38,6 +41,16 @@ */ class FlagEvalLoggingHookTest { + @BeforeEach + void enableFlagEvaluationEnqueue() { + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); + } + + @AfterEach + void resetFlagEvaluationEnqueue() { + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); + } + // ---- helpers ---- /** @@ -283,12 +296,30 @@ void finallyAfterOnlyCallsEnqueueNoOtherWriterMethods() { hook.finallyAfter(null, det, Collections.emptyMap()); - // Exactly one enqueue call, no start/close/aggregate + // Exactly one enqueue call, no start/close. verify(writer, times(1)).enqueue(any(FlagEvalEvent.class)); verify(writer, never()).close(); verify(writer, never()).start(); } + @Test + void enqueueDisabledIsNoOpBeforeWriterLookup() { + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(false); + final AtomicReference writerResolved = new AtomicReference<>(false); + final FlagEvalLoggingHook hook = + new FlagEvalLoggingHook<>( + () -> { + writerResolved.set(true); + throw new AssertionError("writer should not be resolved when enqueue is disabled"); + }); + final FlagEvaluationDetails det = + details("flag", "v", "v", Reason.TARGETING_MATCH.name(), null); + + hook.finallyAfter(hookCtxWithTargetingKey("flag", "user-1"), det, Collections.emptyMap()); + + assertFalse(writerResolved.get()); + } + // ---- test: writer=null -> no-op (killswitch off / not yet started) ---- @Test diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java index 3fd4417d6f1..ef37cdb330b 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java @@ -72,6 +72,7 @@ public void tearDown() { OpenFeatureAPI.getInstance().shutdown(); FeatureFlaggingGateway.dispatch((ServerConfiguration) null); FeatureFlaggingGateway.setFlagEvalWriter(null); + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); } @Test diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java index a0856fc0ab3..61237e12760 100644 --- a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java +++ b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java @@ -29,6 +29,8 @@ public interface ExposureListener extends Consumer {} private static final AtomicReference FLAG_EVAL_WRITER = new AtomicReference<>(); + private static volatile boolean flagEvalEnqueueEnabled = true; + private FeatureFlaggingGateway() {} public static void addConfigListener(final ConfigListener listener) { @@ -71,6 +73,15 @@ public static void setFlagEvalWriter(final FlagEvaluationWriter writer) { FLAG_EVAL_WRITER.set(writer); } + /** + * Enables or disables enqueueing EVP flagevaluation events on the OpenFeature hook path. This is + * populated from {@code DD_FLAGGING_EVALUATION_COUNTS_ENABLED} at feature-flagging startup and + * cleared during shutdown before the writer drains. + */ + public static void setFlagEvaluationEnqueueEnabled(final boolean enabled) { + flagEvalEnqueueEnabled = enabled; + } + /** * Returns the active EVP flagevaluation writer, or {@code null} when disabled (killswitch off or * not yet started). @@ -78,4 +89,9 @@ public static void setFlagEvalWriter(final FlagEvaluationWriter writer) { public static FlagEvaluationWriter getFlagEvalWriter() { return FLAG_EVAL_WRITER.get(); } + + /** Returns whether EVP flagevaluation hook events may be enqueued. */ + public static boolean isFlagEvaluationEnqueueEnabled() { + return flagEvalEnqueueEnabled; + } } diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/FeatureFlaggingGatewayTest.java b/products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/FeatureFlaggingGatewayTest.java index fc04cf2193b..da4a6c18891 100644 --- a/products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/FeatureFlaggingGatewayTest.java +++ b/products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/FeatureFlaggingGatewayTest.java @@ -33,6 +33,8 @@ void setUp() { void tearDown() { FeatureFlaggingGateway.removeConfigListener(configListener); FeatureFlaggingGateway.removeExposureListener(exposureListener); + FeatureFlaggingGateway.setFlagEvalWriter(null); + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); } @Test diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java index 18b04d5c99a..685d85fec79 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java @@ -173,7 +173,8 @@ public void close() { if (!closed.compareAndSet(false, true)) { return; } - // Deregister from the gateway so no new events are enqueued. + // Disable and deregister from the gateway so no new events are enqueued. + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(false); FeatureFlaggingGateway.setFlagEvalWriter(null); if (!this.serializerThread.isAlive()) { return; @@ -198,9 +199,13 @@ public void enqueue(final FlagEvalEvent event) { if (event == null) { return; } + if (isClosedOrEnqueueDisabled()) { + countClosedDropIfClosed(); + return; + } synchronized (lifecycleLock) { - if (closed.get()) { - countMetric(FLAG_EVALUATION_DROPPED_METRIC, 1, DROP_REASON_CLOSED); + if (isClosedOrEnqueueDisabled()) { + countClosedDropIfClosed(); return; } // Non-blocking offer. Count overflow so loss is observable rather than silent; the count is @@ -211,6 +216,16 @@ public void enqueue(final FlagEvalEvent event) { } } + private boolean isClosedOrEnqueueDisabled() { + return closed.get() || !FeatureFlaggingGateway.isFlagEvaluationEnqueueEnabled(); + } + + private void countClosedDropIfClosed() { + if (closed.get()) { + countMetric(FLAG_EVALUATION_DROPPED_METRIC, 1, DROP_REASON_CLOSED); + } + } + /** Returns the count of events dropped due to queue-overflow backpressure (observable). */ long droppedQueueOverflow() { return droppedQueueOverflow.get(); diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java index 6737e5553d6..280b3028803 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java @@ -36,6 +36,7 @@ import datadog.trace.api.telemetry.CoreMetricCollector; import datadog.trace.api.telemetry.MetricCollector; import java.io.IOException; +import java.lang.reflect.Field; import java.util.Collection; import java.util.HashMap; import java.util.Map; @@ -53,12 +54,14 @@ class FlagEvaluationWriterImplTest { @BeforeEach void clearCoreMetricsBefore() { clearCoreMetrics(); + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); } @AfterEach void clearCoreMetricsAfter() { clearCoreMetrics(); FeatureFlaggingGateway.setFlagEvalWriter(null); + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); } @Test @@ -167,6 +170,56 @@ void enqueueAfterCloseIsDroppedAndCounted() { assertNull(writer.pollQueuedEventForTest()); } + @Test + void enqueueDisabledDropsWithoutQueueingOrCountingClosedDrop() { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + when(factory.createBackendApi(any(), any(), eq(false))).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(false); + writer.enqueue(simpleEvent("disabled-flag", "on")); + + final Collection metrics = + CoreMetricCollector.getInstance().drain(); + assertEquals( + 0, + metricSum( + metrics, + FlagEvaluationWriterImpl.FLAG_EVALUATION_DROPPED_METRIC, + "reason:" + FlagEvaluationWriterImpl.DROP_REASON_CLOSED)); + assertNull(writer.pollQueuedEventForTest()); + } + + @Test + void enqueueRechecksEnabledStateAfterTakingLifecycleLock() throws Exception { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + when(factory.createBackendApi(any(), any(), eq(false))).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + final Thread enqueuer = new Thread(() -> writer.enqueue(simpleEvent("race-flag", "on"))); + + final Object lifecycleLock = lifecycleLock(writer); + try { + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); + synchronized (lifecycleLock) { + enqueuer.start(); + awaitThreadState(enqueuer, Thread.State.BLOCKED); + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(false); + } + enqueuer.join(TimeUnit.SECONDS.toMillis(5)); + assertTrue(!enqueuer.isAlive()); + } finally { + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); + } + + assertNull(writer.pollQueuedEventForTest()); + } + @Test void enqueueIgnoresNullEvent() { final BackendApi mockEvp = mock(BackendApi.class); @@ -480,6 +533,21 @@ void splitPostFailureDoesNotRetryAlreadySentPayloads() throws Exception { assertEquals(2, posts.get()); } + private static Object lifecycleLock(final FlagEvaluationWriterImpl writer) throws Exception { + final Field field = FlagEvaluationWriterImpl.class.getDeclaredField("lifecycleLock"); + field.setAccessible(true); + return field.get(writer); + } + + private static void awaitThreadState(final Thread thread, final Thread.State state) + throws InterruptedException { + final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (thread.getState() != state && System.nanoTime() < deadline) { + Thread.sleep(10); + } + assertEquals(state, thread.getState()); + } + private static Map context() { final Map context = new HashMap<>(); context.put("service", "test-service"); From 78d829789db405dfdb5d70f1eea5cecaab8a940f Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 2 Jul 2026 14:27:24 -0400 Subject: [PATCH 25/26] test(feature-flagging): cover flag eval event --- .../flagevaluation/FlagEvalEventTest.java | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEventTest.java diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEventTest.java b/products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEventTest.java new file mode 100644 index 00000000000..7faa19c72b3 --- /dev/null +++ b/products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEventTest.java @@ -0,0 +1,73 @@ +package datadog.trace.api.featureflag.flagevaluation; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; + +class FlagEvalEventTest { + + @Test + void storesFieldsWithEagerContextAttributes() { + final Map attrs = Collections.singletonMap("tier", "gold"); + + final FlagEvalEvent event = + new FlagEvalEvent("my-flag", "on", "allocation-1", "target-1", 123L, attrs); + + assertEquals("my-flag", event.flagKey); + assertEquals("on", event.variant); + assertEquals("allocation-1", event.allocationKey); + assertEquals("target-1", event.targetingKey); + assertNull(event.errorMessage); + assertEquals(123L, event.evalTimeMs); + assertSame(attrs, event.attrs); + assertSame(attrs, event.contextAttributes()); + } + + @Test + void storesErrorMessageAndDefaultsNullEagerContextAttributes() { + final Map attrs = null; + final FlagEvalEvent event = + new FlagEvalEvent("my-flag", null, null, null, "type mismatch", 456L, attrs); + + assertEquals("type mismatch", event.errorMessage); + assertTrue(event.attrs.isEmpty()); + assertTrue(event.contextAttributes().isEmpty()); + } + + @Test + void resolvesLazyContextAttributesOnDemand() { + final AtomicInteger resolutions = new AtomicInteger(); + final Map attrs = Collections.singletonMap("region", "us-east-1"); + final FlagEvalEvent event = + new FlagEvalEvent( + "my-flag", + "on", + "allocation-1", + "target-1", + null, + 789L, + () -> { + resolutions.incrementAndGet(); + return attrs; + }); + + assertTrue(event.attrs.isEmpty()); + assertEquals(0, resolutions.get()); + assertSame(attrs, event.contextAttributes()); + assertEquals(1, resolutions.get()); + } + + @Test + void defaultsNullLazyContextAttributes() { + final FlagEvalEvent event = + new FlagEvalEvent("my-flag", "on", null, null, null, 789L, () -> null); + + assertTrue(event.contextAttributes().isEmpty()); + } +} From 6acbb5efe1dea72033960343a41eb5717e8616ac Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 2 Jul 2026 22:31:33 -0400 Subject: [PATCH 26/26] fix(feature-flagging): make aggregator count atomic --- .../featureflag/FlagEvaluationAggregator.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationAggregator.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationAggregator.java index eb15eb884a5..a146501f825 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationAggregator.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationAggregator.java @@ -5,6 +5,7 @@ import java.util.Map; import java.util.Objects; import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; final class FlagEvaluationAggregator { @@ -39,7 +40,7 @@ final class FlagEvaluationAggregator { final Map degradedTier = new HashMap<>(); final Map perFlagCount = new HashMap<>(); final AtomicLong droppedDegradedOverflow = new AtomicLong(0); - int globalFullCount = 0; + final AtomicInteger globalFullCount = new AtomicInteger(0); void aggregate(final FlagEvalEvent event) { final boolean isDefault = event.variant == null; @@ -54,7 +55,7 @@ void aggregate(final FlagEvalEvent event) { } final int flagCount = perFlagCount.getOrDefault(event.flagKey, 0); - if (globalFullCount < GLOBAL_CAP && flagCount < PER_FLAG_CAP) { + if (globalFullCount.get() < GLOBAL_CAP && flagCount < PER_FLAG_CAP) { fullTier.put( fullKey, new EvalBucket( @@ -66,7 +67,7 @@ void aggregate(final FlagEvalEvent event) { event.evalTimeMs, isDefault, prunedAttrs)); - globalFullCount++; + globalFullCount.incrementAndGet(); perFlagCount.put(event.flagKey, flagCount + 1); return; } @@ -128,7 +129,7 @@ void clear() { fullTier.clear(); degradedTier.clear(); perFlagCount.clear(); - globalFullCount = 0; + globalFullCount.set(0); } AggregatedState snapshot() { @@ -137,12 +138,12 @@ AggregatedState snapshot() { } void simulateFullTierAtCap() { - for (int i = globalFullCount; i < GLOBAL_CAP; i++) { + for (int i = globalFullCount.get(); i < GLOBAL_CAP; i++) { final String key = "synthetic-full-" + i; fullTier.put( new FullKey(key, "on", "alloc", false, null, null, ""), new EvalBucket(key, "on", "alloc", null, null, 1L, false, null)); - globalFullCount++; + globalFullCount.incrementAndGet(); perFlagCount.merge(key, 1, Integer::sum); } }