From 4aa04808b9e44ed9f4f848a09193760baec1c52c Mon Sep 17 00:00:00 2001 From: Munir Abdinur Date: Thu, 5 Nov 2026 16:45:54 -0500 Subject: [PATCH 1/4] Require explicit opt-in for OTLP APM RED metrics computation --- .../common/metrics/MetricsAggregatorFactory.java | 2 +- .../common/metrics/MetricsAggregatorFactoryTest.java | 12 ++++++++++++ .../src/main/java/datadog/trace/api/Config.java | 8 +------- .../src/test/java/datadog/trace/api/ConfigTest.java | 11 ++++------- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricsAggregatorFactory.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricsAggregatorFactory.java index afd25d9566a..67ece192ab3 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricsAggregatorFactory.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricsAggregatorFactory.java @@ -44,7 +44,7 @@ public static MetricsAggregator createMetricsAggregator( return new ClientStatsAggregator( config, sharedCommunicationObjects, healthMetrics, new OtlpStatsMetricWriter(config)); } - if (config.isTracerMetricsEnabled()) { + if (config.isTracerMetricsEnabled() && !config.isTraceOtlpExporterEnabled()) { log.debug("tracer metrics enabled"); return new ClientStatsAggregator(config, sharedCommunicationObjects, healthMetrics); } diff --git a/dd-trace-core/src/test/java/datadog/trace/common/metrics/MetricsAggregatorFactoryTest.java b/dd-trace-core/src/test/java/datadog/trace/common/metrics/MetricsAggregatorFactoryTest.java index e9d981df389..3f499bc51e9 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/metrics/MetricsAggregatorFactoryTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/metrics/MetricsAggregatorFactoryTest.java @@ -3,6 +3,7 @@ import static datadog.trace.api.config.GeneralConfig.TRACE_STATS_COMPUTATION_ENABLED; import static datadog.trace.api.config.OtlpConfig.OTEL_TRACES_SPAN_METRICS_ENABLED; import static datadog.trace.api.config.OtlpConfig.OTLP_METRICS_PROTOCOL; +import static datadog.trace.api.config.OtlpConfig.TRACE_OTEL_EXPORTER; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -70,6 +71,17 @@ void whenNativeTracerMetricsEnabledSerializingWriterSelected() { assertEquals(SECONDS, conflating.reportingIntervalTimeUnit()); } + @Test + void whenOtlpTracesEnabledNoOpAggregatorCreatedByDefault() { + Config config = Config.get(props(TRACE_OTEL_EXPORTER, "otlp")); + + MetricsAggregator aggregator = + MetricsAggregatorFactory.createMetricsAggregator( + config, sharedCommunicationObjects(), HealthMetrics.NO_OP); + + assertInstanceOf(NoOpMetricsAggregator.class, aggregator); + } + @Test void whenOtlpTraceMetricsEnabledOtlpStatsMetricWriterSelected() { Config config = diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 0179089883d..73434247955 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -2250,14 +2250,8 @@ private Config(final ConfigProvider configProvider, final InstrumenterConfig ins OtlpConfig.Temporality.class, OtlpConfig.Temporality.DELTA); - // Tri-state default: when unset, SDK-computed OTLP span metrics are emitted iff OTLP trace - // export and OTLP metrics export are both enabled. otelTracesSpanMetricsEnabled = - configProvider.getBoolean( - OTEL_TRACES_SPAN_METRICS_ENABLED, - isTraceOtlpExporterEnabled() - && isMetricsOtelEnabled() - && isMetricsOtlpExporterEnabled()); + configProvider.getBoolean(OTEL_TRACES_SPAN_METRICS_ENABLED, false); otlpTimeout = configProvider.getInteger(OTLP_TRACES_TIMEOUT, DEFAULT_OTLP_TRACES_TIMEOUT); if (otlpTimeout < 0) { diff --git a/internal-api/src/test/java/datadog/trace/api/ConfigTest.java b/internal-api/src/test/java/datadog/trace/api/ConfigTest.java index c1699c97a7a..838abc1fdba 100644 --- a/internal-api/src/test/java/datadog/trace/api/ConfigTest.java +++ b/internal-api/src/test/java/datadog/trace/api/ConfigTest.java @@ -1074,8 +1074,8 @@ void otelOtlpFallbackKeyEndpoint2() throws Exception { @TableTest({ "scenario | exporter | metricsEnabled | metricsExporter | override | expected", - "metrics defaults to otlp | otlp | true | | | true ", - "otlp traces and metrics | otlp | true | otlp | | true ", + "metrics defaults to otlp | otlp | true | | | false ", + "otlp traces and metrics | otlp | true | otlp | | false ", "metrics exporter explicitly disabled | otlp | true | none | | false ", "metrics feature flag off overrides exporter | otlp | false | otlp | | false ", "no trace export, default metrics exporter | none | true | | | false ", @@ -1083,7 +1083,7 @@ void otelOtlpFallbackKeyEndpoint2() throws Exception { "explicit override wins (enabled) | none | false | none | true | true ", "explicit override wins (disabled) | otlp | true | otlp | false | false " }) - void tracesSpanMetricsTriStateDefault( + void tracesSpanMetricsRequireExplicitOptIn( String exporter, String metricsEnabled, String metricsExporter, @@ -1101,8 +1101,6 @@ void tracesSpanMetricsTriStateDefault( Config config = newConfig(); - // Unset: emit iff OTLP trace export and OTLP metrics export are both on. An explicit - // dd.otel.traces.span.metrics.enabled always wins. assertEquals(expected, config.isOtelTracesSpanMetricsEnabled()); } @@ -1402,8 +1400,7 @@ void specifyOverridesViaSystemProperties() throws Exception { assertEquals(5002, config.getOtlpTracesTimeout()); assertTrue(config.isTraceOtelSemanticsEnabled()); - // tri-state default: OTLP trace export + OTel metrics both on - assertTrue(config.isOtelTracesSpanMetricsEnabled()); + assertFalse(config.isOtelTracesSpanMetricsEnabled()); } @Test From 9ee01dc90d1cfc749e5c0a1493bbc1436426c02d Mon Sep 17 00:00:00 2001 From: Munir Abdinur Date: Wed, 11 Nov 2026 13:04:51 -0500 Subject: [PATCH 2/4] Reuse comma pattern for native metrics writer selection --- internal-api/src/main/java/datadog/trace/api/Config.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 1140ea36ad7..1671275ef3e 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -4074,7 +4074,7 @@ public boolean isNativeTracerMetricsEnabled() { return isAgentWriterType(writerType); } String multiWriterConfig = Strings.replace(writerType, MULTI_WRITER_TYPE + ":", ""); - for (String subWriterType : multiWriterConfig.split(",")) { + for (String subWriterType : COMMA.split(multiWriterConfig)) { if (isAgentWriterType(subWriterType)) { return true; } From a054559d552a20cef132897c179e8cb3e91ed84c Mon Sep 17 00:00:00 2001 From: Munir Abdinur Date: Wed, 11 Nov 2026 22:53:26 -0500 Subject: [PATCH 3/4] Warn about conflicting span metrics only with a native writer --- .../metrics/MetricsAggregatorFactory.java | 2 +- .../metrics/MetricsAggregatorFactoryTest.java | 48 +++++++++++++++++++ .../main/java/datadog/trace/api/Config.java | 7 +-- 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricsAggregatorFactory.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricsAggregatorFactory.java index 3b07ee2722a..f8c9dc0d1a2 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricsAggregatorFactory.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricsAggregatorFactory.java @@ -21,7 +21,7 @@ public static MetricsAggregator createMetricsAggregator( // the same ClientStatsAggregator span selection + DDSketch aggregation, differing only in // the injected MetricWriter. if (config.isOtelTracesSpanMetricsEnabled()) { - if (config.isTracerMetricsEnabled()) { + if (config.isTracerMetricsEnabled() && config.hasAgentWriter()) { log.warn( "Both OTLP trace span metrics and native tracer metrics are enabled; " + "using OTLP export and ignoring native tracer metrics (the two are mutually " diff --git a/dd-trace-core/src/test/java/datadog/trace/common/metrics/MetricsAggregatorFactoryTest.java b/dd-trace-core/src/test/java/datadog/trace/common/metrics/MetricsAggregatorFactoryTest.java index 3f499bc51e9..04842b5bb43 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/metrics/MetricsAggregatorFactoryTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/metrics/MetricsAggregatorFactoryTest.java @@ -4,6 +4,7 @@ import static datadog.trace.api.config.OtlpConfig.OTEL_TRACES_SPAN_METRICS_ENABLED; import static datadog.trace.api.config.OtlpConfig.OTLP_METRICS_PROTOCOL; import static datadog.trace.api.config.OtlpConfig.TRACE_OTEL_EXPORTER; +import static datadog.trace.api.config.TracerConfig.WRITER_TYPE; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -14,6 +15,10 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; import datadog.communication.ddagent.DDAgentFeaturesDiscovery; import datadog.communication.ddagent.SharedCommunicationObjects; import datadog.trace.api.Config; @@ -21,6 +26,8 @@ import java.util.Properties; import okhttp3.HttpUrl; import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; +import org.tabletest.junit.TableTest; /** * Tests the native-vs-OTLP XOR writer selection in {@link MetricsAggregatorFactory}. The selected @@ -98,4 +105,45 @@ void whenOtlpTraceMetricsEnabledOtlpStatsMetricWriterSelected() { assertEquals(config.getTraceStatsInterval(), conflating.reportingInterval()); assertEquals(MILLISECONDS, conflating.reportingIntervalTimeUnit()); } + + @TableTest({ + "writerType | nativeMetrics | expectedWarning", + "OtlpWriter | true | false ", + "DDAgentWriter | true | true ", + "'MultiWriter:OtlpWriter,DDAgentWriter' | true | true ", + "'MultiWriter:OtlpWriter,LoggingWriter' | true | false ", + "DDAgentWriter | false | false " + }) + void warnsAboutConflictingMetricsOnlyWithNativeWriter( + String writerType, boolean nativeMetrics, boolean expectedWarning) { + Config config = + Config.get( + props( + WRITER_TYPE, + writerType, + TRACE_STATS_COMPUTATION_ENABLED, + Boolean.toString(nativeMetrics), + OTEL_TRACES_SPAN_METRICS_ENABLED, + "true")); + Logger logger = (Logger) LoggerFactory.getLogger(MetricsAggregatorFactory.class); + Level previousLevel = logger.getLevel(); + ListAppender appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + logger.setLevel(Level.WARN); + try { + MetricsAggregatorFactory.createMetricsAggregator( + config, sharedCommunicationObjects(), HealthMetrics.NO_OP); + assertEquals( + expectedWarning, + appender.list.stream() + .anyMatch(event -> + event.getLevel() == Level.WARN + && event.getFormattedMessage().startsWith("Both OTLP trace span metrics"))); + } finally { + logger.detachAppender(appender); + logger.setLevel(previousLevel); + appender.stop(); + } + } } diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 1671275ef3e..b16986ee2a8 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -4067,9 +4067,10 @@ public boolean isTracerMetricsEnabled() { } public boolean isNativeTracerMetricsEnabled() { - if (!isTracerMetricsEnabled() || isOtelTracesSpanMetricsEnabled()) { - return false; - } + return isTracerMetricsEnabled() && !isOtelTracesSpanMetricsEnabled() && hasAgentWriter(); + } + + public boolean hasAgentWriter() { if (!writerType.startsWith(MULTI_WRITER_TYPE)) { return isAgentWriterType(writerType); } From 0bc8e48dcd077c7acf1384101df1c66bc3778c23 Mon Sep 17 00:00:00 2001 From: Munir Abdinur Date: Wed, 11 Nov 2026 23:10:22 -0500 Subject: [PATCH 4/4] Record false defaults for OTLP span metrics configuration --- metadata/supported-configurations.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 304d4a02d4b..0e2511f7c92 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -8934,9 +8934,9 @@ ], "DD_OTEL_TRACES_SPAN_METRICS_ENABLED": [ { - "version": "A", + "version": "B", "type": "boolean", - "default": null, + "default": "false", "aliases": [] } ], @@ -12070,9 +12070,9 @@ ], "OTEL_TRACES_SPAN_METRICS_ENABLED": [ { - "version": "A", + "version": "B", "type": "boolean", - "default": null, + "default": "false", "aliases": [] } ],