diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricsAggregatorFactory.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricsAggregatorFactory.java index afd25d9566a..f8c9dc0d1a2 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricsAggregatorFactory.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricsAggregatorFactory.java @@ -21,7 +21,7 @@ public static MetricsAggregator createMetricsAggregator( // the same ClientStatsAggregator span selection + DDSketch aggregation, differing only in // the injected MetricWriter. if (config.isOtelTracesSpanMetricsEnabled()) { - if (config.isTracerMetricsEnabled()) { + if (config.isTracerMetricsEnabled() && config.hasAgentWriter()) { log.warn( "Both OTLP trace span metrics and native tracer metrics are enabled; " + "using OTLP export and ignoring native tracer metrics (the two are mutually " @@ -44,7 +44,7 @@ public static MetricsAggregator createMetricsAggregator( return new ClientStatsAggregator( config, sharedCommunicationObjects, healthMetrics, new OtlpStatsMetricWriter(config)); } - if (config.isTracerMetricsEnabled()) { + if (config.isNativeTracerMetricsEnabled()) { log.debug("tracer metrics enabled"); return new ClientStatsAggregator(config, sharedCommunicationObjects, healthMetrics); } diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/WriterFactory.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/WriterFactory.java index 0ad61b81e39..cc6f1ab073e 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/WriterFactory.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/WriterFactory.java @@ -194,7 +194,7 @@ public static Writer createWriter( commObjects.agentUrl, featuresDiscovery, commObjects.monitoring, - config.isTracerMetricsEnabled()); + config.isNativeTracerMetricsEnabled()); // Register whichever sampler applies agent published rates, regardless of what wraps it. final RateByServiceTraceSampler agentRateSampler = sampler.agentSampler(); diff --git a/dd-trace-core/src/test/java/datadog/trace/common/metrics/MetricsAggregatorFactoryTest.java b/dd-trace-core/src/test/java/datadog/trace/common/metrics/MetricsAggregatorFactoryTest.java index e9d981df389..04842b5bb43 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/metrics/MetricsAggregatorFactoryTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/metrics/MetricsAggregatorFactoryTest.java @@ -3,6 +3,8 @@ import static datadog.trace.api.config.GeneralConfig.TRACE_STATS_COMPUTATION_ENABLED; import static datadog.trace.api.config.OtlpConfig.OTEL_TRACES_SPAN_METRICS_ENABLED; import static datadog.trace.api.config.OtlpConfig.OTLP_METRICS_PROTOCOL; +import static datadog.trace.api.config.OtlpConfig.TRACE_OTEL_EXPORTER; +import static datadog.trace.api.config.TracerConfig.WRITER_TYPE; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -13,6 +15,10 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; import datadog.communication.ddagent.DDAgentFeaturesDiscovery; import datadog.communication.ddagent.SharedCommunicationObjects; import datadog.trace.api.Config; @@ -20,6 +26,8 @@ import java.util.Properties; import okhttp3.HttpUrl; import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; +import org.tabletest.junit.TableTest; /** * Tests the native-vs-OTLP XOR writer selection in {@link MetricsAggregatorFactory}. The selected @@ -70,6 +78,17 @@ void whenNativeTracerMetricsEnabledSerializingWriterSelected() { assertEquals(SECONDS, conflating.reportingIntervalTimeUnit()); } + @Test + void whenOtlpTracesEnabledNoOpAggregatorCreatedByDefault() { + Config config = Config.get(props(TRACE_OTEL_EXPORTER, "otlp")); + + MetricsAggregator aggregator = + MetricsAggregatorFactory.createMetricsAggregator( + config, sharedCommunicationObjects(), HealthMetrics.NO_OP); + + assertInstanceOf(NoOpMetricsAggregator.class, aggregator); + } + @Test void whenOtlpTraceMetricsEnabledOtlpStatsMetricWriterSelected() { Config config = @@ -86,4 +105,45 @@ void whenOtlpTraceMetricsEnabledOtlpStatsMetricWriterSelected() { assertEquals(config.getTraceStatsInterval(), conflating.reportingInterval()); assertEquals(MILLISECONDS, conflating.reportingIntervalTimeUnit()); } + + @TableTest({ + "writerType | nativeMetrics | expectedWarning", + "OtlpWriter | true | false ", + "DDAgentWriter | true | true ", + "'MultiWriter:OtlpWriter,DDAgentWriter' | true | true ", + "'MultiWriter:OtlpWriter,LoggingWriter' | true | false ", + "DDAgentWriter | false | false " + }) + void warnsAboutConflictingMetricsOnlyWithNativeWriter( + String writerType, boolean nativeMetrics, boolean expectedWarning) { + Config config = + Config.get( + props( + WRITER_TYPE, + writerType, + TRACE_STATS_COMPUTATION_ENABLED, + Boolean.toString(nativeMetrics), + OTEL_TRACES_SPAN_METRICS_ENABLED, + "true")); + Logger logger = (Logger) LoggerFactory.getLogger(MetricsAggregatorFactory.class); + Level previousLevel = logger.getLevel(); + ListAppender appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + logger.setLevel(Level.WARN); + try { + MetricsAggregatorFactory.createMetricsAggregator( + config, sharedCommunicationObjects(), HealthMetrics.NO_OP); + assertEquals( + expectedWarning, + appender.list.stream() + .anyMatch(event -> + event.getLevel() == Level.WARN + && event.getFormattedMessage().startsWith("Both OTLP trace span metrics"))); + } finally { + logger.detachAppender(appender); + logger.setLevel(previousLevel); + appender.stop(); + } + } } diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/WriterFactoryTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/WriterFactoryTest.java index 18899be310e..ffd9b8404bf 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/WriterFactoryTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/WriterFactoryTest.java @@ -1,8 +1,14 @@ package datadog.trace.common.writer; +import static datadog.trace.api.config.GeneralConfig.TRACE_STATS_COMPUTATION_ENABLED; +import static datadog.trace.api.config.OtlpConfig.OTEL_TRACES_SPAN_METRICS_ENABLED; +import static datadog.trace.api.config.OtlpConfig.TRACE_OTEL_EXPORTER; import static datadog.trace.api.config.TracerConfig.PRIORITIZATION_TYPE; +import static datadog.trace.api.config.TracerConfig.WRITER_TYPE; +import static datadog.trace.test.junit.utils.config.WithConfigExtension.injectSysConfig; import static java.util.Collections.singletonList; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; @@ -18,6 +24,10 @@ import datadog.trace.api.Config; import datadog.trace.api.config.OtlpConfig; import datadog.trace.api.intake.TrackType; +import datadog.trace.common.metrics.ClientStatsAggregator; +import datadog.trace.common.metrics.MetricsAggregator; +import datadog.trace.common.metrics.MetricsAggregatorFactory; +import datadog.trace.common.metrics.NoOpMetricsAggregator; import datadog.trace.common.sampling.RateByServiceTraceSampler; import datadog.trace.common.sampling.Sampler; import datadog.trace.common.writer.ddagent.DDAgentApi; @@ -254,6 +264,87 @@ void testWriterCreationForOtlpWriter( writer.close(); } + @TableTest({ + "scenario | exporter | writerType | nativeMetrics | spanMetrics | agentSupportsMetrics | statsWriter | agentWriter", + "native default | | | true | false | true | native | true ", + "native disabled | | | false | false | true | none | true ", + "agent without stats | | | true | false | false | native | true ", + "OTLP default | otlp | | true | false | true | none | false ", + "native override | otlp | DDAgentWriter | true | false | true | native | true ", + "native override disabled | otlp | DDAgentWriter | false | false | true | none | true ", + "OTLP writer override | | OtlpWriter | true | false | true | none | false ", + "mixed default exporter | | 'MultiWriter:DDAgentWriter,OtlpWriter' | true | false | true | native | true ", + "mixed OTLP exporter | otlp | 'MultiWriter:OtlpWriter,DDAgentWriter' | true | false | true | native | true ", + "mixed native disabled | otlp | 'MultiWriter:OtlpWriter,DDAgentWriter' | false | false | true | none | true ", + "mixed no native writer | otlp | 'MultiWriter:OtlpWriter,LoggingWriter' | true | false | true | none | false ", + "logging writer | | LoggingWriter | true | false | true | none | false ", + "unknown writer fallback | otlp | unknown | true | false | true | native | true ", + "OTLP stats opt-in | otlp | | true | true | true | otlp | false ", + "mixed OTLP stats opt-in | otlp | 'MultiWriter:OtlpWriter,DDAgentWriter' | true | true | true | otlp | true ", + "OTLP stats native off | otlp | DDAgentWriter | false | true | false | otlp | true " + }) + void statsAggregatorMatchesAgentHeader( + String exporter, + String writerType, + boolean nativeMetrics, + boolean spanMetrics, + boolean agentSupportsMetrics, + String statsWriter, + boolean agentWriter) + throws Exception { + if (exporter != null) injectSysConfig(TRACE_OTEL_EXPORTER, exporter); + if (writerType != null) injectSysConfig(WRITER_TYPE, writerType); + injectSysConfig(TRACE_STATS_COMPUTATION_ENABLED, Boolean.toString(nativeMetrics)); + injectSysConfig(OTEL_TRACES_SPAN_METRICS_ENABLED, Boolean.toString(spanMetrics)); + Config config = Config.get(); + + try (JavaTestHttpServer agent = tracesAgent()) { + DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); + when(discovery.getTraceEndpoint()).thenReturn("v0.4/traces"); + when(discovery.supportsMetrics()).thenReturn(agentSupportsMetrics); + SharedCommunicationObjects comm = new SharedCommunicationObjects(); + comm.agentUrl = HttpUrl.get(agent.getAddress().toString()); + comm.setFeaturesDiscovery(discovery); + comm.createRemaining(config); + + MetricsAggregator aggregator = + MetricsAggregatorFactory.createMetricsAggregator(config, comm, HealthMetrics.NO_OP); + try (Writer writer = + WriterFactory.createWriter( + config, comm, mock(Sampler.class), null, HealthMetrics.NO_OP)) { + if ("none".equals(statsWriter)) { + assertInstanceOf(NoOpMetricsAggregator.class, aggregator); + } else { + assertEquals( + "otlp".equals(statsWriter), + readField( + assertInstanceOf(ClientStatsAggregator.class, aggregator), + "otlpStatsExportEnabled")); + } + Writer[] writers = + writer instanceof MultiWriter + ? (Writer[]) readField(writer, "writers") + : new Writer[] {writer}; + boolean sentToAgent = false; + for (Writer child : writers) { + if (child instanceof DDAgentWriter) { + assertTrue(onlyApi(child).sendSerializedTraces(emptyV04Payload()).success()); + assertEquals( + "otlp".equals(statsWriter) + || ("native".equals(statsWriter) && agentSupportsMetrics), + "true" + .equals( + agent.getLastRequest().getHeaders().get("Datadog-Client-Computed-Stats"))); + sentToAgent = true; + } + } + assertEquals(agentWriter, sentToAgent); + } finally { + aggregator.close(); + } + } + } + @Test void registersAgentSamplerAsResponseListener() throws Exception { try (JavaTestHttpServer agent = tracesAgent()) { diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 10693837cfe..b16986ee2a8 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -748,8 +748,12 @@ import static datadog.trace.api.config.TracerConfig.WRITER_LINKS_INJECT; import static datadog.trace.api.config.TracerConfig.WRITER_TYPE; import static datadog.trace.api.telemetry.LogCollector.SEND_TELEMETRY; +import static datadog.trace.bootstrap.instrumentation.api.WriterConstants.DD_INTAKE_WRITER_TYPE; +import static datadog.trace.bootstrap.instrumentation.api.WriterConstants.LOGGING_WRITER_TYPE; import static datadog.trace.bootstrap.instrumentation.api.WriterConstants.MULTI_WRITER_TYPE; import static datadog.trace.bootstrap.instrumentation.api.WriterConstants.OTLP_WRITER_TYPE; +import static datadog.trace.bootstrap.instrumentation.api.WriterConstants.PRINTING_WRITER_TYPE; +import static datadog.trace.bootstrap.instrumentation.api.WriterConstants.TRACE_STRUCTURE_WRITER_TYPE; import static datadog.trace.util.CollectionUtils.tryMakeImmutableList; import static datadog.trace.util.CollectionUtils.tryMakeImmutableSet; import static datadog.trace.util.ConfigStrings.propertyNameToEnvironmentVariableName; @@ -2251,14 +2255,8 @@ private Config(final ConfigProvider configProvider, final InstrumenterConfig ins OtlpConfig.Temporality.class, OtlpConfig.Temporality.DELTA); - // Tri-state default: when unset, SDK-computed OTLP span metrics are emitted iff OTLP trace - // export and OTLP metrics export are both enabled. OTEL_SDK_DISABLED=true turns them off. otelTracesSpanMetricsEnabled = - configProvider.getBoolean( - OTEL_TRACES_SPAN_METRICS_ENABLED, - isTraceOtlpExporterEnabled() - && isMetricsOtelEnabled() - && isMetricsOtlpExporterEnabled()) + configProvider.getBoolean(OTEL_TRACES_SPAN_METRICS_ENABLED, false) && !configProvider.isOtelSdkDisabled(); otlpTimeout = configProvider.getInteger(OTLP_TRACES_TIMEOUT, DEFAULT_OTLP_TRACES_TIMEOUT); @@ -4068,6 +4066,32 @@ public boolean isTracerMetricsEnabled() { return tracerMetricsEnabled && isApmTracingEnabled(); } + public boolean isNativeTracerMetricsEnabled() { + return isTracerMetricsEnabled() && !isOtelTracesSpanMetricsEnabled() && hasAgentWriter(); + } + + public boolean hasAgentWriter() { + if (!writerType.startsWith(MULTI_WRITER_TYPE)) { + return isAgentWriterType(writerType); + } + String multiWriterConfig = Strings.replace(writerType, MULTI_WRITER_TYPE + ":", ""); + for (String subWriterType : COMMA.split(multiWriterConfig)) { + if (isAgentWriterType(subWriterType)) { + return true; + } + } + return false; + } + + private static boolean isAgentWriterType(String type) { + // WriterFactory falls back to DDAgentWriter for unrecognized writer types. + return !OTLP_WRITER_TYPE.equals(type) + && !DD_INTAKE_WRITER_TYPE.equals(type) + && !LOGGING_WRITER_TYPE.equals(type) + && !PRINTING_WRITER_TYPE.equals(type) + && !type.startsWith(TRACE_STRUCTURE_WRITER_TYPE); + } + public boolean isTracerMetricsIgnoreAgentVersion() { return tracerMetricsIgnoreAgentVersion; } diff --git a/internal-api/src/test/java/datadog/trace/api/ConfigTest.java b/internal-api/src/test/java/datadog/trace/api/ConfigTest.java index 09e8c883444..5f828934872 100644 --- a/internal-api/src/test/java/datadog/trace/api/ConfigTest.java +++ b/internal-api/src/test/java/datadog/trace/api/ConfigTest.java @@ -1078,8 +1078,8 @@ void otelOtlpFallbackKeyEndpoint2() throws Exception { @TableTest({ "scenario | exporter | metricsEnabled | metricsExporter | override | expected", - "metrics defaults to otlp | otlp | true | | | true ", - "otlp traces and metrics | otlp | true | otlp | | true ", + "metrics defaults to otlp | otlp | true | | | false ", + "otlp traces and metrics | otlp | true | otlp | | false ", "metrics exporter explicitly disabled | otlp | true | none | | false ", "metrics feature flag off overrides exporter | otlp | false | otlp | | false ", "no trace export, default metrics exporter | none | true | | | false ", @@ -1087,7 +1087,7 @@ void otelOtlpFallbackKeyEndpoint2() throws Exception { "explicit override wins (enabled) | none | false | none | true | true ", "explicit override wins (disabled) | otlp | true | otlp | false | false " }) - void tracesSpanMetricsTriStateDefault( + void tracesSpanMetricsRequireExplicitOptIn( String exporter, String metricsEnabled, String metricsExporter, @@ -1105,8 +1105,6 @@ void tracesSpanMetricsTriStateDefault( Config config = newConfig(); - // Unset: emit iff OTLP trace export and OTLP metrics export are both on. An explicit - // dd.otel.traces.span.metrics.enabled always wins. assertEquals(expected, config.isOtelTracesSpanMetricsEnabled()); } @@ -1441,8 +1439,7 @@ void specifyOverridesViaSystemProperties() throws Exception { assertEquals(5002, config.getOtlpTracesTimeout()); assertTrue(config.isTraceOtelSemanticsEnabled()); - // tri-state default: OTLP trace export + OTel metrics both on - assertTrue(config.isOtelTracesSpanMetricsEnabled()); + assertFalse(config.isOtelTracesSpanMetricsEnabled()); } @Test diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 304d4a02d4b..0e2511f7c92 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -8934,9 +8934,9 @@ ], "DD_OTEL_TRACES_SPAN_METRICS_ENABLED": [ { - "version": "A", + "version": "B", "type": "boolean", - "default": null, + "default": "false", "aliases": [] } ], @@ -12070,9 +12070,9 @@ ], "OTEL_TRACES_SPAN_METRICS_ENABLED": [ { - "version": "A", + "version": "B", "type": "boolean", - "default": null, + "default": "false", "aliases": [] } ],