Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ public static MetricsAggregator createMetricsAggregator(
// the same ClientStatsAggregator span selection + DDSketch aggregation, differing only in
// the injected MetricWriter.
if (config.isOtelTracesSpanMetricsEnabled()) {
if (config.isTracerMetricsEnabled()) {
if (config.isTracerMetricsEnabled() && config.hasAgentWriter()) {
log.warn(
"Both OTLP trace span metrics and native tracer metrics are enabled; "
+ "using OTLP export and ignoring native tracer metrics (the two are mutually "
Expand All @@ -44,7 +44,7 @@ public static MetricsAggregator createMetricsAggregator(
return new ClientStatsAggregator(
config, sharedCommunicationObjects, healthMetrics, new OtlpStatsMetricWriter(config));
}
if (config.isTracerMetricsEnabled()) {
if (config.isNativeTracerMetricsEnabled()) {
log.debug("tracer metrics enabled");
return new ClientStatsAggregator(config, sharedCommunicationObjects, healthMetrics);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,7 @@ public static Writer createWriter(
commObjects.agentUrl,
featuresDiscovery,
commObjects.monitoring,
config.isTracerMetricsEnabled());
config.isNativeTracerMetricsEnabled());

// Register whichever sampler applies agent published rates, regardless of what wraps it.
final RateByServiceTraceSampler agentRateSampler = sampler.agentSampler();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
import static datadog.trace.api.config.GeneralConfig.TRACE_STATS_COMPUTATION_ENABLED;
import static datadog.trace.api.config.OtlpConfig.OTEL_TRACES_SPAN_METRICS_ENABLED;
import static datadog.trace.api.config.OtlpConfig.OTLP_METRICS_PROTOCOL;
import static datadog.trace.api.config.OtlpConfig.TRACE_OTEL_EXPORTER;
import static datadog.trace.api.config.TracerConfig.WRITER_TYPE;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.junit.jupiter.api.Assertions.assertEquals;
Expand All @@ -13,13 +15,19 @@
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.Logger;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.read.ListAppender;
import datadog.communication.ddagent.DDAgentFeaturesDiscovery;
import datadog.communication.ddagent.SharedCommunicationObjects;
import datadog.trace.api.Config;
import datadog.trace.core.monitor.HealthMetrics;
import java.util.Properties;
import okhttp3.HttpUrl;
import org.junit.jupiter.api.Test;
import org.slf4j.LoggerFactory;
import org.tabletest.junit.TableTest;

/**
* Tests the native-vs-OTLP XOR writer selection in {@link MetricsAggregatorFactory}. The selected
Expand Down Expand Up @@ -70,6 +78,17 @@ void whenNativeTracerMetricsEnabledSerializingWriterSelected() {
assertEquals(SECONDS, conflating.reportingIntervalTimeUnit());
}

@Test
void whenOtlpTracesEnabledNoOpAggregatorCreatedByDefault() {
Config config = Config.get(props(TRACE_OTEL_EXPORTER, "otlp"));

MetricsAggregator aggregator =
MetricsAggregatorFactory.createMetricsAggregator(
config, sharedCommunicationObjects(), HealthMetrics.NO_OP);

assertInstanceOf(NoOpMetricsAggregator.class, aggregator);
}

@Test
void whenOtlpTraceMetricsEnabledOtlpStatsMetricWriterSelected() {
Config config =
Expand All @@ -86,4 +105,45 @@ void whenOtlpTraceMetricsEnabledOtlpStatsMetricWriterSelected() {
assertEquals(config.getTraceStatsInterval(), conflating.reportingInterval());
assertEquals(MILLISECONDS, conflating.reportingIntervalTimeUnit());
}

@TableTest({
"writerType | nativeMetrics | expectedWarning",
"OtlpWriter | true | false ",
"DDAgentWriter | true | true ",
"'MultiWriter:OtlpWriter,DDAgentWriter' | true | true ",
"'MultiWriter:OtlpWriter,LoggingWriter' | true | false ",
"DDAgentWriter | false | false "
})
void warnsAboutConflictingMetricsOnlyWithNativeWriter(
String writerType, boolean nativeMetrics, boolean expectedWarning) {
Config config =
Config.get(
props(
WRITER_TYPE,
writerType,
TRACE_STATS_COMPUTATION_ENABLED,
Boolean.toString(nativeMetrics),
OTEL_TRACES_SPAN_METRICS_ENABLED,
"true"));
Logger logger = (Logger) LoggerFactory.getLogger(MetricsAggregatorFactory.class);
Level previousLevel = logger.getLevel();
ListAppender<ILoggingEvent> appender = new ListAppender<>();
appender.start();
logger.addAppender(appender);
logger.setLevel(Level.WARN);
try {
MetricsAggregatorFactory.createMetricsAggregator(
config, sharedCommunicationObjects(), HealthMetrics.NO_OP);
assertEquals(
expectedWarning,
appender.list.stream()
.anyMatch(event ->
event.getLevel() == Level.WARN
&& event.getFormattedMessage().startsWith("Both OTLP trace span metrics")));
} finally {
logger.detachAppender(appender);
logger.setLevel(previousLevel);
appender.stop();
}
}
}
Original file line number Diff line number Diff line change
@@ -1,8 +1,14 @@
package datadog.trace.common.writer;

import static datadog.trace.api.config.GeneralConfig.TRACE_STATS_COMPUTATION_ENABLED;
import static datadog.trace.api.config.OtlpConfig.OTEL_TRACES_SPAN_METRICS_ENABLED;
import static datadog.trace.api.config.OtlpConfig.TRACE_OTEL_EXPORTER;
import static datadog.trace.api.config.TracerConfig.PRIORITIZATION_TYPE;
import static datadog.trace.api.config.TracerConfig.WRITER_TYPE;
import static datadog.trace.test.junit.utils.config.WithConfigExtension.injectSysConfig;
import static java.util.Collections.singletonList;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
Expand All @@ -18,6 +24,10 @@
import datadog.trace.api.Config;
import datadog.trace.api.config.OtlpConfig;
import datadog.trace.api.intake.TrackType;
import datadog.trace.common.metrics.ClientStatsAggregator;
import datadog.trace.common.metrics.MetricsAggregator;
import datadog.trace.common.metrics.MetricsAggregatorFactory;
import datadog.trace.common.metrics.NoOpMetricsAggregator;
import datadog.trace.common.sampling.RateByServiceTraceSampler;
import datadog.trace.common.sampling.Sampler;
import datadog.trace.common.writer.ddagent.DDAgentApi;
Expand Down Expand Up @@ -254,6 +264,87 @@ void testWriterCreationForOtlpWriter(
writer.close();
}

@TableTest({
"scenario | exporter | writerType | nativeMetrics | spanMetrics | agentSupportsMetrics | statsWriter | agentWriter",
"native default | | | true | false | true | native | true ",
"native disabled | | | false | false | true | none | true ",
"agent without stats | | | true | false | false | native | true ",
"OTLP default | otlp | | true | false | true | none | false ",
"native override | otlp | DDAgentWriter | true | false | true | native | true ",
"native override disabled | otlp | DDAgentWriter | false | false | true | none | true ",
"OTLP writer override | | OtlpWriter | true | false | true | none | false ",
"mixed default exporter | | 'MultiWriter:DDAgentWriter,OtlpWriter' | true | false | true | native | true ",
"mixed OTLP exporter | otlp | 'MultiWriter:OtlpWriter,DDAgentWriter' | true | false | true | native | true ",
"mixed native disabled | otlp | 'MultiWriter:OtlpWriter,DDAgentWriter' | false | false | true | none | true ",
"mixed no native writer | otlp | 'MultiWriter:OtlpWriter,LoggingWriter' | true | false | true | none | false ",
"logging writer | | LoggingWriter | true | false | true | none | false ",
"unknown writer fallback | otlp | unknown | true | false | true | native | true ",
"OTLP stats opt-in | otlp | | true | true | true | otlp | false ",
"mixed OTLP stats opt-in | otlp | 'MultiWriter:OtlpWriter,DDAgentWriter' | true | true | true | otlp | true ",
"OTLP stats native off | otlp | DDAgentWriter | false | true | false | otlp | true "
})
void statsAggregatorMatchesAgentHeader(
String exporter,
String writerType,
boolean nativeMetrics,
boolean spanMetrics,
boolean agentSupportsMetrics,
String statsWriter,
boolean agentWriter)
throws Exception {
if (exporter != null) injectSysConfig(TRACE_OTEL_EXPORTER, exporter);
if (writerType != null) injectSysConfig(WRITER_TYPE, writerType);
injectSysConfig(TRACE_STATS_COMPUTATION_ENABLED, Boolean.toString(nativeMetrics));
injectSysConfig(OTEL_TRACES_SPAN_METRICS_ENABLED, Boolean.toString(spanMetrics));
Config config = Config.get();

try (JavaTestHttpServer agent = tracesAgent()) {
DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class);
when(discovery.getTraceEndpoint()).thenReturn("v0.4/traces");
when(discovery.supportsMetrics()).thenReturn(agentSupportsMetrics);
SharedCommunicationObjects comm = new SharedCommunicationObjects();
comm.agentUrl = HttpUrl.get(agent.getAddress().toString());
comm.setFeaturesDiscovery(discovery);
comm.createRemaining(config);

MetricsAggregator aggregator =
MetricsAggregatorFactory.createMetricsAggregator(config, comm, HealthMetrics.NO_OP);
try (Writer writer =
WriterFactory.createWriter(
config, comm, mock(Sampler.class), null, HealthMetrics.NO_OP)) {
if ("none".equals(statsWriter)) {
assertInstanceOf(NoOpMetricsAggregator.class, aggregator);
} else {
assertEquals(
"otlp".equals(statsWriter),
readField(
assertInstanceOf(ClientStatsAggregator.class, aggregator),
"otlpStatsExportEnabled"));
}
Writer[] writers =
writer instanceof MultiWriter
? (Writer[]) readField(writer, "writers")
: new Writer[] {writer};
boolean sentToAgent = false;
for (Writer child : writers) {
if (child instanceof DDAgentWriter) {
assertTrue(onlyApi(child).sendSerializedTraces(emptyV04Payload()).success());
assertEquals(
"otlp".equals(statsWriter)
|| ("native".equals(statsWriter) && agentSupportsMetrics),
"true"
.equals(
agent.getLastRequest().getHeaders().get("Datadog-Client-Computed-Stats")));
sentToAgent = true;
}
}
assertEquals(agentWriter, sentToAgent);
} finally {
aggregator.close();
}
}
}

@Test
void registersAgentSamplerAsResponseListener() throws Exception {
try (JavaTestHttpServer agent = tracesAgent()) {
Expand Down
38 changes: 31 additions & 7 deletions internal-api/src/main/java/datadog/trace/api/Config.java
Original file line number Diff line number Diff line change
Expand Up @@ -748,8 +748,12 @@
import static datadog.trace.api.config.TracerConfig.WRITER_LINKS_INJECT;
import static datadog.trace.api.config.TracerConfig.WRITER_TYPE;
import static datadog.trace.api.telemetry.LogCollector.SEND_TELEMETRY;
import static datadog.trace.bootstrap.instrumentation.api.WriterConstants.DD_INTAKE_WRITER_TYPE;
import static datadog.trace.bootstrap.instrumentation.api.WriterConstants.LOGGING_WRITER_TYPE;
import static datadog.trace.bootstrap.instrumentation.api.WriterConstants.MULTI_WRITER_TYPE;
import static datadog.trace.bootstrap.instrumentation.api.WriterConstants.OTLP_WRITER_TYPE;
import static datadog.trace.bootstrap.instrumentation.api.WriterConstants.PRINTING_WRITER_TYPE;
import static datadog.trace.bootstrap.instrumentation.api.WriterConstants.TRACE_STRUCTURE_WRITER_TYPE;
import static datadog.trace.util.CollectionUtils.tryMakeImmutableList;
import static datadog.trace.util.CollectionUtils.tryMakeImmutableSet;
import static datadog.trace.util.ConfigStrings.propertyNameToEnvironmentVariableName;
Expand Down Expand Up @@ -2251,14 +2255,8 @@ private Config(final ConfigProvider configProvider, final InstrumenterConfig ins
OtlpConfig.Temporality.class,
OtlpConfig.Temporality.DELTA);

// Tri-state default: when unset, SDK-computed OTLP span metrics are emitted iff OTLP trace
// export and OTLP metrics export are both enabled. OTEL_SDK_DISABLED=true turns them off.
otelTracesSpanMetricsEnabled =
configProvider.getBoolean(
OTEL_TRACES_SPAN_METRICS_ENABLED,
isTraceOtlpExporterEnabled()
&& isMetricsOtelEnabled()
&& isMetricsOtlpExporterEnabled())
configProvider.getBoolean(OTEL_TRACES_SPAN_METRICS_ENABLED, false)
Comment thread
mabdinur marked this conversation as resolved.
Resolved
&& !configProvider.isOtelSdkDisabled();

otlpTimeout = configProvider.getInteger(OTLP_TRACES_TIMEOUT, DEFAULT_OTLP_TRACES_TIMEOUT);
Expand Down Expand Up @@ -4068,6 +4066,32 @@ public boolean isTracerMetricsEnabled() {
return tracerMetricsEnabled && isApmTracingEnabled();
}

public boolean isNativeTracerMetricsEnabled() {
return isTracerMetricsEnabled() && !isOtelTracesSpanMetricsEnabled() && hasAgentWriter();
}

public boolean hasAgentWriter() {
if (!writerType.startsWith(MULTI_WRITER_TYPE)) {
return isAgentWriterType(writerType);
}
String multiWriterConfig = Strings.replace(writerType, MULTI_WRITER_TYPE + ":", "");
for (String subWriterType : COMMA.split(multiWriterConfig)) {
if (isAgentWriterType(subWriterType)) {
return true;
}
}
return false;
}

private static boolean isAgentWriterType(String type) {
// WriterFactory falls back to DDAgentWriter for unrecognized writer types.
return !OTLP_WRITER_TYPE.equals(type)
&& !DD_INTAKE_WRITER_TYPE.equals(type)
&& !LOGGING_WRITER_TYPE.equals(type)
&& !PRINTING_WRITER_TYPE.equals(type)
&& !type.startsWith(TRACE_STRUCTURE_WRITER_TYPE);
}

public boolean isTracerMetricsIgnoreAgentVersion() {
return tracerMetricsIgnoreAgentVersion;
}
Expand Down
11 changes: 4 additions & 7 deletions internal-api/src/test/java/datadog/trace/api/ConfigTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -1078,16 +1078,16 @@ void otelOtlpFallbackKeyEndpoint2() throws Exception {

@TableTest({
"scenario | exporter | metricsEnabled | metricsExporter | override | expected",
"metrics defaults to otlp | otlp | true | | | true ",
"otlp traces and metrics | otlp | true | otlp | | true ",
"metrics defaults to otlp | otlp | true | | | false ",
"otlp traces and metrics | otlp | true | otlp | | false ",
"metrics exporter explicitly disabled | otlp | true | none | | false ",
"metrics feature flag off overrides exporter | otlp | false | otlp | | false ",
"no trace export, default metrics exporter | none | true | | | false ",
"no trace export, otlp metrics exporter | none | true | otlp | | false ",
"explicit override wins (enabled) | none | false | none | true | true ",
"explicit override wins (disabled) | otlp | true | otlp | false | false "
})
void tracesSpanMetricsTriStateDefault(
void tracesSpanMetricsRequireExplicitOptIn(
String exporter,
String metricsEnabled,
String metricsExporter,
Expand All @@ -1105,8 +1105,6 @@ void tracesSpanMetricsTriStateDefault(

Config config = newConfig();

// Unset: emit iff OTLP trace export and OTLP metrics export are both on. An explicit
// dd.otel.traces.span.metrics.enabled always wins.
assertEquals(expected, config.isOtelTracesSpanMetricsEnabled());
}

Expand Down Expand Up @@ -1441,8 +1439,7 @@ void specifyOverridesViaSystemProperties() throws Exception {
assertEquals(5002, config.getOtlpTracesTimeout());

assertTrue(config.isTraceOtelSemanticsEnabled());
// tri-state default: OTLP trace export + OTel metrics both on
assertTrue(config.isOtelTracesSpanMetricsEnabled());
assertFalse(config.isOtelTracesSpanMetricsEnabled());
}

@Test
Expand Down
8 changes: 4 additions & 4 deletions metadata/supported-configurations.json
Original file line number Diff line number Diff line change
Expand Up @@ -8934,9 +8934,9 @@
],
"DD_OTEL_TRACES_SPAN_METRICS_ENABLED": [
{
"version": "A",
"version": "B",
"type": "boolean",
"default": null,
"default": "false",
"aliases": []
}
],
Expand Down Expand Up @@ -12070,9 +12070,9 @@
],
"OTEL_TRACES_SPAN_METRICS_ENABLED": [
{
"version": "A",
"version": "B",
"type": "boolean",
"default": null,
"default": "false",
"aliases": []
}
],
Expand Down
Loading