Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package datadog.trace.common.writer.ddagent;

import static datadog.communication.http.OkHttpUtils.msgpackRequestBodyOf;
import static datadog.trace.api.DDTags.SPAN_EVENTS;
import static datadog.trace.api.cache.RadixTreeCache.UNSET_STATUS;
import static datadog.trace.core.CoreSpan.ROOT_PROPAGATION_TAGS;
import static datadog.trace.core.CoreSpan.STRUCTURED_EVENTS;
import static datadog.trace.core.CoreSpan.STRUCTURED_LINKS;
Expand All @@ -19,12 +17,14 @@
import datadog.trace.api.Config;
import datadog.trace.api.DDTags;
import datadog.trace.api.DDTraceId;
import datadog.trace.api.KnownTags;
import datadog.trace.api.ProcessTags;
import datadog.trace.api.TagMap;
import datadog.trace.api.sampling.SamplingMechanism;
import datadog.trace.bootstrap.instrumentation.api.AgentSpanLink;
import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags;
import datadog.trace.bootstrap.instrumentation.api.Tags;
import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString;
import datadog.trace.common.writer.Payload;
import datadog.trace.core.CoreSpan;
import datadog.trace.core.DDSpanEvent;
Expand Down Expand Up @@ -72,6 +72,11 @@ public final class TraceMapperV1 implements TraceMapper {
private final MsgPackWriter metaStructWriter;
private final ByteBuffer header;

/**
* Scratch state for {@link #countTagAttributes}, reset for each span on the serialization thread.
*/
private int tagAttributeCount;

public TraceMapperV1(int bufferSize) {
this.bufferSize = bufferSize;
this.stringTable = new StringTable();
Expand Down Expand Up @@ -171,13 +176,13 @@ private void encodeSpans(Writable writable, int fieldId, List<? extends CoreSpan
// events = 12, a collection of events that occurred during this span
encodeSpanEvents(writable, 12, meta.getSpanEvents());
// env = 13, the optional string environment of this span
encodeString(writable, 13, tags.getString(Tags.ENV));
encodeString(writable, 13, tags.loadString(KnownTags.ENV));
// version = 14, the optional string version of this span
encodeString(writable, 14, tags.getString(Tags.VERSION));
encodeString(writable, 14, tags.loadString(KnownTags.VERSION));
// component = 15, the string component name of this span
encodeString(writable, 15, tags.getString(Tags.COMPONENT));
encodeString(writable, 15, tags.loadString(KnownTags.COMPONENT));
// kind = 16, the SpanKind of this span as defined in the OTEL Specification, uint32
encodeInt(writable, 16, getSpanKindValue(tags.getString(Tags.SPAN_KIND)));
encodeInt(writable, 16, getSpanKindValue(tags.loadString(KnownTags.SPAN_KIND)));

meta = null; // Proceed to next span metadata.
}
Expand Down Expand Up @@ -282,7 +287,7 @@ private void writeEventAttributeValue(Writable writable, Object value) {
return;
}
writable.writeInt(VALUE_TYPE_STRING);
writeStreamingString(writable, String.valueOf(value));
writeStreamingString(writable, value);
}

private boolean isEncodableEventArrayItem(Object item) {
Expand All @@ -300,7 +305,7 @@ private void writeEventArrayItemValue(Writable writable, Object item) {
return;
}
writable.writeInt(VALUE_TYPE_STRING);
writeStreamingString(writable, String.valueOf(item));
writeStreamingString(writable, item);
}

private void writeEventNumberValue(Writable writable, Number number) {
Expand All @@ -321,17 +326,16 @@ private void encodeSpanAttributes(
Writable writable, int fieldId, Metadata meta, Map<String, Object> metaStruct) {
TagMap tags = meta.getTags();
Map<String, String> baggage = meta.getBaggage();
String httpStatusCode =
meta.getHttpStatusCode() == UNSET_STATUS ? null : meta.getHttpStatusCodeString().toString();
boolean writeHttpStatus = httpStatusCode != null && tags.getString(HTTP_STATUS) == null;
UTF8BytesString httpStatusCode = meta.getHttpStatusCodeString();
boolean writeHttpStatus =
httpStatusCode != null && tags.getEntryReader(KnownTags.HTTP_STATUS_CODE) == null;
boolean writeTopLevel = meta.topLevel();
int tagCount = 0;
for (TagMap.EntryReader entry : tags) {
if (SPAN_EVENTS.equals(entry.tag())) {
continue;
}
tagCount += getFlatAttributeCount(entry);
}

// Count flattened tags before writing the array header. Explicit callback state keeps both
// tag passes free of TagMap iterators and capturing lambdas.
tagAttributeCount = 0;
tags.forEach(this, TraceMapperV1::countTagAttributes);
int tagCount = tagAttributeCount;

writable.writeInt(fieldId);
writable.startArray(
Expand All @@ -346,26 +350,42 @@ private void encodeSpanAttributes(
writeAttribute(writable, DDTags.THREAD_ID, meta.getThreadId());
writeAttribute(writable, DDTags.THREAD_NAME, meta.getThreadName());

for (Map.Entry<String, String> entry : baggage.entrySet()) {
writeAttribute(writable, entry.getKey(), entry.getValue());
}

for (TagMap.EntryReader entry : tags) {
if (SPAN_EVENTS.equals(entry.tag())) {
continue;
// Skip entry-set views and iterators for empty baggage and meta-struct maps.
if (!baggage.isEmpty()) {
for (Map.Entry<String, String> entry : baggage.entrySet()) {
writeAttribute(writable, entry.getKey(), entry.getValue());
}
writeFlattenedTagAttribute(writable, entry);
}

tags.forEach(this, writable, TraceMapperV1::writeTagAttribute);

if (writeHttpStatus) {
writeAttribute(writable, HTTP_STATUS, httpStatusCode);
}

if (writeTopLevel) {
writeAttribute(writable, InstrumentationTags.DD_TOP_LEVEL.toString(), 1);
}
for (Map.Entry<String, Object> metaStructField : metaStruct.entrySet()) {
writeStreamingString(writable, metaStructField.getKey());
writable.writeInt(VALUE_TYPE_BYTES);
writable.writeBinary(serializeMetaStructValue(metaStructField.getValue()));

if (!metaStruct.isEmpty()) {
for (Map.Entry<String, Object> metaStructField : metaStruct.entrySet()) {
writeStreamingString(writable, metaStructField.getKey());
writable.writeInt(VALUE_TYPE_BYTES);
writable.writeBinary(serializeMetaStructValue(metaStructField.getValue()));
}
}
}

private static void countTagAttributes(TraceMapperV1 mapper, TagMap.EntryReader entry) {
if (entry.tagId() != KnownTags.EVENTS) {
mapper.tagAttributeCount += mapper.getFlatAttributeCount(entry);
}
}

private static void writeTagAttribute(
TraceMapperV1 mapper, Writable writable, TagMap.EntryReader entry) {
if (entry.tagId() != KnownTags.EVENTS) {
mapper.writeFlattenedTagAttribute(writable, entry);
}
}

Expand Down Expand Up @@ -452,7 +472,7 @@ private void writeAttribute(Writable writable, String key, Object value) {
}

writable.writeInt(VALUE_TYPE_STRING);
writeStreamingString(writable, value == null ? "" : value.toString());
writeStreamingString(writable, value);
}

private byte[] serializeMetaStructValue(Object value) {
Expand Down Expand Up @@ -499,16 +519,24 @@ private void encodeTraceId(Writable writable, int fieldId, DDTraceId traceId) {
writable.writeBinary(traceId.toHighOrderLong(), traceId.toLong());
}

/** Writes a string using the streaming string table encoding. */
private void writeStreamingString(Writable writable, CharSequence value) {
/**
* Writes a string-table reference or a new string, treating null as empty.
*
* <p>String keys let equal text share an index across value types. On a miss, preserve a {@link
* UTF8BytesString} to reuse its cached encoding; other values reuse the String computed for
* lookup.
*/
private void writeStreamingString(Writable writable, Object value) {
String str = value == null ? "" : value.toString();
Integer index = stringTable.get(str);
if (index != null) {
// String already in table, write index
writable.writeInt(index);
} else {
// New string, write it directly and add to table
writable.writeString(str, null);
if (value instanceof UTF8BytesString) {
writable.writeUTF8((UTF8BytesString) value);
} else {
writable.writeString(str, null);
}
stringTable.add(str);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import static datadog.trace.api.DDTags.PROCESS_TAGS;
import static datadog.trace.api.DDTags.SDK_OTLP_EXPORT;
import static datadog.trace.api.DDTags.SPAN_EVENTS;
import static datadog.trace.api.DDTags.THREAD_ID;
import static datadog.trace.api.DDTags.THREAD_NAME;
import static datadog.trace.api.config.TracerConfig.WRITER_TYPE;
Expand Down Expand Up @@ -42,6 +43,8 @@
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;

import datadog.communication.serialization.ByteBufferConsumer;
import datadog.communication.serialization.FlushingBuffer;
Expand All @@ -50,11 +53,13 @@
import datadog.trace.api.DDSpanId;
import datadog.trace.api.DDTraceId;
import datadog.trace.api.ProcessTags;
import datadog.trace.api.cache.RadixTreeCache;
import datadog.trace.api.sampling.SamplingMechanism;
import datadog.trace.bootstrap.instrumentation.api.AgentSpanLink;
import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags;
import datadog.trace.bootstrap.instrumentation.api.SpanAttributes;
import datadog.trace.bootstrap.instrumentation.api.SpanLink;
import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString;
import datadog.trace.common.writer.Payload;
import datadog.trace.common.writer.TraceGenerator.PojoSpan;
import datadog.trace.common.writer.ddagent.V1PayloadReader.ChunkField;
Expand Down Expand Up @@ -185,6 +190,63 @@ void spanKindValueConversion(String scenario, String spanKind, int expectedKind)
assertEquals(expectedKind, TraceMapperV1.getSpanKindValue(spanKind));
}

@Test
void preEncodedAttributeUsesItsUtf8Bytes() throws IOException {
UTF8BytesString url = UTF8BytesString.create("http://localhost/é/東京");
SpiedSerialization serialized = serializeV1PayloadWithSpy(span(singletonMap(HTTP_URL, url)));

verify(serialized.packer).writeUTF8(url);
assertEquals(url.toString(), readFirstSpan(serialized.payload).getAttributes().get(HTTP_URL));
}

@Test
void preEncodedEventAttributeUsesItsUtf8Bytes() throws IOException {
UTF8BytesString value = UTF8BytesString.create("event-value-é");
UTF8BytesString item = UTF8BytesString.create("event-item-東京");
Map<String, Object> attributes = new HashMap<>();
attributes.put("str", value);
attributes.put("arr", singletonList(item));
List<DDSpanEvent> spanEvents = singletonList(new DDSpanEvent("event", 5L, attributes));

SpiedSerialization serialized = serializeV1PayloadWithSpy(spanWithEvents(spanEvents));

verify(serialized.packer).writeUTF8(value);
verify(serialized.packer).writeUTF8(item);
Map<String, Object> decoded =
readFirstSpan(serialized.payload).getEvents().get(0).getAttributes();
assertEquals(value.toString(), decoded.get("str"));
assertEquals(singletonList(item.toString()), decoded.get("arr"));
}

@Test
void httpStatusAttributeUsesCachedUtf8Bytes() throws IOException {
UTF8BytesString status = RadixTreeCache.HTTP_STATUSES.get(404);
SpiedSerialization serialized = serializeV1PayloadWithSpy(span(emptyMap(), 404));

verify(serialized.packer).writeUTF8(status);
assertEquals("404", readFirstSpan(serialized.payload).getAttributes().get(HTTP_STATUS));
}

@Test
void explicitHttpStatusTagOverridesInterceptedStatus() throws IOException {
PojoSpan span = span(singletonMap(HTTP_STATUS, 503), 200);

Map<String, Object> attributes = readFirstSpan(serializeV1Payload(span)).getAttributes();

assertAttributeValueEquals(503, attributes.get(HTTP_STATUS), HTTP_STATUS);
assertEquals(3, attributes.size(), "Only thread metadata and one HTTP status should remain");
}

@Test
void nullHttpStatusTagDoesNotSuppressInterceptedStatus() throws IOException {
PojoSpan span = span(singletonMap(HTTP_STATUS, null), 200);

Map<String, Object> attributes = readFirstSpan(serializeV1Payload(span)).getAttributes();

assertEquals("200", attributes.get(HTTP_STATUS));
assertEquals(3, attributes.size(), "Only thread metadata and one HTTP status should remain");
}

@Test
void payloadContainsExpectedHeaderAndChunkFields() throws IOException {
Map<String, Object> tags = new HashMap<>();
Expand Down Expand Up @@ -398,6 +460,16 @@ void missingSpanLinksEncodeEmptyLinks() throws IOException {
assertTrue(readFirstSpan(encoded).getLinks().isEmpty());
}

@Test
void legacySpanEventsTagIsExcludedFromAttributes() throws IOException {
PojoSpan span = span(singletonMap(SPAN_EVENTS, "legacy-events"), 0);

Map<String, Object> attributes = readFirstSpan(serializeV1Payload(span)).getAttributes();

assertFalse(attributes.containsKey(SPAN_EVENTS));
assertEquals(2, attributes.size(), "Only thread metadata should remain");
}

@Test
void spanEventsAreEncodedFromStructuredSpanEvents() throws IOException {
List<DDSpanEvent> spanEvents = new ArrayList<>();
Expand Down Expand Up @@ -892,6 +964,32 @@ private static byte[] serializeV1Payload(List<PojoSpan> trace) {
return capturedBody.payloadBytes;
}

/**
* Exposes writer calls alongside payload bytes: round-trip assertions alone cannot detect
* redundant UTF-8 encoding.
*/
private static SpiedSerialization serializeV1PayloadWithSpy(PojoSpan span) {
TraceMapperV1 mapper = new TraceMapperV1();
CapturedBody capturedBody = new CapturedBody(mapper);
MsgPackWriter packer = spy(new MsgPackWriter(new FlushingBuffer(2 << 20, capturedBody)));

assertTrue(packer.format(singletonList(span), mapper));
packer.flush();

assertNotNull(capturedBody.payloadBytes);
return new SpiedSerialization(packer, capturedBody.payloadBytes);
}

private static final class SpiedSerialization {
private final MsgPackWriter packer;
private final byte[] payload;

private SpiedSerialization(MsgPackWriter packer, byte[] payload) {
this.packer = packer;
this.payload = payload;
}
}

private static PojoSpan span(Map<String, Object> tags) {
return span(123L, 0L, emptyMap(), tags, 200, emptyList(), emptyList());
}
Expand Down
Loading