From 824bf4851a85c1204cfa6aa91d2eead7356ebd9a Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 16 Nov 2026 15:19:58 -0500 Subject: [PATCH 1/3] Avoid redundant allocations in V1 trace serialization --- .../common/writer/ddagent/TraceMapperV1.java | 81 ++++++++++++------- .../ddagent/TraceMapperV1PayloadTest.java | 67 +++++++++++++++ 2 files changed, 118 insertions(+), 30 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV1.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV1.java index 8797df5f740..050414e704a 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV1.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV1.java @@ -2,7 +2,6 @@ import static datadog.communication.http.OkHttpUtils.msgpackRequestBodyOf; import static datadog.trace.api.DDTags.SPAN_EVENTS; -import static datadog.trace.api.cache.RadixTreeCache.UNSET_STATUS; import static datadog.trace.core.CoreSpan.ROOT_PROPAGATION_TAGS; import static datadog.trace.core.CoreSpan.STRUCTURED_EVENTS; import static datadog.trace.core.CoreSpan.STRUCTURED_LINKS; @@ -25,6 +24,7 @@ import datadog.trace.bootstrap.instrumentation.api.AgentSpanLink; import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags; import datadog.trace.bootstrap.instrumentation.api.Tags; +import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.common.writer.Payload; import datadog.trace.core.CoreSpan; import datadog.trace.core.DDSpanEvent; @@ -72,6 +72,9 @@ public final class TraceMapperV1 implements TraceMapper { private final MsgPackWriter metaStructWriter; private final ByteBuffer header; + // Scratch state for {@link #countTagAttributes}, reset for each span on the serialization thread. + private int tagAttributeCount; + public TraceMapperV1(int bufferSize) { this.bufferSize = bufferSize; this.stringTable = new StringTable(); @@ -282,7 +285,7 @@ private void writeEventAttributeValue(Writable writable, Object value) { return; } writable.writeInt(VALUE_TYPE_STRING); - writeStreamingString(writable, String.valueOf(value)); + writeStreamingString(writable, value); } private boolean isEncodableEventArrayItem(Object item) { @@ -300,7 +303,7 @@ private void writeEventArrayItemValue(Writable writable, Object item) { return; } writable.writeInt(VALUE_TYPE_STRING); - writeStreamingString(writable, String.valueOf(item)); + writeStreamingString(writable, item); } private void writeEventNumberValue(Writable writable, Number number) { @@ -321,17 +324,15 @@ private void encodeSpanAttributes( Writable writable, int fieldId, Metadata meta, Map metaStruct) { TagMap tags = meta.getTags(); Map baggage = meta.getBaggage(); - String httpStatusCode = - meta.getHttpStatusCode() == UNSET_STATUS ? null : meta.getHttpStatusCodeString().toString(); + UTF8BytesString httpStatusCode = meta.getHttpStatusCodeString(); boolean writeHttpStatus = httpStatusCode != null && tags.getString(HTTP_STATUS) == null; boolean writeTopLevel = meta.topLevel(); - int tagCount = 0; - for (TagMap.EntryReader entry : tags) { - if (SPAN_EVENTS.equals(entry.tag())) { - continue; - } - tagCount += getFlatAttributeCount(entry); - } + + // Count flattened tags before writing the array header. Explicit callback state keeps both + // tag passes free of TagMap iterators and capturing lambdas. + tagAttributeCount = 0; + tags.forEach(this, TraceMapperV1::countTagAttributes); + int tagCount = tagAttributeCount; writable.writeInt(fieldId); writable.startArray( @@ -346,26 +347,42 @@ private void encodeSpanAttributes( writeAttribute(writable, DDTags.THREAD_ID, meta.getThreadId()); writeAttribute(writable, DDTags.THREAD_NAME, meta.getThreadName()); - for (Map.Entry entry : baggage.entrySet()) { - writeAttribute(writable, entry.getKey(), entry.getValue()); - } - - for (TagMap.EntryReader entry : tags) { - if (SPAN_EVENTS.equals(entry.tag())) { - continue; + // Skip entry-set views and iterators for empty baggage and meta-struct maps. + if (!baggage.isEmpty()) { + for (Map.Entry entry : baggage.entrySet()) { + writeAttribute(writable, entry.getKey(), entry.getValue()); } - writeFlattenedTagAttribute(writable, entry); } + + tags.forEach(this, writable, TraceMapperV1::writeTagAttribute); + if (writeHttpStatus) { writeAttribute(writable, HTTP_STATUS, httpStatusCode); } + if (writeTopLevel) { writeAttribute(writable, InstrumentationTags.DD_TOP_LEVEL.toString(), 1); } - for (Map.Entry metaStructField : metaStruct.entrySet()) { - writeStreamingString(writable, metaStructField.getKey()); - writable.writeInt(VALUE_TYPE_BYTES); - writable.writeBinary(serializeMetaStructValue(metaStructField.getValue())); + + if (!metaStruct.isEmpty()) { + for (Map.Entry metaStructField : metaStruct.entrySet()) { + writeStreamingString(writable, metaStructField.getKey()); + writable.writeInt(VALUE_TYPE_BYTES); + writable.writeBinary(serializeMetaStructValue(metaStructField.getValue())); + } + } + } + + private static void countTagAttributes(TraceMapperV1 mapper, TagMap.EntryReader entry) { + if (!SPAN_EVENTS.equals(entry.tag())) { + mapper.tagAttributeCount += mapper.getFlatAttributeCount(entry); + } + } + + private static void writeTagAttribute( + TraceMapperV1 mapper, Writable writable, TagMap.EntryReader entry) { + if (!SPAN_EVENTS.equals(entry.tag())) { + mapper.writeFlattenedTagAttribute(writable, entry); } } @@ -452,7 +469,7 @@ private void writeAttribute(Writable writable, String key, Object value) { } writable.writeInt(VALUE_TYPE_STRING); - writeStreamingString(writable, value == null ? "" : value.toString()); + writeStreamingString(writable, value); } private byte[] serializeMetaStructValue(Object value) { @@ -499,16 +516,20 @@ private void encodeTraceId(Writable writable, int fieldId, DDTraceId traceId) { writable.writeBinary(traceId.toHighOrderLong(), traceId.toLong()); } - /** Writes a string using the streaming string table encoding. */ - private void writeStreamingString(Writable writable, CharSequence value) { + /** + * Writes a string-table reference or a new string, treating null as empty. + * + *

String keys let equal text share an index across value types. On a miss, preserve a {@link + * UTF8BytesString} to reuse its cached encoding; other values reuse the String computed for + * lookup. + */ + private void writeStreamingString(Writable writable, Object value) { String str = value == null ? "" : value.toString(); Integer index = stringTable.get(str); if (index != null) { - // String already in table, write index writable.writeInt(index); } else { - // New string, write it directly and add to table - writable.writeString(str, null); + writable.writeString(value instanceof UTF8BytesString ? (UTF8BytesString) value : str, null); stringTable.add(str); } } diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV1PayloadTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV1PayloadTest.java index 9920ada43a4..4b20d0006a0 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV1PayloadTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV1PayloadTest.java @@ -42,6 +42,8 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; import datadog.communication.serialization.ByteBufferConsumer; import datadog.communication.serialization.FlushingBuffer; @@ -50,11 +52,13 @@ import datadog.trace.api.DDSpanId; import datadog.trace.api.DDTraceId; import datadog.trace.api.ProcessTags; +import datadog.trace.api.cache.RadixTreeCache; import datadog.trace.api.sampling.SamplingMechanism; import datadog.trace.bootstrap.instrumentation.api.AgentSpanLink; import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags; import datadog.trace.bootstrap.instrumentation.api.SpanAttributes; import datadog.trace.bootstrap.instrumentation.api.SpanLink; +import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.common.writer.Payload; import datadog.trace.common.writer.TraceGenerator.PojoSpan; import datadog.trace.common.writer.ddagent.V1PayloadReader.ChunkField; @@ -185,6 +189,43 @@ void spanKindValueConversion(String scenario, String spanKind, int expectedKind) assertEquals(expectedKind, TraceMapperV1.getSpanKindValue(spanKind)); } + @Test + void preEncodedAttributeUsesItsUtf8Bytes() throws IOException { + UTF8BytesString url = UTF8BytesString.create("http://localhost/é/東京"); + SpiedSerialization serialized = serializeV1PayloadWithSpy(span(singletonMap(HTTP_URL, url))); + + verify(serialized.packer).writeUTF8(url); + assertEquals(url.toString(), readFirstSpan(serialized.payload).getAttributes().get(HTTP_URL)); + } + + @Test + void preEncodedEventAttributeUsesItsUtf8Bytes() throws IOException { + UTF8BytesString value = UTF8BytesString.create("event-value-é"); + UTF8BytesString item = UTF8BytesString.create("event-item-東京"); + Map attributes = new HashMap<>(); + attributes.put("str", value); + attributes.put("arr", singletonList(item)); + List spanEvents = singletonList(new DDSpanEvent("event", 5L, attributes)); + + SpiedSerialization serialized = serializeV1PayloadWithSpy(spanWithEvents(spanEvents)); + + verify(serialized.packer).writeUTF8(value); + verify(serialized.packer).writeUTF8(item); + Map decoded = + readFirstSpan(serialized.payload).getEvents().get(0).getAttributes(); + assertEquals(value.toString(), decoded.get("str")); + assertEquals(singletonList(item.toString()), decoded.get("arr")); + } + + @Test + void httpStatusAttributeUsesCachedUtf8Bytes() throws IOException { + UTF8BytesString status = RadixTreeCache.HTTP_STATUSES.get(404); + SpiedSerialization serialized = serializeV1PayloadWithSpy(span(emptyMap(), 404)); + + verify(serialized.packer).writeUTF8(status); + assertEquals("404", readFirstSpan(serialized.payload).getAttributes().get(HTTP_STATUS)); + } + @Test void payloadContainsExpectedHeaderAndChunkFields() throws IOException { Map tags = new HashMap<>(); @@ -892,6 +933,32 @@ private static byte[] serializeV1Payload(List trace) { return capturedBody.payloadBytes; } + /** + * Exposes writer calls alongside payload bytes: round-trip assertions alone cannot detect + * redundant UTF-8 encoding. + */ + private static SpiedSerialization serializeV1PayloadWithSpy(PojoSpan span) { + TraceMapperV1 mapper = new TraceMapperV1(); + CapturedBody capturedBody = new CapturedBody(mapper); + MsgPackWriter packer = spy(new MsgPackWriter(new FlushingBuffer(2 << 20, capturedBody))); + + assertTrue(packer.format(singletonList(span), mapper)); + packer.flush(); + + assertNotNull(capturedBody.payloadBytes); + return new SpiedSerialization(packer, capturedBody.payloadBytes); + } + + private static final class SpiedSerialization { + private final MsgPackWriter packer; + private final byte[] payload; + + private SpiedSerialization(MsgPackWriter packer, byte[] payload) { + this.packer = packer; + this.payload = payload; + } + } + private static PojoSpan span(Map tags) { return span(123L, 0L, emptyMap(), tags, 200, emptyList(), emptyList()); } From f544f76580189c05661197cb7a1879f9dafc643a Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 16 Nov 2026 16:47:53 -0500 Subject: [PATCH 2/3] Address V1 serializer review feedback --- .../trace/common/writer/ddagent/TraceMapperV1.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV1.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV1.java index 050414e704a..1eb665f00f9 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV1.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV1.java @@ -72,7 +72,9 @@ public final class TraceMapperV1 implements TraceMapper { private final MsgPackWriter metaStructWriter; private final ByteBuffer header; - // Scratch state for {@link #countTagAttributes}, reset for each span on the serialization thread. + /** + * Scratch state for {@link #countTagAttributes}, reset for each span on the serialization thread. + */ private int tagAttributeCount; public TraceMapperV1(int bufferSize) { @@ -529,7 +531,11 @@ private void writeStreamingString(Writable writable, Object value) { if (index != null) { writable.writeInt(index); } else { - writable.writeString(value instanceof UTF8BytesString ? (UTF8BytesString) value : str, null); + if (value instanceof UTF8BytesString) { + writable.writeUTF8((UTF8BytesString) value); + } else { + writable.writeString(str, null); + } stringTable.add(str); } } From 6f6affa9c6842b33f599692a94a728c5e9cf604f Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 16 Nov 2026 17:36:54 -0500 Subject: [PATCH 3/3] Use known tag IDs in V1 trace serialization --- .../common/writer/ddagent/TraceMapperV1.java | 17 +++++----- .../ddagent/TraceMapperV1PayloadTest.java | 31 +++++++++++++++++++ 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV1.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV1.java index 1eb665f00f9..1c301a5d358 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV1.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV1.java @@ -1,7 +1,6 @@ package datadog.trace.common.writer.ddagent; import static datadog.communication.http.OkHttpUtils.msgpackRequestBodyOf; -import static datadog.trace.api.DDTags.SPAN_EVENTS; import static datadog.trace.core.CoreSpan.ROOT_PROPAGATION_TAGS; import static datadog.trace.core.CoreSpan.STRUCTURED_EVENTS; import static datadog.trace.core.CoreSpan.STRUCTURED_LINKS; @@ -18,6 +17,7 @@ import datadog.trace.api.Config; import datadog.trace.api.DDTags; import datadog.trace.api.DDTraceId; +import datadog.trace.api.KnownTags; import datadog.trace.api.ProcessTags; import datadog.trace.api.TagMap; import datadog.trace.api.sampling.SamplingMechanism; @@ -176,13 +176,13 @@ private void encodeSpans(Writable writable, int fieldId, List baggage = meta.getBaggage(); UTF8BytesString httpStatusCode = meta.getHttpStatusCodeString(); - boolean writeHttpStatus = httpStatusCode != null && tags.getString(HTTP_STATUS) == null; + boolean writeHttpStatus = + httpStatusCode != null && tags.getEntryReader(KnownTags.HTTP_STATUS_CODE) == null; boolean writeTopLevel = meta.topLevel(); // Count flattened tags before writing the array header. Explicit callback state keeps both @@ -376,14 +377,14 @@ private void encodeSpanAttributes( } private static void countTagAttributes(TraceMapperV1 mapper, TagMap.EntryReader entry) { - if (!SPAN_EVENTS.equals(entry.tag())) { + if (entry.tagId() != KnownTags.EVENTS) { mapper.tagAttributeCount += mapper.getFlatAttributeCount(entry); } } private static void writeTagAttribute( TraceMapperV1 mapper, Writable writable, TagMap.EntryReader entry) { - if (!SPAN_EVENTS.equals(entry.tag())) { + if (entry.tagId() != KnownTags.EVENTS) { mapper.writeFlattenedTagAttribute(writable, entry); } } diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV1PayloadTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV1PayloadTest.java index 4b20d0006a0..9e7a0fa1863 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV1PayloadTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV1PayloadTest.java @@ -2,6 +2,7 @@ import static datadog.trace.api.DDTags.PROCESS_TAGS; import static datadog.trace.api.DDTags.SDK_OTLP_EXPORT; +import static datadog.trace.api.DDTags.SPAN_EVENTS; import static datadog.trace.api.DDTags.THREAD_ID; import static datadog.trace.api.DDTags.THREAD_NAME; import static datadog.trace.api.config.TracerConfig.WRITER_TYPE; @@ -226,6 +227,26 @@ void httpStatusAttributeUsesCachedUtf8Bytes() throws IOException { assertEquals("404", readFirstSpan(serialized.payload).getAttributes().get(HTTP_STATUS)); } + @Test + void explicitHttpStatusTagOverridesInterceptedStatus() throws IOException { + PojoSpan span = span(singletonMap(HTTP_STATUS, 503), 200); + + Map attributes = readFirstSpan(serializeV1Payload(span)).getAttributes(); + + assertAttributeValueEquals(503, attributes.get(HTTP_STATUS), HTTP_STATUS); + assertEquals(3, attributes.size(), "Only thread metadata and one HTTP status should remain"); + } + + @Test + void nullHttpStatusTagDoesNotSuppressInterceptedStatus() throws IOException { + PojoSpan span = span(singletonMap(HTTP_STATUS, null), 200); + + Map attributes = readFirstSpan(serializeV1Payload(span)).getAttributes(); + + assertEquals("200", attributes.get(HTTP_STATUS)); + assertEquals(3, attributes.size(), "Only thread metadata and one HTTP status should remain"); + } + @Test void payloadContainsExpectedHeaderAndChunkFields() throws IOException { Map tags = new HashMap<>(); @@ -439,6 +460,16 @@ void missingSpanLinksEncodeEmptyLinks() throws IOException { assertTrue(readFirstSpan(encoded).getLinks().isEmpty()); } + @Test + void legacySpanEventsTagIsExcludedFromAttributes() throws IOException { + PojoSpan span = span(singletonMap(SPAN_EVENTS, "legacy-events"), 0); + + Map attributes = readFirstSpan(serializeV1Payload(span)).getAttributes(); + + assertFalse(attributes.containsKey(SPAN_EVENTS)); + assertEquals(2, attributes.size(), "Only thread metadata should remain"); + } + @Test void spanEventsAreEncodedFromStructuredSpanEvents() throws IOException { List spanEvents = new ArrayList<>();