Encode tag ids with a colored slot and collapse dense presence to one occupancy long (phase 2)

dd-trace-api/src/main/java/datadog/trace/api/config/TracerConfig.java

@@ -173,5 +173,12 @@ public final class TracerConfig {
   public static final String TRACE_ORG_GUARD_STRICT = "trace.org.guard.strict";
   public static final String TRACE_ORG_GUARD_TRUSTED_OPMS = "trace.org.guard.trusted.opms";
 
+  /**
+   * Routes known tags through the dense (id-keyed) tag store instead of per-tag entries.
+   * Experimental, OFF by default. The {@code KnownTagCodec} is registered regardless; this flag
+   * only selects whether tags take the dense storage path.
+   */
+  public static final String TRACE_DENSE_TAGS_ENABLED = "trace.experimental.dense.tags.enabled";
+
   private TracerConfig() {}
 }

dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java

@@ -40,6 +40,7 @@
 import datadog.trace.api.EndpointTracker;
 import datadog.trace.api.IdGenerationStrategy;
 import datadog.trace.api.InstrumenterConfig;
+import datadog.trace.api.KnownTags;
 import datadog.trace.api.Pair;
 import datadog.trace.api.TagMap;
 import datadog.trace.api.TraceConfig;
@@ -656,6 +657,13 @@ private CoreTracer(
     // preload this enum to avoid triggering classloading on the hot path
     TraceCollector.PublishState.values();
 
+    // Register the KnownTagCodec resolver unconditionally so tag-id name resolution (keyOf/nameOf,
+    // OTel name mapping) is always live. Whether known tags actually take the dense store is a
+    // separate, const-folded decision (KnownTagCodec.DENSE_STORE, from
+    // trace.experimental.dense.tags.enabled); when that flag is off, tag storage is byte-identical
+    // to the bucket-only behavior.
+    KnownTags.init();
+
     if (reportInTracerFlare) {
       TracerFlare.addReporter(this);
     }
@@ -2207,18 +2215,31 @@ protected static final DDSpanContext buildSpanContext(
               propagationTags,
               tracer.profilingContextIntegration,
               tracer.injectBaggageAsTags,
-              tracer.injectLinksAsTags);
+              tracer.injectLinksAsTags,
+              mergedTracerTagsNeedsIntercept ? null : mergedTracerTags);
 
       // By setting the tags on the context we apply decorators to any tags that have been set via
       // the builder. This is the order that the tags were added previously, but maybe the `tags`
       // set in the builder should come last, so that they override other tags.
-      context.setAllTags(mergedTracerTags, mergedTracerTagsNeedsIntercept);
+      //
+      // mergedTracerTags is trace-level shared state and the precedence floor (everything below
+      // overrides it). When it carries no interceptable tags it is attached as a read-through
+      // PARENT at construction (shared by reference, no per-span copy). When it does need
+      // interception, copy its entries in (the interceptor's per-span side-effects can't be
+      // shared by reference).
+      if (mergedTracerTagsNeedsIntercept) {
+        context.setAllTags(mergedTracerTags, true);
+      }
       context.setAllTags(tagLedger);
       context.setAllTags(coreTags, coreTagsNeedsIntercept);
       context.setAllTags(rootSpanTags, rootSpanTagsNeedsIntercept);
       context.setAllTags(contextualTags);
-      // remove version here since will be done later on the postProcessor.
-      // it will allow knowing if it will be set manually or not
+      // Version is added later by the postProcessor (InternalTagsAdder), only if not already set
+      // during the request. Config version is kept out of the trace-level bundle (see
+      // withTracerTags), so this removal now only wipes a version set via the span builder —
+      // keeping
+      // the existing semantics where a builder-set version is replaced by the config version. Under
+      // read-through this is a cheap local removal (version isn't in the parent, so no tombstone).
       context.removeTag(Tags.VERSION);
       return context;
     }
@@ -2449,6 +2470,25 @@ static TagMap withTracerTags(
       Map<String, ?> userSpanTags, Config config, TraceConfig traceConfig) {
     final TagMap result = TagMap.create(userSpanTags.size() + 5);
     result.putAll(userSpanTags);
+    // Version is conditionally managed by InternalTagsAdder (added only when service == DD_SERVICE
+    // and not set during the request), so keep it OUT of the trace-level bundle. This matters under
+    // read-through: the bundle becomes a shared parent, and a per-span removeTag(VERSION) on a key
+    // that lived in the parent would mint a per-span tombstone. With version excluded here, the
+    // per-span removeTag (retained, to wipe a builder-set version) is a cheap local op, never a
+    // tombstone.
+    //
+    // EXCEPTION: when `version` is a split-service tag, the TagInterceptor derives the service name
+    // from it, so it must reach the interceptor. Keeping it in the bundle forces the intercepting
+    // seed path (a split tag makes the bundle needsIntercept=true -> copied, not a read-through
+    // parent), where the retained removeTag(VERSION) still deletes only a local copy -- so the
+    // split
+    // side-effect fires and no per-span tombstone is minted either way.
+    //
+    // Cold path: withTracerTags runs at setup / config-change, not per span (mergedTracerTags is
+    // cached on the config snapshot), so this getSplitByTags() lookup needn't be hoisted.
+    if (config == null || !config.getSplitByTags().contains(Tags.VERSION)) {
+      result.remove(Tags.VERSION);
+    }
     if (null != config) { // static
       if (!config.getEnv().isEmpty()) {
         result.set("env", config.getEnv());

dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java

@@ -243,7 +243,8 @@ public DDSpanContext(
         propagationTags,
         ProfilingContextIntegration.NoOp.INSTANCE,
         true,
-        true);
+        true,
+        null);
   }
 
   public DDSpanContext(
@@ -293,9 +294,11 @@ public DDSpanContext(
         propagationTags,
         ProfilingContextIntegration.NoOp.INSTANCE,
         injectBaggageAsTags,
-        injectLinksAsTags);
+        injectLinksAsTags,
+        null);
   }
 
+  /** Back-compat ctor (no read-through parent); delegates with a null parent. */
   public DDSpanContext(
       final DDTraceId traceId,
       final long spanId,
@@ -322,6 +325,62 @@ public DDSpanContext(
       final ProfilingContextIntegration profilingContextIntegration,
       final boolean injectBaggageAsTags,
       final boolean injectLinksAsTags) {
+    this(
+        traceId,
+        spanId,
+        parentId,
+        parentServiceName,
+        serviceNameSource,
+        serviceName,
+        operationName,
+        resourceName,
+        samplingPriority,
+        origin,
+        baggageItems,
+        w3cBaggage,
+        errorFlag,
+        spanType,
+        tagsSize,
+        traceCollector,
+        requestContextDataAppSec,
+        requestContextDataIast,
+        CiVisibilityContextData,
+        pathwayContext,
+        disableSamplingMechanismValidation,
+        propagationTags,
+        profilingContextIntegration,
+        injectBaggageAsTags,
+        injectLinksAsTags,
+        null);
+  }
+
+  public DDSpanContext(
+      final DDTraceId traceId,
+      final long spanId,
+      final long parentId,
+      final CharSequence parentServiceName,
+      final CharSequence serviceNameSource,
+      final String serviceName,
+      final CharSequence operationName,
+      final CharSequence resourceName,
+      final int samplingPriority,
+      final CharSequence origin,
+      final Map<String, String> baggageItems,
+      final Baggage w3cBaggage,
+      final boolean errorFlag,
+      final CharSequence spanType,
+      final int tagsSize,
+      final TraceCollector traceCollector,
+      final Object requestContextDataAppSec,
+      final Object requestContextDataIast,
+      final Object CiVisibilityContextData,
+      final PathwayContext pathwayContext,
+      final boolean disableSamplingMechanismValidation,
+      final PropagationTags propagationTags,
+      final ProfilingContextIntegration profilingContextIntegration,
+      final boolean injectBaggageAsTags,
+      final boolean injectLinksAsTags,
+      final TagMap readThroughParent) {
 
     assert traceCollector != null;
     this.traceCollector = traceCollector;
@@ -350,7 +409,10 @@ public DDSpanContext(
     // The +1 is the magic number from the tags below that we set at the end,
     // and "* 4 / 3" is to make sure that we don't resize immediately
     final int capacity = Math.max((tagsSize <= 0 ? 3 : (tagsSize + 1)) * 4 / 3, 8);
-    this.unsafeTags = TagMap.create(capacity);
+    this.unsafeTags =
+        readThroughParent != null
+            ? TagMap.createFromParent(readThroughParent)
+            : TagMap.create(capacity);
 
     // must set this before setting the service and resource names below
     this.profilingContextIntegration = profilingContextIntegration;

dd-trace-core/src/test/java/datadog/trace/core/WithTracerTagsVersionTest.java

@@ -0,0 +1,42 @@
+package datadog.trace.core;
+
+import static datadog.trace.api.config.TracerConfig.SPLIT_BY_TAGS;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import datadog.trace.api.Config;
+import datadog.trace.api.TagMap;
+import datadog.trace.bootstrap.instrumentation.api.Tags;
+import datadog.trace.test.junit.utils.config.WithConfig;
+import org.junit.jupiter.api.Test;
+
+/**
+ * {@code withTracerTags} keeps {@code version} OUT of the trace-level bundle so a per-span {@code
+ * removeTag(VERSION)} doesn't mint a read-through tombstone -- EXCEPT when {@code version} is a
+ * split-service tag, where the {@code TagInterceptor} must still see it to derive the service name
+ * (regression guard for the level-split consumer).
+ */
+class WithTracerTagsVersionTest extends DDCoreJavaSpecification {
+
+  private static TagMap tracerTagsWithVersion(Config config) {
+    TagMap userTags = TagMap.create();
+    userTags.set(Tags.VERSION, "1.2.3");
+    return CoreTracer.withTracerTags(userTags, config, null);
+  }
+
+  @Test
+  void versionStrippedFromBundleByDefault() {
+    assertNull(
+        tracerTagsWithVersion(Config.get()).getString(Tags.VERSION),
+        "version is kept out of the trace-level bundle by default (avoids per-span tombstone)");
+  }
+
+  @Test
+  @WithConfig(key = SPLIT_BY_TAGS, value = "version")
+  void versionKeptInBundleWhenSplitByVersion() {
+    assertEquals(
+        "1.2.3",
+        tracerTagsWithVersion(Config.get()).getString(Tags.VERSION),
+        "version must stay in the bundle so split-by-tags can derive the service name");
+  }
+}

gradle/libs.versions.toml

@@ -42,6 +42,7 @@ jmh = "1.37"
 # Profiling
 jmc = "8.1.0"
 jafar = "0.16.0"
+jol = "0.17"
 
 # Web & Network
 jnr-unixsocket = "0.38.25"
@@ -125,6 +126,7 @@ instrument-java = { module = "com.datadoghq:dd-instrument-java", version.ref = "
 jmc-common = { module = "org.openjdk.jmc:common", version.ref = "jmc" }
 jmc-flightrecorder = { module = "org.openjdk.jmc:flightrecorder", version.ref = "jmc" }
 jafar-tools = { module = "io.btrace:jafar-tools", version.ref = "jafar" }
+jol-core = { module = "org.openjdk.jol:jol-core", version.ref = "jol" }
 
 # Web & Network
 okio = { module = "com.datadoghq.okio:okio", version.ref = "okio" }

internal-api/build.gradle.kts

@@ -282,6 +282,7 @@ dependencies {
   testImplementation("org.junit.vintage:junit-vintage-engine:${libs.versions.junit5.get()}")
   testImplementation(libs.commons.math)
   testImplementation(libs.bundles.mockito)
+  testImplementation(libs.jol.core)
 }
 
 jmh {