From 509c917f1566ce08f8b77e77da0617c7cc0fe7e4 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Mon, 17 Aug 2026 19:17:53 -0400 Subject: [PATCH 1/6] Generate KnownTags from tag-conventions via graph coloring Introduce the tag-registry code generator (buildSrc plugin + tag-conventions YAML) and use it to assign the colored slots from the previous commit by graph-coloring the tag co-occurrence graph (each concrete span type's resolved set plus the clique), packing all stored tags into SLOT_COUNT=16 slots. Relocates the hand-maintained KnownTags from src/main to the generated src/generated source set and renames the codec accessors to their generated form (serialNum/makeTagId). Adds the generator verify task so CI fails if the checked-in KnownTags drifts from the conventions. Co-Authored-By: Claude Opus 4.8 --- buildSrc/build.gradle.kts | 6 + .../plugin/tags/GenerateKnownTagsTask.kt | 38 ++++ .../gradle/plugin/tags/KnownTagsEmitter.kt | 154 ++++++++++++++++ .../gradle/plugin/tags/TagConventions.kt | 173 ++++++++++++++++++ .../datadog/gradle/plugin/tags/TagRegistry.kt | 153 ++++++++++++++++ .../plugin/tags/TagRegistryGenerator.kt | 172 +++++++++++++++++ .../plugin/tags/TagRegistryGeneratorPlugin.kt | 41 +++++ .../gradle/plugin/tags/VerifyKnownTagsTask.kt | 67 +++++++ gradle/spotless.gradle | 3 +- internal-api/build.gradle.kts | 16 ++ internal-api/src/generated/folded-types.txt | 93 ++++++++++ .../java/datadog/trace/api/KnownTags.java | 4 +- internal-api/src/generated/layout-by-type.txt | 91 +++++++++ internal-api/src/generated/resolved-tags.txt | 77 ++++++++ internal-api/src/generated/tag-assignment.txt | 69 +++++++ tag-conventions.java.yaml | 36 ++++ tag-conventions.yaml | 134 ++++++++++++++ 17 files changed, 1323 insertions(+), 4 deletions(-) create mode 100644 buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/GenerateKnownTagsTask.kt create mode 100644 buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt create mode 100644 buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagConventions.kt create mode 100644 buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistry.kt create mode 100644 buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt create mode 100644 buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGeneratorPlugin.kt create mode 100644 buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/VerifyKnownTagsTask.kt create mode 100644 internal-api/src/generated/folded-types.txt rename internal-api/src/{main => generated}/java/datadog/trace/api/KnownTags.java (99%) create mode 100644 internal-api/src/generated/layout-by-type.txt create mode 100644 internal-api/src/generated/resolved-tags.txt create mode 100644 internal-api/src/generated/tag-assignment.txt create mode 100644 tag-conventions.java.yaml create mode 100644 tag-conventions.yaml diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 991ca899596..b7eacb3cd95 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -59,6 +59,11 @@ gradlePlugin { implementationClass = "datadog.gradle.plugin.config.SupportedConfigPlugin" } + create("tag-registry-generator") { + id = "dd-trace-java.tag-registry-generator" + implementationClass = "datadog.gradle.plugin.tags.TagRegistryGeneratorPlugin" + } + create("supported-config-linter") { id = "dd-trace-java.config-inversion-linter" implementationClass = "datadog.gradle.plugin.config.ConfigInversionLinter" @@ -107,6 +112,7 @@ dependencies { implementation("com.fasterxml.jackson.core:jackson-databind") implementation("com.fasterxml.jackson.core:jackson-annotations") implementation("com.fasterxml.jackson.core:jackson-core") + implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml") compileOnly(libs.develocity) } diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/GenerateKnownTagsTask.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/GenerateKnownTagsTask.kt new file mode 100644 index 00000000000..ab1c2631c60 --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/GenerateKnownTagsTask.kt @@ -0,0 +1,38 @@ +package datadog.gradle.plugin.tags + +import javax.inject.Inject +import org.gradle.api.DefaultTask +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.model.ObjectFactory +import org.gradle.api.tasks.CacheableTask +import org.gradle.api.tasks.InputFile +import org.gradle.api.tasks.OutputDirectory +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity +import org.gradle.api.tasks.TaskAction + +/** + * Generates the committed tag registry (KnownTags.java + layout reports) from the language-agnostic + * {@code tag-conventions.yaml} + the Java overlay. The actual emit lives in [TagRegistryGenerator]; + * this task just wires the inputs/outputs so Gradle can cache and up-to-date-check it. + */ +@CacheableTask +abstract class GenerateKnownTagsTask @Inject constructor(objects: ObjectFactory) : DefaultTask() { + @get:InputFile + @get:PathSensitive(PathSensitivity.NONE) + val domainYaml: RegularFileProperty = objects.fileProperty() + + @get:InputFile + @get:PathSensitive(PathSensitivity.NONE) + val overlayYaml: RegularFileProperty = objects.fileProperty() + + @get:OutputDirectory val destinationDirectory: DirectoryProperty = objects.directoryProperty() + + @TaskAction + fun generate() { + val outDir = destinationDirectory.get().asFile + TagRegistryGenerator.generate(domainYaml.get().asFile, overlayYaml.get().asFile, outDir) + logger.lifecycle("tag-registry: generated -> $outDir") + } +} diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt new file mode 100644 index 00000000000..7ee0c7f458e --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt @@ -0,0 +1,154 @@ +package datadog.gradle.plugin.tags + +/** + * Emits the generated `KnownTags.java` from a [TagRegistry]. Public API first — per-tag + * `_NAME` (string) + `_ID` (encoded long, literal) couplets with a trailing `// makeTagId(...)` + * derivation comment — then the package-private `_SERIAL_NUM` constants, the + * `StringIndex.EmbeddingSupport` keyOf table, the `serialNum` switch `nameOf`, and resolver + * registration. + */ +object KnownTagsEmitter { + + fun emit(reg: TagRegistry, pkg: String, className: String): String { + // Sanitize tag names into unique Java constant identifiers. + val used = HashSet() + val cname = HashMap() + fun mk(name: String): String { + var c = name.uppercase().replace(Regex("[^A-Za-z0-9]"), "_").replace(Regex("_+"), "_").trim('_') + if (c.isEmpty() || c[0].isDigit()) c = "T_$c" + var u = c + var n = 2 + while (u in used) { + u = "${c}_$n"; n++ + } + used.add(u) + cname[name] = u + return u + } + reg.reserved.forEach { mk(it.name) } + reg.stored.forEach { mk(it.name) } + + // Constant names. Collapse a duplicated trailing token so e.g. "resource.name" yields NAME + // (not NAME_NAME) and "_dd.parent_id" yields ID (not ID_ID); the non-duplicating pairs + // (ID + _NAME -> ID_NAME, NAME + _ID -> NAME_ID) are kept as-is. + fun withSuffix(base: String, suffix: String) = if (base.endsWith(suffix)) base else "$base$suffix" + fun nameC(name: String) = withSuffix(cname[name]!!, "_NAME") + fun idC(name: String) = withSuffix(cname[name]!!, "_ID") + fun serialC(name: String) = withSuffix(cname[name]!!, "_SERIAL_NUM") + + val order = reg.reserved.map { it.name } + reg.stored.map { it.name } // stable emit order + val b = StringBuilder() + b.appendLine("package $pkg;") + b.appendLine() + b.appendLine("import datadog.trace.util.StringIndex;") + b.appendLine() + b.appendLine("// GENERATED by the tag-registry code generator (dd-trace-java.tag-registry-generator).") + b.appendLine("// DO NOT EDIT. Source: tag-conventions.yaml + tag-conventions.java.yaml.") + b.appendLine("public final class $className {") + b.appendLine(" static final int SLOT_COUNT = ${reg.slotCount};") + b.appendLine() + + // Public API first (name + encoded id couplets), so readers see the useful parts up top; the + // serial ids and keyOf/resolver machinery follow below. Derivation is in the trailing comment. + b.appendLine(" // ---- reserved (routed to span fields or directives; not stored) ----") + for (v in reg.reserved) { + b.appendLine(" public static final String ${nameC(v.name)} = \"${v.name}\";") + b.appendLine(" public static final long ${idC(v.name)} = ${hex(v.id)};") + b.appendLine(" // makeTagId(serial=${v.serial}, slot=NO_SLOT) + intercepted [${v.kind}${v.field?.let { " -> $it" } ?: ""}]") + b.appendLine() + } + + b.appendLine(" // ---- stored (dense colored slot, or bucketed when slot=NO_SLOT) ----") + for (t in reg.stored) { + val slot = if (t.slotted) t.slot.toString() else "NO_SLOT" + b.appendLine(" public static final String ${nameC(t.name)} = \"${t.name}\";") + b.appendLine(" public static final long ${idC(t.name)} = ${hex(t.id)};") + b.appendLine(" // makeTagId(serial=${t.serial}, slot=$slot)${if (t.intercepted) " + intercepted" else ""}${if (t.traceLevel) " + trace-level" else ""} <${t.required}>") + b.appendLine() + } + + // Serial numbers (globalSerial per tag) — package-private, consumed by the resolver switch. + b.appendLine(" // ---- serial numbers ----") + for (v in reg.reserved) { + b.appendLine(" static final int ${serialC(v.name)} = ${v.serial};") + } + for (t in reg.stored) { + b.appendLine(" static final int ${serialC(t.name)} = ${t.serial};") + } + b.appendLine() + + // keyOf table (open-addressed, via StringIndex.EmbeddingSupport). + b.appendLine(" private static final String[] KEYOF_NAMES = {") + order.forEach { b.appendLine(" ${nameC(it)},") } + b.appendLine(" };") + b.appendLine(" private static final long[] KEYOF_VALUES = {") + order.forEach { b.appendLine(" ${idC(it)},") } + b.appendLine(" };") + b.appendLine(" private static final int[] KEYOF_HASHES;") + b.appendLine(" private static final String[] KEYOF_KEYS;") + b.appendLine(" private static final long[] KEYOF_IDS;") + b.appendLine(" static {") + b.appendLine(" StringIndex.Data data = StringIndex.EmbeddingSupport.create(KEYOF_NAMES);") + b.appendLine(" long[] ids = new long[data.names.length];") + b.appendLine(" for (int j = 0; j < KEYOF_NAMES.length; j++) {") + b.appendLine(" ids[StringIndex.EmbeddingSupport.indexOf(data.hashes, data.names, KEYOF_NAMES[j])] = KEYOF_VALUES[j];") + b.appendLine(" }") + b.appendLine(" KEYOF_HASHES = data.hashes;") + b.appendLine(" KEYOF_KEYS = data.names;") + b.appendLine(" KEYOF_IDS = ids;") + b.appendLine(" }") + b.appendLine() + + // Resolver. + b.appendLine(" static final KnownTagCodec.Resolver RESOLVER =") + b.appendLine(" new KnownTagCodec.Resolver() {") + b.appendLine(" @Override") + b.appendLine(" public String nameOf(long tagId) {") + b.appendLine(" switch (KnownTagCodec.serialNum(tagId)) {") + for (name in order) { + b.appendLine(" case ${serialC(name)}:") + b.appendLine(" return ${nameC(name)};") + } + b.appendLine(" default:") + b.appendLine(" return null;") + b.appendLine(" }") + b.appendLine(" }") + b.appendLine() + b.appendLine(" @Override") + b.appendLine(" public int slotCount() {") + b.appendLine(" return SLOT_COUNT;") + b.appendLine(" }") + b.appendLine() + b.appendLine(" @Override") + b.appendLine(" public long keyOf(String name) {") + b.appendLine(" int slot = StringIndex.EmbeddingSupport.indexOf(KEYOF_HASHES, KEYOF_KEYS, name);") + b.appendLine(" return slot < 0 ? 0L : KEYOF_IDS[slot];") + b.appendLine(" }") + b.appendLine(" };") + b.appendLine() + b.appendLine(" static {") + b.appendLine(" KnownTagCodec.register(RESOLVER);") + b.appendLine(" }") + b.appendLine() + b.appendLine(" /**") + b.appendLine( + " * Forces resolver registration. Merely invoking this static method runs {@code } (which") + b.appendLine( + " * registers {@link #RESOLVER}), so calling it once at tracer init makes tag-id name resolution") + b.appendLine( + " * ({@code keyOf}/{@code nameOf}) live; idempotent. Whether known tags then take the dense store") + b.appendLine( + " * is a separate, const-folded decision ({@link KnownTagCodec#DENSE_STORE}). Until something") + b.appendLine( + " * references this class the registry stays dormant and {@code keyOf} returns 0, so tag storage is") + b.appendLine(" * byte-identical to the bucket-only behavior.") + b.appendLine(" */") + b.appendLine(" public static void init() {}") + b.appendLine() + b.appendLine(" private $className() {}") + b.appendLine("}") + return b.toString() + } + + private fun hex(id: Long): String = "0x%016XL".format(id) +} diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagConventions.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagConventions.kt new file mode 100644 index 00000000000..a01da7bc75c --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagConventions.kt @@ -0,0 +1,173 @@ +package datadog.gradle.plugin.tags + +/** + * Parsed tag-conventions domain model + the per-type tag-set resolver. Language-agnostic: it knows + * only structure (extends / include / applies) and per-tag semantics (name / type / required / + * source). Id assignment and emission are layered on top of the resolved sets. + */ +class TagConventions +private constructor( + private val spanTypes: Map, + private val mixins: Map, + private val traceLevel: List, +) { + /** A tag declaration (domain semantics only). */ + data class Tag( + val name: String, + val type: String, + val required: String, + ) + + data class SpanType( + val name: String, + val abstract: Boolean, + val extends: String?, + val include: List, + val tags: List, + ) + + data class Mixin( + val name: String, + val appliesAll: Boolean, + val appliesTo: Set, + val tags: List, + ) + + /** Concrete (instantiable) span types — the ones a layout is computed for. */ + fun concreteTypes(): List = + spanTypes.values.filter { !it.abstract }.map { it.name }.sorted() + + /** + * resolved(type) = own tags + tags up the `extends` chain (incl. base) + tags of every mixin the + * type or an ancestor `include`s + tags of every mixin whose `applies` matches. De-duped by tag + * name (first occurrence wins). Base-first order, so it is stable across runs. + */ + fun resolve(typeName: String): List { + val result = LinkedHashMap() + fun add(t: Tag) = result.putIfAbsent(t.name, t) + + val chain = ArrayList() + var cur: SpanType? = spanTypes[typeName] + while (cur != null) { + chain.add(cur) + cur = cur.extends?.let { spanTypes[it] } + } + for (st in chain.asReversed()) { + st.tags.forEach { add(it) } + for (mixinName in st.include) mixins[mixinName]?.tags?.forEach { add(it) } + } + val chainNames = chain.map { it.name }.toSet() + for (mx in mixins.values) { + if (mx.appliesAll || mx.appliesTo.any { it in chainNames }) mx.tags.forEach { add(it) } + } + return result.values.toList() + } + + /** The explicit trace-level tier tags (their own TagMap "type" on the TraceSegment). */ + fun traceLevelTags(): List = traceLevel + + /** A declaration group: the source that *declares* a set of tags (its own `tags:` list). */ + data class Group(val name: String, val kind: String, val tags: List) + + /** + * The declaration groups, in a stable order: the trace-level tier first, then every span type + * (abstract included — `base`/`http` declare real tags) sorted by name, then every mixin sorted by + * name. Each maps to one `group-decl`. A tag is *declared* once (in its own container's `tags:`); + * the same tag reached via extends/include/applies is not re-declared, so first-declaration (in + * this order) is its home group. Groups with no declared tags are omitted. + */ + fun declarationGroups(): List { + val groups = ArrayList() + if (traceLevel.isNotEmpty()) groups.add(Group(TRACE_LAYER, "trace", traceLevel)) + for (name in spanTypes.keys.sorted()) { + val st = spanTypes.getValue(name) + if (st.tags.isNotEmpty()) groups.add(Group(name, "span_type", st.tags)) + } + for (name in mixins.keys.sorted()) { + val mx = mixins.getValue(name) + if (mx.tags.isNotEmpty()) groups.add(Group(name, "mixin", mx.tags)) + } + return groups + } + + /** Full stored-tag universe (concrete span types' resolves + trace-level), de-duped by name. */ + fun allStoredTags(): List { + val union = LinkedHashMap() + for (type in concreteTypes()) for (t in resolve(type)) union.putIfAbsent(t.name, t) + for (t in traceLevel) union.putIfAbsent(t.name, t) + return union.values.toList() + } + + /** + * Full composition for a type as (origin, tag) pairs, in composition order and NOT de-duped, so a + * tag contributed by more than one source shows up more than once. Origin is the contributing + * span type (via extends), `incl:` (via include), or `appl:` (via applies). + */ + fun compose(typeName: String): List> { + val out = ArrayList>() + val chain = ArrayList() + var cur: SpanType? = spanTypes[typeName] + while (cur != null) { + chain.add(cur) + cur = cur.extends?.let { spanTypes[it] } + } + for (st in chain.asReversed()) { + st.tags.forEach { out.add(st.name to it) } + for (mixinName in st.include) mixins[mixinName]?.tags?.forEach { out.add("incl:$mixinName" to it) } + } + val chainNames = chain.map { it.name }.toSet() + for (mx in mixins.values) { + if (mx.appliesAll || mx.appliesTo.any { it in chainNames }) { + mx.tags.forEach { out.add("appl:${mx.name}" to it) } + } + } + return out + } + + companion object { + /** Group name of the trace-level tier (its own TagMap layer on the TraceSegment). */ + const val TRACE_LAYER = "" + + @Suppress("UNCHECKED_CAST") + fun parse(root: Map): TagConventions { + val spanTypesRaw = (root["span_types"] as? Map) ?: emptyMap() + val spanTypes = + spanTypesRaw.mapValues { (name, v) -> + val m = v as Map + SpanType( + name = name, + abstract = (m["abstract"] as? Boolean) ?: false, + extends = m["extends"] as? String, + include = (m["include"] as? List) ?: emptyList(), + tags = tagList(m["tags"]), + ) + } + + val mixinsRaw = (root["mixins"] as? Map) ?: emptyMap() + val mixins = + mixinsRaw.mapValues { (name, v) -> + val m = v as Map + val applies = m["applies"] + Mixin( + name = name, + appliesAll = applies == "all", + appliesTo = if (applies is List<*>) applies.map { it.toString() }.toSet() else emptySet(), + tags = tagList(m["tags"]), + ) + } + + val traceLevel = tagList((root["trace_level"] as? Map)?.get("tags")) + return TagConventions(spanTypes, mixins, traceLevel) + } + + @Suppress("UNCHECKED_CAST") + private fun tagList(tags: Any?): List = + (tags as? List>)?.map { m -> + Tag( + name = m["tag"].toString(), + type = (m["type"] as? String) ?: "string", + required = (m["required"] as? String) ?: "optional", + ) + } ?: emptyList() + } +} diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistry.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistry.kt new file mode 100644 index 00000000000..0eebcbda629 --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistry.kt @@ -0,0 +1,153 @@ +package datadog.gradle.plugin.tags + +/** + * Assigns tag ids from a parsed [TagConventions] plus the Java overlay (intercepted set + reserved + * registry). The id encoding mirrors KnownTagCodec: [63 intercepted][62-48 serial][47-32 slot][31-0 + * zero] (known ids carry no nameHash — they are dense-store addressed). + * + *

The `slot` is a single globally stable coordinate from GRAPH COLORING the tag co-occurrence + * graph. Each concrete span type's resolved tag set (see [TagConventions.resolve]) is a clique — its + * tags all appear together on one span, so they must get distinct slots. The trace-level tier is its + * own clique (a separate TagMap on the TraceSegment), so it may reuse slot numbers freely with the + * span layers. Slots are shared only between tags that never co-occur, so slotCount stays bounded by + * the largest clique (≤ 64) — small enough that the dense store's presence fast path is a single + * occupancy `long` (`1L << slot`), which is exactly why the earlier two-tier (group + field bloom) + * scheme could collapse to one word. Correctness never depends on the coloring (the dense scan is + * authoritative); only the fast-path hit rate does. + * + *

Slotting (does a tag get a slot / dense presence bit) is derived from the domain `required` + * level: required/conditional/recommended tags are colored (slotted), the rest are NO_SLOT + * (bucketed) and carry no slot bit. + */ +class TagRegistry +private constructor( + val stored: List, + val reserved: List, + val slotCount: Int, +) { + data class StoredTag( + val name: String, + val type: String, + val required: String, + val serial: Int, + val intercepted: Boolean, + val slot: Int, + val traceLevel: Boolean, + val id: Long, + ) { + val slotted: Boolean + get() = slot != NO_SLOT + } + + data class ReservedTag( + val name: String, + val kind: String, + val field: String?, + val serial: Int, + val id: Long, + ) + + /** Java overlay: intercepted tag names + the reserved/special-key registry. */ + class Overlay(val intercepted: Set, val reserved: List) { + data class ReservedDef(val name: String, val kind: String, val field: String?) + + companion object { + @Suppress("UNCHECKED_CAST") + fun parse(root: Map): Overlay { + val intercepted = (root["intercepted"] as? List)?.toSet() ?: emptySet() + val reserved = + (root["reserved"] as? List>)?.map { m -> + ReservedDef(m["tag"].toString(), (m["kind"] as? String) ?: "directive", m["field"] as? String) + } ?: emptyList() + return Overlay(intercepted, reserved) + } + } + } + + companion object { + const val FIRST_STORED_SERIAL = 256 + const val NO_SLOT = 0xFFFF // slot all-ones sentinel (16 bits); mirrors KnownTagCodec.NO_SLOT + const val MAX_SLOT = 63 // one occupancy long: colored slots must fit in [0, 63] + const val LEVEL_TRACE = 1L shl 2 // low-32 carve bit 2; mirrors KnownTagCodec.LEVEL_TRACE + const val TRACE_LAYER = "" + + // Domain `required` levels that get a colored slot (the rest are bucketed with NO_SLOT). + val COLORABLE = setOf("required", "conditional", "recommended") + + /** + * Mirrors KnownTagCodec.makeTagId(serial, slot) + intercepted()/traceLevel() — must stay in + * sync. slot [47-32], LEVEL_TRACE at bit 2, other low bits zero. + */ + fun encode(serial: Int, intercepted: Boolean, slot: Int, traceLevel: Boolean): Long { + var id = (serial.toLong() shl 48) or ((slot.toLong() and 0xFFFF) shl 32) + if (traceLevel) id = id or LEVEL_TRACE + if (intercepted) id = id or Long.MIN_VALUE + return id + } + + fun build(conv: TagConventions, overlay: Overlay): TagRegistry { + val all = conv.allStoredTags() + val traceNames = conv.traceLevelTags().map { it.name }.toSet() + val colorable = all.filter { it.required in COLORABLE }.map { it.name }.toSet() + + // Co-occurrence cliques: each concrete type's resolved colorable tags, plus the trace-level + // tier as its own clique (a separate TagMap -> free to reuse span slot numbers). Tags in the + // same clique must get distinct colors; tags never sharing a clique may share a color. + val cliques = ArrayList>() + for (type in conv.concreteTypes()) { + cliques.add(conv.resolve(type).map { it.name }.filter { it in colorable }.toSet()) + } + cliques.add(traceNames.filter { it in colorable }.toSet()) + + // Adjacency: an edge between every pair of tags that co-occur in some clique. + val adj = HashMap>() + colorable.forEach { adj[it] = HashSet() } + for (clique in cliques) { + val members = clique.toList() + for (i in members.indices) for (j in i + 1 until members.size) { + adj.getValue(members[i]).add(members[j]) + adj.getValue(members[j]).add(members[i]) + } + } + + // Greedy coloring, most-constrained-first (by clique membership count, then name for a stable + // tie-break). Each tag takes the smallest color not used by an already-colored neighbor. + val cliqueCount = colorable.associateWith { n -> cliques.count { n in it } } + val order = colorable.sortedWith(compareByDescending { cliqueCount.getValue(it) }.thenBy { it }) + val color = HashMap() + for (n in order) { + val used = adj.getValue(n).mapNotNull { color[it] }.toSet() + var c = 0 + while (c in used) c++ + color[n] = c + } + val slotCount = (color.values.maxOrNull() ?: -1) + 1 + require(slotCount <= MAX_SLOT + 1) { + "coloring produced $slotCount slots; the single occupancy long holds at most ${MAX_SLOT + 1}" + } + + val reserved = + overlay.reserved.mapIndexed { i, v -> + val serial = 1 + i + ReservedTag( + v.name, v.kind, v.field, serial, + encode(serial, intercepted = true, slot = NO_SLOT, traceLevel = false)) + } + + // Stored tags in a stable order (by name); serials are a dense global counter from + // FIRST_STORED_SERIAL. slot comes from the coloring (NO_SLOT for non-colorable/bucketed tags). + val stored = + all.sortedBy { it.name }.mapIndexed { i, t -> + val serial = FIRST_STORED_SERIAL + i + val intercepted = t.name in overlay.intercepted + val slot = color[t.name] ?: NO_SLOT + val traceLevel = t.name in traceNames + StoredTag( + t.name, t.type, t.required, serial, intercepted, slot, traceLevel, + id = encode(serial, intercepted, slot, traceLevel)) + } + + return TagRegistry(stored, reserved, slotCount) + } + } +} diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt new file mode 100644 index 00000000000..c57a055c405 --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt @@ -0,0 +1,172 @@ +package datadog.gradle.plugin.tags + +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import java.io.File + +/** + * Turns the language-agnostic {@code tag-conventions.yaml} + the Java overlay into the generated tag + * registry: {@code KnownTags.java} (under {@code java/}) plus verification report dumps + * (resolved-tags / tag-assignment / layout-by-type / folded-types) at the destination root. + * + * Pure function of its inputs (deterministic ordering throughout), so the same inputs always produce + * byte-identical output -- which is what the {@code verifyKnownTags} freshness gate relies on. + */ +object TagRegistryGenerator { + /** Parses the two YAML files and writes the full generated tree under [outDir]. */ + fun generate(domainYaml: File, overlayYaml: File, outDir: File) { + val mapper = ObjectMapper(YAMLFactory()) + val domain: Map = + domainYaml.inputStream().use { + mapper.readValue(it, object : TypeReference>() {}) + } + val overlayMap: Map = + overlayYaml.inputStream().use { + mapper.readValue(it, object : TypeReference>() {}) + } + + outDir.mkdirs() + // KnownTags.java goes under java/ (added as a srcDir); the .txt reports sit at the root. + val javaPkg = File(outDir, "java/datadog/trace/api").apply { mkdirs() } + + val conv = TagConventions.parse(domain) + val overlay = TagRegistry.Overlay.parse(overlayMap) + val reg = TagRegistry.build(conv, overlay) + + File(outDir, "resolved-tags.txt").writeText(resolvedReport(conv)) + File(outDir, "tag-assignment.txt").writeText(assignmentReport(conv, reg)) + File(outDir, "layout-by-type.txt").writeText(layoutByTypeReport(conv, reg)) + File(outDir, "folded-types.txt").writeText(foldedTypesReport(conv, reg)) + File(javaPkg, "KnownTags.java") + .writeText(KnownTagsEmitter.emit(reg, "datadog.trace.api", "KnownTags")) + } + + /** resolved-tags.txt — the per-type resolved sets (composition check). */ + private fun resolvedReport(conv: TagConventions): String { + val resolved = StringBuilder() + resolved.appendLine("# Resolved per-type tag sets (concrete span types).") + for (type in conv.concreteTypes()) { + val tags = conv.resolve(type) + resolved.appendLine() + resolved.appendLine("$type (${tags.size} tags):") + for (t in tags) resolved.appendLine(" - ${t.name}") + } + return resolved.toString() + } + + /** tag-assignment.txt — serials, colored slots, ids, per-type slot sets (coloring check). */ + private fun assignmentReport(conv: TagConventions, reg: TagRegistry): String { + val byName = reg.stored.associateBy { it.name } + val a = StringBuilder() + a.appendLine( + "# Tag id assignment. slotCount=${reg.slotCount} stored=${reg.stored.size} reserved=${reg.reserved.size}") + a.appendLine() + a.appendLine("# STORED serial slot int lvl id required name") + for (t in reg.stored) { + a.appendLine( + " %6d %5s %s %s %-18s %-12s %s".format( + t.serial, + if (t.slotted) t.slot.toString() else "-", + if (t.intercepted) "I" else "-", + if (t.traceLevel) "T" else "-", + "0x%016X".format(t.id), + t.required, + t.name)) + } + a.appendLine() + a.appendLine("# RESERVED serial id kind name") + for (v in reg.reserved) { + a.appendLine( + " %6d %-18s %-12s %s%s".format( + v.serial, "0x%016X".format(v.id), v.kind, v.name, v.field?.let { " -> $it" } ?: "")) + } + a.appendLine() + a.appendLine("# PER-TYPE colored slots. Slots within a type must be DISTINCT (a valid coloring of the") + a.appendLine("# co-occurrence clique); is its own clique and freely reuses span slot numbers.") + for (type in conv.concreteTypes()) { + val slots = + conv.resolve(type).mapNotNull { byName[it.name] } + .filter { it.slotted && !it.traceLevel } + .map { it.slot } + .sorted() + a.appendLine(" %-14s count=%-3d slots=%s".format(type, slots.size, slots)) + } + val traceSlots = + reg.stored.filter { it.traceLevel && it.slotted }.map { it.slot }.sorted() + a.appendLine(" %-14s count=%-3d slots=%s".format("", traceSlots.size, traceSlots)) + return a.toString() + } + + /** + * layout-by-type.txt — full composition per type (origins shown, NOT de-duped), each tag annotated + * with its slot/tier: s = colored slot, trace s = trace-level layer, bkt = bucket. + */ + private fun layoutByTypeReport(conv: TagConventions, reg: TagRegistry): String { + val byName = reg.stored.associateBy { it.name } + val lay = StringBuilder() + lay.appendLine("# Full tag composition per concrete span type (after extends/include/applies).") + lay.appendLine("# Not de-duped: a tag from >1 source appears >1 time.") + lay.appendLine("# annotation: [s colored slot | trace s trace layer | bkt bucketed] I=intercepted") + for (type in conv.concreteTypes()) { + val comp = conv.compose(type) + val distinct = comp.map { it.second.name }.distinct().size + lay.appendLine() + lay.appendLine("$type (${comp.size} contributions, $distinct distinct):") + val byOrigin = LinkedHashMap>() + for ((origin, tag) in comp) byOrigin.getOrPut(origin) { ArrayList() }.add(tag) + for ((origin, tags) in byOrigin) { + lay.appendLine(" [$origin]") + for (t in tags) { + val st = byName[t.name] + val field = + when { + st == null -> "?" + st.traceLevel && st.slotted -> "trace s${st.slot}" + st.slotted -> "s${st.slot}" + else -> "bkt" + } + lay.appendLine( + " %-26s %-12s %-12s %s".format(t.name, field, t.required, if (st?.intercepted == true) "I" else "")) + } + } + } + return lay.toString() + } + + /** + * folded-types.txt — each type's full resolved set (extends + include + applies, DE-DUPED) with its + * slot; plus the type. This is the "type with everything folded in" view. + */ + private fun foldedTypesReport(conv: TagConventions, reg: TagRegistry): String { + val byName = reg.stored.associateBy { it.name } + fun tierField(st: TagRegistry.StoredTag?): String = + when { + st == null -> "?" + st.traceLevel -> if (st.slotted) "trace s${st.slot}" else "trace-bkt" + st.slotted -> "s${st.slot}" + else -> "bkt" + } + val f = StringBuilder() + f.appendLine("# Folded tag set per type (extends + include + applies, de-duped), with colored slots.") + f.appendLine( + "# field: s=colored slot trace s=trace-level layer bkt=bucketed trace-bkt=trace-level bucketed I=intercepted") + for (type in conv.concreteTypes()) { + val tags = conv.resolve(type) + f.appendLine() + f.appendLine("$type (${tags.size} tags):") + for (t in tags) { + val st = byName[t.name] + f.appendLine(" %-12s %-26s %s".format(tierField(st), t.name, if (st?.intercepted == true) "I" else "")) + } + } + val traceTags = + reg.stored.filter { it.traceLevel }.sortedWith(compareBy({ !it.slotted }, { it.slot }, { it.name })) + f.appendLine() + f.appendLine(" (${traceTags.size} tags):") + for (st in traceTags) { + f.appendLine(" %-12s %-26s %s".format(tierField(st), st.name, if (st.intercepted) "I" else "")) + } + return f.toString() + } +} diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGeneratorPlugin.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGeneratorPlugin.kt new file mode 100644 index 00000000000..313bd859068 --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGeneratorPlugin.kt @@ -0,0 +1,41 @@ +package datadog.gradle.plugin.tags + +import javax.inject.Inject +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.model.ObjectFactory + +/** Extension configuring the tag-registry generator inputs/outputs. */ +abstract class TagRegistryExtension @Inject constructor(objects: ObjectFactory) { + val domainYaml: RegularFileProperty = objects.fileProperty() + val overlayYaml: RegularFileProperty = objects.fileProperty() + val destinationDirectory: DirectoryProperty = objects.directoryProperty() +} + +/** + * Registers {@code generateKnownTags} (emits the committed tag registry) and {@code verifyKnownTags} + * (a freshness gate that regenerates and byte-compares against the committed output). The verify task + * is wired into {@code check} so stale generated sources fail CI. + */ +class TagRegistryGeneratorPlugin : Plugin { + override fun apply(project: Project) { + val ext = project.extensions.create("tagRegistry", TagRegistryExtension::class.java) + project.tasks.register("generateKnownTags", GenerateKnownTagsTask::class.java) { + domainYaml.set(ext.domainYaml) + overlayYaml.set(ext.overlayYaml) + destinationDirectory.set(ext.destinationDirectory) + } + val verify = + project.tasks.register("verifyKnownTags", VerifyKnownTagsTask::class.java) { + domainYaml.set(ext.domainYaml) + overlayYaml.set(ext.overlayYaml) + committedDirectory.set(ext.destinationDirectory) + } + // `check` is contributed by lifecycle-base (via java-library); wait for it before wiring. + project.pluginManager.withPlugin("lifecycle-base") { + project.tasks.named("check").configure { dependsOn(verify) } + } + } +} diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/VerifyKnownTagsTask.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/VerifyKnownTagsTask.kt new file mode 100644 index 00000000000..e990e15e957 --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/VerifyKnownTagsTask.kt @@ -0,0 +1,67 @@ +package datadog.gradle.plugin.tags + +import java.io.File +import javax.inject.Inject +import org.gradle.api.DefaultTask +import org.gradle.api.GradleException +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.model.ObjectFactory +import org.gradle.api.tasks.InputDirectory +import org.gradle.api.tasks.InputFile +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity +import org.gradle.api.tasks.TaskAction + +/** + * Freshness gate: regenerates the tag registry into a scratch dir and byte-compares it against the + * committed [committedDirectory]. Fails (pointing at {@code generateKnownTags}) if they differ, so a + * stale commit of the generated sources can't slip through CI. Not cacheable -- it must actually run + * the generator to catch drift, and it is cheap. + */ +abstract class VerifyKnownTagsTask @Inject constructor(objects: ObjectFactory) : DefaultTask() { + @get:InputFile + @get:PathSensitive(PathSensitivity.NONE) + val domainYaml: RegularFileProperty = objects.fileProperty() + + @get:InputFile + @get:PathSensitive(PathSensitivity.NONE) + val overlayYaml: RegularFileProperty = objects.fileProperty() + + @get:InputDirectory + @get:PathSensitive(PathSensitivity.RELATIVE) + val committedDirectory: DirectoryProperty = objects.directoryProperty() + + @TaskAction + fun verify() { + val committed = committedDirectory.get().asFile + val scratch = File(temporaryDir, "generated") + scratch.deleteRecursively() + TagRegistryGenerator.generate(domainYaml.get().asFile, overlayYaml.get().asFile, scratch) + + val diffs = ArrayList() + val freshFiles = scratch.walkTopDown().filter { it.isFile }.toList() + for (fresh in freshFiles) { + val rel = fresh.relativeTo(scratch).path + val committedFile = File(committed, rel) + when { + !committedFile.exists() -> diffs.add("missing (not committed): $rel") + committedFile.readText() != fresh.readText() -> diffs.add("out of date: $rel") + } + } + val freshRel = freshFiles.map { it.relativeTo(scratch).path }.toSet() + for (committedFile in committed.walkTopDown().filter { it.isFile }) { + val rel = committedFile.relativeTo(committed).path + if (rel !in freshRel) diffs.add("stale (no longer generated): $rel") + } + + if (diffs.isNotEmpty()) { + throw GradleException( + buildString { + appendLine("Generated tag registry is out of date with tag-conventions.yaml:") + diffs.forEach { appendLine(" - $it") } + append("Run `./gradlew :internal-api:generateKnownTags` and commit the result.") + }) + } + } +} diff --git a/gradle/spotless.gradle b/gradle/spotless.gradle index 93a817e6452..769e153ddd9 100644 --- a/gradle/spotless.gradle +++ b/gradle/spotless.gradle @@ -21,7 +21,8 @@ spotless { // set explicit target to workaround https://github.com/diffplug/spotless/issues/1163 target 'src/**/*.java' // ignore embedded test projects and everything in build dir, e.g. generated sources - targetExclude('src/test/resources/**', buildDirectoryFiles) + // src/generated/** is emitted by code generators (e.g. the tag registry) — verified by their own freshness gate + targetExclude('src/test/resources/**', 'src/generated/**', buildDirectoryFiles) tableTestFormatter('1.1.1') googleJavaFormat('1.35.0') } diff --git a/internal-api/build.gradle.kts b/internal-api/build.gradle.kts index 4d48a434c19..f1ab6486aa2 100644 --- a/internal-api/build.gradle.kts +++ b/internal-api/build.gradle.kts @@ -5,6 +5,7 @@ import groovy.lang.Closure plugins { `java-library` id("me.champeau.jmh") + id("dd-trace-java.tag-registry-generator") } apply(from = "$rootDir/gradle/java.gradle") @@ -32,6 +33,9 @@ extra["minimumBranchCoverage"] = 0.7 extra["minimumInstructionCoverage"] = 0.8 extra["excludedClassesCoverage"] = listOf( + // Generated by the tag-registry code generator (verified fresh via verifyKnownTags). + "datadog.trace.api.KnownTags", + "datadog.trace.api.KnownTags.*", "datadog.trace.api.ClassloaderConfigurationOverrides", "datadog.trace.api.ClassloaderConfigurationOverrides.Lazy", // Interface @@ -261,6 +265,18 @@ extra["excludedClassesBranchCoverage"] = listOf( extra["excludedClassesInstructionCoverage"] = listOf("datadog.trace.util.stacktrace.StackWalkerFactory") +// Tag registry: generated KnownTags is committed under src/generated (audited via git); the srcDir +// puts it on the main compile path and `verifyKnownTags` (wired into `check`) fails CI if it drifts +// from tag-conventions.yaml. Generation is run on demand (`./gradlew :internal-api:generateKnownTags`), +// not on every build, so the committed source stays the source of truth for the compiler. +tagRegistry { + domainYaml.set(rootProject.layout.projectDirectory.file("tag-conventions.yaml")) + overlayYaml.set(rootProject.layout.projectDirectory.file("tag-conventions.java.yaml")) + destinationDirectory.set(layout.projectDirectory.dir("src/generated")) +} + +sourceSets["main"].java.srcDir("src/generated/java") + dependencies { // references TraceScope and Continuation from public api api(project(":dd-trace-api")) diff --git a/internal-api/src/generated/folded-types.txt b/internal-api/src/generated/folded-types.txt new file mode 100644 index 00000000000..48b631bf5a5 --- /dev/null +++ b/internal-api/src/generated/folded-types.txt @@ -0,0 +1,93 @@ +# Folded tag set per type (extends + include + applies, de-duped), with colored slots. +# field: s=colored slot trace s=trace-level layer bkt=bucketed trace-bkt=trace-level bucketed I=intercepted + +db.client (21 tags): + s1 _dd.parent_id + s2 component + s6 span.kind I + s0 _dd.integration + bkt _dd.svc_src + s5 error.type + s3 error.message + s4 error.stack + s12 db.type + s9 db.instance + s10 db.operation + s15 db.user + bkt db.pool.name + s11 db.statement I + s14 peer.service I + s8 _dd.peer.service.source + s7 _dd.peer.service.remapped_from + s13 peer.hostname + bkt peer.ipv4 + bkt peer.ipv6 + bkt peer.port + +http.client (20 tags): + s1 _dd.parent_id + s2 component + s6 span.kind I + s0 _dd.integration + bkt _dd.svc_src + s5 error.type + s3 error.message + s4 error.stack + s9 http.method I + s10 http.status_code + s12 network.protocol.version + s11 http.url I + s15 http.resend_count + s14 peer.service I + s8 _dd.peer.service.source + s7 _dd.peer.service.remapped_from + s13 peer.hostname + bkt peer.ipv4 + bkt peer.ipv6 + bkt peer.port + +http.server (18 tags): + s1 _dd.parent_id + s2 component + s6 span.kind I + s0 _dd.integration + bkt _dd.svc_src + s5 error.type + s3 error.message + s4 error.stack + s9 http.method I + s10 http.status_code + s12 network.protocol.version + s11 http.url I + s13 http.route + s7 http.hostname + s14 http.useragent + s8 http.query.string + bkt servlet.path + bkt servlet.context I + +view.render (9 tags): + s1 _dd.parent_id + s2 component + s6 span.kind I + s0 _dd.integration + bkt _dd.svc_src + s5 error.type + s3 error.message + s4 error.stack + s7 view.name + + (13 tags): + trace s0 _dd.appsec.enabled + trace s1 _dd.base_service + trace s2 _dd.civisibility.enabled + trace s3 _dd.djm.enabled + trace s4 _dd.dsm.enabled + trace s5 _dd.git.commit.sha + trace s6 _dd.git.repository_url + trace s7 _dd.profiling.enabled + trace s8 _dd.tracer_host + trace s9 env + trace s10 language + trace s11 runtime-id + trace s12 version diff --git a/internal-api/src/main/java/datadog/trace/api/KnownTags.java b/internal-api/src/generated/java/datadog/trace/api/KnownTags.java similarity index 99% rename from internal-api/src/main/java/datadog/trace/api/KnownTags.java rename to internal-api/src/generated/java/datadog/trace/api/KnownTags.java index e8646593297..62842fbe2a0 100644 --- a/internal-api/src/main/java/datadog/trace/api/KnownTags.java +++ b/internal-api/src/generated/java/datadog/trace/api/KnownTags.java @@ -410,13 +410,11 @@ public final class KnownTags { private static final int[] KEYOF_HASHES; private static final String[] KEYOF_KEYS; private static final long[] KEYOF_IDS; - static { StringIndex.Data data = StringIndex.EmbeddingSupport.create(KEYOF_NAMES); long[] ids = new long[data.names.length]; for (int j = 0; j < KEYOF_NAMES.length; j++) { - ids[StringIndex.EmbeddingSupport.indexOf(data.hashes, data.names, KEYOF_NAMES[j])] = - KEYOF_VALUES[j]; + ids[StringIndex.EmbeddingSupport.indexOf(data.hashes, data.names, KEYOF_NAMES[j])] = KEYOF_VALUES[j]; } KEYOF_HASHES = data.hashes; KEYOF_KEYS = data.names; diff --git a/internal-api/src/generated/layout-by-type.txt b/internal-api/src/generated/layout-by-type.txt new file mode 100644 index 00000000000..309bdfcf8e3 --- /dev/null +++ b/internal-api/src/generated/layout-by-type.txt @@ -0,0 +1,91 @@ +# Full tag composition per concrete span type (after extends/include/applies). +# Not de-duped: a tag from >1 source appears >1 time. +# annotation: [s colored slot | trace s trace layer | bkt bucketed] I=intercepted + +db.client (21 contributions, 21 distinct): + [base] + _dd.parent_id s1 required + component s2 required + span.kind s6 required I + _dd.integration s0 recommended + _dd.svc_src bkt optional + error.type s5 recommended + error.message s3 recommended + error.stack s4 recommended + [db.client] + db.type s12 required + db.instance s9 recommended + db.operation s10 recommended + db.user s15 recommended + db.pool.name bkt optional + db.statement s11 recommended I + [incl:peer] + peer.service s14 recommended I + _dd.peer.service.source s8 recommended + _dd.peer.service.remapped_from s7 recommended + peer.hostname s13 recommended + peer.ipv4 bkt optional + peer.ipv6 bkt optional + peer.port bkt optional + +http.client (20 contributions, 20 distinct): + [base] + _dd.parent_id s1 required + component s2 required + span.kind s6 required I + _dd.integration s0 recommended + _dd.svc_src bkt optional + error.type s5 recommended + error.message s3 recommended + error.stack s4 recommended + [http] + http.method s9 required I + http.status_code s10 conditional + network.protocol.version s12 recommended + [http.client] + http.url s11 required I + http.resend_count s15 recommended + [incl:peer] + peer.service s14 recommended I + _dd.peer.service.source s8 recommended + _dd.peer.service.remapped_from s7 recommended + peer.hostname s13 recommended + peer.ipv4 bkt optional + peer.ipv6 bkt optional + peer.port bkt optional + +http.server (18 contributions, 18 distinct): + [base] + _dd.parent_id s1 required + component s2 required + span.kind s6 required I + _dd.integration s0 recommended + _dd.svc_src bkt optional + error.type s5 recommended + error.message s3 recommended + error.stack s4 recommended + [http] + http.method s9 required I + http.status_code s10 conditional + network.protocol.version s12 recommended + [http.server] + http.url s11 required I + http.route s13 conditional + http.hostname s7 required + http.useragent s14 recommended + http.query.string s8 recommended + servlet.path bkt optional + servlet.context bkt optional I + +view.render (9 contributions, 9 distinct): + [base] + _dd.parent_id s1 required + component s2 required + span.kind s6 required I + _dd.integration s0 recommended + _dd.svc_src bkt optional + error.type s5 recommended + error.message s3 recommended + error.stack s4 recommended + [view.render] + view.name s7 recommended diff --git a/internal-api/src/generated/resolved-tags.txt b/internal-api/src/generated/resolved-tags.txt new file mode 100644 index 00000000000..0edb3479608 --- /dev/null +++ b/internal-api/src/generated/resolved-tags.txt @@ -0,0 +1,77 @@ +# Resolved per-type tag sets (concrete span types). + +db.client (21 tags): + - _dd.parent_id + - component + - span.kind + - _dd.integration + - _dd.svc_src + - error.type + - error.message + - error.stack + - db.type + - db.instance + - db.operation + - db.user + - db.pool.name + - db.statement + - peer.service + - _dd.peer.service.source + - _dd.peer.service.remapped_from + - peer.hostname + - peer.ipv4 + - peer.ipv6 + - peer.port + +http.client (20 tags): + - _dd.parent_id + - component + - span.kind + - _dd.integration + - _dd.svc_src + - error.type + - error.message + - error.stack + - http.method + - http.status_code + - network.protocol.version + - http.url + - http.resend_count + - peer.service + - _dd.peer.service.source + - _dd.peer.service.remapped_from + - peer.hostname + - peer.ipv4 + - peer.ipv6 + - peer.port + +http.server (18 tags): + - _dd.parent_id + - component + - span.kind + - _dd.integration + - _dd.svc_src + - error.type + - error.message + - error.stack + - http.method + - http.status_code + - network.protocol.version + - http.url + - http.route + - http.hostname + - http.useragent + - http.query.string + - servlet.path + - servlet.context + +view.render (9 tags): + - _dd.parent_id + - component + - span.kind + - _dd.integration + - _dd.svc_src + - error.type + - error.message + - error.stack + - view.name diff --git a/internal-api/src/generated/tag-assignment.txt b/internal-api/src/generated/tag-assignment.txt new file mode 100644 index 00000000000..c1516085a3b --- /dev/null +++ b/internal-api/src/generated/tag-assignment.txt @@ -0,0 +1,69 @@ +# Tag id assignment. slotCount=16 stored=46 reserved=10 + +# STORED serial slot int lvl id required name + 256 0 - T 0x0100000000000004 recommended _dd.appsec.enabled + 257 1 - T 0x0101000100000004 required _dd.base_service + 258 2 - T 0x0102000200000004 recommended _dd.civisibility.enabled + 259 3 - T 0x0103000300000004 recommended _dd.djm.enabled + 260 4 - T 0x0104000400000004 recommended _dd.dsm.enabled + 261 5 - T 0x0105000500000004 recommended _dd.git.commit.sha + 262 6 - T 0x0106000600000004 recommended _dd.git.repository_url + 263 0 - - 0x0107000000000000 recommended _dd.integration + 264 1 - - 0x0108000100000000 required _dd.parent_id + 265 7 - - 0x0109000700000000 recommended _dd.peer.service.remapped_from + 266 8 - - 0x010A000800000000 recommended _dd.peer.service.source + 267 7 - T 0x010B000700000004 recommended _dd.profiling.enabled + 268 - - - 0x010CFFFF00000000 optional _dd.svc_src + 269 8 - T 0x010D000800000004 recommended _dd.tracer_host + 270 2 - - 0x010E000200000000 required component + 271 9 - - 0x010F000900000000 recommended db.instance + 272 10 - - 0x0110000A00000000 recommended db.operation + 273 - - - 0x0111FFFF00000000 optional db.pool.name + 274 11 I - 0x8112000B00000000 recommended db.statement + 275 12 - - 0x0113000C00000000 required db.type + 276 15 - - 0x0114000F00000000 recommended db.user + 277 9 - T 0x0115000900000004 recommended env + 278 3 - - 0x0116000300000000 recommended error.message + 279 4 - - 0x0117000400000000 recommended error.stack + 280 5 - - 0x0118000500000000 recommended error.type + 281 7 - - 0x0119000700000000 required http.hostname + 282 9 I - 0x811A000900000000 required http.method + 283 8 - - 0x011B000800000000 recommended http.query.string + 284 15 - - 0x011C000F00000000 recommended http.resend_count + 285 13 - - 0x011D000D00000000 conditional http.route + 286 10 - - 0x011E000A00000000 conditional http.status_code + 287 11 I - 0x811F000B00000000 required http.url + 288 14 - - 0x0120000E00000000 recommended http.useragent + 289 10 - T 0x0121000A00000004 required language + 290 12 - - 0x0122000C00000000 recommended network.protocol.version + 291 13 - - 0x0123000D00000000 recommended peer.hostname + 292 - - - 0x0124FFFF00000000 optional peer.ipv4 + 293 - - - 0x0125FFFF00000000 optional peer.ipv6 + 294 - - - 0x0126FFFF00000000 optional peer.port + 295 14 I - 0x8127000E00000000 recommended peer.service + 296 11 - T 0x0128000B00000004 required runtime-id + 297 - I - 0x8129FFFF00000000 optional servlet.context + 298 - - - 0x012AFFFF00000000 optional servlet.path + 299 6 I - 0x812B000600000000 required span.kind + 300 12 - T 0x012C000C00000004 recommended version + 301 7 - - 0x012D000700000000 recommended view.name + +# RESERVED serial id kind name + 1 0x8001FFFF00000000 structural error -> error + 2 0x8002FFFF00000000 structural service -> service + 3 0x8003FFFF00000000 structural resource.name -> resource + 4 0x8004FFFF00000000 structural span.type -> type + 5 0x8005FFFF00000000 structural origin -> origin + 6 0x8006FFFF00000000 directive sampling.priority + 7 0x8007FFFF00000000 directive manual.keep + 8 0x8008FFFF00000000 directive manual.drop + 9 0x8009FFFF00000000 directive measured + 10 0x800AFFFF00000000 directive analytics.sample_rate + +# PER-TYPE colored slots. Slots within a type must be DISTINCT (a valid coloring of the +# co-occurrence clique); is its own clique and freely reuses span slot numbers. + db.client count=16 slots=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + http.client count=16 slots=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + http.server count=15 slots=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] + view.render count=8 slots=[0, 1, 2, 3, 4, 5, 6, 7] + count=13 slots=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] diff --git a/tag-conventions.java.yaml b/tag-conventions.java.yaml new file mode 100644 index 00000000000..ab28b64545a --- /dev/null +++ b/tag-conventions.java.yaml @@ -0,0 +1,36 @@ +# dd-trace-java overlay — impl hints + special-tag registry. +# Consumed alongside the language-agnostic tag-conventions.yaml. Keyed by tag name +# (these are tag-intrinsic, not per-type). Only exceptions are listed. +# --------------------------------------------------------------------------- +# NOTE: the id coordinate (group-decl / field-decl) is NOT here — the generator assigns it from the +# declaration groups; the dense-vs-bucket split derives from the domain `required` level. + +# Tags whose set-path is handled by the Java TagInterceptor (side-effecting, but still stored). +# Generated ids carry the intercepted flag (bit 63). +intercepted: + - span.kind + - http.method + - http.url + - servlet.context + - db.statement + - peer.service + +# Reserved / special keys: accepted by the public setTag(...) API but ROUTED to a span field or a +# trace directive instead of tag storage. Reserved-tier ids (serial < FIRST_STORED_SERIAL, no slot). +# "Reserved" names the shared mechanism (the tracer reserves the key and handles it); the two kinds +# split on whether a value exists: structural has one (in a span/trace field), directive has none. +# The generated id->handler dispatch table is the data-driven replacement for the imperative +# TagInterceptor chain. +# kind: structural -> sets a span/trace field (`field:` names it) +# kind: directive -> triggers sampling/trace behavior +reserved: + - { tag: error, kind: structural, field: error } + - { tag: service, kind: structural, field: service, aliases: [service.name] } + - { tag: resource.name, kind: structural, field: resource } + - { tag: span.type, kind: structural, field: type } + - { tag: origin, kind: structural, field: origin } # trace-level field + - { tag: sampling.priority, kind: directive } + - { tag: manual.keep, kind: directive } + - { tag: manual.drop, kind: directive } + - { tag: measured, kind: directive } + - { tag: analytics.sample_rate, kind: directive } # legacy diff --git a/tag-conventions.yaml b/tag-conventions.yaml new file mode 100644 index 00000000000..2eff24f1784 --- /dev/null +++ b/tag-conventions.yaml @@ -0,0 +1,134 @@ +# Tag conventions — LANGUAGE-AGNOSTIC domain spec (structure + semantics only) +# --------------------------------------------------------------------------- +# The code generator consumes THIS file (domain) plus a per-language overlay +# (tag-conventions..yaml: impl hints like `intercepted`, and the reserved/ +# special-tag registry) to emit each language's tag-id constants, id<->name +# resolver, and slot (bitmask-bit) assignment. +# +# TRACE-LEVEL is its own thing (its own TagMap "type" on the TraceSegment) — the process/trace +# constants + product flags that are set once per trace, NOT per span. Declared explicitly in the +# `trace_level` section below (a distinct tier), never inferred from `source`. +# +# SPAN TYPES compose three ways: +# extends — structural is-a inheritance (http.server is-a http is-a base). `base` is implicitly +# in every span; abstract layers exist only to be extended. +# include — a span type PULLS in a mixin it intrinsically has (has-a; core-owned). +# applies — a mixin PUSHES itself onto span types, gated by `enabled_by`. +# resolved_tags(type) = own + extends-chain (incl base) + included mixins + applied mixins (de-duped). +# +# tag fields (DOMAIN only): tag | type (string|int|long|boolean|double) +# | required (required|conditional|recommended|optional|opt_in) | aliases. +# The id coordinate (group-decl / field-decl) is NOT authored here — the generator assigns it: each +# declaration source (the trace-level tier, each span type, each mixin) is a group, and within a +# group `field-decl` numbers the dense (required/conditional/recommended) tags; the rest are +# bucketed. See the design doc. +# --------------------------------------------------------------------------- + +# Trace-level tier: its own TagMap on the TraceSegment. Set once per trace, not per span. +trace_level: + tags: + - { tag: _dd.base_service, type: string, required: required } + - { tag: version, type: string, required: recommended } + - { tag: env, type: string, required: recommended } + - { tag: language, type: string, required: required } + - { tag: runtime-id, type: string, required: required } + - { tag: _dd.tracer_host, type: string, required: recommended } + - { tag: _dd.git.commit.sha, type: string, required: recommended } + - { tag: _dd.git.repository_url, type: string, required: recommended } + # product .enabled flags — process-constant; present on the trace segment regardless of whether + # the product is enabled (the flag carries the state), so always-present => recommended. + - { tag: _dd.profiling.enabled, type: boolean, required: recommended } + - { tag: _dd.dsm.enabled, type: boolean, required: recommended } + - { tag: _dd.appsec.enabled, type: boolean, required: recommended } + - { tag: _dd.djm.enabled, type: boolean, required: recommended } + - { tag: _dd.civisibility.enabled, type: boolean, required: recommended } + +span_types: + # root: per-span tags every span has (incl. the per-span core tags parent_id / integration / svc_src + # — core-set but per-span, so NOT trace-level). + base: + abstract: true + tags: + - { tag: _dd.parent_id, type: string, required: required } + - { tag: component, type: string, required: required } + - { tag: span.kind, type: string, required: required } + - { tag: _dd.integration, type: string, required: recommended } + - { tag: _dd.svc_src, type: string, required: optional } + - { tag: error.type, type: string, required: recommended } + - { tag: error.message, type: string, required: recommended } + - { tag: error.stack, type: string, required: recommended } + + http: + abstract: true + extends: base + tags: + - { tag: http.method, type: string, required: required, aliases: [http.request.method] } + - { tag: http.status_code, type: int, required: conditional, aliases: [http.response.status_code] } + - { tag: network.protocol.version, type: string, required: recommended } + + http.server: + extends: http + tags: + - { tag: http.url, type: string, required: required, aliases: [url.full] } + - { tag: http.route, type: string, required: conditional } + - { tag: http.hostname, type: string, required: required, aliases: [server.address] } + - { tag: http.useragent, type: string, required: recommended } + - { tag: http.query.string, type: string, required: recommended, aliases: [url.query] } + - { tag: servlet.path, type: string, required: optional } + - { tag: servlet.context, type: string, required: optional } + + http.client: + extends: http + include: [ peer ] + tags: + - { tag: http.url, type: string, required: required, aliases: [url.full] } + - { tag: http.resend_count, type: int, required: recommended } + + db.client: + extends: base + include: [ peer ] + tags: + - { tag: db.type, type: string, required: required, aliases: [db.system] } + - { tag: db.instance, type: string, required: recommended } + - { tag: db.operation, type: string, required: recommended, aliases: [db.operation.name] } + - { tag: db.user, type: string, required: recommended } + - { tag: db.pool.name, type: string, required: optional } + - { tag: db.statement, type: string, required: recommended, aliases: [db.query.text] } + + view.render: + extends: base + tags: + - { tag: view.name, type: string, required: recommended } + +mixins: + # peer — outbound/remote-peer capability, PULLED via `include` by client span types. + peer: + tags: + - { tag: peer.service, type: string, required: recommended } + - { tag: _dd.peer.service.source, type: string, required: recommended } + - { tag: _dd.peer.service.remapped_from, type: string, required: recommended } + - { tag: peer.hostname, type: string, required: recommended } + - { tag: peer.ipv4, type: string } + - { tag: peer.ipv6, type: string } + - { tag: peer.port, type: int } + + # ci_visibility — per-span test tags. Its capability flag (_dd.civisibility.enabled) lives in + # trace_level, outside this mixin (general rule: capability flags are trace-level, mixins hold the + # per-span tags). Applies to the `test` span type (not modeled here yet). + ci_visibility: + enabled_by: dd.civisibility.enabled + applies: [ test ] + tags: + - { tag: test.name, type: string, required: recommended } + - { tag: test.suite, type: string, required: recommended } + - { tag: test.status, type: string, required: recommended } + - { tag: test.framework, type: string, required: recommended } + +# --------------------------------------------------------------------------- +# Notes +# - Product .enabled flags moved to `trace_level` (process-constant) — the old product mixins held +# only those flags, so they dissolved. `enabled_by`/attachment gating is a runtime concern. +# - span.kind enumerates: server | client | producer | consumer | internal | broker. +# - Reserved/special keys (service, resource.name, error, sampling.priority, ...) route to span +# fields/directives, not tag storage — they live in the per-language overlay, not here. +# --------------------------------------------------------------------------- From e1d9d8e6574729835a6666611cbbf97d6b7de35d Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 18 Aug 2026 10:43:35 -0400 Subject: [PATCH 2/6] Add lazy tagId() to TagMap.Entry and EntryReader Expose the known-tag id on EntryReader: TagMap.Entry resolves it lazily via KnownTagCodec.keyOf (same memoized-field idiom as lazyTagHash; 0L means unknown tag / inactive codec, so the sentinel is Long.MIN_VALUE). The dense reader flyweight already has the id in hand at emit time and records it directly, skipping the keyOf resolve. Co-Authored-By: Claude Opus 4.8 --- .../main/java/datadog/trace/api/TagMap.java | 51 ++++++++++++++++++- .../trace/api/TagMapDenseForkedTest.java | 26 ++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/internal-api/src/main/java/datadog/trace/api/TagMap.java b/internal-api/src/main/java/datadog/trace/api/TagMap.java index b7a5b62a58d..63fe695afef 100644 --- a/internal-api/src/main/java/datadog/trace/api/TagMap.java +++ b/internal-api/src/main/java/datadog/trace/api/TagMap.java @@ -51,6 +51,10 @@ public final class TagMap implements Map, Iterable. public static final TagMap EMPTY = new TagMap(new Object[1 << 4], 0); + // Sentinel for a not-yet-resolved lazy tag id. Cannot be 0L: 0L is a valid keyOf result (the tag + // is not a known tag, or the codec is inactive). Shared by Entry and EntryReadingHelper. + static final long TAG_ID_NOT_COMPUTED = Long.MIN_VALUE; + /** Creates a new mutable TagMap that contains the contents of map */ public static final TagMap fromMap(@Nonnull Map map) { TagMap tagMap = TagMap.create(map.size()); @@ -165,6 +169,12 @@ public interface EntryReader { String tag(); + /** + * The known-tag id for this entry's tag, or {@code 0L} when the tag is not a known tag (or the + * {@link KnownTagCodec} is inactive). Resolved via {@link KnownTagCodec#keyOf(String)}. + */ + long tagId(); + byte type(); boolean is(byte type); @@ -313,6 +323,13 @@ static Entry newDoubleEntry(String tag, Double box) { */ int lazyTagHash; + /* + * Known-tag id, lazily resolved using the same trick as lazyTagHash. TAG_ID_NOT_COMPUTED marks + * "not yet resolved" (0L is a valid result -- unknown tag / inactive codec -- so it cannot be + * the sentinel). Only pays off on the dense-OFF path; dense-ON known tags never become Entry-s. + */ + long lazyTagId = TAG_ID_NOT_COMPUTED; + // To optimize construction of Entry around boxed primitives and Object entries, // no type checks are done during construction. // Any Object entries are initially marked as type ANY, prim set to 0, and the Object put into @@ -353,6 +370,17 @@ int hash() { return hash; } + @Override + public long tagId() { + // Same lazy idiom as hash(): a benign race just recomputes keyOf, which is deterministic. + long id = this.lazyTagId; + if (id != TAG_ID_NOT_COMPUTED) return id; + + id = KnownTagCodec.keyOf(this.tag); + this.lazyTagId = id; + return id; + } + @Override public Entry entry() { return this; @@ -2668,7 +2696,7 @@ private EntryReader emitDense(long tagId, Object value) { if (reader == null) { reader = this.denseReader = new EntryReadingHelper(); } - reader.set(KnownTagCodec.nameOf(tagId), value); + reader.set(KnownTagCodec.nameOf(tagId), value, tagId); return reader; } @@ -3307,17 +3335,28 @@ final class EntryReadingHelper implements TagMap.EntryReader { private Map.Entry mapEntry; private String tag; private Object value; + private long tagId; void set(String tag, Object value) { this.mapEntry = null; this.tag = tag; this.value = value; + this.tagId = TagMap.TAG_ID_NOT_COMPUTED; // resolve lazily via keyOf on first tagId() access + } + + /** Dense emit: the id is known directly, so record it and skip the keyOf resolve. */ + void set(String tag, Object value, long tagId) { + this.mapEntry = null; + this.tag = tag; + this.value = value; + this.tagId = tagId; } void set(Map.Entry mapEntry) { this.mapEntry = mapEntry; this.tag = mapEntry.getKey(); this.value = mapEntry.getValue(); + this.tagId = TagMap.TAG_ID_NOT_COMPUTED; // resolve lazily via keyOf on first tagId() access } @Override @@ -3325,6 +3364,16 @@ public String tag() { return this.tag; } + @Override + public long tagId() { + long id = this.tagId; + if (id != TagMap.TAG_ID_NOT_COMPUTED) return id; + + id = KnownTagCodec.keyOf(this.tag); + this.tagId = id; + return id; + } + @Override public byte type() { return TagValueConversions.typeOf(this.value); diff --git a/internal-api/src/test/java/datadog/trace/api/TagMapDenseForkedTest.java b/internal-api/src/test/java/datadog/trace/api/TagMapDenseForkedTest.java index 712b2281eee..41380771594 100644 --- a/internal-api/src/test/java/datadog/trace/api/TagMapDenseForkedTest.java +++ b/internal-api/src/test/java/datadog/trace/api/TagMapDenseForkedTest.java @@ -281,4 +281,30 @@ void removingParentDenseKeyTombstonesIt() { assertFalse(union.containsKey(BASE_SERVICE)); child.checkIntegrity(); } + + @Test + void denseReaderExposesTagId() { + TagMap map = map(); + map.set(BASE_SERVICE, "billing"); // dense-routed + map.set(CUSTOM_A, "alpha"); // bucket-routed + + Map idsByTag = new HashMap<>(); + map.forEach(reader -> idsByTag.put(reader.tag(), reader.tagId())); + + // dense entry: the reader carries the real known-tag id directly + assertEquals(KnownTagCodec.keyOf(BASE_SERVICE), idsByTag.get(BASE_SERVICE).longValue()); + assertTrue(idsByTag.get(BASE_SERVICE) != 0L, "known tag has a non-zero id"); + // bucket entry for a custom tag: unknown -> 0L + assertEquals(0L, idsByTag.get(CUSTOM_A).longValue()); + } + + @Test + void bucketEntryResolvesTagIdLazily() { + TagMap map = map(); + map.set(CUSTOM_A, "alpha"); // custom tag stays a bucket Entry + + TagMap.Entry entry = map.getEntry(CUSTOM_A); + assertEquals(0L, entry.tagId()); // unknown tag -> 0L + assertEquals(0L, entry.tagId()); // second read hits the memoized field + } } From c2c365581037d87070780494bf6d2cca87e89a5c Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 18 Aug 2026 11:03:11 -0400 Subject: [PATCH 3/6] Harden tag-registry generator: clear output tree + Locale.ROOT formatting Address two Codex robustness findings on the generator: - generateKnownTags now clears its owned destination tree before regenerating, so a report/source file retired by a later revision cannot linger and fail verifyKnownTags with no way for the fix task to remove it. - All String.format calls in the report/emitter use Locale.ROOT, so %d output no longer localizes on machines whose default locale uses non-ASCII digits, preserving the byte-identical-output invariant verifyKnownTags relies on. Co-Authored-By: Claude Opus 4.8 --- .../gradle/plugin/tags/KnownTagsEmitter.kt | 4 ++- .../plugin/tags/TagRegistryGenerator.kt | 31 ++++++++++++++----- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt index 7ee0c7f458e..dff8f940222 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt @@ -1,5 +1,7 @@ package datadog.gradle.plugin.tags +import java.util.Locale + /** * Emits the generated `KnownTags.java` from a [TagRegistry]. Public API first — per-tag * `_NAME` (string) + `_ID` (encoded long, literal) couplets with a trailing `// makeTagId(...)` @@ -150,5 +152,5 @@ object KnownTagsEmitter { return b.toString() } - private fun hex(id: Long): String = "0x%016XL".format(id) + private fun hex(id: Long): String = "0x%016XL".format(Locale.ROOT, id) } diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt index c57a055c405..9cfda68246e 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt @@ -4,6 +4,7 @@ import com.fasterxml.jackson.core.type.TypeReference import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.dataformat.yaml.YAMLFactory import java.io.File +import java.util.Locale /** * Turns the language-agnostic {@code tag-conventions.yaml} + the Java overlay into the generated tag @@ -26,6 +27,10 @@ object TagRegistryGenerator { mapper.readValue(it, object : TypeReference>() {}) } + // Clear the owned destination tree first, so a report/source file retired by a later generator + // revision doesn't linger: otherwise verifyKnownTags flags it as stale while telling developers + // to rerun generateKnownTags, which (without this) can't actually remove it. + outDir.deleteRecursively() outDir.mkdirs() // KnownTags.java goes under java/ (added as a srcDir); the .txt reports sit at the root. val javaPkg = File(outDir, "java/datadog/trace/api").apply { mkdirs() } @@ -66,11 +71,12 @@ object TagRegistryGenerator { for (t in reg.stored) { a.appendLine( " %6d %5s %s %s %-18s %-12s %s".format( + Locale.ROOT, t.serial, if (t.slotted) t.slot.toString() else "-", if (t.intercepted) "I" else "-", if (t.traceLevel) "T" else "-", - "0x%016X".format(t.id), + "0x%016X".format(Locale.ROOT, t.id), t.required, t.name)) } @@ -79,7 +85,12 @@ object TagRegistryGenerator { for (v in reg.reserved) { a.appendLine( " %6d %-18s %-12s %s%s".format( - v.serial, "0x%016X".format(v.id), v.kind, v.name, v.field?.let { " -> $it" } ?: "")) + Locale.ROOT, + v.serial, + "0x%016X".format(Locale.ROOT, v.id), + v.kind, + v.name, + v.field?.let { " -> $it" } ?: "")) } a.appendLine() a.appendLine("# PER-TYPE colored slots. Slots within a type must be DISTINCT (a valid coloring of the") @@ -90,11 +101,12 @@ object TagRegistryGenerator { .filter { it.slotted && !it.traceLevel } .map { it.slot } .sorted() - a.appendLine(" %-14s count=%-3d slots=%s".format(type, slots.size, slots)) + a.appendLine(" %-14s count=%-3d slots=%s".format(Locale.ROOT, type, slots.size, slots)) } val traceSlots = reg.stored.filter { it.traceLevel && it.slotted }.map { it.slot }.sorted() - a.appendLine(" %-14s count=%-3d slots=%s".format("", traceSlots.size, traceSlots)) + a.appendLine( + " %-14s count=%-3d slots=%s".format(Locale.ROOT, "", traceSlots.size, traceSlots)) return a.toString() } @@ -127,7 +139,8 @@ object TagRegistryGenerator { else -> "bkt" } lay.appendLine( - " %-26s %-12s %-12s %s".format(t.name, field, t.required, if (st?.intercepted == true) "I" else "")) + " %-26s %-12s %-12s %s".format( + Locale.ROOT, t.name, field, t.required, if (st?.intercepted == true) "I" else "")) } } } @@ -157,7 +170,9 @@ object TagRegistryGenerator { f.appendLine("$type (${tags.size} tags):") for (t in tags) { val st = byName[t.name] - f.appendLine(" %-12s %-26s %s".format(tierField(st), t.name, if (st?.intercepted == true) "I" else "")) + f.appendLine( + " %-12s %-26s %s".format( + Locale.ROOT, tierField(st), t.name, if (st?.intercepted == true) "I" else "")) } } val traceTags = @@ -165,7 +180,9 @@ object TagRegistryGenerator { f.appendLine() f.appendLine(" (${traceTags.size} tags):") for (st in traceTags) { - f.appendLine(" %-12s %-26s %s".format(tierField(st), st.name, if (st.intercepted) "I" else "")) + f.appendLine( + " %-12s %-26s %s".format( + Locale.ROOT, tierField(st), st.name, if (st.intercepted) "I" else "")) } return f.toString() } From eb24d3f06e366ba6f0fb9ee8e474d00f1b20cd27 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 18 Aug 2026 11:15:15 -0400 Subject: [PATCH 4/6] Log dense-tags flag in Config.toString when set away from default Address the Codex finding that the new experimental dense-tags flag was stored but never surfaced in Config.toString (repo convention requires new configs appear there for diagnostics). Surface it only when it diverges from the default, so normal config dumps stay uncluttered, and compare against a new DEFAULT_TRACE_DENSE_TAGS_ENABLED constant (also now the getBoolean default) so the condition stays correct if the default ever changes. Co-Authored-By: Claude Opus 4.8 --- .../src/main/java/datadog/trace/api/ConfigDefaults.java | 1 + internal-api/src/main/java/datadog/trace/api/Config.java | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java index a57e5d37882..49c9377dff9 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java @@ -305,6 +305,7 @@ public final class ConfigDefaults { public static final int DEFAULT_TRACE_X_DATADOG_TAGS_MAX_LENGTH = 512; static final boolean DEFAULT_TRACE_HTTP_RESOURCE_REMOVE_TRAILING_SLASH = false; + static final boolean DEFAULT_TRACE_DENSE_TAGS_ENABLED = false; static final boolean DEFAULT_TRACE_LONG_RUNNING_ENABLED = false; static final long DEFAULT_TRACE_LONG_RUNNING_INITIAL_FLUSH_INTERVAL = 20; // seconds static final long DEFAULT_TRACE_LONG_RUNNING_FLUSH_INTERVAL = 120; // seconds -> 2 minutes diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 75e618e514b..6e7fbe2d6c0 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -175,6 +175,7 @@ import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_BAGGAGE_MAX_ITEMS; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_BAGGAGE_TAG_KEYS; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_CLOUD_PAYLOAD_TAGGING_SERVICES; +import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_DENSE_TAGS_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_EXPERIMENTAL_FEATURES_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_HTTP_RESOURCE_REMOVE_TRAILING_SLASH; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_KEEP_LATENCY_THRESHOLD_MS; @@ -3303,7 +3304,8 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) this.spanBuilderReuseEnabled = configProvider.getBoolean(GeneralConfig.SPAN_BUILDER_REUSE_ENABLED, true); this.traceDenseTagsEnabled = - configProvider.getBoolean(TracerConfig.TRACE_DENSE_TAGS_ENABLED, false); + configProvider.getBoolean( + TracerConfig.TRACE_DENSE_TAGS_ENABLED, DEFAULT_TRACE_DENSE_TAGS_ENABLED); this.tagNameUtf8CacheSize = Math.max(configProvider.getInteger(GeneralConfig.TAG_NAME_UTF8_CACHE_SIZE, 128), 0); this.tagValueUtf8CacheSize = @@ -6828,6 +6830,11 @@ public String toString() { + sqsInjectDatadogAttributeEnabled + ", snsInjectDatadogAttributeEnabled=" + snsInjectDatadogAttributeEnabled + // Experimental: surfaced only when set away from the default, keeping normal dumps clean. + // Compared to the default constant (not a literal) so it survives a default change. + + (traceDenseTagsEnabled != DEFAULT_TRACE_DENSE_TAGS_ENABLED + ? ", traceDenseTagsEnabled=" + traceDenseTagsEnabled + : "") + '}'; } } From 899db8e920e7f8c840165111fbf9d0a49703eb2a Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 18 Aug 2026 11:21:46 -0400 Subject: [PATCH 5/6] Hold generated tag registry to the formatting standard Address the Codex finding that gradle/spotless.gradle excluded src/generated/**, bypassing google-java-format on committed generated code. Drop the exclude and make KnownTagsEmitter emit google-java-format-clean output (blank line before the keyOf static initializer; wrap the long id-assignment), so spotlessCheck and the byte-identical verifyKnownTags gate both pass on the emitted file. internal-api is the only module with a committed src/generated Java tree, so the exclude removal is otherwise a no-op. Co-Authored-By: Claude Opus 4.8 --- .../kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt | 4 +++- gradle/spotless.gradle | 8 +++++--- .../src/generated/java/datadog/trace/api/KnownTags.java | 4 +++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt index dff8f940222..ec5eacdf86b 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt @@ -89,11 +89,13 @@ object KnownTagsEmitter { b.appendLine(" private static final int[] KEYOF_HASHES;") b.appendLine(" private static final String[] KEYOF_KEYS;") b.appendLine(" private static final long[] KEYOF_IDS;") + b.appendLine() b.appendLine(" static {") b.appendLine(" StringIndex.Data data = StringIndex.EmbeddingSupport.create(KEYOF_NAMES);") b.appendLine(" long[] ids = new long[data.names.length];") b.appendLine(" for (int j = 0; j < KEYOF_NAMES.length; j++) {") - b.appendLine(" ids[StringIndex.EmbeddingSupport.indexOf(data.hashes, data.names, KEYOF_NAMES[j])] = KEYOF_VALUES[j];") + b.appendLine(" ids[StringIndex.EmbeddingSupport.indexOf(data.hashes, data.names, KEYOF_NAMES[j])] =") + b.appendLine(" KEYOF_VALUES[j];") b.appendLine(" }") b.appendLine(" KEYOF_HASHES = data.hashes;") b.appendLine(" KEYOF_KEYS = data.names;") diff --git a/gradle/spotless.gradle b/gradle/spotless.gradle index 769e153ddd9..f27408c7cec 100644 --- a/gradle/spotless.gradle +++ b/gradle/spotless.gradle @@ -20,9 +20,11 @@ spotless { toggleOffOn() // set explicit target to workaround https://github.com/diffplug/spotless/issues/1163 target 'src/**/*.java' - // ignore embedded test projects and everything in build dir, e.g. generated sources - // src/generated/** is emitted by code generators (e.g. the tag registry) — verified by their own freshness gate - targetExclude('src/test/resources/**', 'src/generated/**', buildDirectoryFiles) + // ignore embedded test projects and everything in build dir, e.g. generated sources. + // src/generated/** is committed generated code (e.g. the tag registry) and IS held to the + // formatting standard: emitters must produce google-java-format-clean output, and their + // freshness gate byte-compares against these formatted files. + targetExclude('src/test/resources/**', buildDirectoryFiles) tableTestFormatter('1.1.1') googleJavaFormat('1.35.0') } diff --git a/internal-api/src/generated/java/datadog/trace/api/KnownTags.java b/internal-api/src/generated/java/datadog/trace/api/KnownTags.java index 62842fbe2a0..e8646593297 100644 --- a/internal-api/src/generated/java/datadog/trace/api/KnownTags.java +++ b/internal-api/src/generated/java/datadog/trace/api/KnownTags.java @@ -410,11 +410,13 @@ public final class KnownTags { private static final int[] KEYOF_HASHES; private static final String[] KEYOF_KEYS; private static final long[] KEYOF_IDS; + static { StringIndex.Data data = StringIndex.EmbeddingSupport.create(KEYOF_NAMES); long[] ids = new long[data.names.length]; for (int j = 0; j < KEYOF_NAMES.length; j++) { - ids[StringIndex.EmbeddingSupport.indexOf(data.hashes, data.names, KEYOF_NAMES[j])] = KEYOF_VALUES[j]; + ids[StringIndex.EmbeddingSupport.indexOf(data.hashes, data.names, KEYOF_NAMES[j])] = + KEYOF_VALUES[j]; } KEYOF_HASHES = data.hashes; KEYOF_KEYS = data.names; From 382a1eb18286c5b15571b36f7a627dd432a2143f Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 18 Aug 2026 11:56:48 -0400 Subject: [PATCH 6/6] Resolve OpenTelemetry tag names through the registry Adds a per-namespace name to the tag registry: each tag may declare an `open-telemetry-name` (replacing the vestigial `aliases` list). The generator now: - parses it off both YAMLs into the model, - validates it (an OTel name may not collide with a canonical tag name nor be claimed by two tags -> build fails loudly rather than silently picking one), - emits it into the keyOf table so keyOf(otelName) resolves to the canonical tag's id (inbound, many->one), and - emits a reverse switch so openTelemetryNameOf(tagId) recovers it (outbound). KnownTagCodec gains datadogTagOf(tagId) (== nameOf, the canonical name) and openTelemetryTagOf(tagId) (the OTel name, or null). nameOf is unchanged and still returns the Datadog name -- outbound is namespace-specific, not normalized. Serializer applicability (when a span renders under OTel names, fallback policy) and additional namespaces are a follow-on concern. Co-Authored-By: Claude Opus 4.8 --- .../gradle/plugin/tags/KnownTagsEmitter.kt | 33 +++++++++++++- .../gradle/plugin/tags/TagConventions.kt | 7 +++ .../datadog/gradle/plugin/tags/TagRegistry.kt | 43 ++++++++++++++++-- .../plugin/tags/TagRegistryGenerator.kt | 10 +++++ .../java/datadog/trace/api/KnownTags.java | 44 +++++++++++++++++++ internal-api/src/generated/tag-assignment.txt | 12 +++++ .../java/datadog/trace/api/KnownTagCodec.java | 23 ++++++++++ .../java/datadog/trace/api/KnownTagsTest.java | 41 +++++++++++++++++ .../trace/api/TagMapDenseFuzzForkedTest.java | 5 +++ tag-conventions.java.yaml | 2 +- tag-conventions.yaml | 20 ++++----- 11 files changed, 224 insertions(+), 16 deletions(-) diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt index ec5eacdf86b..cce17f15feb 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt @@ -39,6 +39,11 @@ object KnownTagsEmitter { fun serialC(name: String) = withSuffix(cname[name]!!, "_SERIAL_NUM") val order = reg.reserved.map { it.name } + reg.stored.map { it.name } // stable emit order + // canonical name -> OpenTelemetry name, for the reverse (openTelemetryNameOf) switch. + val otelName = + (reg.reserved.mapNotNull { v -> v.otelName?.let { v.name to it } } + + reg.stored.mapNotNull { t -> t.otelName?.let { t.name to it } }) + .toMap() val b = StringBuilder() b.appendLine("package $pkg;") b.appendLine() @@ -79,12 +84,23 @@ object KnownTagsEmitter { } b.appendLine() - // keyOf table (open-addressed, via StringIndex.EmbeddingSupport). + // OpenTelemetry name -> canonical tag name, for the tags that declare one. Deterministic order + // (by OTel name) so output stays byte-identical. + val otelByCanonical = + (reg.stored.mapNotNull { t -> t.otelName?.let { it to t.name } } + + reg.reserved.mapNotNull { v -> v.otelName?.let { it to v.name } }) + .sortedBy { it.first } + + // keyOf table (open-addressed, via StringIndex.EmbeddingSupport). Canonical names first, then + // OpenTelemetry names -- an OTel name resolves to its canonical tag's id (there is no distinct id + // for it), so keyOf(otelName) == keyOf(canonical); nameOf still returns the canonical name. b.appendLine(" private static final String[] KEYOF_NAMES = {") order.forEach { b.appendLine(" ${nameC(it)},") } + otelByCanonical.forEach { (otel, _) -> b.appendLine(" \"$otel\",") } b.appendLine(" };") b.appendLine(" private static final long[] KEYOF_VALUES = {") order.forEach { b.appendLine(" ${idC(it)},") } + otelByCanonical.forEach { (_, canonical) -> b.appendLine(" ${idC(canonical)},") } b.appendLine(" };") b.appendLine(" private static final int[] KEYOF_HASHES;") b.appendLine(" private static final String[] KEYOF_KEYS;") @@ -118,6 +134,21 @@ object KnownTagsEmitter { b.appendLine(" }") b.appendLine(" }") b.appendLine() + // openTelemetryNameOf: canonical id -> OTel-namespace name, null when the tag has none. The + // caller (a serializer) owns any fall-back-to-Datadog-name policy; this stays a pure lookup. + b.appendLine(" @Override") + b.appendLine(" public String openTelemetryNameOf(long tagId) {") + b.appendLine(" switch (KnownTagCodec.serialNum(tagId)) {") + for (name in order) { + val otel = otelName[name] ?: continue + b.appendLine(" case ${serialC(name)}:") + b.appendLine(" return \"$otel\";") + } + b.appendLine(" default:") + b.appendLine(" return null;") + b.appendLine(" }") + b.appendLine(" }") + b.appendLine() b.appendLine(" @Override") b.appendLine(" public int slotCount() {") b.appendLine(" return SLOT_COUNT;") diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagConventions.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagConventions.kt index a01da7bc75c..180d2b63a08 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagConventions.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagConventions.kt @@ -16,6 +16,12 @@ private constructor( val name: String, val type: String, val required: String, + /** + * The tag's OpenTelemetry-namespace name, if it has one. keyOf resolves it to this tag's + * canonical id (inbound, many->one); openTelemetryNameOf recovers it (outbound). Further + * namespaces and serializer applicability are a follow-on concern. + */ + val otelName: String? = null, ) data class SpanType( @@ -167,6 +173,7 @@ private constructor( name = m["tag"].toString(), type = (m["type"] as? String) ?: "string", required = (m["required"] as? String) ?: "optional", + otelName = m["open-telemetry-name"] as? String, ) } ?: emptyList() } diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistry.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistry.kt index 0eebcbda629..4c256e1221e 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistry.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistry.kt @@ -34,6 +34,7 @@ private constructor( val slot: Int, val traceLevel: Boolean, val id: Long, + val otelName: String? = null, ) { val slotted: Boolean get() = slot != NO_SLOT @@ -45,11 +46,17 @@ private constructor( val field: String?, val serial: Int, val id: Long, + val otelName: String? = null, ) /** Java overlay: intercepted tag names + the reserved/special-key registry. */ class Overlay(val intercepted: Set, val reserved: List) { - data class ReservedDef(val name: String, val kind: String, val field: String?) + data class ReservedDef( + val name: String, + val kind: String, + val field: String?, + val otelName: String? = null, + ) companion object { @Suppress("UNCHECKED_CAST") @@ -57,7 +64,11 @@ private constructor( val intercepted = (root["intercepted"] as? List)?.toSet() ?: emptySet() val reserved = (root["reserved"] as? List>)?.map { m -> - ReservedDef(m["tag"].toString(), (m["kind"] as? String) ?: "directive", m["field"] as? String) + ReservedDef( + m["tag"].toString(), + (m["kind"] as? String) ?: "directive", + m["field"] as? String, + m["open-telemetry-name"] as? String) } ?: emptyList() return Overlay(intercepted, reserved) } @@ -131,7 +142,8 @@ private constructor( val serial = 1 + i ReservedTag( v.name, v.kind, v.field, serial, - encode(serial, intercepted = true, slot = NO_SLOT, traceLevel = false)) + encode(serial, intercepted = true, slot = NO_SLOT, traceLevel = false), + v.otelName) } // Stored tags in a stable order (by name); serials are a dense global counter from @@ -144,10 +156,33 @@ private constructor( val traceLevel = t.name in traceNames StoredTag( t.name, t.type, t.required, serial, intercepted, slot, traceLevel, - id = encode(serial, intercepted, slot, traceLevel)) + id = encode(serial, intercepted, slot, traceLevel), + otelName = t.otelName) } + validateOtelNames(stored, reserved) return TagRegistry(stored, reserved, slotCount) } + + /** + * An OpenTelemetry name must be unambiguous: it may not collide with any canonical tag name, nor + * be claimed by two different tags. Otherwise keyOf(otelName) would have no single right answer. + * Fail the build loudly rather than silently pick a winner. + */ + private fun validateOtelNames(stored: List, reserved: List) { + val canonical = (stored.map { it.name } + reserved.map { it.name }).toSet() + val owner = HashMap() + val check = { name: String, otel: String? -> + if (otel != null) { + require(otel !in canonical) { + "OpenTelemetry name '$otel' (of '$name') collides with canonical tag name '$otel'" + } + val prev = owner.put(otel, name) + require(prev == null) { "OpenTelemetry name '$otel' is claimed by both '$prev' and '$name'" } + } + } + stored.forEach { check(it.name, it.otelName) } + reserved.forEach { check(it.name, it.otelName) } + } } } diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt index 9cfda68246e..477e2e9d245 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt @@ -107,6 +107,16 @@ object TagRegistryGenerator { reg.stored.filter { it.traceLevel && it.slotted }.map { it.slot }.sorted() a.appendLine( " %-14s count=%-3d slots=%s".format(Locale.ROOT, "", traceSlots.size, traceSlots)) + a.appendLine() + a.appendLine("# OPENTELEMETRY NAMES. keyOf(otelName) resolves to the canonical tag's id; nameOf still") + a.appendLine("# returns the Datadog name, openTelemetryNameOf returns the name below. (No distinct id.)") + val otelPairs = + (reg.stored.mapNotNull { t -> t.otelName?.let { it to t.name } } + + reg.reserved.mapNotNull { v -> v.otelName?.let { it to v.name } }) + .sortedBy { it.first } + for ((otel, canonical) in otelPairs) { + a.appendLine(" %-30s -> %s".format(Locale.ROOT, otel, canonical)) + } return a.toString() } diff --git a/internal-api/src/generated/java/datadog/trace/api/KnownTags.java b/internal-api/src/generated/java/datadog/trace/api/KnownTags.java index e8646593297..5912ed63bf0 100644 --- a/internal-api/src/generated/java/datadog/trace/api/KnownTags.java +++ b/internal-api/src/generated/java/datadog/trace/api/KnownTags.java @@ -348,6 +348,15 @@ public final class KnownTags { SPAN_KIND_NAME, VERSION_NAME, VIEW_NAME, + "db.operation.name", + "db.query.text", + "db.system", + "http.request.method", + "http.response.status_code", + "server.address", + "service.name", + "url.full", + "url.query", }; private static final long[] KEYOF_VALUES = { ERROR_ID, @@ -406,6 +415,15 @@ public final class KnownTags { SPAN_KIND_ID, VERSION_ID, VIEW_NAME_ID, + DB_OPERATION_ID, + DB_STATEMENT_ID, + DB_TYPE_ID, + HTTP_METHOD_ID, + HTTP_STATUS_CODE_ID, + HTTP_HOSTNAME_ID, + SERVICE_ID, + HTTP_URL_ID, + HTTP_QUERY_STRING_ID, }; private static final int[] KEYOF_HASHES; private static final String[] KEYOF_KEYS; @@ -545,6 +563,32 @@ public String nameOf(long tagId) { } } + @Override + public String openTelemetryNameOf(long tagId) { + switch (KnownTagCodec.serialNum(tagId)) { + case SERVICE_SERIAL_NUM: + return "service.name"; + case DB_OPERATION_SERIAL_NUM: + return "db.operation.name"; + case DB_STATEMENT_SERIAL_NUM: + return "db.query.text"; + case DB_TYPE_SERIAL_NUM: + return "db.system"; + case HTTP_HOSTNAME_SERIAL_NUM: + return "server.address"; + case HTTP_METHOD_SERIAL_NUM: + return "http.request.method"; + case HTTP_QUERY_STRING_SERIAL_NUM: + return "url.query"; + case HTTP_STATUS_CODE_SERIAL_NUM: + return "http.response.status_code"; + case HTTP_URL_SERIAL_NUM: + return "url.full"; + default: + return null; + } + } + @Override public int slotCount() { return SLOT_COUNT; diff --git a/internal-api/src/generated/tag-assignment.txt b/internal-api/src/generated/tag-assignment.txt index c1516085a3b..9045eb8cde9 100644 --- a/internal-api/src/generated/tag-assignment.txt +++ b/internal-api/src/generated/tag-assignment.txt @@ -67,3 +67,15 @@ http.server count=15 slots=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] view.render count=8 slots=[0, 1, 2, 3, 4, 5, 6, 7] count=13 slots=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + +# OPENTELEMETRY NAMES. keyOf(otelName) resolves to the canonical tag's id; nameOf still +# returns the Datadog name, openTelemetryNameOf returns the name below. (No distinct id.) + db.operation.name -> db.operation + db.query.text -> db.statement + db.system -> db.type + http.request.method -> http.method + http.response.status_code -> http.status_code + server.address -> http.hostname + service.name -> service + url.full -> http.url + url.query -> http.query.string diff --git a/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java b/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java index 925e01c13a5..0f5873389d8 100644 --- a/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java +++ b/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java @@ -213,6 +213,9 @@ public static int slotCount() { public interface Resolver { String nameOf(long tagId); + /** The tag's OpenTelemetry-namespace name, or {@code null} when it declares none. */ + String openTelemetryNameOf(long tagId); + long keyOf(String name); /** Number of positional slots this provider uses: (max stored fieldPos) + 1. */ @@ -233,6 +236,11 @@ public String nameOf(long tagId) { return null; } + @Override + public String openTelemetryNameOf(long tagId) { + return null; + } + @Override public long keyOf(String name) { return 0L; @@ -286,6 +294,21 @@ public static String nameOf(long tagId) { return null; } + /** The tag's Datadog-namespace (canonical) name — the same value as {@link #nameOf}. */ + public static String datadogNameOf(long tagId) { + return nameOf(tagId); + } + + /** + * The tag's OpenTelemetry-namespace name, or {@code null} when it declares none (or no resolver + * is registered). A serializer owns any fall-back-to-Datadog-name policy; this is a pure lookup. + */ + public static String openTelemetryNameOf(long tagId) { + if (!active) return null; + Resolver r = resolver; + return r != null ? r.openTelemetryNameOf(tagId) : null; + } + public static long keyOf(String name) { if (active) { return resolver.keyOf(name); diff --git a/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java b/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java index 3e0dccaae56..c70e5ff65fa 100644 --- a/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java +++ b/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java @@ -58,6 +58,24 @@ static Stream knownTags() { Arguments.of(Tags.DB_POOL_NAME, KnownTags.DB_POOL_NAME_ID)); } + /** + * (otelName, canonicalId, datadogName) — the OpenTelemetry name resolves (keyOf) to the canonical + * tag's id; datadogNameOf returns the Datadog name and openTelemetryNameOf returns the OTel name. + */ + static Stream otelNamedTags() { + return Stream.of( + Arguments.of("http.request.method", KnownTags.HTTP_METHOD_ID, "http.method"), + Arguments.of( + "http.response.status_code", KnownTags.HTTP_STATUS_CODE_ID, "http.status_code"), + Arguments.of("url.full", KnownTags.HTTP_URL_ID, "http.url"), + Arguments.of("server.address", KnownTags.HTTP_HOSTNAME_ID, "http.hostname"), + Arguments.of("url.query", KnownTags.HTTP_QUERY_STRING_ID, "http.query.string"), + Arguments.of("db.system", KnownTags.DB_TYPE_ID, "db.type"), + Arguments.of("db.operation.name", KnownTags.DB_OPERATION_ID, "db.operation"), + Arguments.of("db.query.text", KnownTags.DB_STATEMENT_ID, "db.statement"), + Arguments.of("service.name", KnownTags.SERVICE_ID, "service")); + } + /** * The subset flagged INTERCEPTED (sign bit) — must agree with the interceptor's needsIntercept. */ @@ -120,6 +138,29 @@ void nameOfResolvesIdToName(String name, long id) { assertEquals(name, KnownTagCodec.nameOf(id), "nameOf(" + name + ")"); } + @ParameterizedTest + @MethodSource("otelNamedTags") + void otelNameResolvesToCanonicalId(String otelName, long id, String datadogName) { + // Inbound (keyOf) is many->one: both names land on the same canonical id. + assertEquals(id, KnownTagCodec.keyOf(otelName), "keyOf(" + otelName + ")"); + assertEquals(id, KnownTagCodec.keyOf(datadogName), "keyOf(" + datadogName + ")"); + } + + @ParameterizedTest + @MethodSource("otelNamedTags") + void namespaceAccessorsReturnPerNamespaceName(String otelName, long id, String datadogName) { + assertEquals(datadogName, KnownTagCodec.datadogNameOf(id), "datadogNameOf"); + assertEquals(otelName, KnownTagCodec.openTelemetryNameOf(id), "openTelemetryNameOf"); + // nameOf stays the Datadog name -- outbound is namespace-specific, not normalized to OTel. + assertEquals(datadogName, KnownTagCodec.nameOf(id), "nameOf stays Datadog"); + } + + @Test + void tagsWithoutOtelNameReturnNull() { + assertNull(KnownTagCodec.openTelemetryNameOf(KnownTags.HTTP_ROUTE_ID)); // no OTel name declared + assertNull(KnownTagCodec.openTelemetryNameOf(0L)); // unknown id + } + @ParameterizedTest @MethodSource("interceptedTags") void interceptedTagsCarryFlag(long id) { diff --git a/internal-api/src/test/java/datadog/trace/api/TagMapDenseFuzzForkedTest.java b/internal-api/src/test/java/datadog/trace/api/TagMapDenseFuzzForkedTest.java index 9ff3fd46419..8100ca464a7 100644 --- a/internal-api/src/test/java/datadog/trace/api/TagMapDenseFuzzForkedTest.java +++ b/internal-api/src/test/java/datadog/trace/api/TagMapDenseFuzzForkedTest.java @@ -78,6 +78,11 @@ public String nameOf(long tagId) { : null; } + @Override + public String openTelemetryNameOf(long tagId) { + return null; // synthetic resolver declares no OpenTelemetry names + } + @Override public int slotCount() { return 0; // positional unused diff --git a/tag-conventions.java.yaml b/tag-conventions.java.yaml index ab28b64545a..fcb91ac04f0 100644 --- a/tag-conventions.java.yaml +++ b/tag-conventions.java.yaml @@ -25,7 +25,7 @@ intercepted: # kind: directive -> triggers sampling/trace behavior reserved: - { tag: error, kind: structural, field: error } - - { tag: service, kind: structural, field: service, aliases: [service.name] } + - { tag: service, kind: structural, field: service, open-telemetry-name: service.name } - { tag: resource.name, kind: structural, field: resource } - { tag: span.type, kind: structural, field: type } - { tag: origin, kind: structural, field: origin } # trace-level field diff --git a/tag-conventions.yaml b/tag-conventions.yaml index 2eff24f1784..1639e0e8afc 100644 --- a/tag-conventions.yaml +++ b/tag-conventions.yaml @@ -17,7 +17,7 @@ # resolved_tags(type) = own + extends-chain (incl base) + included mixins + applied mixins (de-duped). # # tag fields (DOMAIN only): tag | type (string|int|long|boolean|double) -# | required (required|conditional|recommended|optional|opt_in) | aliases. +# | required (required|conditional|recommended|optional|opt_in) | open-telemetry-name. # The id coordinate (group-decl / field-decl) is NOT authored here — the generator assigns it: each # declaration source (the trace-level tier, each span type, each mixin) is a group, and within a # group `field-decl` numbers the dense (required/conditional/recommended) tags; the rest are @@ -62,18 +62,18 @@ span_types: abstract: true extends: base tags: - - { tag: http.method, type: string, required: required, aliases: [http.request.method] } - - { tag: http.status_code, type: int, required: conditional, aliases: [http.response.status_code] } + - { tag: http.method, type: string, required: required, open-telemetry-name: http.request.method } + - { tag: http.status_code, type: int, required: conditional, open-telemetry-name: http.response.status_code } - { tag: network.protocol.version, type: string, required: recommended } http.server: extends: http tags: - - { tag: http.url, type: string, required: required, aliases: [url.full] } + - { tag: http.url, type: string, required: required, open-telemetry-name: url.full } - { tag: http.route, type: string, required: conditional } - - { tag: http.hostname, type: string, required: required, aliases: [server.address] } + - { tag: http.hostname, type: string, required: required, open-telemetry-name: server.address } - { tag: http.useragent, type: string, required: recommended } - - { tag: http.query.string, type: string, required: recommended, aliases: [url.query] } + - { tag: http.query.string, type: string, required: recommended, open-telemetry-name: url.query } - { tag: servlet.path, type: string, required: optional } - { tag: servlet.context, type: string, required: optional } @@ -81,19 +81,19 @@ span_types: extends: http include: [ peer ] tags: - - { tag: http.url, type: string, required: required, aliases: [url.full] } + - { tag: http.url, type: string, required: required, open-telemetry-name: url.full } - { tag: http.resend_count, type: int, required: recommended } db.client: extends: base include: [ peer ] tags: - - { tag: db.type, type: string, required: required, aliases: [db.system] } + - { tag: db.type, type: string, required: required, open-telemetry-name: db.system } - { tag: db.instance, type: string, required: recommended } - - { tag: db.operation, type: string, required: recommended, aliases: [db.operation.name] } + - { tag: db.operation, type: string, required: recommended, open-telemetry-name: db.operation.name } - { tag: db.user, type: string, required: recommended } - { tag: db.pool.name, type: string, required: optional } - - { tag: db.statement, type: string, required: recommended, aliases: [db.query.text] } + - { tag: db.statement, type: string, required: recommended, open-telemetry-name: db.query.text } view.render: extends: base