diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 991ca899596..b7eacb3cd95 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -59,6 +59,11 @@ gradlePlugin { implementationClass = "datadog.gradle.plugin.config.SupportedConfigPlugin" } + create("tag-registry-generator") { + id = "dd-trace-java.tag-registry-generator" + implementationClass = "datadog.gradle.plugin.tags.TagRegistryGeneratorPlugin" + } + create("supported-config-linter") { id = "dd-trace-java.config-inversion-linter" implementationClass = "datadog.gradle.plugin.config.ConfigInversionLinter" @@ -107,6 +112,7 @@ dependencies { implementation("com.fasterxml.jackson.core:jackson-databind") implementation("com.fasterxml.jackson.core:jackson-annotations") implementation("com.fasterxml.jackson.core:jackson-core") + implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml") compileOnly(libs.develocity) } diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/GenerateKnownTagsTask.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/GenerateKnownTagsTask.kt new file mode 100644 index 00000000000..ab1c2631c60 --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/GenerateKnownTagsTask.kt @@ -0,0 +1,38 @@ +package datadog.gradle.plugin.tags + +import javax.inject.Inject +import org.gradle.api.DefaultTask +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.model.ObjectFactory +import org.gradle.api.tasks.CacheableTask +import org.gradle.api.tasks.InputFile +import org.gradle.api.tasks.OutputDirectory +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity +import org.gradle.api.tasks.TaskAction + +/** + * Generates the committed tag registry (KnownTags.java + layout reports) from the language-agnostic + * {@code tag-conventions.yaml} + the Java overlay. The actual emit lives in [TagRegistryGenerator]; + * this task just wires the inputs/outputs so Gradle can cache and up-to-date-check it. + */ +@CacheableTask +abstract class GenerateKnownTagsTask @Inject constructor(objects: ObjectFactory) : DefaultTask() { + @get:InputFile + @get:PathSensitive(PathSensitivity.NONE) + val domainYaml: RegularFileProperty = objects.fileProperty() + + @get:InputFile + @get:PathSensitive(PathSensitivity.NONE) + val overlayYaml: RegularFileProperty = objects.fileProperty() + + @get:OutputDirectory val destinationDirectory: DirectoryProperty = objects.directoryProperty() + + @TaskAction + fun generate() { + val outDir = destinationDirectory.get().asFile + TagRegistryGenerator.generate(domainYaml.get().asFile, overlayYaml.get().asFile, outDir) + logger.lifecycle("tag-registry: generated -> $outDir") + } +} diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt new file mode 100644 index 00000000000..cce17f15feb --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt @@ -0,0 +1,189 @@ +package datadog.gradle.plugin.tags + +import java.util.Locale + +/** + * Emits the generated `KnownTags.java` from a [TagRegistry]. Public API first — per-tag + * `_NAME` (string) + `_ID` (encoded long, literal) couplets with a trailing `// makeTagId(...)` + * derivation comment — then the package-private `_SERIAL_NUM` constants, the + * `StringIndex.EmbeddingSupport` keyOf table, the `serialNum` switch `nameOf`, and resolver + * registration. + */ +object KnownTagsEmitter { + + fun emit(reg: TagRegistry, pkg: String, className: String): String { + // Sanitize tag names into unique Java constant identifiers. + val used = HashSet() + val cname = HashMap() + fun mk(name: String): String { + var c = name.uppercase().replace(Regex("[^A-Za-z0-9]"), "_").replace(Regex("_+"), "_").trim('_') + if (c.isEmpty() || c[0].isDigit()) c = "T_$c" + var u = c + var n = 2 + while (u in used) { + u = "${c}_$n"; n++ + } + used.add(u) + cname[name] = u + return u + } + reg.reserved.forEach { mk(it.name) } + reg.stored.forEach { mk(it.name) } + + // Constant names. Collapse a duplicated trailing token so e.g. "resource.name" yields NAME + // (not NAME_NAME) and "_dd.parent_id" yields ID (not ID_ID); the non-duplicating pairs + // (ID + _NAME -> ID_NAME, NAME + _ID -> NAME_ID) are kept as-is. + fun withSuffix(base: String, suffix: String) = if (base.endsWith(suffix)) base else "$base$suffix" + fun nameC(name: String) = withSuffix(cname[name]!!, "_NAME") + fun idC(name: String) = withSuffix(cname[name]!!, "_ID") + fun serialC(name: String) = withSuffix(cname[name]!!, "_SERIAL_NUM") + + val order = reg.reserved.map { it.name } + reg.stored.map { it.name } // stable emit order + // canonical name -> OpenTelemetry name, for the reverse (openTelemetryNameOf) switch. + val otelName = + (reg.reserved.mapNotNull { v -> v.otelName?.let { v.name to it } } + + reg.stored.mapNotNull { t -> t.otelName?.let { t.name to it } }) + .toMap() + val b = StringBuilder() + b.appendLine("package $pkg;") + b.appendLine() + b.appendLine("import datadog.trace.util.StringIndex;") + b.appendLine() + b.appendLine("// GENERATED by the tag-registry code generator (dd-trace-java.tag-registry-generator).") + b.appendLine("// DO NOT EDIT. Source: tag-conventions.yaml + tag-conventions.java.yaml.") + b.appendLine("public final class $className {") + b.appendLine(" static final int SLOT_COUNT = ${reg.slotCount};") + b.appendLine() + + // Public API first (name + encoded id couplets), so readers see the useful parts up top; the + // serial ids and keyOf/resolver machinery follow below. Derivation is in the trailing comment. + b.appendLine(" // ---- reserved (routed to span fields or directives; not stored) ----") + for (v in reg.reserved) { + b.appendLine(" public static final String ${nameC(v.name)} = \"${v.name}\";") + b.appendLine(" public static final long ${idC(v.name)} = ${hex(v.id)};") + b.appendLine(" // makeTagId(serial=${v.serial}, slot=NO_SLOT) + intercepted [${v.kind}${v.field?.let { " -> $it" } ?: ""}]") + b.appendLine() + } + + b.appendLine(" // ---- stored (dense colored slot, or bucketed when slot=NO_SLOT) ----") + for (t in reg.stored) { + val slot = if (t.slotted) t.slot.toString() else "NO_SLOT" + b.appendLine(" public static final String ${nameC(t.name)} = \"${t.name}\";") + b.appendLine(" public static final long ${idC(t.name)} = ${hex(t.id)};") + b.appendLine(" // makeTagId(serial=${t.serial}, slot=$slot)${if (t.intercepted) " + intercepted" else ""}${if (t.traceLevel) " + trace-level" else ""} <${t.required}>") + b.appendLine() + } + + // Serial numbers (globalSerial per tag) — package-private, consumed by the resolver switch. + b.appendLine(" // ---- serial numbers ----") + for (v in reg.reserved) { + b.appendLine(" static final int ${serialC(v.name)} = ${v.serial};") + } + for (t in reg.stored) { + b.appendLine(" static final int ${serialC(t.name)} = ${t.serial};") + } + b.appendLine() + + // OpenTelemetry name -> canonical tag name, for the tags that declare one. Deterministic order + // (by OTel name) so output stays byte-identical. + val otelByCanonical = + (reg.stored.mapNotNull { t -> t.otelName?.let { it to t.name } } + + reg.reserved.mapNotNull { v -> v.otelName?.let { it to v.name } }) + .sortedBy { it.first } + + // keyOf table (open-addressed, via StringIndex.EmbeddingSupport). Canonical names first, then + // OpenTelemetry names -- an OTel name resolves to its canonical tag's id (there is no distinct id + // for it), so keyOf(otelName) == keyOf(canonical); nameOf still returns the canonical name. + b.appendLine(" private static final String[] KEYOF_NAMES = {") + order.forEach { b.appendLine(" ${nameC(it)},") } + otelByCanonical.forEach { (otel, _) -> b.appendLine(" \"$otel\",") } + b.appendLine(" };") + b.appendLine(" private static final long[] KEYOF_VALUES = {") + order.forEach { b.appendLine(" ${idC(it)},") } + otelByCanonical.forEach { (_, canonical) -> b.appendLine(" ${idC(canonical)},") } + b.appendLine(" };") + b.appendLine(" private static final int[] KEYOF_HASHES;") + b.appendLine(" private static final String[] KEYOF_KEYS;") + b.appendLine(" private static final long[] KEYOF_IDS;") + b.appendLine() + b.appendLine(" static {") + b.appendLine(" StringIndex.Data data = StringIndex.EmbeddingSupport.create(KEYOF_NAMES);") + b.appendLine(" long[] ids = new long[data.names.length];") + b.appendLine(" for (int j = 0; j < KEYOF_NAMES.length; j++) {") + b.appendLine(" ids[StringIndex.EmbeddingSupport.indexOf(data.hashes, data.names, KEYOF_NAMES[j])] =") + b.appendLine(" KEYOF_VALUES[j];") + b.appendLine(" }") + b.appendLine(" KEYOF_HASHES = data.hashes;") + b.appendLine(" KEYOF_KEYS = data.names;") + b.appendLine(" KEYOF_IDS = ids;") + b.appendLine(" }") + b.appendLine() + + // Resolver. + b.appendLine(" static final KnownTagCodec.Resolver RESOLVER =") + b.appendLine(" new KnownTagCodec.Resolver() {") + b.appendLine(" @Override") + b.appendLine(" public String nameOf(long tagId) {") + b.appendLine(" switch (KnownTagCodec.serialNum(tagId)) {") + for (name in order) { + b.appendLine(" case ${serialC(name)}:") + b.appendLine(" return ${nameC(name)};") + } + b.appendLine(" default:") + b.appendLine(" return null;") + b.appendLine(" }") + b.appendLine(" }") + b.appendLine() + // openTelemetryNameOf: canonical id -> OTel-namespace name, null when the tag has none. The + // caller (a serializer) owns any fall-back-to-Datadog-name policy; this stays a pure lookup. + b.appendLine(" @Override") + b.appendLine(" public String openTelemetryNameOf(long tagId) {") + b.appendLine(" switch (KnownTagCodec.serialNum(tagId)) {") + for (name in order) { + val otel = otelName[name] ?: continue + b.appendLine(" case ${serialC(name)}:") + b.appendLine(" return \"$otel\";") + } + b.appendLine(" default:") + b.appendLine(" return null;") + b.appendLine(" }") + b.appendLine(" }") + b.appendLine() + b.appendLine(" @Override") + b.appendLine(" public int slotCount() {") + b.appendLine(" return SLOT_COUNT;") + b.appendLine(" }") + b.appendLine() + b.appendLine(" @Override") + b.appendLine(" public long keyOf(String name) {") + b.appendLine(" int slot = StringIndex.EmbeddingSupport.indexOf(KEYOF_HASHES, KEYOF_KEYS, name);") + b.appendLine(" return slot < 0 ? 0L : KEYOF_IDS[slot];") + b.appendLine(" }") + b.appendLine(" };") + b.appendLine() + b.appendLine(" static {") + b.appendLine(" KnownTagCodec.register(RESOLVER);") + b.appendLine(" }") + b.appendLine() + b.appendLine(" /**") + b.appendLine( + " * Forces resolver registration. Merely invoking this static method runs {@code } (which") + b.appendLine( + " * registers {@link #RESOLVER}), so calling it once at tracer init makes tag-id name resolution") + b.appendLine( + " * ({@code keyOf}/{@code nameOf}) live; idempotent. Whether known tags then take the dense store") + b.appendLine( + " * is a separate, const-folded decision ({@link KnownTagCodec#DENSE_STORE}). Until something") + b.appendLine( + " * references this class the registry stays dormant and {@code keyOf} returns 0, so tag storage is") + b.appendLine(" * byte-identical to the bucket-only behavior.") + b.appendLine(" */") + b.appendLine(" public static void init() {}") + b.appendLine() + b.appendLine(" private $className() {}") + b.appendLine("}") + return b.toString() + } + + private fun hex(id: Long): String = "0x%016XL".format(Locale.ROOT, id) +} diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagConventions.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagConventions.kt new file mode 100644 index 00000000000..180d2b63a08 --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagConventions.kt @@ -0,0 +1,180 @@ +package datadog.gradle.plugin.tags + +/** + * Parsed tag-conventions domain model + the per-type tag-set resolver. Language-agnostic: it knows + * only structure (extends / include / applies) and per-tag semantics (name / type / required / + * source). Id assignment and emission are layered on top of the resolved sets. + */ +class TagConventions +private constructor( + private val spanTypes: Map, + private val mixins: Map, + private val traceLevel: List, +) { + /** A tag declaration (domain semantics only). */ + data class Tag( + val name: String, + val type: String, + val required: String, + /** + * The tag's OpenTelemetry-namespace name, if it has one. keyOf resolves it to this tag's + * canonical id (inbound, many->one); openTelemetryNameOf recovers it (outbound). Further + * namespaces and serializer applicability are a follow-on concern. + */ + val otelName: String? = null, + ) + + data class SpanType( + val name: String, + val abstract: Boolean, + val extends: String?, + val include: List, + val tags: List, + ) + + data class Mixin( + val name: String, + val appliesAll: Boolean, + val appliesTo: Set, + val tags: List, + ) + + /** Concrete (instantiable) span types — the ones a layout is computed for. */ + fun concreteTypes(): List = + spanTypes.values.filter { !it.abstract }.map { it.name }.sorted() + + /** + * resolved(type) = own tags + tags up the `extends` chain (incl. base) + tags of every mixin the + * type or an ancestor `include`s + tags of every mixin whose `applies` matches. De-duped by tag + * name (first occurrence wins). Base-first order, so it is stable across runs. + */ + fun resolve(typeName: String): List { + val result = LinkedHashMap() + fun add(t: Tag) = result.putIfAbsent(t.name, t) + + val chain = ArrayList() + var cur: SpanType? = spanTypes[typeName] + while (cur != null) { + chain.add(cur) + cur = cur.extends?.let { spanTypes[it] } + } + for (st in chain.asReversed()) { + st.tags.forEach { add(it) } + for (mixinName in st.include) mixins[mixinName]?.tags?.forEach { add(it) } + } + val chainNames = chain.map { it.name }.toSet() + for (mx in mixins.values) { + if (mx.appliesAll || mx.appliesTo.any { it in chainNames }) mx.tags.forEach { add(it) } + } + return result.values.toList() + } + + /** The explicit trace-level tier tags (their own TagMap "type" on the TraceSegment). */ + fun traceLevelTags(): List = traceLevel + + /** A declaration group: the source that *declares* a set of tags (its own `tags:` list). */ + data class Group(val name: String, val kind: String, val tags: List) + + /** + * The declaration groups, in a stable order: the trace-level tier first, then every span type + * (abstract included — `base`/`http` declare real tags) sorted by name, then every mixin sorted by + * name. Each maps to one `group-decl`. A tag is *declared* once (in its own container's `tags:`); + * the same tag reached via extends/include/applies is not re-declared, so first-declaration (in + * this order) is its home group. Groups with no declared tags are omitted. + */ + fun declarationGroups(): List { + val groups = ArrayList() + if (traceLevel.isNotEmpty()) groups.add(Group(TRACE_LAYER, "trace", traceLevel)) + for (name in spanTypes.keys.sorted()) { + val st = spanTypes.getValue(name) + if (st.tags.isNotEmpty()) groups.add(Group(name, "span_type", st.tags)) + } + for (name in mixins.keys.sorted()) { + val mx = mixins.getValue(name) + if (mx.tags.isNotEmpty()) groups.add(Group(name, "mixin", mx.tags)) + } + return groups + } + + /** Full stored-tag universe (concrete span types' resolves + trace-level), de-duped by name. */ + fun allStoredTags(): List { + val union = LinkedHashMap() + for (type in concreteTypes()) for (t in resolve(type)) union.putIfAbsent(t.name, t) + for (t in traceLevel) union.putIfAbsent(t.name, t) + return union.values.toList() + } + + /** + * Full composition for a type as (origin, tag) pairs, in composition order and NOT de-duped, so a + * tag contributed by more than one source shows up more than once. Origin is the contributing + * span type (via extends), `incl:` (via include), or `appl:` (via applies). + */ + fun compose(typeName: String): List> { + val out = ArrayList>() + val chain = ArrayList() + var cur: SpanType? = spanTypes[typeName] + while (cur != null) { + chain.add(cur) + cur = cur.extends?.let { spanTypes[it] } + } + for (st in chain.asReversed()) { + st.tags.forEach { out.add(st.name to it) } + for (mixinName in st.include) mixins[mixinName]?.tags?.forEach { out.add("incl:$mixinName" to it) } + } + val chainNames = chain.map { it.name }.toSet() + for (mx in mixins.values) { + if (mx.appliesAll || mx.appliesTo.any { it in chainNames }) { + mx.tags.forEach { out.add("appl:${mx.name}" to it) } + } + } + return out + } + + companion object { + /** Group name of the trace-level tier (its own TagMap layer on the TraceSegment). */ + const val TRACE_LAYER = "" + + @Suppress("UNCHECKED_CAST") + fun parse(root: Map): TagConventions { + val spanTypesRaw = (root["span_types"] as? Map) ?: emptyMap() + val spanTypes = + spanTypesRaw.mapValues { (name, v) -> + val m = v as Map + SpanType( + name = name, + abstract = (m["abstract"] as? Boolean) ?: false, + extends = m["extends"] as? String, + include = (m["include"] as? List) ?: emptyList(), + tags = tagList(m["tags"]), + ) + } + + val mixinsRaw = (root["mixins"] as? Map) ?: emptyMap() + val mixins = + mixinsRaw.mapValues { (name, v) -> + val m = v as Map + val applies = m["applies"] + Mixin( + name = name, + appliesAll = applies == "all", + appliesTo = if (applies is List<*>) applies.map { it.toString() }.toSet() else emptySet(), + tags = tagList(m["tags"]), + ) + } + + val traceLevel = tagList((root["trace_level"] as? Map)?.get("tags")) + return TagConventions(spanTypes, mixins, traceLevel) + } + + @Suppress("UNCHECKED_CAST") + private fun tagList(tags: Any?): List = + (tags as? List>)?.map { m -> + Tag( + name = m["tag"].toString(), + type = (m["type"] as? String) ?: "string", + required = (m["required"] as? String) ?: "optional", + otelName = m["open-telemetry-name"] as? String, + ) + } ?: emptyList() + } +} diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistry.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistry.kt new file mode 100644 index 00000000000..4c256e1221e --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistry.kt @@ -0,0 +1,188 @@ +package datadog.gradle.plugin.tags + +/** + * Assigns tag ids from a parsed [TagConventions] plus the Java overlay (intercepted set + reserved + * registry). The id encoding mirrors KnownTagCodec: [63 intercepted][62-48 serial][47-32 slot][31-0 + * zero] (known ids carry no nameHash — they are dense-store addressed). + * + *

The `slot` is a single globally stable coordinate from GRAPH COLORING the tag co-occurrence + * graph. Each concrete span type's resolved tag set (see [TagConventions.resolve]) is a clique — its + * tags all appear together on one span, so they must get distinct slots. The trace-level tier is its + * own clique (a separate TagMap on the TraceSegment), so it may reuse slot numbers freely with the + * span layers. Slots are shared only between tags that never co-occur, so slotCount stays bounded by + * the largest clique (≤ 64) — small enough that the dense store's presence fast path is a single + * occupancy `long` (`1L << slot`), which is exactly why the earlier two-tier (group + field bloom) + * scheme could collapse to one word. Correctness never depends on the coloring (the dense scan is + * authoritative); only the fast-path hit rate does. + * + *

Slotting (does a tag get a slot / dense presence bit) is derived from the domain `required` + * level: required/conditional/recommended tags are colored (slotted), the rest are NO_SLOT + * (bucketed) and carry no slot bit. + */ +class TagRegistry +private constructor( + val stored: List, + val reserved: List, + val slotCount: Int, +) { + data class StoredTag( + val name: String, + val type: String, + val required: String, + val serial: Int, + val intercepted: Boolean, + val slot: Int, + val traceLevel: Boolean, + val id: Long, + val otelName: String? = null, + ) { + val slotted: Boolean + get() = slot != NO_SLOT + } + + data class ReservedTag( + val name: String, + val kind: String, + val field: String?, + val serial: Int, + val id: Long, + val otelName: String? = null, + ) + + /** Java overlay: intercepted tag names + the reserved/special-key registry. */ + class Overlay(val intercepted: Set, val reserved: List) { + data class ReservedDef( + val name: String, + val kind: String, + val field: String?, + val otelName: String? = null, + ) + + companion object { + @Suppress("UNCHECKED_CAST") + fun parse(root: Map): Overlay { + val intercepted = (root["intercepted"] as? List)?.toSet() ?: emptySet() + val reserved = + (root["reserved"] as? List>)?.map { m -> + ReservedDef( + m["tag"].toString(), + (m["kind"] as? String) ?: "directive", + m["field"] as? String, + m["open-telemetry-name"] as? String) + } ?: emptyList() + return Overlay(intercepted, reserved) + } + } + } + + companion object { + const val FIRST_STORED_SERIAL = 256 + const val NO_SLOT = 0xFFFF // slot all-ones sentinel (16 bits); mirrors KnownTagCodec.NO_SLOT + const val MAX_SLOT = 63 // one occupancy long: colored slots must fit in [0, 63] + const val LEVEL_TRACE = 1L shl 2 // low-32 carve bit 2; mirrors KnownTagCodec.LEVEL_TRACE + const val TRACE_LAYER = "" + + // Domain `required` levels that get a colored slot (the rest are bucketed with NO_SLOT). + val COLORABLE = setOf("required", "conditional", "recommended") + + /** + * Mirrors KnownTagCodec.makeTagId(serial, slot) + intercepted()/traceLevel() — must stay in + * sync. slot [47-32], LEVEL_TRACE at bit 2, other low bits zero. + */ + fun encode(serial: Int, intercepted: Boolean, slot: Int, traceLevel: Boolean): Long { + var id = (serial.toLong() shl 48) or ((slot.toLong() and 0xFFFF) shl 32) + if (traceLevel) id = id or LEVEL_TRACE + if (intercepted) id = id or Long.MIN_VALUE + return id + } + + fun build(conv: TagConventions, overlay: Overlay): TagRegistry { + val all = conv.allStoredTags() + val traceNames = conv.traceLevelTags().map { it.name }.toSet() + val colorable = all.filter { it.required in COLORABLE }.map { it.name }.toSet() + + // Co-occurrence cliques: each concrete type's resolved colorable tags, plus the trace-level + // tier as its own clique (a separate TagMap -> free to reuse span slot numbers). Tags in the + // same clique must get distinct colors; tags never sharing a clique may share a color. + val cliques = ArrayList>() + for (type in conv.concreteTypes()) { + cliques.add(conv.resolve(type).map { it.name }.filter { it in colorable }.toSet()) + } + cliques.add(traceNames.filter { it in colorable }.toSet()) + + // Adjacency: an edge between every pair of tags that co-occur in some clique. + val adj = HashMap>() + colorable.forEach { adj[it] = HashSet() } + for (clique in cliques) { + val members = clique.toList() + for (i in members.indices) for (j in i + 1 until members.size) { + adj.getValue(members[i]).add(members[j]) + adj.getValue(members[j]).add(members[i]) + } + } + + // Greedy coloring, most-constrained-first (by clique membership count, then name for a stable + // tie-break). Each tag takes the smallest color not used by an already-colored neighbor. + val cliqueCount = colorable.associateWith { n -> cliques.count { n in it } } + val order = colorable.sortedWith(compareByDescending { cliqueCount.getValue(it) }.thenBy { it }) + val color = HashMap() + for (n in order) { + val used = adj.getValue(n).mapNotNull { color[it] }.toSet() + var c = 0 + while (c in used) c++ + color[n] = c + } + val slotCount = (color.values.maxOrNull() ?: -1) + 1 + require(slotCount <= MAX_SLOT + 1) { + "coloring produced $slotCount slots; the single occupancy long holds at most ${MAX_SLOT + 1}" + } + + val reserved = + overlay.reserved.mapIndexed { i, v -> + val serial = 1 + i + ReservedTag( + v.name, v.kind, v.field, serial, + encode(serial, intercepted = true, slot = NO_SLOT, traceLevel = false), + v.otelName) + } + + // Stored tags in a stable order (by name); serials are a dense global counter from + // FIRST_STORED_SERIAL. slot comes from the coloring (NO_SLOT for non-colorable/bucketed tags). + val stored = + all.sortedBy { it.name }.mapIndexed { i, t -> + val serial = FIRST_STORED_SERIAL + i + val intercepted = t.name in overlay.intercepted + val slot = color[t.name] ?: NO_SLOT + val traceLevel = t.name in traceNames + StoredTag( + t.name, t.type, t.required, serial, intercepted, slot, traceLevel, + id = encode(serial, intercepted, slot, traceLevel), + otelName = t.otelName) + } + + validateOtelNames(stored, reserved) + return TagRegistry(stored, reserved, slotCount) + } + + /** + * An OpenTelemetry name must be unambiguous: it may not collide with any canonical tag name, nor + * be claimed by two different tags. Otherwise keyOf(otelName) would have no single right answer. + * Fail the build loudly rather than silently pick a winner. + */ + private fun validateOtelNames(stored: List, reserved: List) { + val canonical = (stored.map { it.name } + reserved.map { it.name }).toSet() + val owner = HashMap() + val check = { name: String, otel: String? -> + if (otel != null) { + require(otel !in canonical) { + "OpenTelemetry name '$otel' (of '$name') collides with canonical tag name '$otel'" + } + val prev = owner.put(otel, name) + require(prev == null) { "OpenTelemetry name '$otel' is claimed by both '$prev' and '$name'" } + } + } + stored.forEach { check(it.name, it.otelName) } + reserved.forEach { check(it.name, it.otelName) } + } + } +} diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt new file mode 100644 index 00000000000..477e2e9d245 --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt @@ -0,0 +1,199 @@ +package datadog.gradle.plugin.tags + +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import java.io.File +import java.util.Locale + +/** + * Turns the language-agnostic {@code tag-conventions.yaml} + the Java overlay into the generated tag + * registry: {@code KnownTags.java} (under {@code java/}) plus verification report dumps + * (resolved-tags / tag-assignment / layout-by-type / folded-types) at the destination root. + * + * Pure function of its inputs (deterministic ordering throughout), so the same inputs always produce + * byte-identical output -- which is what the {@code verifyKnownTags} freshness gate relies on. + */ +object TagRegistryGenerator { + /** Parses the two YAML files and writes the full generated tree under [outDir]. */ + fun generate(domainYaml: File, overlayYaml: File, outDir: File) { + val mapper = ObjectMapper(YAMLFactory()) + val domain: Map = + domainYaml.inputStream().use { + mapper.readValue(it, object : TypeReference>() {}) + } + val overlayMap: Map = + overlayYaml.inputStream().use { + mapper.readValue(it, object : TypeReference>() {}) + } + + // Clear the owned destination tree first, so a report/source file retired by a later generator + // revision doesn't linger: otherwise verifyKnownTags flags it as stale while telling developers + // to rerun generateKnownTags, which (without this) can't actually remove it. + outDir.deleteRecursively() + outDir.mkdirs() + // KnownTags.java goes under java/ (added as a srcDir); the .txt reports sit at the root. + val javaPkg = File(outDir, "java/datadog/trace/api").apply { mkdirs() } + + val conv = TagConventions.parse(domain) + val overlay = TagRegistry.Overlay.parse(overlayMap) + val reg = TagRegistry.build(conv, overlay) + + File(outDir, "resolved-tags.txt").writeText(resolvedReport(conv)) + File(outDir, "tag-assignment.txt").writeText(assignmentReport(conv, reg)) + File(outDir, "layout-by-type.txt").writeText(layoutByTypeReport(conv, reg)) + File(outDir, "folded-types.txt").writeText(foldedTypesReport(conv, reg)) + File(javaPkg, "KnownTags.java") + .writeText(KnownTagsEmitter.emit(reg, "datadog.trace.api", "KnownTags")) + } + + /** resolved-tags.txt — the per-type resolved sets (composition check). */ + private fun resolvedReport(conv: TagConventions): String { + val resolved = StringBuilder() + resolved.appendLine("# Resolved per-type tag sets (concrete span types).") + for (type in conv.concreteTypes()) { + val tags = conv.resolve(type) + resolved.appendLine() + resolved.appendLine("$type (${tags.size} tags):") + for (t in tags) resolved.appendLine(" - ${t.name}") + } + return resolved.toString() + } + + /** tag-assignment.txt — serials, colored slots, ids, per-type slot sets (coloring check). */ + private fun assignmentReport(conv: TagConventions, reg: TagRegistry): String { + val byName = reg.stored.associateBy { it.name } + val a = StringBuilder() + a.appendLine( + "# Tag id assignment. slotCount=${reg.slotCount} stored=${reg.stored.size} reserved=${reg.reserved.size}") + a.appendLine() + a.appendLine("# STORED serial slot int lvl id required name") + for (t in reg.stored) { + a.appendLine( + " %6d %5s %s %s %-18s %-12s %s".format( + Locale.ROOT, + t.serial, + if (t.slotted) t.slot.toString() else "-", + if (t.intercepted) "I" else "-", + if (t.traceLevel) "T" else "-", + "0x%016X".format(Locale.ROOT, t.id), + t.required, + t.name)) + } + a.appendLine() + a.appendLine("# RESERVED serial id kind name") + for (v in reg.reserved) { + a.appendLine( + " %6d %-18s %-12s %s%s".format( + Locale.ROOT, + v.serial, + "0x%016X".format(Locale.ROOT, v.id), + v.kind, + v.name, + v.field?.let { " -> $it" } ?: "")) + } + a.appendLine() + a.appendLine("# PER-TYPE colored slots. Slots within a type must be DISTINCT (a valid coloring of the") + a.appendLine("# co-occurrence clique); is its own clique and freely reuses span slot numbers.") + for (type in conv.concreteTypes()) { + val slots = + conv.resolve(type).mapNotNull { byName[it.name] } + .filter { it.slotted && !it.traceLevel } + .map { it.slot } + .sorted() + a.appendLine(" %-14s count=%-3d slots=%s".format(Locale.ROOT, type, slots.size, slots)) + } + val traceSlots = + reg.stored.filter { it.traceLevel && it.slotted }.map { it.slot }.sorted() + a.appendLine( + " %-14s count=%-3d slots=%s".format(Locale.ROOT, "", traceSlots.size, traceSlots)) + a.appendLine() + a.appendLine("# OPENTELEMETRY NAMES. keyOf(otelName) resolves to the canonical tag's id; nameOf still") + a.appendLine("# returns the Datadog name, openTelemetryNameOf returns the name below. (No distinct id.)") + val otelPairs = + (reg.stored.mapNotNull { t -> t.otelName?.let { it to t.name } } + + reg.reserved.mapNotNull { v -> v.otelName?.let { it to v.name } }) + .sortedBy { it.first } + for ((otel, canonical) in otelPairs) { + a.appendLine(" %-30s -> %s".format(Locale.ROOT, otel, canonical)) + } + return a.toString() + } + + /** + * layout-by-type.txt — full composition per type (origins shown, NOT de-duped), each tag annotated + * with its slot/tier: s = colored slot, trace s = trace-level layer, bkt = bucket. + */ + private fun layoutByTypeReport(conv: TagConventions, reg: TagRegistry): String { + val byName = reg.stored.associateBy { it.name } + val lay = StringBuilder() + lay.appendLine("# Full tag composition per concrete span type (after extends/include/applies).") + lay.appendLine("# Not de-duped: a tag from >1 source appears >1 time.") + lay.appendLine("# annotation: [s colored slot | trace s trace layer | bkt bucketed] I=intercepted") + for (type in conv.concreteTypes()) { + val comp = conv.compose(type) + val distinct = comp.map { it.second.name }.distinct().size + lay.appendLine() + lay.appendLine("$type (${comp.size} contributions, $distinct distinct):") + val byOrigin = LinkedHashMap>() + for ((origin, tag) in comp) byOrigin.getOrPut(origin) { ArrayList() }.add(tag) + for ((origin, tags) in byOrigin) { + lay.appendLine(" [$origin]") + for (t in tags) { + val st = byName[t.name] + val field = + when { + st == null -> "?" + st.traceLevel && st.slotted -> "trace s${st.slot}" + st.slotted -> "s${st.slot}" + else -> "bkt" + } + lay.appendLine( + " %-26s %-12s %-12s %s".format( + Locale.ROOT, t.name, field, t.required, if (st?.intercepted == true) "I" else "")) + } + } + } + return lay.toString() + } + + /** + * folded-types.txt — each type's full resolved set (extends + include + applies, DE-DUPED) with its + * slot; plus the type. This is the "type with everything folded in" view. + */ + private fun foldedTypesReport(conv: TagConventions, reg: TagRegistry): String { + val byName = reg.stored.associateBy { it.name } + fun tierField(st: TagRegistry.StoredTag?): String = + when { + st == null -> "?" + st.traceLevel -> if (st.slotted) "trace s${st.slot}" else "trace-bkt" + st.slotted -> "s${st.slot}" + else -> "bkt" + } + val f = StringBuilder() + f.appendLine("# Folded tag set per type (extends + include + applies, de-duped), with colored slots.") + f.appendLine( + "# field: s=colored slot trace s=trace-level layer bkt=bucketed trace-bkt=trace-level bucketed I=intercepted") + for (type in conv.concreteTypes()) { + val tags = conv.resolve(type) + f.appendLine() + f.appendLine("$type (${tags.size} tags):") + for (t in tags) { + val st = byName[t.name] + f.appendLine( + " %-12s %-26s %s".format( + Locale.ROOT, tierField(st), t.name, if (st?.intercepted == true) "I" else "")) + } + } + val traceTags = + reg.stored.filter { it.traceLevel }.sortedWith(compareBy({ !it.slotted }, { it.slot }, { it.name })) + f.appendLine() + f.appendLine(" (${traceTags.size} tags):") + for (st in traceTags) { + f.appendLine( + " %-12s %-26s %s".format( + Locale.ROOT, tierField(st), st.name, if (st.intercepted) "I" else "")) + } + return f.toString() + } +} diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGeneratorPlugin.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGeneratorPlugin.kt new file mode 100644 index 00000000000..313bd859068 --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGeneratorPlugin.kt @@ -0,0 +1,41 @@ +package datadog.gradle.plugin.tags + +import javax.inject.Inject +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.model.ObjectFactory + +/** Extension configuring the tag-registry generator inputs/outputs. */ +abstract class TagRegistryExtension @Inject constructor(objects: ObjectFactory) { + val domainYaml: RegularFileProperty = objects.fileProperty() + val overlayYaml: RegularFileProperty = objects.fileProperty() + val destinationDirectory: DirectoryProperty = objects.directoryProperty() +} + +/** + * Registers {@code generateKnownTags} (emits the committed tag registry) and {@code verifyKnownTags} + * (a freshness gate that regenerates and byte-compares against the committed output). The verify task + * is wired into {@code check} so stale generated sources fail CI. + */ +class TagRegistryGeneratorPlugin : Plugin { + override fun apply(project: Project) { + val ext = project.extensions.create("tagRegistry", TagRegistryExtension::class.java) + project.tasks.register("generateKnownTags", GenerateKnownTagsTask::class.java) { + domainYaml.set(ext.domainYaml) + overlayYaml.set(ext.overlayYaml) + destinationDirectory.set(ext.destinationDirectory) + } + val verify = + project.tasks.register("verifyKnownTags", VerifyKnownTagsTask::class.java) { + domainYaml.set(ext.domainYaml) + overlayYaml.set(ext.overlayYaml) + committedDirectory.set(ext.destinationDirectory) + } + // `check` is contributed by lifecycle-base (via java-library); wait for it before wiring. + project.pluginManager.withPlugin("lifecycle-base") { + project.tasks.named("check").configure { dependsOn(verify) } + } + } +} diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/VerifyKnownTagsTask.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/VerifyKnownTagsTask.kt new file mode 100644 index 00000000000..e990e15e957 --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/VerifyKnownTagsTask.kt @@ -0,0 +1,67 @@ +package datadog.gradle.plugin.tags + +import java.io.File +import javax.inject.Inject +import org.gradle.api.DefaultTask +import org.gradle.api.GradleException +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.model.ObjectFactory +import org.gradle.api.tasks.InputDirectory +import org.gradle.api.tasks.InputFile +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity +import org.gradle.api.tasks.TaskAction + +/** + * Freshness gate: regenerates the tag registry into a scratch dir and byte-compares it against the + * committed [committedDirectory]. Fails (pointing at {@code generateKnownTags}) if they differ, so a + * stale commit of the generated sources can't slip through CI. Not cacheable -- it must actually run + * the generator to catch drift, and it is cheap. + */ +abstract class VerifyKnownTagsTask @Inject constructor(objects: ObjectFactory) : DefaultTask() { + @get:InputFile + @get:PathSensitive(PathSensitivity.NONE) + val domainYaml: RegularFileProperty = objects.fileProperty() + + @get:InputFile + @get:PathSensitive(PathSensitivity.NONE) + val overlayYaml: RegularFileProperty = objects.fileProperty() + + @get:InputDirectory + @get:PathSensitive(PathSensitivity.RELATIVE) + val committedDirectory: DirectoryProperty = objects.directoryProperty() + + @TaskAction + fun verify() { + val committed = committedDirectory.get().asFile + val scratch = File(temporaryDir, "generated") + scratch.deleteRecursively() + TagRegistryGenerator.generate(domainYaml.get().asFile, overlayYaml.get().asFile, scratch) + + val diffs = ArrayList() + val freshFiles = scratch.walkTopDown().filter { it.isFile }.toList() + for (fresh in freshFiles) { + val rel = fresh.relativeTo(scratch).path + val committedFile = File(committed, rel) + when { + !committedFile.exists() -> diffs.add("missing (not committed): $rel") + committedFile.readText() != fresh.readText() -> diffs.add("out of date: $rel") + } + } + val freshRel = freshFiles.map { it.relativeTo(scratch).path }.toSet() + for (committedFile in committed.walkTopDown().filter { it.isFile }) { + val rel = committedFile.relativeTo(committed).path + if (rel !in freshRel) diffs.add("stale (no longer generated): $rel") + } + + if (diffs.isNotEmpty()) { + throw GradleException( + buildString { + appendLine("Generated tag registry is out of date with tag-conventions.yaml:") + diffs.forEach { appendLine(" - $it") } + append("Run `./gradlew :internal-api:generateKnownTags` and commit the result.") + }) + } + } +} diff --git a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java index a57e5d37882..49c9377dff9 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java @@ -305,6 +305,7 @@ public final class ConfigDefaults { public static final int DEFAULT_TRACE_X_DATADOG_TAGS_MAX_LENGTH = 512; static final boolean DEFAULT_TRACE_HTTP_RESOURCE_REMOVE_TRAILING_SLASH = false; + static final boolean DEFAULT_TRACE_DENSE_TAGS_ENABLED = false; static final boolean DEFAULT_TRACE_LONG_RUNNING_ENABLED = false; static final long DEFAULT_TRACE_LONG_RUNNING_INITIAL_FLUSH_INTERVAL = 20; // seconds static final long DEFAULT_TRACE_LONG_RUNNING_FLUSH_INTERVAL = 120; // seconds -> 2 minutes diff --git a/gradle/spotless.gradle b/gradle/spotless.gradle index 93a817e6452..f27408c7cec 100644 --- a/gradle/spotless.gradle +++ b/gradle/spotless.gradle @@ -20,7 +20,10 @@ spotless { toggleOffOn() // set explicit target to workaround https://github.com/diffplug/spotless/issues/1163 target 'src/**/*.java' - // ignore embedded test projects and everything in build dir, e.g. generated sources + // ignore embedded test projects and everything in build dir, e.g. generated sources. + // src/generated/** is committed generated code (e.g. the tag registry) and IS held to the + // formatting standard: emitters must produce google-java-format-clean output, and their + // freshness gate byte-compares against these formatted files. targetExclude('src/test/resources/**', buildDirectoryFiles) tableTestFormatter('1.1.1') googleJavaFormat('1.35.0') diff --git a/internal-api/build.gradle.kts b/internal-api/build.gradle.kts index 4d48a434c19..f1ab6486aa2 100644 --- a/internal-api/build.gradle.kts +++ b/internal-api/build.gradle.kts @@ -5,6 +5,7 @@ import groovy.lang.Closure plugins { `java-library` id("me.champeau.jmh") + id("dd-trace-java.tag-registry-generator") } apply(from = "$rootDir/gradle/java.gradle") @@ -32,6 +33,9 @@ extra["minimumBranchCoverage"] = 0.7 extra["minimumInstructionCoverage"] = 0.8 extra["excludedClassesCoverage"] = listOf( + // Generated by the tag-registry code generator (verified fresh via verifyKnownTags). + "datadog.trace.api.KnownTags", + "datadog.trace.api.KnownTags.*", "datadog.trace.api.ClassloaderConfigurationOverrides", "datadog.trace.api.ClassloaderConfigurationOverrides.Lazy", // Interface @@ -261,6 +265,18 @@ extra["excludedClassesBranchCoverage"] = listOf( extra["excludedClassesInstructionCoverage"] = listOf("datadog.trace.util.stacktrace.StackWalkerFactory") +// Tag registry: generated KnownTags is committed under src/generated (audited via git); the srcDir +// puts it on the main compile path and `verifyKnownTags` (wired into `check`) fails CI if it drifts +// from tag-conventions.yaml. Generation is run on demand (`./gradlew :internal-api:generateKnownTags`), +// not on every build, so the committed source stays the source of truth for the compiler. +tagRegistry { + domainYaml.set(rootProject.layout.projectDirectory.file("tag-conventions.yaml")) + overlayYaml.set(rootProject.layout.projectDirectory.file("tag-conventions.java.yaml")) + destinationDirectory.set(layout.projectDirectory.dir("src/generated")) +} + +sourceSets["main"].java.srcDir("src/generated/java") + dependencies { // references TraceScope and Continuation from public api api(project(":dd-trace-api")) diff --git a/internal-api/src/generated/folded-types.txt b/internal-api/src/generated/folded-types.txt new file mode 100644 index 00000000000..48b631bf5a5 --- /dev/null +++ b/internal-api/src/generated/folded-types.txt @@ -0,0 +1,93 @@ +# Folded tag set per type (extends + include + applies, de-duped), with colored slots. +# field: s=colored slot trace s=trace-level layer bkt=bucketed trace-bkt=trace-level bucketed I=intercepted + +db.client (21 tags): + s1 _dd.parent_id + s2 component + s6 span.kind I + s0 _dd.integration + bkt _dd.svc_src + s5 error.type + s3 error.message + s4 error.stack + s12 db.type + s9 db.instance + s10 db.operation + s15 db.user + bkt db.pool.name + s11 db.statement I + s14 peer.service I + s8 _dd.peer.service.source + s7 _dd.peer.service.remapped_from + s13 peer.hostname + bkt peer.ipv4 + bkt peer.ipv6 + bkt peer.port + +http.client (20 tags): + s1 _dd.parent_id + s2 component + s6 span.kind I + s0 _dd.integration + bkt _dd.svc_src + s5 error.type + s3 error.message + s4 error.stack + s9 http.method I + s10 http.status_code + s12 network.protocol.version + s11 http.url I + s15 http.resend_count + s14 peer.service I + s8 _dd.peer.service.source + s7 _dd.peer.service.remapped_from + s13 peer.hostname + bkt peer.ipv4 + bkt peer.ipv6 + bkt peer.port + +http.server (18 tags): + s1 _dd.parent_id + s2 component + s6 span.kind I + s0 _dd.integration + bkt _dd.svc_src + s5 error.type + s3 error.message + s4 error.stack + s9 http.method I + s10 http.status_code + s12 network.protocol.version + s11 http.url I + s13 http.route + s7 http.hostname + s14 http.useragent + s8 http.query.string + bkt servlet.path + bkt servlet.context I + +view.render (9 tags): + s1 _dd.parent_id + s2 component + s6 span.kind I + s0 _dd.integration + bkt _dd.svc_src + s5 error.type + s3 error.message + s4 error.stack + s7 view.name + + (13 tags): + trace s0 _dd.appsec.enabled + trace s1 _dd.base_service + trace s2 _dd.civisibility.enabled + trace s3 _dd.djm.enabled + trace s4 _dd.dsm.enabled + trace s5 _dd.git.commit.sha + trace s6 _dd.git.repository_url + trace s7 _dd.profiling.enabled + trace s8 _dd.tracer_host + trace s9 env + trace s10 language + trace s11 runtime-id + trace s12 version diff --git a/internal-api/src/main/java/datadog/trace/api/KnownTags.java b/internal-api/src/generated/java/datadog/trace/api/KnownTags.java similarity index 94% rename from internal-api/src/main/java/datadog/trace/api/KnownTags.java rename to internal-api/src/generated/java/datadog/trace/api/KnownTags.java index e8646593297..5912ed63bf0 100644 --- a/internal-api/src/main/java/datadog/trace/api/KnownTags.java +++ b/internal-api/src/generated/java/datadog/trace/api/KnownTags.java @@ -348,6 +348,15 @@ public final class KnownTags { SPAN_KIND_NAME, VERSION_NAME, VIEW_NAME, + "db.operation.name", + "db.query.text", + "db.system", + "http.request.method", + "http.response.status_code", + "server.address", + "service.name", + "url.full", + "url.query", }; private static final long[] KEYOF_VALUES = { ERROR_ID, @@ -406,6 +415,15 @@ public final class KnownTags { SPAN_KIND_ID, VERSION_ID, VIEW_NAME_ID, + DB_OPERATION_ID, + DB_STATEMENT_ID, + DB_TYPE_ID, + HTTP_METHOD_ID, + HTTP_STATUS_CODE_ID, + HTTP_HOSTNAME_ID, + SERVICE_ID, + HTTP_URL_ID, + HTTP_QUERY_STRING_ID, }; private static final int[] KEYOF_HASHES; private static final String[] KEYOF_KEYS; @@ -545,6 +563,32 @@ public String nameOf(long tagId) { } } + @Override + public String openTelemetryNameOf(long tagId) { + switch (KnownTagCodec.serialNum(tagId)) { + case SERVICE_SERIAL_NUM: + return "service.name"; + case DB_OPERATION_SERIAL_NUM: + return "db.operation.name"; + case DB_STATEMENT_SERIAL_NUM: + return "db.query.text"; + case DB_TYPE_SERIAL_NUM: + return "db.system"; + case HTTP_HOSTNAME_SERIAL_NUM: + return "server.address"; + case HTTP_METHOD_SERIAL_NUM: + return "http.request.method"; + case HTTP_QUERY_STRING_SERIAL_NUM: + return "url.query"; + case HTTP_STATUS_CODE_SERIAL_NUM: + return "http.response.status_code"; + case HTTP_URL_SERIAL_NUM: + return "url.full"; + default: + return null; + } + } + @Override public int slotCount() { return SLOT_COUNT; diff --git a/internal-api/src/generated/layout-by-type.txt b/internal-api/src/generated/layout-by-type.txt new file mode 100644 index 00000000000..309bdfcf8e3 --- /dev/null +++ b/internal-api/src/generated/layout-by-type.txt @@ -0,0 +1,91 @@ +# Full tag composition per concrete span type (after extends/include/applies). +# Not de-duped: a tag from >1 source appears >1 time. +# annotation: [s colored slot | trace s trace layer | bkt bucketed] I=intercepted + +db.client (21 contributions, 21 distinct): + [base] + _dd.parent_id s1 required + component s2 required + span.kind s6 required I + _dd.integration s0 recommended + _dd.svc_src bkt optional + error.type s5 recommended + error.message s3 recommended + error.stack s4 recommended + [db.client] + db.type s12 required + db.instance s9 recommended + db.operation s10 recommended + db.user s15 recommended + db.pool.name bkt optional + db.statement s11 recommended I + [incl:peer] + peer.service s14 recommended I + _dd.peer.service.source s8 recommended + _dd.peer.service.remapped_from s7 recommended + peer.hostname s13 recommended + peer.ipv4 bkt optional + peer.ipv6 bkt optional + peer.port bkt optional + +http.client (20 contributions, 20 distinct): + [base] + _dd.parent_id s1 required + component s2 required + span.kind s6 required I + _dd.integration s0 recommended + _dd.svc_src bkt optional + error.type s5 recommended + error.message s3 recommended + error.stack s4 recommended + [http] + http.method s9 required I + http.status_code s10 conditional + network.protocol.version s12 recommended + [http.client] + http.url s11 required I + http.resend_count s15 recommended + [incl:peer] + peer.service s14 recommended I + _dd.peer.service.source s8 recommended + _dd.peer.service.remapped_from s7 recommended + peer.hostname s13 recommended + peer.ipv4 bkt optional + peer.ipv6 bkt optional + peer.port bkt optional + +http.server (18 contributions, 18 distinct): + [base] + _dd.parent_id s1 required + component s2 required + span.kind s6 required I + _dd.integration s0 recommended + _dd.svc_src bkt optional + error.type s5 recommended + error.message s3 recommended + error.stack s4 recommended + [http] + http.method s9 required I + http.status_code s10 conditional + network.protocol.version s12 recommended + [http.server] + http.url s11 required I + http.route s13 conditional + http.hostname s7 required + http.useragent s14 recommended + http.query.string s8 recommended + servlet.path bkt optional + servlet.context bkt optional I + +view.render (9 contributions, 9 distinct): + [base] + _dd.parent_id s1 required + component s2 required + span.kind s6 required I + _dd.integration s0 recommended + _dd.svc_src bkt optional + error.type s5 recommended + error.message s3 recommended + error.stack s4 recommended + [view.render] + view.name s7 recommended diff --git a/internal-api/src/generated/resolved-tags.txt b/internal-api/src/generated/resolved-tags.txt new file mode 100644 index 00000000000..0edb3479608 --- /dev/null +++ b/internal-api/src/generated/resolved-tags.txt @@ -0,0 +1,77 @@ +# Resolved per-type tag sets (concrete span types). + +db.client (21 tags): + - _dd.parent_id + - component + - span.kind + - _dd.integration + - _dd.svc_src + - error.type + - error.message + - error.stack + - db.type + - db.instance + - db.operation + - db.user + - db.pool.name + - db.statement + - peer.service + - _dd.peer.service.source + - _dd.peer.service.remapped_from + - peer.hostname + - peer.ipv4 + - peer.ipv6 + - peer.port + +http.client (20 tags): + - _dd.parent_id + - component + - span.kind + - _dd.integration + - _dd.svc_src + - error.type + - error.message + - error.stack + - http.method + - http.status_code + - network.protocol.version + - http.url + - http.resend_count + - peer.service + - _dd.peer.service.source + - _dd.peer.service.remapped_from + - peer.hostname + - peer.ipv4 + - peer.ipv6 + - peer.port + +http.server (18 tags): + - _dd.parent_id + - component + - span.kind + - _dd.integration + - _dd.svc_src + - error.type + - error.message + - error.stack + - http.method + - http.status_code + - network.protocol.version + - http.url + - http.route + - http.hostname + - http.useragent + - http.query.string + - servlet.path + - servlet.context + +view.render (9 tags): + - _dd.parent_id + - component + - span.kind + - _dd.integration + - _dd.svc_src + - error.type + - error.message + - error.stack + - view.name diff --git a/internal-api/src/generated/tag-assignment.txt b/internal-api/src/generated/tag-assignment.txt new file mode 100644 index 00000000000..9045eb8cde9 --- /dev/null +++ b/internal-api/src/generated/tag-assignment.txt @@ -0,0 +1,81 @@ +# Tag id assignment. slotCount=16 stored=46 reserved=10 + +# STORED serial slot int lvl id required name + 256 0 - T 0x0100000000000004 recommended _dd.appsec.enabled + 257 1 - T 0x0101000100000004 required _dd.base_service + 258 2 - T 0x0102000200000004 recommended _dd.civisibility.enabled + 259 3 - T 0x0103000300000004 recommended _dd.djm.enabled + 260 4 - T 0x0104000400000004 recommended _dd.dsm.enabled + 261 5 - T 0x0105000500000004 recommended _dd.git.commit.sha + 262 6 - T 0x0106000600000004 recommended _dd.git.repository_url + 263 0 - - 0x0107000000000000 recommended _dd.integration + 264 1 - - 0x0108000100000000 required _dd.parent_id + 265 7 - - 0x0109000700000000 recommended _dd.peer.service.remapped_from + 266 8 - - 0x010A000800000000 recommended _dd.peer.service.source + 267 7 - T 0x010B000700000004 recommended _dd.profiling.enabled + 268 - - - 0x010CFFFF00000000 optional _dd.svc_src + 269 8 - T 0x010D000800000004 recommended _dd.tracer_host + 270 2 - - 0x010E000200000000 required component + 271 9 - - 0x010F000900000000 recommended db.instance + 272 10 - - 0x0110000A00000000 recommended db.operation + 273 - - - 0x0111FFFF00000000 optional db.pool.name + 274 11 I - 0x8112000B00000000 recommended db.statement + 275 12 - - 0x0113000C00000000 required db.type + 276 15 - - 0x0114000F00000000 recommended db.user + 277 9 - T 0x0115000900000004 recommended env + 278 3 - - 0x0116000300000000 recommended error.message + 279 4 - - 0x0117000400000000 recommended error.stack + 280 5 - - 0x0118000500000000 recommended error.type + 281 7 - - 0x0119000700000000 required http.hostname + 282 9 I - 0x811A000900000000 required http.method + 283 8 - - 0x011B000800000000 recommended http.query.string + 284 15 - - 0x011C000F00000000 recommended http.resend_count + 285 13 - - 0x011D000D00000000 conditional http.route + 286 10 - - 0x011E000A00000000 conditional http.status_code + 287 11 I - 0x811F000B00000000 required http.url + 288 14 - - 0x0120000E00000000 recommended http.useragent + 289 10 - T 0x0121000A00000004 required language + 290 12 - - 0x0122000C00000000 recommended network.protocol.version + 291 13 - - 0x0123000D00000000 recommended peer.hostname + 292 - - - 0x0124FFFF00000000 optional peer.ipv4 + 293 - - - 0x0125FFFF00000000 optional peer.ipv6 + 294 - - - 0x0126FFFF00000000 optional peer.port + 295 14 I - 0x8127000E00000000 recommended peer.service + 296 11 - T 0x0128000B00000004 required runtime-id + 297 - I - 0x8129FFFF00000000 optional servlet.context + 298 - - - 0x012AFFFF00000000 optional servlet.path + 299 6 I - 0x812B000600000000 required span.kind + 300 12 - T 0x012C000C00000004 recommended version + 301 7 - - 0x012D000700000000 recommended view.name + +# RESERVED serial id kind name + 1 0x8001FFFF00000000 structural error -> error + 2 0x8002FFFF00000000 structural service -> service + 3 0x8003FFFF00000000 structural resource.name -> resource + 4 0x8004FFFF00000000 structural span.type -> type + 5 0x8005FFFF00000000 structural origin -> origin + 6 0x8006FFFF00000000 directive sampling.priority + 7 0x8007FFFF00000000 directive manual.keep + 8 0x8008FFFF00000000 directive manual.drop + 9 0x8009FFFF00000000 directive measured + 10 0x800AFFFF00000000 directive analytics.sample_rate + +# PER-TYPE colored slots. Slots within a type must be DISTINCT (a valid coloring of the +# co-occurrence clique); is its own clique and freely reuses span slot numbers. + db.client count=16 slots=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + http.client count=16 slots=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + http.server count=15 slots=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] + view.render count=8 slots=[0, 1, 2, 3, 4, 5, 6, 7] + count=13 slots=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + +# OPENTELEMETRY NAMES. keyOf(otelName) resolves to the canonical tag's id; nameOf still +# returns the Datadog name, openTelemetryNameOf returns the name below. (No distinct id.) + db.operation.name -> db.operation + db.query.text -> db.statement + db.system -> db.type + http.request.method -> http.method + http.response.status_code -> http.status_code + server.address -> http.hostname + service.name -> service + url.full -> http.url + url.query -> http.query.string diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 75e618e514b..6e7fbe2d6c0 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -175,6 +175,7 @@ import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_BAGGAGE_MAX_ITEMS; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_BAGGAGE_TAG_KEYS; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_CLOUD_PAYLOAD_TAGGING_SERVICES; +import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_DENSE_TAGS_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_EXPERIMENTAL_FEATURES_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_HTTP_RESOURCE_REMOVE_TRAILING_SLASH; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_KEEP_LATENCY_THRESHOLD_MS; @@ -3303,7 +3304,8 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) this.spanBuilderReuseEnabled = configProvider.getBoolean(GeneralConfig.SPAN_BUILDER_REUSE_ENABLED, true); this.traceDenseTagsEnabled = - configProvider.getBoolean(TracerConfig.TRACE_DENSE_TAGS_ENABLED, false); + configProvider.getBoolean( + TracerConfig.TRACE_DENSE_TAGS_ENABLED, DEFAULT_TRACE_DENSE_TAGS_ENABLED); this.tagNameUtf8CacheSize = Math.max(configProvider.getInteger(GeneralConfig.TAG_NAME_UTF8_CACHE_SIZE, 128), 0); this.tagValueUtf8CacheSize = @@ -6828,6 +6830,11 @@ public String toString() { + sqsInjectDatadogAttributeEnabled + ", snsInjectDatadogAttributeEnabled=" + snsInjectDatadogAttributeEnabled + // Experimental: surfaced only when set away from the default, keeping normal dumps clean. + // Compared to the default constant (not a literal) so it survives a default change. + + (traceDenseTagsEnabled != DEFAULT_TRACE_DENSE_TAGS_ENABLED + ? ", traceDenseTagsEnabled=" + traceDenseTagsEnabled + : "") + '}'; } } diff --git a/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java b/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java index 925e01c13a5..0f5873389d8 100644 --- a/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java +++ b/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java @@ -213,6 +213,9 @@ public static int slotCount() { public interface Resolver { String nameOf(long tagId); + /** The tag's OpenTelemetry-namespace name, or {@code null} when it declares none. */ + String openTelemetryNameOf(long tagId); + long keyOf(String name); /** Number of positional slots this provider uses: (max stored fieldPos) + 1. */ @@ -233,6 +236,11 @@ public String nameOf(long tagId) { return null; } + @Override + public String openTelemetryNameOf(long tagId) { + return null; + } + @Override public long keyOf(String name) { return 0L; @@ -286,6 +294,21 @@ public static String nameOf(long tagId) { return null; } + /** The tag's Datadog-namespace (canonical) name — the same value as {@link #nameOf}. */ + public static String datadogNameOf(long tagId) { + return nameOf(tagId); + } + + /** + * The tag's OpenTelemetry-namespace name, or {@code null} when it declares none (or no resolver + * is registered). A serializer owns any fall-back-to-Datadog-name policy; this is a pure lookup. + */ + public static String openTelemetryNameOf(long tagId) { + if (!active) return null; + Resolver r = resolver; + return r != null ? r.openTelemetryNameOf(tagId) : null; + } + public static long keyOf(String name) { if (active) { return resolver.keyOf(name); diff --git a/internal-api/src/main/java/datadog/trace/api/TagMap.java b/internal-api/src/main/java/datadog/trace/api/TagMap.java index b7a5b62a58d..63fe695afef 100644 --- a/internal-api/src/main/java/datadog/trace/api/TagMap.java +++ b/internal-api/src/main/java/datadog/trace/api/TagMap.java @@ -51,6 +51,10 @@ public final class TagMap implements Map, Iterable. public static final TagMap EMPTY = new TagMap(new Object[1 << 4], 0); + // Sentinel for a not-yet-resolved lazy tag id. Cannot be 0L: 0L is a valid keyOf result (the tag + // is not a known tag, or the codec is inactive). Shared by Entry and EntryReadingHelper. + static final long TAG_ID_NOT_COMPUTED = Long.MIN_VALUE; + /** Creates a new mutable TagMap that contains the contents of map */ public static final TagMap fromMap(@Nonnull Map map) { TagMap tagMap = TagMap.create(map.size()); @@ -165,6 +169,12 @@ public interface EntryReader { String tag(); + /** + * The known-tag id for this entry's tag, or {@code 0L} when the tag is not a known tag (or the + * {@link KnownTagCodec} is inactive). Resolved via {@link KnownTagCodec#keyOf(String)}. + */ + long tagId(); + byte type(); boolean is(byte type); @@ -313,6 +323,13 @@ static Entry newDoubleEntry(String tag, Double box) { */ int lazyTagHash; + /* + * Known-tag id, lazily resolved using the same trick as lazyTagHash. TAG_ID_NOT_COMPUTED marks + * "not yet resolved" (0L is a valid result -- unknown tag / inactive codec -- so it cannot be + * the sentinel). Only pays off on the dense-OFF path; dense-ON known tags never become Entry-s. + */ + long lazyTagId = TAG_ID_NOT_COMPUTED; + // To optimize construction of Entry around boxed primitives and Object entries, // no type checks are done during construction. // Any Object entries are initially marked as type ANY, prim set to 0, and the Object put into @@ -353,6 +370,17 @@ int hash() { return hash; } + @Override + public long tagId() { + // Same lazy idiom as hash(): a benign race just recomputes keyOf, which is deterministic. + long id = this.lazyTagId; + if (id != TAG_ID_NOT_COMPUTED) return id; + + id = KnownTagCodec.keyOf(this.tag); + this.lazyTagId = id; + return id; + } + @Override public Entry entry() { return this; @@ -2668,7 +2696,7 @@ private EntryReader emitDense(long tagId, Object value) { if (reader == null) { reader = this.denseReader = new EntryReadingHelper(); } - reader.set(KnownTagCodec.nameOf(tagId), value); + reader.set(KnownTagCodec.nameOf(tagId), value, tagId); return reader; } @@ -3307,17 +3335,28 @@ final class EntryReadingHelper implements TagMap.EntryReader { private Map.Entry mapEntry; private String tag; private Object value; + private long tagId; void set(String tag, Object value) { this.mapEntry = null; this.tag = tag; this.value = value; + this.tagId = TagMap.TAG_ID_NOT_COMPUTED; // resolve lazily via keyOf on first tagId() access + } + + /** Dense emit: the id is known directly, so record it and skip the keyOf resolve. */ + void set(String tag, Object value, long tagId) { + this.mapEntry = null; + this.tag = tag; + this.value = value; + this.tagId = tagId; } void set(Map.Entry mapEntry) { this.mapEntry = mapEntry; this.tag = mapEntry.getKey(); this.value = mapEntry.getValue(); + this.tagId = TagMap.TAG_ID_NOT_COMPUTED; // resolve lazily via keyOf on first tagId() access } @Override @@ -3325,6 +3364,16 @@ public String tag() { return this.tag; } + @Override + public long tagId() { + long id = this.tagId; + if (id != TagMap.TAG_ID_NOT_COMPUTED) return id; + + id = KnownTagCodec.keyOf(this.tag); + this.tagId = id; + return id; + } + @Override public byte type() { return TagValueConversions.typeOf(this.value); diff --git a/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java b/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java index 3e0dccaae56..c70e5ff65fa 100644 --- a/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java +++ b/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java @@ -58,6 +58,24 @@ static Stream knownTags() { Arguments.of(Tags.DB_POOL_NAME, KnownTags.DB_POOL_NAME_ID)); } + /** + * (otelName, canonicalId, datadogName) — the OpenTelemetry name resolves (keyOf) to the canonical + * tag's id; datadogNameOf returns the Datadog name and openTelemetryNameOf returns the OTel name. + */ + static Stream otelNamedTags() { + return Stream.of( + Arguments.of("http.request.method", KnownTags.HTTP_METHOD_ID, "http.method"), + Arguments.of( + "http.response.status_code", KnownTags.HTTP_STATUS_CODE_ID, "http.status_code"), + Arguments.of("url.full", KnownTags.HTTP_URL_ID, "http.url"), + Arguments.of("server.address", KnownTags.HTTP_HOSTNAME_ID, "http.hostname"), + Arguments.of("url.query", KnownTags.HTTP_QUERY_STRING_ID, "http.query.string"), + Arguments.of("db.system", KnownTags.DB_TYPE_ID, "db.type"), + Arguments.of("db.operation.name", KnownTags.DB_OPERATION_ID, "db.operation"), + Arguments.of("db.query.text", KnownTags.DB_STATEMENT_ID, "db.statement"), + Arguments.of("service.name", KnownTags.SERVICE_ID, "service")); + } + /** * The subset flagged INTERCEPTED (sign bit) — must agree with the interceptor's needsIntercept. */ @@ -120,6 +138,29 @@ void nameOfResolvesIdToName(String name, long id) { assertEquals(name, KnownTagCodec.nameOf(id), "nameOf(" + name + ")"); } + @ParameterizedTest + @MethodSource("otelNamedTags") + void otelNameResolvesToCanonicalId(String otelName, long id, String datadogName) { + // Inbound (keyOf) is many->one: both names land on the same canonical id. + assertEquals(id, KnownTagCodec.keyOf(otelName), "keyOf(" + otelName + ")"); + assertEquals(id, KnownTagCodec.keyOf(datadogName), "keyOf(" + datadogName + ")"); + } + + @ParameterizedTest + @MethodSource("otelNamedTags") + void namespaceAccessorsReturnPerNamespaceName(String otelName, long id, String datadogName) { + assertEquals(datadogName, KnownTagCodec.datadogNameOf(id), "datadogNameOf"); + assertEquals(otelName, KnownTagCodec.openTelemetryNameOf(id), "openTelemetryNameOf"); + // nameOf stays the Datadog name -- outbound is namespace-specific, not normalized to OTel. + assertEquals(datadogName, KnownTagCodec.nameOf(id), "nameOf stays Datadog"); + } + + @Test + void tagsWithoutOtelNameReturnNull() { + assertNull(KnownTagCodec.openTelemetryNameOf(KnownTags.HTTP_ROUTE_ID)); // no OTel name declared + assertNull(KnownTagCodec.openTelemetryNameOf(0L)); // unknown id + } + @ParameterizedTest @MethodSource("interceptedTags") void interceptedTagsCarryFlag(long id) { diff --git a/internal-api/src/test/java/datadog/trace/api/TagMapDenseForkedTest.java b/internal-api/src/test/java/datadog/trace/api/TagMapDenseForkedTest.java index 712b2281eee..41380771594 100644 --- a/internal-api/src/test/java/datadog/trace/api/TagMapDenseForkedTest.java +++ b/internal-api/src/test/java/datadog/trace/api/TagMapDenseForkedTest.java @@ -281,4 +281,30 @@ void removingParentDenseKeyTombstonesIt() { assertFalse(union.containsKey(BASE_SERVICE)); child.checkIntegrity(); } + + @Test + void denseReaderExposesTagId() { + TagMap map = map(); + map.set(BASE_SERVICE, "billing"); // dense-routed + map.set(CUSTOM_A, "alpha"); // bucket-routed + + Map idsByTag = new HashMap<>(); + map.forEach(reader -> idsByTag.put(reader.tag(), reader.tagId())); + + // dense entry: the reader carries the real known-tag id directly + assertEquals(KnownTagCodec.keyOf(BASE_SERVICE), idsByTag.get(BASE_SERVICE).longValue()); + assertTrue(idsByTag.get(BASE_SERVICE) != 0L, "known tag has a non-zero id"); + // bucket entry for a custom tag: unknown -> 0L + assertEquals(0L, idsByTag.get(CUSTOM_A).longValue()); + } + + @Test + void bucketEntryResolvesTagIdLazily() { + TagMap map = map(); + map.set(CUSTOM_A, "alpha"); // custom tag stays a bucket Entry + + TagMap.Entry entry = map.getEntry(CUSTOM_A); + assertEquals(0L, entry.tagId()); // unknown tag -> 0L + assertEquals(0L, entry.tagId()); // second read hits the memoized field + } } diff --git a/internal-api/src/test/java/datadog/trace/api/TagMapDenseFuzzForkedTest.java b/internal-api/src/test/java/datadog/trace/api/TagMapDenseFuzzForkedTest.java index 9ff3fd46419..8100ca464a7 100644 --- a/internal-api/src/test/java/datadog/trace/api/TagMapDenseFuzzForkedTest.java +++ b/internal-api/src/test/java/datadog/trace/api/TagMapDenseFuzzForkedTest.java @@ -78,6 +78,11 @@ public String nameOf(long tagId) { : null; } + @Override + public String openTelemetryNameOf(long tagId) { + return null; // synthetic resolver declares no OpenTelemetry names + } + @Override public int slotCount() { return 0; // positional unused diff --git a/tag-conventions.java.yaml b/tag-conventions.java.yaml new file mode 100644 index 00000000000..fcb91ac04f0 --- /dev/null +++ b/tag-conventions.java.yaml @@ -0,0 +1,36 @@ +# dd-trace-java overlay — impl hints + special-tag registry. +# Consumed alongside the language-agnostic tag-conventions.yaml. Keyed by tag name +# (these are tag-intrinsic, not per-type). Only exceptions are listed. +# --------------------------------------------------------------------------- +# NOTE: the id coordinate (group-decl / field-decl) is NOT here — the generator assigns it from the +# declaration groups; the dense-vs-bucket split derives from the domain `required` level. + +# Tags whose set-path is handled by the Java TagInterceptor (side-effecting, but still stored). +# Generated ids carry the intercepted flag (bit 63). +intercepted: + - span.kind + - http.method + - http.url + - servlet.context + - db.statement + - peer.service + +# Reserved / special keys: accepted by the public setTag(...) API but ROUTED to a span field or a +# trace directive instead of tag storage. Reserved-tier ids (serial < FIRST_STORED_SERIAL, no slot). +# "Reserved" names the shared mechanism (the tracer reserves the key and handles it); the two kinds +# split on whether a value exists: structural has one (in a span/trace field), directive has none. +# The generated id->handler dispatch table is the data-driven replacement for the imperative +# TagInterceptor chain. +# kind: structural -> sets a span/trace field (`field:` names it) +# kind: directive -> triggers sampling/trace behavior +reserved: + - { tag: error, kind: structural, field: error } + - { tag: service, kind: structural, field: service, open-telemetry-name: service.name } + - { tag: resource.name, kind: structural, field: resource } + - { tag: span.type, kind: structural, field: type } + - { tag: origin, kind: structural, field: origin } # trace-level field + - { tag: sampling.priority, kind: directive } + - { tag: manual.keep, kind: directive } + - { tag: manual.drop, kind: directive } + - { tag: measured, kind: directive } + - { tag: analytics.sample_rate, kind: directive } # legacy diff --git a/tag-conventions.yaml b/tag-conventions.yaml new file mode 100644 index 00000000000..1639e0e8afc --- /dev/null +++ b/tag-conventions.yaml @@ -0,0 +1,134 @@ +# Tag conventions — LANGUAGE-AGNOSTIC domain spec (structure + semantics only) +# --------------------------------------------------------------------------- +# The code generator consumes THIS file (domain) plus a per-language overlay +# (tag-conventions..yaml: impl hints like `intercepted`, and the reserved/ +# special-tag registry) to emit each language's tag-id constants, id<->name +# resolver, and slot (bitmask-bit) assignment. +# +# TRACE-LEVEL is its own thing (its own TagMap "type" on the TraceSegment) — the process/trace +# constants + product flags that are set once per trace, NOT per span. Declared explicitly in the +# `trace_level` section below (a distinct tier), never inferred from `source`. +# +# SPAN TYPES compose three ways: +# extends — structural is-a inheritance (http.server is-a http is-a base). `base` is implicitly +# in every span; abstract layers exist only to be extended. +# include — a span type PULLS in a mixin it intrinsically has (has-a; core-owned). +# applies — a mixin PUSHES itself onto span types, gated by `enabled_by`. +# resolved_tags(type) = own + extends-chain (incl base) + included mixins + applied mixins (de-duped). +# +# tag fields (DOMAIN only): tag | type (string|int|long|boolean|double) +# | required (required|conditional|recommended|optional|opt_in) | open-telemetry-name. +# The id coordinate (group-decl / field-decl) is NOT authored here — the generator assigns it: each +# declaration source (the trace-level tier, each span type, each mixin) is a group, and within a +# group `field-decl` numbers the dense (required/conditional/recommended) tags; the rest are +# bucketed. See the design doc. +# --------------------------------------------------------------------------- + +# Trace-level tier: its own TagMap on the TraceSegment. Set once per trace, not per span. +trace_level: + tags: + - { tag: _dd.base_service, type: string, required: required } + - { tag: version, type: string, required: recommended } + - { tag: env, type: string, required: recommended } + - { tag: language, type: string, required: required } + - { tag: runtime-id, type: string, required: required } + - { tag: _dd.tracer_host, type: string, required: recommended } + - { tag: _dd.git.commit.sha, type: string, required: recommended } + - { tag: _dd.git.repository_url, type: string, required: recommended } + # product .enabled flags — process-constant; present on the trace segment regardless of whether + # the product is enabled (the flag carries the state), so always-present => recommended. + - { tag: _dd.profiling.enabled, type: boolean, required: recommended } + - { tag: _dd.dsm.enabled, type: boolean, required: recommended } + - { tag: _dd.appsec.enabled, type: boolean, required: recommended } + - { tag: _dd.djm.enabled, type: boolean, required: recommended } + - { tag: _dd.civisibility.enabled, type: boolean, required: recommended } + +span_types: + # root: per-span tags every span has (incl. the per-span core tags parent_id / integration / svc_src + # — core-set but per-span, so NOT trace-level). + base: + abstract: true + tags: + - { tag: _dd.parent_id, type: string, required: required } + - { tag: component, type: string, required: required } + - { tag: span.kind, type: string, required: required } + - { tag: _dd.integration, type: string, required: recommended } + - { tag: _dd.svc_src, type: string, required: optional } + - { tag: error.type, type: string, required: recommended } + - { tag: error.message, type: string, required: recommended } + - { tag: error.stack, type: string, required: recommended } + + http: + abstract: true + extends: base + tags: + - { tag: http.method, type: string, required: required, open-telemetry-name: http.request.method } + - { tag: http.status_code, type: int, required: conditional, open-telemetry-name: http.response.status_code } + - { tag: network.protocol.version, type: string, required: recommended } + + http.server: + extends: http + tags: + - { tag: http.url, type: string, required: required, open-telemetry-name: url.full } + - { tag: http.route, type: string, required: conditional } + - { tag: http.hostname, type: string, required: required, open-telemetry-name: server.address } + - { tag: http.useragent, type: string, required: recommended } + - { tag: http.query.string, type: string, required: recommended, open-telemetry-name: url.query } + - { tag: servlet.path, type: string, required: optional } + - { tag: servlet.context, type: string, required: optional } + + http.client: + extends: http + include: [ peer ] + tags: + - { tag: http.url, type: string, required: required, open-telemetry-name: url.full } + - { tag: http.resend_count, type: int, required: recommended } + + db.client: + extends: base + include: [ peer ] + tags: + - { tag: db.type, type: string, required: required, open-telemetry-name: db.system } + - { tag: db.instance, type: string, required: recommended } + - { tag: db.operation, type: string, required: recommended, open-telemetry-name: db.operation.name } + - { tag: db.user, type: string, required: recommended } + - { tag: db.pool.name, type: string, required: optional } + - { tag: db.statement, type: string, required: recommended, open-telemetry-name: db.query.text } + + view.render: + extends: base + tags: + - { tag: view.name, type: string, required: recommended } + +mixins: + # peer — outbound/remote-peer capability, PULLED via `include` by client span types. + peer: + tags: + - { tag: peer.service, type: string, required: recommended } + - { tag: _dd.peer.service.source, type: string, required: recommended } + - { tag: _dd.peer.service.remapped_from, type: string, required: recommended } + - { tag: peer.hostname, type: string, required: recommended } + - { tag: peer.ipv4, type: string } + - { tag: peer.ipv6, type: string } + - { tag: peer.port, type: int } + + # ci_visibility — per-span test tags. Its capability flag (_dd.civisibility.enabled) lives in + # trace_level, outside this mixin (general rule: capability flags are trace-level, mixins hold the + # per-span tags). Applies to the `test` span type (not modeled here yet). + ci_visibility: + enabled_by: dd.civisibility.enabled + applies: [ test ] + tags: + - { tag: test.name, type: string, required: recommended } + - { tag: test.suite, type: string, required: recommended } + - { tag: test.status, type: string, required: recommended } + - { tag: test.framework, type: string, required: recommended } + +# --------------------------------------------------------------------------- +# Notes +# - Product .enabled flags moved to `trace_level` (process-constant) — the old product mixins held +# only those flags, so they dissolved. `enabled_by`/attachment gating is a runtime concern. +# - span.kind enumerates: server | client | producer | consumer | internal | broker. +# - Reserved/special keys (service, resource.name, error, sampling.priority, ...) route to span +# fields/directives, not tag storage — they live in the per-language overlay, not here. +# ---------------------------------------------------------------------------