diff --git a/apps/api/src/chat/prompts.ts b/apps/api/src/chat/prompts.ts index 74a1b9776..a44885ee1 100644 --- a/apps/api/src/chat/prompts.ts +++ b/apps/api/src/chat/prompts.ts @@ -121,7 +121,7 @@ If you write "unknown" in \`suspectedCause\`, you MUST populate \`ruledOut\` wit The same applies when you DO name a cause: \`ruledOut\` is what makes the named cause believable. A responder reading your report should be able to see what else you considered. -Never report a bare label as a cause. "Unknown Error" is a grouping label for spans with no exception and no status message — it is the *name* of the thing you were asked to explain, not an explanation of it. +Never report a bare label as a cause. "Unknown Error" is a grouping label for spans with no exception event, no exception.*/error.* attributes and no status message — it is the *name* of the thing you were asked to explain, not an explanation of it. ## After diagnosing Stay in the conversation. Answer follow-up questions using the same tools, referencing the evidence you already gathered. When the user asks you to act — create an alert, transition an issue, propose a fix — call the matching mutating tool; it is approval-gated (see below). diff --git a/apps/api/src/services/warehouse/error-events-attribute-fallback.clickhouse.e2e.test.ts b/apps/api/src/services/warehouse/error-events-attribute-fallback.clickhouse.e2e.test.ts new file mode 100644 index 000000000..12dd82ff2 --- /dev/null +++ b/apps/api/src/services/warehouse/error-events-attribute-fallback.clickhouse.e2e.test.ts @@ -0,0 +1,331 @@ +// SAFETY-FILE: JSON in this test is emitted by the fixture or unit under test before its fields are asserted. +// Synthetic exporter shapes exercise the real warehouse projection and migration. +// They are not captured Cloudflare payloads; exporter compatibility needs its own fixture. + +import { afterAll, assert, beforeAll, describe, it } from "@effect/vitest" +import { migrations } from "@maple/domain/clickhouse" +import { Schema } from "effect" +import { msToDate } from "../../platform/time" +import { + applyRealMigrations, + clickhouseE2eEnabled, + clickhouseExec, + uniqueDatabase, +} from "./clickhouse-e2e-support" + +const database = uniqueDatabase("maple_error_events_attrs_e2e") +const ORG_ID = "org_error_events_attrs" +const SERVICE = "cf-worker" + +/** + * Now-relative, not a fixed date: `traces` and both error tables enforce a TTL + * at insert time, and a hardcoded timestamp silently drops every seed once it + * ages past the horizon, leaving the suite comparing nothing to nothing. + */ +const SEED_MS = Date.now() - 60 * 60 * 1000 +const chDateTime = (epochMs: number): string => msToDate(epochMs).toISOString().replace("T", " ").slice(0, 19) +const SEED_TS = chDateTime(SEED_MS) + +const quote = (value: string): string => `'${value.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'` +const chMap = (entries: Readonly>): string => { + const pairs = Object.entries(entries).flatMap(([key, value]) => [quote(key), quote(value)]) + return pairs.length === 0 ? "map()" : `map(${pairs.join(", ")})` +} + +interface ExceptionEvent { + readonly type: string + readonly message: string + readonly stacktrace: string +} + +interface SeedSpan { + readonly spanId: string + readonly kind: "Client" | "Server" + readonly statusMessage: string + readonly spanAttributes: Readonly> + readonly exceptionEvent?: ExceptionEvent +} + +const SEED_SPANS: ReadonlyArray = [ + // The case this file exists for: no event, no status description, only + // semconv error.* attributes. + { + spanId: "cf-error-type", + kind: "Server", + statusMessage: "", + spanAttributes: { + "error.type": "TypeError", + "error.message": "Cannot load account 1234567890", + "http.request.method": "GET", + }, + }, + // Same type, different message: must be a different issue, not one + // "TypeError" bucket per service. + { + spanId: "cf-error-type-other-bug", + kind: "Server", + statusMessage: "", + spanAttributes: { + "error.type": "TypeError", + "error.message": "Cannot read properties of null (reading 'headers')", + }, + }, + // Same bug, different id in the message: the redacted signature groups them. + { + spanId: "cf-error-type-same-bug", + kind: "Server", + statusMessage: "", + spanAttributes: { + "error.type": "TypeError", + "error.message": "Cannot load account 9876543210", + "user.id": "u_1234567890", + }, + }, + // exception.* attributes win over error.*, and the stacktrace attribute + // feeds the frame portion of the hash. + { + spanId: "cf-exception-attrs", + kind: "Server", + statusMessage: "", + spanAttributes: { + "exception.type": "RangeError", + "exception.message": "offset 4096 is out of range", + "exception.stacktrace": + "RangeError: offset 4096 is out of range\n at slice (worker.js:1542:13655)", + "error.type": "LosesToException", + "error.message": "must not be read", + }, + }, + // A real exception event keeps exactly the precedence it always had, even + // when attributes disagree with it. + { + spanId: "event-wins", + kind: "Server", + statusMessage: "status text", + spanAttributes: { + "exception.type": "AttrError", + "error.type": "AttrError2", + "error.message": "attr", + }, + exceptionEvent: { + type: "EventError", + message: "from the event", + stacktrace: " at handler (/app/src/routes/user.ts:17:21)", + }, + }, + // The StatusMessage fallback is unchanged for spans with nothing else. + { + spanId: "status-only", + kind: "Server", + statusMessage: "DatabaseError: connection reset", + spanAttributes: {}, + }, + // StatusMessage still supplies the message text when it is set, so an + // event-less span that already had one keeps its hash. + { + spanId: "status-and-error-type", + kind: "Server", + statusMessage: "connection reset", + spanAttributes: { "error.type": "TimeoutError", "error.message": "attribute message" }, + }, + { + spanId: "status-and-exception-attrs", + kind: "Server", + statusMessage: "connection reset", + spanAttributes: { + "exception.type": "TimeoutError", + "exception.message": "attribute message", + "exception.stacktrace": "TimeoutError: reset\n at connect (/app/db.ts:12:4)", + }, + }, + { + spanId: "empty-event", + kind: "Server", + statusMessage: "", + spanAttributes: { "exception.type": "IgnoredError", "error.message": "ignored" }, + exceptionEvent: { type: "", message: "", stacktrace: "" }, + }, + // Nothing carries an exception: still the Unknown Error bucket. + { + spanId: "unknown", + kind: "Server", + statusMessage: "", + spanAttributes: { "http.request.method": "GET" }, + }, + // The 0016 guard: a 4xx client span whose only error.type is the status code + // (HTTP semconv sets that on any non-2xx response) is bot noise, not an error. + { + spanId: "bot-404", + kind: "Client", + statusMessage: "", + spanAttributes: { "http.response.status_code": "404", "error.type": "404", "url.path": "/wp-admin" }, + }, + // ...but a 4xx carrying a real exception type is still an error. + { + spanId: "real-4xx", + kind: "Client", + statusMessage: "", + spanAttributes: { "http.response.status_code": "400", "error.type": "ValidationError" }, + }, +] + +const seed = async (): Promise => { + const rows = SEED_SPANS.map((row) => { + const resource = chMap({ + "service.version": "e2e", + "deployment.environment.name": "production", + }) + const events = + row.exceptionEvent === undefined + ? "[], [], []" + : `[toDateTime64(${quote(SEED_TS)}, 9)], ['exception'], [${chMap({ + "exception.type": row.exceptionEvent.type, + "exception.message": row.exceptionEvent.message, + "exception.stacktrace": row.exceptionEvent.stacktrace, + })}]` + return `(${quote(ORG_ID)}, ${quote(SEED_TS)}, ${quote(`trace-${row.spanId}`)}, ${quote(row.spanId)}, '', 'GET /', ${quote(row.kind)}, ${quote(SERVICE)}, 1000000, 'Error', ${quote(row.statusMessage)}, ${chMap(row.spanAttributes)}, ${resource}, ${events})` + }).join(",\n") + + await clickhouseExec( + `INSERT INTO traces + (OrgId, Timestamp, TraceId, SpanId, ParentSpanId, SpanName, SpanKind, ServiceName, Duration, StatusCode, StatusMessage, SpanAttributes, ResourceAttributes, EventsTimestamp, EventsName, EventsAttributes) + VALUES\n${rows}`, + database, + ) +} + +const ErrorEventRow = Schema.Struct({ + SpanId: Schema.String, + ErrorLabel: Schema.String, + ExceptionType: Schema.String, + ExceptionMessage: Schema.String, + ExceptionStacktrace: Schema.String, + TopFrame: Schema.String, + FingerprintHash: Schema.String, +}) +type ErrorEventRow = typeof ErrorEventRow.Type +const decodeErrorEvent = Schema.decodeUnknownSync(Schema.fromJsonString(ErrorEventRow)) + +const readErrorEvents = async (from: string): Promise> => { + const body = await clickhouseExec( + `SELECT SpanId, ErrorLabel, ExceptionType, ExceptionMessage, ExceptionStacktrace, TopFrame, toString(FingerprintHash) AS FingerprintHash + FROM ${from} + WHERE OrgId = ${quote(ORG_ID)} + ORDER BY SpanId + FORMAT JSONEachRow`, + database, + ) + const rows = body + .split("\n") + .filter((line) => line.trim().length > 0) + .map(decodeErrorEvent) + const bySpan = new Map(rows.map((row) => [row.SpanId, row])) + assert.strictEqual(bySpan.size, rows.length, "unexpected duplicate occurrences") + return bySpan +} + +const mustGet = (rows: Map, spanId: string): ErrorEventRow => { + const row = rows.get(spanId) + assert.isDefined(row, `expected ${spanId} to be materialized into error_events`) + return row +} + +describe.skipIf(!clickhouseE2eEnabled)("error_events attribute fallback (ClickHouse e2e)", () => { + let rows: Map + + beforeAll(async () => { + await clickhouseExec(`CREATE DATABASE IF NOT EXISTS ${database}`) + await applyRealMigrations(database) + await seed() + rows = await readErrorEvents("error_events") + }, 180_000) + + afterAll(async () => { + await clickhouseExec(`DROP DATABASE IF EXISTS ${database}`) + }) + + it("labels an attribute-only span from its error.* attributes", () => { + const row = mustGet(rows, "cf-error-type") + assert.strictEqual(row.ErrorLabel, "TypeError") + assert.strictEqual(row.ExceptionType, "TypeError") + assert.strictEqual(row.ExceptionMessage, "Cannot load account 1234567890") + assert.strictEqual(row.ExceptionStacktrace, "") + assert.notStrictEqual(row.FingerprintHash, mustGet(rows, "unknown").FingerprintHash) + }) + + it("separates two bugs of the same type in one Worker, and groups one bug across ids", () => { + const bug = mustGet(rows, "cf-error-type") + assert.notStrictEqual(bug.FingerprintHash, mustGet(rows, "cf-error-type-other-bug").FingerprintHash) + assert.strictEqual(bug.FingerprintHash, mustGet(rows, "cf-error-type-same-bug").FingerprintHash) + }) + + it("reads exception.* attributes ahead of error.*, stacktrace included", () => { + const row = mustGet(rows, "cf-exception-attrs") + assert.strictEqual(row.ErrorLabel, "RangeError") + assert.strictEqual(row.ExceptionMessage, "offset 4096 is out of range") + assert.include(row.ExceptionStacktrace, "at slice") + assert.strictEqual(row.TopFrame, " at slice (worker.js)") + }) + + it("keeps the exception event's precedence when a span has one", () => { + const row = mustGet(rows, "event-wins") + assert.strictEqual(row.ErrorLabel, "EventError") + assert.strictEqual(row.ExceptionMessage, "from the event") + assert.strictEqual(row.TopFrame, " at handler (/app/src/routes/user.ts)") + }) + + it("keeps the StatusMessage and Unknown Error fallbacks for everything else", () => { + assert.strictEqual(mustGet(rows, "status-only").ErrorLabel, "DatabaseError") + assert.strictEqual(mustGet(rows, "status-only").ExceptionMessage, "DatabaseError: connection reset") + assert.strictEqual(mustGet(rows, "unknown").ErrorLabel, "Unknown Error") + }) + + it("preserves pre-migration hashes and details for events and status messages", async () => { + // Migration 0020 is the last deployed definition of these views before 0030. + // Evaluate its frozen SELECT against identical spans; do not reimplement hashing. + const previousView = migrations + .find((migration) => migration.version === 20) + ?.statements.find( + (statement) => + typeof statement === "string" && + statement.startsWith("CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv "), + ) + assert.isString(previousView) + const previousSelect = previousView.slice(previousView.indexOf(" AS\n") + 4) + const previousRows = await readErrorEvents(`(${previousSelect})`) + for (const spanId of [ + "event-wins", + "empty-event", + "status-only", + "status-and-error-type", + "status-and-exception-attrs", + "unknown", + ]) { + assert.deepStrictEqual(mustGet(rows, spanId), mustGet(previousRows, spanId), spanId) + } + }) + + it("preserves stored occurrences when the migration is reapplied", async () => { + const migration = migrations.find((entry) => entry.version === 30) + assert.isDefined(migration) + for (const statement of migration.statements) { + assert.isString(statement) + await clickhouseExec(statement, database) + } + assert.deepStrictEqual(await readErrorEvents("error_events"), rows) + assert.deepStrictEqual(await readErrorEvents("error_events_by_time"), rows) + }) + + it("still drops a 4xx client span whose only error.type is the status code", () => { + assert.isUndefined(rows.get("bot-404")) + assert.strictEqual(mustGet(rows, "real-4xx").ErrorLabel, "ValidationError") + }) + + it("writes the same projection to error_events_by_time", async () => { + const byTime = await readErrorEvents("error_events_by_time") + assert.deepStrictEqual([...byTime.keys()], [...rows.keys()]) + for (const [spanId, row] of rows) { + assert.deepStrictEqual(byTime.get(spanId), row, `error_events_by_time disagrees on ${spanId}`) + } + }) +}) diff --git a/apps/cli/src/server/local-schema-history.ts b/apps/cli/src/server/local-schema-history.ts index f5bc1083d..fb4250844 100644 --- a/apps/cli/src/server/local-schema-history.ts +++ b/apps/cli/src/server/local-schema-history.ts @@ -226,4 +226,19 @@ export const LOCAL_SCHEMA_HISTORY: ReadonlyArray = Obje manifestDigest: "7887f63cadd66a33e495dc3277dc55059285799a1d044a1f1d9bb614f38af3bd", projectRevision: "ed74788ef292834069e0ea6ee3b22d68fc604fb66cb54d2d551db67ce8d20b3a", }), + Object.freeze({ + // v20 rebuilds error_events_mv / error_events_by_time_mv so a span with + // no `exception` event is labelled from its exception.* / error.* span + // attributes (ClickHouse migration 0030). No part is rewritten and no row + // moves; rows already materialized keep their 'Unknown Error' label. + // + // projectRevision is carried forward deliberately — it is a hardcoded + // constant that no longer tracks the generator's header, and the identity + // this gate compares is the fingerprint/digest pair. + version: 20, + fingerprint: "ad8e854c9e2bb021", + digest: "ad8e854c9e2bb02184ace30e6b6eb483c978626f8a452f54293ee66c358ad1c5", + manifestDigest: "caec674c22441b89a3294d9158ca1b4b1d7b3b1b41f9cef3853b9570ad1db7b8", + projectRevision: "ed74788ef292834069e0ea6ee3b22d68fc604fb66cb54d2d551db67ce8d20b3a", + }), ] as const) diff --git a/apps/cli/src/server/local-schema-version.ts b/apps/cli/src/server/local-schema-version.ts index 5c8b5c9b7..eecfa31d1 100644 --- a/apps/cli/src/server/local-schema-version.ts +++ b/apps/cli/src/server/local-schema-version.ts @@ -1,4 +1,4 @@ // Increment this value for every structural change to the generated local // schema. The compatibility manifest and migration registry must be updated in // the same change before a new value can ship. -export const LOCAL_SCHEMA_VERSION = 19 as const +export const LOCAL_SCHEMA_VERSION = 20 as const diff --git a/apps/cli/src/server/local-store-migrations.ts b/apps/cli/src/server/local-store-migrations.ts index f47ca57a6..88d299a91 100644 --- a/apps/cli/src/server/local-store-migrations.ts +++ b/apps/cli/src/server/local-store-migrations.ts @@ -55,6 +55,7 @@ import { v15ToV16AiTraceIndexFilterColumnsModule } from "./local-store-migration import { v16ToV17AuditLogModule } from "./local-store-migrations/v16-to-v17-audit-log" import { v17ToV18ProductEventsFromTracesModule } from "./local-store-migrations/v17-to-v18-product-events-from-traces" import { v18ToV19AiTraceIndexUsageConventionsModule } from "./local-store-migrations/v18-to-v19-ai-trace-index-usage-conventions" +import { v19ToV20ErrorEventsAttributeFallbackModule } from "./local-store-migrations/v19-to-v20-error-events-attribute-fallback" import type { AnyLocalStoreMigrationModule, LocalStoreMigration, @@ -131,6 +132,7 @@ export const localStoreMigrations: ReadonlyArray = v16ToV17AuditLogModule, v17ToV18ProductEventsFromTracesModule, v18ToV19AiTraceIndexUsageConventionsModule, + v19ToV20ErrorEventsAttributeFallbackModule, ] export const validateMigrationRegistry = ( diff --git a/apps/cli/src/server/local-store-migrations/v19-to-v20-error-events-attribute-fallback.ts b/apps/cli/src/server/local-store-migrations/v19-to-v20-error-events-attribute-fallback.ts new file mode 100644 index 000000000..6d7a3791e --- /dev/null +++ b/apps/cli/src/server/local-store-migrations/v19-to-v20-error-events-attribute-fallback.ts @@ -0,0 +1,208 @@ +// SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. +import { resolve } from "node:path" +import { Schema } from "effect" +import { + cloneStoreForStaging, + decodeInstalledProgress, + makeRawRowsState, + type InstalledProgress, + RAW_TABLES, + rawRowCounts, + expectedManifest, + UnsignedDecimal, +} from "./journal-codecs" +import { readRawTelemetryRetentionDays } from "../chdb" +import type { + LocalStoreMigrationModule, + MigrationModuleContext, + MigrationOperation, + StateDispositionEntry, +} from "../local-store-migration-module" +import { + LOCAL_SCHEMA_V19, + LOCAL_SCHEMA_V19_MANIFEST, + LOCAL_SCHEMA_V19_SQL, + LOCAL_SCHEMA_V20, + LOCAL_SCHEMA_V20_MANIFEST, + LOCAL_SCHEMA_V20_SQL, +} from "../schema-identity" +import { assertPhysicalSchema } from "../schema-physical" + +/** Stamped into the journal and matched on the way back out. */ +const MODULE_ID = "local-0019-to-0020-error-events-attribute-fallback" as const + +class RowCountMismatch extends Schema.TaggedError()("@maple/cli/RowCountMismatch", { + message: Schema.String, + moduleId: Schema.String, + table: Schema.String, + expected: UnsignedDecimal, + actual: UnsignedDecimal, +}) {} + +const V19ToV20StateCodec = makeRawRowsState(MODULE_ID) + +type V19ToV20State = typeof V19ToV20StateCodec.schema.Type +type V19ToV20Progress = InstalledProgress + +const decodeState = V19ToV20StateCodec.decode +const decodeProgress = decodeInstalledProgress + +/** Replace the two view definitions from migration 0030, preserving stored rows. */ +const preflight = async (context: MigrationModuleContext): Promise => { + await context.ensureCapacity() + const retentionDays = readRawTelemetryRetentionDays(context.dataDir) + const rawRows = await context.openSource( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V19_MANIFEST, retentionDays)) + return rawRowCounts(db) + }, + { schemaSql: LOCAL_SCHEMA_V19_SQL, bootstrapSchema: false }, + ) + // Two literals rather than a conditional spread: `retentionDays` is an + // `optionalKey`, so an absent floor has to be an absent key, not a present + // `undefined`. + return retentionDays === undefined + ? { module: MODULE_ID, version: 1, rawRows } + : { module: MODULE_ID, version: 1, rawRows, retentionDays } +} + +const prepareTarget = async ( + context: MigrationModuleContext, + state: V19ToV20State, +): Promise => { + await context.closeStores() + const source = resolve(context.sourceDataDir) + const target = resolve(context.targetDataDir) + if (source !== target) { + await cloneStoreForStaging(source, target) + } + return state +} + +/** + * Like v7 -> v8, this edge replaces the body of two existing views rather than + * adding anything. A materialized view's SELECT is frozen at creation and the + * bundled DDL uses `CREATE ... IF NOT EXISTS`, so both views must be dropped + * before the v20 schema can install its versions. Dropping a view never touches + * rows already in its target table. + */ +const apply = async (context: MigrationModuleContext): Promise => { + await context.openTarget( + (db) => { + db.exec("DROP TABLE IF EXISTS error_events_mv") + db.exec("DROP TABLE IF EXISTS error_events_by_time_mv") + }, + { schemaSql: LOCAL_SCHEMA_V19_SQL, bootstrapSchema: false }, + ) + return context.openTarget(() => ({ installed: true }), { + schemaSql: LOCAL_SCHEMA_V20_SQL, + bootstrapSchema: true, + }) +} + +const verify = async ( + context: MigrationModuleContext, + state: V19ToV20State, + _progress: V19ToV20Progress, +): Promise => { + await context.openTarget( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V20_MANIFEST, state.retentionDays)) + const targetRows = rawRowCounts(db) + for (const table of RAW_TABLES) { + if (targetRows[table] !== state.rawRows[table]) + throw new RowCountMismatch({ + message: `v19 -> v20 raw telemetry verification failed for ${table}`, + moduleId: MODULE_ID, + table, + expected: state.rawRows[table] ?? "0", + actual: targetRows[table] ?? "0", + }) + } + }, + { schemaSql: LOCAL_SCHEMA_V20_SQL, bootstrapSchema: false }, + ) +} + +const operations: ReadonlyArray = [ + { + id: "clone-v19-store", + description: "Clone the stopped v19 store into the staged migration target", + requiresQuiescence: true, + phase: "target-created", + }, + { + id: "rebuild-error-events-views", + description: + "Drop and recreate the error-events views so an exception-less span is labelled from its exception.* / error.* attributes", + requiresQuiescence: true, + phase: "copying", + }, + { + id: "verify-v20-schema", + description: "Verify the v20 physical schema and retained raw telemetry counts", + requiresQuiescence: true, + phase: "copy-verified", + }, +] + +const dispositions: ReadonlyArray = [ + { + name: "local store", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: "The clean stopped v19 store is cloned byte-for-byte before the views are replaced.", + }, + { + name: "traces", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: + "The source of the replaced views is neither read nor rewritten; only the view definitions change.", + }, + { + // Rows already materialized keep their 'Unknown Error' label and hash — + // error_events holds no span attributes to re-derive them from, and + // recomputing hashes would re-bucket every existing issue. Forward-only, + // and bounded by the tables' 90-day TTL. + name: "error_events", + classification: "derived", + disposition: "preserve-exact", + guarantee: + "Existing rows are preserved untouched; the attribute fallback applies to events materialized after the migration and converges as the retention window rolls.", + preservationInterval: "error retention horizon", + sourceRetentionDays: 90, + targetRetentionDays: 90, + }, + { + name: "error_events_by_time", + classification: "derived", + disposition: "preserve-exact", + guarantee: + "Same projection as error_events and treated identically: preserved rows, forward-only correction.", + preservationInterval: "error retention horizon", + sourceRetentionDays: 90, + targetRetentionDays: 90, + }, +] + +export const v19ToV20ErrorEventsAttributeFallbackModule: LocalStoreMigrationModule< + V19ToV20State, + V19ToV20Progress +> = { + id: MODULE_ID, + moduleVersion: 1, + description: + "Rebuild the error-events views so an exception-less span is labelled from its exception.* / error.* attributes", + from: LOCAL_SCHEMA_V19, + to: LOCAL_SCHEMA_V20, + operations, + dispositions, + decodeState, + decodeProgress, + preflight, + prepareTarget, + apply, + verify, + recover: async (_context, state, progress) => ({ state, progress }), +} diff --git a/apps/cli/src/server/schema-identity.ts b/apps/cli/src/server/schema-identity.ts index 0252d2f06..f4e81a9a0 100644 --- a/apps/cli/src/server/schema-identity.ts +++ b/apps/cli/src/server/schema-identity.ts @@ -18,6 +18,7 @@ import schemaV16Sql from "./schema/local-schema-v16.sql" with { type: "text" } import schemaV17Sql from "./schema/local-schema-v17.sql" with { type: "text" } import schemaV18Sql from "./schema/local-schema-v18.sql" with { type: "text" } import schemaV19Sql from "./schema/local-schema-v19.sql" with { type: "text" } +import schemaV20Sql from "./schema/local-schema-v20.sql" with { type: "text" } import { schemaDigest as digestSchema, schemaFingerprint as fingerprintSchema } from "./store-version" import { buildLocalSchemaManifest, type LocalSchemaManifest } from "./schema-manifest" import { LOCAL_SCHEMA_VERSION } from "./local-schema-version" @@ -81,6 +82,7 @@ const SNAPSHOT_SQL: ReadonlyArray = [ schemaV17Sql, schemaV18Sql, schemaV19Sql, + schemaV20Sql, ] export interface LocalSchemaSnapshot { @@ -143,6 +145,8 @@ export const LOCAL_SCHEMA_V18_SQL = snapshotAt(18).sql export const LOCAL_SCHEMA_V18_MANIFEST = snapshotAt(18).manifest export const LOCAL_SCHEMA_V19_SQL = snapshotAt(19).sql export const LOCAL_SCHEMA_V19_MANIFEST = snapshotAt(19).manifest +export const LOCAL_SCHEMA_V20_SQL = snapshotAt(20).sql +export const LOCAL_SCHEMA_V20_MANIFEST = snapshotAt(20).manifest export interface LocalSchemaIdentity { readonly version: number @@ -192,6 +196,7 @@ export const LOCAL_SCHEMA_V16 = identityAt(16) export const LOCAL_SCHEMA_V17 = identityAt(17) export const LOCAL_SCHEMA_V18 = identityAt(18) export const LOCAL_SCHEMA_V19 = identityAt(19) +export const LOCAL_SCHEMA_V20 = identityAt(20) export const CURRENT_LOCAL_SCHEMA: LocalSchemaIdentity = Object.freeze({ version: LOCAL_SCHEMA_VERSION, diff --git a/apps/cli/src/server/schema/local-inserts.json b/apps/cli/src/server/schema/local-inserts.json index f5e667e87..c91d480d4 100644 --- a/apps/cli/src/server/schema/local-inserts.json +++ b/apps/cli/src/server/schema/local-inserts.json @@ -1,5 +1,5 @@ { - "projectRevision": "46001bd069c07ec8d4e47fd7aa9a5ed09b550f78ee28b93682a3d8242dc67766", + "projectRevision": "f87dd560d4607f017adc21f444f7bb1d2f4a4c988c6edbe9b44dab806978c87c", "orgPlaceholder": "__ORG__", "datasources": { "traces": { diff --git a/apps/cli/src/server/schema/local-schema-v20.sql b/apps/cli/src/server/schema/local-schema-v20.sql new file mode 100644 index 000000000..47786af04 --- /dev/null +++ b/apps/cli/src/server/schema/local-schema-v20.sql @@ -0,0 +1,2045 @@ +-- This file is generated by scripts/generate-clickhouse-schema-sql.ts +-- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. +-- projectRevision: f87dd560d4607f017adc21f444f7bb1d2f4a4c988c6edbe9b44dab806978c87c +-- localSchemaVersion: 20 + +CREATE TABLE IF NOT EXISTS ai_trace_index ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SessionId String, + VendorId LowCardinality(String), + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + Model LowCardinality(String), + AgentName LowCardinality(String), + ToolName LowCardinality(String), + SpanId String, + ParentSpanId String, + Duration UInt64, + IsError UInt8, + IsLlmCall UInt8, + IsToolCall UInt8, + Tokens Float64, + Cost Float64, + ResponseId String +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS alert_checks ( + OrgId LowCardinality(String), + RuleId String, + GroupKey String, + Timestamp DateTime64(3), + Status LowCardinality(String), + SignalType LowCardinality(String), + Comparator LowCardinality(String), + Threshold Float64, + ObservedValue Nullable(Float64), + SampleCount UInt32, + WindowMinutes UInt16, + WindowStart DateTime64(3), + WindowEnd DateTime64(3), + ConsecutiveBreaches UInt16, + ConsecutiveHealthy UInt16, + IncidentId Nullable(String), + IncidentTransition LowCardinality(String), + EvaluationDurationMs UInt32, + ErrorMessage Nullable(String), + ErrorCategory LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, RuleId, GroupKey, Timestamp) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS attribute_keys_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, Hour, AttributeKey) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS attribute_values_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeValue String, + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, AttributeKey, Hour, AttributeValue) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS audit_log ( + OrgId LowCardinality(String), + Id String, + OccurredAt DateTime64(3), + RecordedAt DateTime64(3), + ActorType LowCardinality(String), + UserId String, + ApiKeyId String, + ActorId String, + ActorLabel String, + AffectedUserId String, + Source LowCardinality(String), + Action LowCardinality(String), + Outcome LowCardinality(String), + DenialReason String, + ResourceType LowCardinality(String), + ResourceId String, + ChangedFields Array(String), + Changes String, + Metadata String, + RequestId String, + OriginIp String, + OriginCountry LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toYYYYMM(OccurredAt) +ORDER BY (OrgId, OccurredAt, Id) +TTL toDate(OccurredAt) + INTERVAL 2190 DAY; + +CREATE TABLE IF NOT EXISTS error_events ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, FingerprintHash, Timestamp) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events_by_time ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, FingerprintHash) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_fingerprints_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + FingerprintHash UInt64, + ServiceName SimpleAggregateFunction(anyLast, String), + ExceptionType SimpleAggregateFunction(anyLast, String), + ExceptionMessage SimpleAggregateFunction(anyLast, String), + ErrorLabel SimpleAggregateFunction(anyLast, String), + TopFrame SimpleAggregateFunction(anyLast, String), + OccurrenceCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime), + ServiceVersions SimpleAggregateFunction(groupUniqArrayArray, Array(String)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Minute) +ORDER BY (OrgId, Minute, FingerprintHash) +TTL Minute + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS identity_links ( + OrgId LowCardinality(String), + VisitorId String, + UserId String, + FirstSeen SimpleAggregateFunction(min, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY tuple() +ORDER BY (OrgId, VisitorId, UserId) +TTL toDate(FirstSeen) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS logs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TimestampTime DateTime, + TraceId String, + SpanId String, + TraceFlags UInt8, + SeverityText LowCardinality(String), + SeverityNumber UInt8, + ServiceName LowCardinality(String), + Body String, + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + LogAttributes Map(LowCardinality(String), String), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + LogAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(LogAttributes), mapValues(LogAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(TimestampTime) +ORDER BY (OrgId, toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) +TTL toDate(TimestampTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS logs_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SeverityText LowCardinality(String), + DeploymentEnv LowCardinality(String), + Count SimpleAggregateFunction(sum, UInt64), + SizeBytes SimpleAggregateFunction(sum, UInt64), + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS metric_catalog ( + OrgId LowCardinality(String), + Hour DateTime, + MetricType LowCardinality(String), + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription SimpleAggregateFunction(anyLast, String), + MetricUnit SimpleAggregateFunction(anyLast, String), + IsMonotonic SimpleAggregateFunction(anyLast, UInt8), + DataPointCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, MetricType, ServiceName, MetricName, Hour) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_exponential_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + Scale Int32, + ZeroCount UInt64, + PositiveOffset Int32, + PositiveBucketCounts Array(UInt64), + NegativeOffset Int32, + NegativeBucketCounts Array(UInt64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_gauge ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + BucketCounts Array(UInt64), + ExplicitBounds Array(Float64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_sum ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + AggregationTemporality Int32, + IsMonotonic Bool +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS product_events ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + Source LowCardinality(String) DEFAULT 'browser', + SessionId String DEFAULT '', + Seq UInt32 DEFAULT 0, + VisitorId String DEFAULT '', + UserId String DEFAULT '', + GroupId String DEFAULT '', + Kind LowCardinality(String), + EventName String, + Host LowCardinality(String) DEFAULT '', + PagePath String DEFAULT '', + Url String DEFAULT '', + ServiceName LowCardinality(String) DEFAULT '', + Attributes Map(String, String) DEFAULT map(), + TraceId String DEFAULT '', + SpanId String DEFAULT '', + INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4, + INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4, + INDEX idx_trace_id TraceId TYPE bloom_filter GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, VisitorId, SessionId, Seq) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_address_resolutions_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + ParentServerAddress String, + ResolvedTargetService LowCardinality(String), + DeploymentEnv LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_external_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + TargetType LowCardinality(String), + TargetSystem LowCardinality(String), + TargetName String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_children ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, ParentSpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DbNamespace LowCardinality(String), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_query_shapes_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + QueryKey String, + QueryLabel SimpleAggregateFunction(any, String), + SampleStatement SimpleAggregateFunction(any, String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedCount SimpleAggregateFunction(sum, Float64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSumMs SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace, QueryKey) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, TargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly_ingest ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount UInt64, + ErrorCount UInt64, + DurationSumMs Float64, + MaxDurationMs Float64, + SampledSpanCount UInt64, + UnsampledSpanCount UInt64, + SampleRateSum Float64 +) +ENGINE = Null; + +CREATE TABLE IF NOT EXISTS service_map_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64), + ClassifiedSpanCount SimpleAggregateFunction(sum, UInt64), + ServerSpanCount SimpleAggregateFunction(sum, UInt64), + RoutedSpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Hour, SpanName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64), + ClassifiedSpanCount SimpleAggregateFunction(sum, UInt64), + ServerSpanCount SimpleAggregateFunction(sum, UInt64), + RoutedSpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Minute, SpanName) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, Hour, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, Minute, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + ServiceName LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String), + CommitSha LowCardinality(String), + SampleRate Float64 DEFAULT 1, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_platforms_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + K8sCluster SimpleAggregateFunction(max, String), + K8sPodName SimpleAggregateFunction(max, String), + K8sDeploymentName SimpleAggregateFunction(max, String), + K8sStatefulSetName SimpleAggregateFunction(max, String), + K8sDaemonSetName SimpleAggregateFunction(max, String), + K8sNamespaceName SimpleAggregateFunction(max, String), + CloudPlatform SimpleAggregateFunction(max, String), + CloudProvider SimpleAggregateFunction(max, String), + FaasName SimpleAggregateFunction(max, String), + MapleSdkType SimpleAggregateFunction(max, String), + ProcessRuntimeName SimpleAggregateFunction(max, String), + SpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_usage ( + OrgId LowCardinality(String), + ServiceName LowCardinality(String), + Hour DateTime, + LogCount UInt64, + LogSizeBytes UInt64, + TraceCount UInt64, + TraceSizeBytes UInt64, + SumMetricCount UInt64, + SumMetricSizeBytes UInt64, + GaugeMetricCount UInt64, + GaugeMetricSizeBytes UInt64, + HistogramMetricCount UInt64, + HistogramMetricSizeBytes UInt64, + ExpHistogramMetricCount UInt64, + ExpHistogramMetricSizeBytes UInt64 +) +ENGINE = SummingMergeTree +ORDER BY (OrgId, ServiceName, Hour) +TTL Hour + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS session_events ( + OrgId LowCardinality(String), + SessionId String, + Timestamp DateTime64(9), + Seq UInt32 DEFAULT 0, + Type LowCardinality(String), + Url String DEFAULT '', + TraceId String DEFAULT '', + Level LowCardinality(String) DEFAULT '', + Message String DEFAULT '', + TargetSelector String DEFAULT '', + TargetText String DEFAULT '', + NetMethod LowCardinality(String) DEFAULT '', + NetUrl String DEFAULT '', + NetStatus UInt16 DEFAULT 0, + NetDurationMs UInt32 DEFAULT 0, + ErrorStack String DEFAULT '', + Attributes Map(String, String), + VisitorId String DEFAULT '', + UserId String DEFAULT '', + GroupId String DEFAULT '', + INDEX idx_type Type TYPE set(16) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, Timestamp, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replay_events ( + OrgId LowCardinality(String), + SessionId String, + ChunkSeq UInt32, + Timestamp DateTime64(9), + DurationMs UInt32 DEFAULT 0, + EventCount UInt32 DEFAULT 0, + ByteSize UInt32 DEFAULT 0, + Events String, + IsCheckpoint UInt8 DEFAULT 0 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, ChunkSeq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replays ( + OrgId LowCardinality(String), + SessionId String, + StartTime DateTime64(9), + EndTime Nullable(DateTime64(9)), + DurationMs Nullable(UInt32), + Status LowCardinality(String), + UserId String, + UrlInitial String, + UserAgent String, + BrowserName LowCardinality(String), + OsName LowCardinality(String), + DeviceType LowCardinality(String), + Country LowCardinality(String) DEFAULT '', + ServiceName LowCardinality(String), + PageViews UInt32 DEFAULT 0, + ClickCount UInt32 DEFAULT 0, + ErrorCount UInt32 DEFAULT 0, + TraceIds Array(String) DEFAULT [], + ResourceAttributes Map(LowCardinality(String), String), + Version UInt32, + VisitorId String DEFAULT '', + VisitorIsNew UInt8 DEFAULT 0, + UserEmail String DEFAULT '', + UserName String DEFAULT '', + GroupId String DEFAULT '', + GroupName String DEFAULT '', + UserTraits Map(String, String) DEFAULT map(), + Referrer String DEFAULT '', + ReferrerHost LowCardinality(String) DEFAULT '', + UtmSource LowCardinality(String) DEFAULT '', + UtmMedium LowCardinality(String) DEFAULT '', + UtmCampaign LowCardinality(String) DEFAULT '', + UtmTerm String DEFAULT '', + UtmContent String DEFAULT '', + Host LowCardinality(String) DEFAULT '', + EntryPath String DEFAULT '', + ExitPath String DEFAULT '', + Language LowCardinality(String) DEFAULT '', + LastActivityAt Nullable(DateTime64(9)) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(StartTime) +ORDER BY (OrgId, SessionId) +TTL toDate(StartTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS span_metrics_calls_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + SpanKind LowCardinality(String), + AttrFingerprint UInt64, + ResourceFingerprint UInt64, + StartTimeUnix DateTime64(9), + LastValue AggregateFunction(argMax, Float64, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS trace_detail_spans ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + ResourceAttributes Map(LowCardinality(String), String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS trace_list_mv ( + OrgId LowCardinality(String), + TraceId String, + Timestamp DateTime, + ServiceName LowCardinality(String), + SpanName String, + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + HttpMethod LowCardinality(String), + HttpRoute String, + HttpStatusCode LowCardinality(String), + DeploymentEnv LowCardinality(String), + HasError UInt8, + TraceState String, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + TraceState String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)), + LinksTraceId Array(String), + LinksSpanId Array(String), + LinksTraceState Array(String), + LinksAttributes Array(Map(LowCardinality(String), String)), + SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0), + IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp)) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + StatusCode LowCardinality(String), + IsEntryPoint UInt8, + DeploymentEnv LowCardinality(String), + WeightedCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSum SimpleAggregateFunction(sum, Float64), + WeightedErrorCount SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32), + DurationMin SimpleAggregateFunction(min, UInt64), + DurationMax SimpleAggregateFunction(max, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE MATERIALIZED VIEW IF NOT EXISTS ai_trace_index_mv TO ai_trace_index AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanAttributes['maple_ai.session.id'] AS SessionId, + SpanAttributes['maple_ai.vendor.id'] AS VendorId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + coalesce(nullIf(SpanAttributes['gen_ai.response.model'], ''), nullIf(SpanAttributes['gen_ai.request.model'], ''), nullIf(SpanAttributes['ai.response.model'], ''), nullIf(SpanAttributes['ai.model.id'], ''), SpanAttributes['llm.model_name']) AS Model, + coalesce(nullIf(SpanAttributes['gen_ai.agent.name'], ''), SpanAttributes['ai.telemetry.functionId']) AS AgentName, + coalesce(nullIf(SpanAttributes['gen_ai.tool.name'], ''), nullIf(SpanAttributes['ai.toolCall.name'], ''), SpanAttributes['tool.name']) AS ToolName, + SpanId, + ParentSpanId, + Duration, + toUInt8(((StatusCode = 'Error' OR SpanAttributes['error.type'] != '') OR SpanAttributes['gen_ai.response.status'] IN ('failed', 'error'))) AS IsError, + toUInt8((coalesce(nullIf(SpanAttributes['gen_ai.operation.name'], ''), multiIf(SpanAttributes['openinference.span.kind'] = 'LLM', 'chat', SpanAttributes['openinference.span.kind'] = 'TOOL', 'execute_tool', SpanAttributes['openinference.span.kind'] = 'AGENT', 'invoke_agent', SpanAttributes['openinference.span.kind'] = 'EMBEDDING', 'embeddings', SpanAttributes['openinference.span.kind'] = 'RETRIEVER', 'retrieval', '')) IN ('chat', 'generate_content', 'text_completion', 'fetch_response') OR (((coalesce(nullIf(SpanAttributes['gen_ai.operation.name'], ''), multiIf(SpanAttributes['openinference.span.kind'] = 'LLM', 'chat', SpanAttributes['openinference.span.kind'] = 'TOOL', 'execute_tool', SpanAttributes['openinference.span.kind'] = 'AGENT', 'invoke_agent', SpanAttributes['openinference.span.kind'] = 'EMBEDDING', 'embeddings', SpanAttributes['openinference.span.kind'] = 'RETRIEVER', 'retrieval', '')) NOT IN ('chat', 'generate_content', 'text_completion', 'fetch_response', 'embeddings', 'retrieval', 'execute_tool', 'invoke_agent', 'create_agent', 'invoke_workflow', 'plan', 'agent_step') AND NOT ((coalesce(nullIf(SpanAttributes['gen_ai.tool.name'], ''), nullIf(SpanAttributes['ai.toolCall.name'], ''), SpanAttributes['tool.name']) != '' OR lower(SpanName) LIKE '%tool%'))) AND NOT ((lower(SpanName) LIKE '%agent%' OR lower(SpanName) LIKE '%workflow%'))) AND (coalesce(nullIf(SpanAttributes['gen_ai.response.model'], ''), nullIf(SpanAttributes['gen_ai.request.model'], ''), nullIf(SpanAttributes['ai.response.model'], ''), nullIf(SpanAttributes['ai.model.id'], ''), SpanAttributes['llm.model_name']) != '' OR (lower(SpanName) LIKE '%chat%' OR lower(SpanName) LIKE '%completion%'))))) AS IsLlmCall, + toUInt8((coalesce(nullIf(SpanAttributes['gen_ai.operation.name'], ''), multiIf(SpanAttributes['openinference.span.kind'] = 'LLM', 'chat', SpanAttributes['openinference.span.kind'] = 'TOOL', 'execute_tool', SpanAttributes['openinference.span.kind'] = 'AGENT', 'invoke_agent', SpanAttributes['openinference.span.kind'] = 'EMBEDDING', 'embeddings', SpanAttributes['openinference.span.kind'] = 'RETRIEVER', 'retrieval', '')) IN ('execute_tool') OR (coalesce(nullIf(SpanAttributes['gen_ai.operation.name'], ''), multiIf(SpanAttributes['openinference.span.kind'] = 'LLM', 'chat', SpanAttributes['openinference.span.kind'] = 'TOOL', 'execute_tool', SpanAttributes['openinference.span.kind'] = 'AGENT', 'invoke_agent', SpanAttributes['openinference.span.kind'] = 'EMBEDDING', 'embeddings', SpanAttributes['openinference.span.kind'] = 'RETRIEVER', 'retrieval', '')) NOT IN ('chat', 'generate_content', 'text_completion', 'fetch_response', 'embeddings', 'retrieval', 'execute_tool', 'invoke_agent', 'create_agent', 'invoke_workflow', 'plan', 'agent_step') AND (coalesce(nullIf(SpanAttributes['gen_ai.tool.name'], ''), nullIf(SpanAttributes['ai.toolCall.name'], ''), SpanAttributes['tool.name']) != '' OR lower(SpanName) LIKE '%tool%')))) AS IsToolCall, + multiIf(SpanAttributes['maple_ai.vendor.id'] IN ('vercel_ai_sdk', 'maple'), greatest(toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.prompt_tokens'], ''), nullIf(SpanAttributes['ai.usage.inputTokens'], ''), nullIf(SpanAttributes['ai.usage.promptTokens'], ''), SpanAttributes['llm.token_count.prompt'])), toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.cache_read.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.input_tokens.cached'], ''), nullIf(SpanAttributes['ai.usage.cachedInputTokens'], ''), nullIf(SpanAttributes['ai.usage.inputTokenDetails.cacheReadTokens'], ''), SpanAttributes['llm.token_count.prompt_details.cache_read'])) + toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.cache_creation.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.cache_write.input_tokens'], ''), SpanAttributes['ai.usage.inputTokenDetails.cacheWriteTokens']))), coalesce(nullIf(SpanAttributes['gen_ai.provider.name'], ''), nullIf(SpanAttributes['gen_ai.system'], ''), nullIf(SpanAttributes['ai.model.provider'], ''), nullIf(SpanAttributes['llm.provider'], ''), SpanAttributes['llm.system']) IN ('openai', 'gcp.gemini', 'gemini', 'gcp.vertex_ai', 'vertex_ai', 'openrouter'), greatest(toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.prompt_tokens'], ''), nullIf(SpanAttributes['ai.usage.inputTokens'], ''), nullIf(SpanAttributes['ai.usage.promptTokens'], ''), SpanAttributes['llm.token_count.prompt'])), toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.cache_read.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.input_tokens.cached'], ''), nullIf(SpanAttributes['ai.usage.cachedInputTokens'], ''), nullIf(SpanAttributes['ai.usage.inputTokenDetails.cacheReadTokens'], ''), SpanAttributes['llm.token_count.prompt_details.cache_read'])) + toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.cache_creation.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.cache_write.input_tokens'], ''), SpanAttributes['ai.usage.inputTokenDetails.cacheWriteTokens']))), coalesce(nullIf(SpanAttributes['gen_ai.provider.name'], ''), nullIf(SpanAttributes['gen_ai.system'], ''), nullIf(SpanAttributes['ai.model.provider'], ''), nullIf(SpanAttributes['llm.provider'], ''), SpanAttributes['llm.system']) IN ('anthropic'), toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.prompt_tokens'], ''), nullIf(SpanAttributes['ai.usage.inputTokens'], ''), nullIf(SpanAttributes['ai.usage.promptTokens'], ''), SpanAttributes['llm.token_count.prompt'])) + toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.cache_read.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.input_tokens.cached'], ''), nullIf(SpanAttributes['ai.usage.cachedInputTokens'], ''), nullIf(SpanAttributes['ai.usage.inputTokenDetails.cacheReadTokens'], ''), SpanAttributes['llm.token_count.prompt_details.cache_read'])) + toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.cache_creation.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.cache_write.input_tokens'], ''), SpanAttributes['ai.usage.inputTokenDetails.cacheWriteTokens'])), greatest(toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.prompt_tokens'], ''), nullIf(SpanAttributes['ai.usage.inputTokens'], ''), nullIf(SpanAttributes['ai.usage.promptTokens'], ''), SpanAttributes['llm.token_count.prompt'])), toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.cache_read.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.input_tokens.cached'], ''), nullIf(SpanAttributes['ai.usage.cachedInputTokens'], ''), nullIf(SpanAttributes['ai.usage.inputTokenDetails.cacheReadTokens'], ''), SpanAttributes['llm.token_count.prompt_details.cache_read'])) + toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.cache_creation.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.cache_write.input_tokens'], ''), SpanAttributes['ai.usage.inputTokenDetails.cacheWriteTokens'])))) + multiIf(SpanAttributes['maple_ai.vendor.id'] IN ('vercel_ai_sdk', 'maple'), greatest(toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.output_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.completion_tokens'], ''), nullIf(SpanAttributes['ai.usage.outputTokens'], ''), nullIf(SpanAttributes['ai.usage.completionTokens'], ''), SpanAttributes['llm.token_count.completion'])), toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.reasoning.output_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.output_tokens.reasoning'], ''), nullIf(SpanAttributes['ai.usage.reasoningTokens'], ''), nullIf(SpanAttributes['ai.usage.outputTokenDetails.reasoningTokens'], ''), SpanAttributes['llm.token_count.completion_details.reasoning']))), coalesce(nullIf(SpanAttributes['gen_ai.provider.name'], ''), nullIf(SpanAttributes['gen_ai.system'], ''), nullIf(SpanAttributes['ai.model.provider'], ''), nullIf(SpanAttributes['llm.provider'], ''), SpanAttributes['llm.system']) IN ('anthropic', 'openai', 'openrouter'), greatest(toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.output_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.completion_tokens'], ''), nullIf(SpanAttributes['ai.usage.outputTokens'], ''), nullIf(SpanAttributes['ai.usage.completionTokens'], ''), SpanAttributes['llm.token_count.completion'])), toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.reasoning.output_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.output_tokens.reasoning'], ''), nullIf(SpanAttributes['ai.usage.reasoningTokens'], ''), nullIf(SpanAttributes['ai.usage.outputTokenDetails.reasoningTokens'], ''), SpanAttributes['llm.token_count.completion_details.reasoning']))), coalesce(nullIf(SpanAttributes['gen_ai.provider.name'], ''), nullIf(SpanAttributes['gen_ai.system'], ''), nullIf(SpanAttributes['ai.model.provider'], ''), nullIf(SpanAttributes['llm.provider'], ''), SpanAttributes['llm.system']) IN ('gcp.gemini', 'gemini', 'gcp.vertex_ai', 'vertex_ai'), toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.output_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.completion_tokens'], ''), nullIf(SpanAttributes['ai.usage.outputTokens'], ''), nullIf(SpanAttributes['ai.usage.completionTokens'], ''), SpanAttributes['llm.token_count.completion'])) + toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.reasoning.output_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.output_tokens.reasoning'], ''), nullIf(SpanAttributes['ai.usage.reasoningTokens'], ''), nullIf(SpanAttributes['ai.usage.outputTokenDetails.reasoningTokens'], ''), SpanAttributes['llm.token_count.completion_details.reasoning'])), greatest(toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.output_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.completion_tokens'], ''), nullIf(SpanAttributes['ai.usage.outputTokens'], ''), nullIf(SpanAttributes['ai.usage.completionTokens'], ''), SpanAttributes['llm.token_count.completion'])), toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.reasoning.output_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.output_tokens.reasoning'], ''), nullIf(SpanAttributes['ai.usage.reasoningTokens'], ''), nullIf(SpanAttributes['ai.usage.outputTokenDetails.reasoningTokens'], ''), SpanAttributes['llm.token_count.completion_details.reasoning'])))) AS Tokens, + toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.cost'], ''), nullIf(SpanAttributes['gen_ai.usage.total_cost'], ''), SpanAttributes['llm.cost.total'])) AS Cost, + coalesce(nullIf(SpanAttributes['gen_ai.response.id'], ''), SpanAttributes['ai.response.id']) AS ResponseId + FROM traces + WHERE SpanAttributes['maple_ai.vendor.id'] != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + -- Only fill the old Unknown Error bucket. Event values (including + -- empty fields) and spans with StatusMessage keep every hash input. + _ei = 0 AND StatusMessage = '' AS _useAttrs, + if( + _ei > 0, EventsAttributes[_ei]['exception.type'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '') + ) AS _exType, + if( + _ei > 0, EventsAttributes[_ei]['exception.message'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage) + ) AS _exMsg, + if( + _ei > 0, EventsAttributes[_ei]['exception.stacktrace'], + if(_useAttrs, SpanAttributes['exception.stacktrace'], '') + ) AS _exStack, + if(_useAttrs, _exMsg, StatusMessage) AS _msgText, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arraySlice( + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection for the message signature below. + isValidJSON(_msgText) AS _isJson, + _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(_msgText) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + 1, 120 + ) + ) AS _msgSig, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'), + JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'), + JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'), + JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'), + JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'), + JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'), + JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + _msgText = '', 'Unknown Error', + position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0, + if( + extract(_msgText, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(_msgText, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel, + left(_msgText, multiIf( + position(_msgText, ': ') > 3, toInt64(position(_msgText, ': ')) - 1, + position(_msgText, ' (') > 3, toInt64(position(_msgText, ' (')) - 1, + position(_msgText, '\n') > 3, toInt64(position(_msgText, '\n')) - 1, + least(toInt64(length(_msgText)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all hold: 4xx, no exception event, no exception.type + -- attribute, and no error.type beyond the status code itself (HTTP + -- semconv sets error.type to the bare status on a non-2xx response, + -- which carries no exception). 5xx and anything carrying a real + -- exception still count, and SpanKind is deliberately not consulted — + -- these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND SpanAttributes['exception.type'] = '' + AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus)) + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + -- Only fill the old Unknown Error bucket. Event values (including + -- empty fields) and spans with StatusMessage keep every hash input. + _ei = 0 AND StatusMessage = '' AS _useAttrs, + if( + _ei > 0, EventsAttributes[_ei]['exception.type'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '') + ) AS _exType, + if( + _ei > 0, EventsAttributes[_ei]['exception.message'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage) + ) AS _exMsg, + if( + _ei > 0, EventsAttributes[_ei]['exception.stacktrace'], + if(_useAttrs, SpanAttributes['exception.stacktrace'], '') + ) AS _exStack, + if(_useAttrs, _exMsg, StatusMessage) AS _msgText, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arraySlice( + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection for the message signature below. + isValidJSON(_msgText) AS _isJson, + _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(_msgText) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + 1, 120 + ) + ) AS _msgSig, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'), + JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'), + JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'), + JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'), + JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'), + JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'), + JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + _msgText = '', 'Unknown Error', + position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0, + if( + extract(_msgText, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(_msgText, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel, + left(_msgText, multiIf( + position(_msgText, ': ') > 3, toInt64(position(_msgText, ': ')) - 1, + position(_msgText, ' (') > 3, toInt64(position(_msgText, ' (')) - 1, + position(_msgText, '\n') > 3, toInt64(position(_msgText, '\n')) - 1, + least(toInt64(length(_msgText)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all hold: 4xx, no exception event, no exception.type + -- attribute, and no error.type beyond the status code itself (HTTP + -- semconv sets error.type to the bare status on a non-2xx response, + -- which carries no exception). 5xx and anything carrying a real + -- exception still count, and SpanKind is deliberately not consulted — + -- these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND SpanAttributes['exception.type'] = '' + AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus)) + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS +SELECT + OrgId, + toStartOfMinute(Timestamp) AS Minute, + FingerprintHash, + anyLast(ServiceName) AS ServiceName, + anyLast(ExceptionType) AS ExceptionType, + anyLast(ExceptionMessage) AS ExceptionMessage, + anyLast(ErrorLabel) AS ErrorLabel, + anyLast(TopFrame) AS TopFrame, + count() AS OccurrenceCount, + min(Timestamp) AS FirstSeen, + max(Timestamp) AS LastSeen, + -- Distinct builds, not a sample: see ServiceVersions on the datasource. + groupUniqArray(ServiceVersion) AS ServiceVersions + FROM error_events + GROUP BY OrgId, Minute, FingerprintHash; + +CREATE MATERIALIZED VIEW IF NOT EXISTS identity_links_mv TO identity_links AS +SELECT + OrgId, + VisitorId, + UserId, + StartTime AS FirstSeen + FROM session_replays + WHERE VisitorId != '' AND UserId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(LogAttributes)) AS AttributeKey, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + WHERE LogAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + ARRAY JOIN + mapKeys(LogAttributes) AS AttributeKey, + mapValues(LogAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS logs_aggregates_hourly_mv TO logs_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(TimestampTime) AS Hour, + ServiceName, + SeverityText, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS Count, + sum(length(Body) + 200) AS SizeBytes, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM logs + GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + arrayJoin(mapKeys(Attributes)) AS AttributeKey, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + WHERE Attributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + AttributeKey, + AttributeValue, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + ARRAY JOIN + mapKeys(Attributes) AS AttributeKey, + mapValues(Attributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_exp_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'exponential_histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_exponential_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_gauge_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'gauge' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_gauge + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'sum' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + anyLast(toUInt8(IsMonotonic)) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_sum + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_mv TO product_events AS +SELECT + OrgId, + Timestamp, + 'browser' AS Source, + SessionId, + Seq, + VisitorId, + UserId, + GroupId, + Type AS Kind, + if(Type = 'navigation', '$pageview', Message) AS EventName, + domain(Url) AS Host, + path(Url) AS PagePath, + Url, + '' AS ServiceName, + Attributes, + '' AS TraceId, + '' AS SpanId + FROM session_events + WHERE Type IN ('navigation', 'custom'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_traces_mv TO product_events AS +SELECT + OrgId, + Timestamp, + 'trace' AS Source, + SpanAttributes['session.id'] AS SessionId, + 0 AS Seq, + SpanAttributes['maple.product_event.visitor_id'] AS VisitorId, + SpanAttributes['maple.product_event.user_id'] AS UserId, + SpanAttributes['maple.product_event.group_id'] AS GroupId, + 'custom' AS Kind, + SpanAttributes['maple.product_event.name'] AS EventName, + domain(SpanAttributes['maple.product_event.url']) AS Host, + path(SpanAttributes['maple.product_event.url']) AS PagePath, + SpanAttributes['maple.product_event.url'] AS Url, + ServiceName, + mapUpdate( + CAST( + mapFilter( + (k, v) -> NOT startsWith(k, 'maple.product_event.') + AND ( + NOT has(mapKeys(SpanAttributes), 'maple.product_event.include') + OR has( + arrayMap( + key -> trimBoth(key), + splitByChar(',', SpanAttributes['maple.product_event.include']) + ), + k + ) + ), + SpanAttributes + ), + 'Map(String, String)' + ), + mapApply( + (k, v) -> (substring(k, 26), v), + mapFilter((k, v) -> startsWith(k, 'maple.product_event.prop.'), SpanAttributes) + ) + ) AS Attributes, + TraceId, + SpanId + FROM traces + WHERE SpanAttributes['maple.product_event.name'] != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', 'messaging', + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc', + 'http' + ) AS TargetType, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'], + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'], + '' + ) AS TargetSystem, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', + if(coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '', coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']), SpanAttributes['messaging.system']), + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', + if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']), + if(SpanAttributes['server.address'] != '', + SpanAttributes['server.address'], + if(SpanAttributes['http.host'] != '', + SpanAttributes['http.host'], + SpanAttributes['url.authority'])) + ) AS TargetName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + sum(SampleRate) AS SampleRateSum, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND SpanAttributes['db.system.name'] = '' + AND ServiceName != '' + AND ( + SpanAttributes['server.address'] != '' + OR SpanAttributes['http.host'] != '' + OR SpanAttributes['url.authority'] != '' + OR coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' + OR SpanAttributes['messaging.system'] != '' + OR SpanAttributes['rpc.service'] != '' + OR SpanAttributes['rpc.system'] != '' + ) + GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv + HAVING TargetName != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') + AND ParentSpanId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + countIf(TraceState LIKE '%th:%') AS SampledSpanCount, + countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount, + sum(SampleRate) AS SampleRateSum, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_query_shapes_hourly_mv TO service_map_db_query_shapes_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + coalesce( + nullIf(SpanAttributes['db.query.fingerprint'], ''), + nullIf(SpanAttributes['db.statement.fingerprint'], ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\'[^\']*\'', '?'), '\\bin\\s*\\([^)]*\\)', 'in (?)'), '[0-9]+(\\.[0-9]+)?', '?'), '\\s+', ' '), '^\\s+|\\s+$', ''))), ''), ''), + toString(cityHash64(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +))) +) AS QueryKey, + any(substring(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +), 1, 220)) AS QueryLabel, + any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(SampleRate) AS EstimatedCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv, QueryKey; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_edges_hourly_ingest_mv TO service_map_edges_hourly AS +SELECT + OrgId, + Hour, + SourceService, + TargetService, + DeploymentEnv, + CallCount, + ErrorCount, + DurationSumMs, + MaxDurationMs, + SampledSpanCount, + UnsampledSpanCount, + SampleRateSum + FROM service_map_edges_hourly_ingest; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_spans_mv TO service_map_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_hourly_mv TO service_operations_hourly AS +SELECT + OrgId, + toStartOfHour(Minute) AS Hour, + ServiceName, + DeploymentEnv, + SpanName, + sum(SpanCount) AS SpanCount, + sum(EstimatedSpanCount) AS EstimatedSpanCount, + sum(ErrorCount) AS ErrorCount, + sum(EstimatedErrorCount) AS EstimatedErrorCount, + sum(DurationSum) AS DurationSum, + quantilesTDigestMergeState(0.5, 0.95)(DurationQuantiles) AS DurationQuantiles, + sum(ClassifiedSpanCount) AS ClassifiedSpanCount, + sum(ServerSpanCount) AS ServerSpanCount, + sum(RoutedSpanCount) AS RoutedSpanCount + FROM service_operations_minutely + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_minutely_mv TO service_operations_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles, + count() AS ClassifiedSpanCount, + countIf(SpanKind IN ('Server', 'Consumer')) AS ServerSpanCount, + countIf(SpanAttributes['http.route'] != '') AS RoutedSpanCount + FROM traces + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_hourly_mv TO service_overview_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['vcs.ref.head.revision'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_minutely_mv TO service_overview_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['vcs.ref.head.revision'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['vcs.ref.head.revision'] AS CommitSha, + SampleRate, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster, + max(ResourceAttributes['k8s.pod.name']) AS K8sPodName, + max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName, + max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName, + max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName, + max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName, + max(ResourceAttributes['cloud.platform']) AS CloudPlatform, + max(ResourceAttributes['cloud.provider']) AS CloudProvider, + max(ResourceAttributes['faas.name']) AS FaasName, + max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType, + max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName, + count() AS SpanCount + FROM traces + WHERE ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_logs_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(TimestampTime) AS Hour, + count() AS LogCount, + sum(length(Body) + 200) AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM logs + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_exp_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + count() AS ExpHistogramMetricCount, + count() * 300 AS ExpHistogramMetricSizeBytes + FROM metrics_exponential_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_gauge_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + count() AS GaugeMetricCount, + count() * 150 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_gauge + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + count() AS HistogramMetricCount, + count() * 250 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_sum_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + count() AS SumMetricCount, + count() * 150 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_sum + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_traces_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS TraceCount, + sum(length(SpanName) + 300) AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM traces + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS span_metrics_calls_hourly_mv TO span_metrics_calls_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + ServiceName, + MetricName, + Attributes['span.kind'] AS SpanKind, + cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint, + cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint, + StartTimeUnix, + argMaxState(Value, TimeUnix) AS LastValue + FROM metrics_sum + -- 'traces.span.metrics.calls' is the name the collector actually emits: + -- spanmetricsconnector output is namespaced by the pipeline it is attached + -- to. Without it this MV matched nothing and the target sat at 0 rows since + -- it was created, while ~880k rows / 2 days of the real counter flowed past + -- into metrics_sum and every read fell back to the raw window-function scan + -- (~7s p95 -- see queries/metrics.ts). Keep this list in sync with + -- SPAN_METRICS_CALLS_NAMES on the read side. + WHERE MetricName IN ('span.metrics.calls', 'calls', 'traces.span.metrics.calls') AND IsMonotonic + GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_detail_spans_mv TO trace_detail_spans AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanId, + ParentSpanId, + SpanName, + SpanKind, + ServiceName, + Duration, + StatusCode, + StatusMessage, + SpanAttributes, + ResourceAttributes + FROM traces; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_list_mv_mv TO trace_list_mv AS +SELECT + OrgId, + TraceId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + if( + (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS')) + AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''), + concat( + if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), + ' ', + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path']) + ), + SpanName + ) AS SpanName, + SpanKind, + Duration, + StatusCode, + if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod, + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute, + if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + toUInt8( + StatusCode = 'Error' + OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500) + OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500) + ) AS HasError, + TraceState, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(ResourceAttributes)) AS AttributeKey, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE ResourceAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(ResourceAttributes) AS AttributeKey, + mapValues(ResourceAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(SpanAttributes)) AS AttributeKey, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE SpanAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(SpanAttributes) AS AttributeKey, + mapValues(SpanAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS traces_aggregates_hourly_mv TO traces_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + SpanName, + SpanKind, + StatusCode, + IsEntryPoint, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + sum(SampleRate) AS WeightedCount, + sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum, + sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount, + quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles, + min(Duration) AS DurationMin, + max(Duration) AS DurationMax + FROM traces + GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv; diff --git a/apps/cli/src/server/schema/local-schema.sql b/apps/cli/src/server/schema/local-schema.sql index e4acfcaa6..47786af04 100644 --- a/apps/cli/src/server/schema/local-schema.sql +++ b/apps/cli/src/server/schema/local-schema.sql @@ -1,7 +1,7 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: 46001bd069c07ec8d4e47fd7aa9a5ed09b550f78ee28b93682a3d8242dc67766 --- localSchemaVersion: 19 +-- projectRevision: f87dd560d4607f017adc21f444f7bb1d2f4a4c988c6edbe9b44dab806978c87c +-- localSchemaVersion: 20 CREATE TABLE IF NOT EXISTS ai_trace_index ( OrgId LowCardinality(String), @@ -948,9 +948,22 @@ SELECT CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS WITH arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, - if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, - if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, - if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + -- Only fill the old Unknown Error bucket. Event values (including + -- empty fields) and spans with StatusMessage keep every hash input. + _ei = 0 AND StatusMessage = '' AS _useAttrs, + if( + _ei > 0, EventsAttributes[_ei]['exception.type'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '') + ) AS _exType, + if( + _ei > 0, EventsAttributes[_ei]['exception.message'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage) + ) AS _exMsg, + if( + _ei > 0, EventsAttributes[_ei]['exception.stacktrace'], + if(_useAttrs, SpanAttributes['exception.stacktrace'], '') + ) AS _exStack, + if(_useAttrs, _exMsg, StatusMessage) AS _msgText, -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old -- rule accepted any line with a colon-digit, which let non-frame lines -- in: Drizzle's `params: ` line, and the `Type: message` @@ -983,8 +996,8 @@ WITH if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, arrayStringConcat(_topFrames, '\n') AS _fpFrames, -- JSON detection for the message signature below. - isValidJSON(StatusMessage) AS _isJson, - _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + isValidJSON(_msgText) AS _isJson, + _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj, -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level -- keys, redact volatile tokens (long hex / numbers) in each raw value, then -- sort by "key=value" so key order & whitespace don't matter. No assumption @@ -994,7 +1007,7 @@ WITH arraySort( arrayMap( kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), - JSONExtractKeysAndValuesRaw(StatusMessage) + JSONExtractKeysAndValuesRaw(_msgText) ) ), '|' @@ -1012,7 +1025,7 @@ WITH multiIf( _isJsonObj, _jsonSig, substringUTF8( - replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), 1, 120 ) ) AS _msgSig, @@ -1020,29 +1033,29 @@ WITH -- many labels may map to one hash). The broad key list here is a DISPLAY -- heuristic only; the fingerprint above makes no key-name assumption. multiIf( - JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), - JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), - JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), - JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), - JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), - JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), - JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'), + JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'), + JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'), + JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'), + JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'), + JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'), + JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'), 'JSON error' ) AS _jsonLabel, multiIf( - StatusMessage = '', 'Unknown Error', - position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + _msgText = '', 'Unknown Error', + position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0, if( - extract(StatusMessage, 'readonly (\\w+)') != '', - concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + extract(_msgText, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(_msgText, 'readonly (\\w+)')), 'Schema parse error' ), - _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, - left(StatusMessage, multiIf( - position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, - position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, - position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, - least(toInt64(length(StatusMessage)), 150) + _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel, + left(_msgText, multiIf( + position(_msgText, ': ') > 3, toInt64(position(_msgText, ': ')) - 1, + position(_msgText, ' (') > 3, toInt64(position(_msgText, ' (')) - 1, + position(_msgText, '\n') > 3, toInt64(position(_msgText, '\n')) - 1, + least(toInt64(length(_msgText)), 150) )) ) AS _statusLabel, if(_exType != '', _exType, _statusLabel) AS _errorLabel, @@ -1076,21 +1089,38 @@ WITH -- Client-side runtimes (notably the native Cloudflare Workers -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a - -- span only when all three hold: 4xx, no exception event, and no - -- exception type. 5xx and anything carrying an exception still count, - -- and SpanKind is deliberately not consulted — these are Client spans. + -- span only when all hold: 4xx, no exception event, no exception.type + -- attribute, and no error.type beyond the status code itself (HTTP + -- semconv sets error.type to the bare status on a non-2xx response, + -- which carries no exception). 5xx and anything carrying a real + -- exception still count, and SpanKind is deliberately not consulted — + -- these are Client spans. AND NOT ( _httpStatus >= 400 AND _httpStatus < 500 AND _ei = 0 - AND _exType = '' + AND SpanAttributes['exception.type'] = '' + AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus)) ); CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS WITH arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, - if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, - if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, - if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + -- Only fill the old Unknown Error bucket. Event values (including + -- empty fields) and spans with StatusMessage keep every hash input. + _ei = 0 AND StatusMessage = '' AS _useAttrs, + if( + _ei > 0, EventsAttributes[_ei]['exception.type'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '') + ) AS _exType, + if( + _ei > 0, EventsAttributes[_ei]['exception.message'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage) + ) AS _exMsg, + if( + _ei > 0, EventsAttributes[_ei]['exception.stacktrace'], + if(_useAttrs, SpanAttributes['exception.stacktrace'], '') + ) AS _exStack, + if(_useAttrs, _exMsg, StatusMessage) AS _msgText, -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old -- rule accepted any line with a colon-digit, which let non-frame lines -- in: Drizzle's `params: ` line, and the `Type: message` @@ -1123,8 +1153,8 @@ WITH if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, arrayStringConcat(_topFrames, '\n') AS _fpFrames, -- JSON detection for the message signature below. - isValidJSON(StatusMessage) AS _isJson, - _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + isValidJSON(_msgText) AS _isJson, + _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj, -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level -- keys, redact volatile tokens (long hex / numbers) in each raw value, then -- sort by "key=value" so key order & whitespace don't matter. No assumption @@ -1134,7 +1164,7 @@ WITH arraySort( arrayMap( kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), - JSONExtractKeysAndValuesRaw(StatusMessage) + JSONExtractKeysAndValuesRaw(_msgText) ) ), '|' @@ -1152,7 +1182,7 @@ WITH multiIf( _isJsonObj, _jsonSig, substringUTF8( - replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), 1, 120 ) ) AS _msgSig, @@ -1160,29 +1190,29 @@ WITH -- many labels may map to one hash). The broad key list here is a DISPLAY -- heuristic only; the fingerprint above makes no key-name assumption. multiIf( - JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), - JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), - JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), - JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), - JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), - JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), - JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'), + JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'), + JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'), + JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'), + JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'), + JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'), + JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'), 'JSON error' ) AS _jsonLabel, multiIf( - StatusMessage = '', 'Unknown Error', - position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + _msgText = '', 'Unknown Error', + position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0, if( - extract(StatusMessage, 'readonly (\\w+)') != '', - concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + extract(_msgText, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(_msgText, 'readonly (\\w+)')), 'Schema parse error' ), - _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, - left(StatusMessage, multiIf( - position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, - position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, - position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, - least(toInt64(length(StatusMessage)), 150) + _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel, + left(_msgText, multiIf( + position(_msgText, ': ') > 3, toInt64(position(_msgText, ': ')) - 1, + position(_msgText, ' (') > 3, toInt64(position(_msgText, ' (')) - 1, + position(_msgText, '\n') > 3, toInt64(position(_msgText, '\n')) - 1, + least(toInt64(length(_msgText)), 150) )) ) AS _statusLabel, if(_exType != '', _exType, _statusLabel) AS _errorLabel, @@ -1216,13 +1246,17 @@ WITH -- Client-side runtimes (notably the native Cloudflare Workers -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a - -- span only when all three hold: 4xx, no exception event, and no - -- exception type. 5xx and anything carrying an exception still count, - -- and SpanKind is deliberately not consulted — these are Client spans. + -- span only when all hold: 4xx, no exception event, no exception.type + -- attribute, and no error.type beyond the status code itself (HTTP + -- semconv sets error.type to the bare status on a non-2xx response, + -- which carries no exception). 5xx and anything carrying a real + -- exception still count, and SpanKind is deliberately not consulted — + -- these are Client spans. AND NOT ( _httpStatus >= 400 AND _httpStatus < 500 AND _ei = 0 - AND _exType = '' + AND SpanAttributes['exception.type'] = '' + AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus)) ); CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS diff --git a/apps/cli/test/local-store-migrations.test.ts b/apps/cli/test/local-store-migrations.test.ts index b8911f47e..c1f045705 100644 --- a/apps/cli/test/local-store-migrations.test.ts +++ b/apps/cli/test/local-store-migrations.test.ts @@ -18,22 +18,13 @@ import { LOCAL_SCHEMA_V5_MANIFEST, LOCAL_SCHEMA_V7_MANIFEST, LOCAL_SCHEMA_V6, - LOCAL_SCHEMA_V7, - LOCAL_SCHEMA_V8, LOCAL_SCHEMA_V10, LOCAL_SCHEMA_V10_MANIFEST, LOCAL_SCHEMA_V11, LOCAL_SCHEMA_V11_MANIFEST, - LOCAL_SCHEMA_V12, LOCAL_SCHEMA_V12_MANIFEST, - LOCAL_SCHEMA_V13, LOCAL_SCHEMA_V13_MANIFEST, - LOCAL_SCHEMA_V14, - LOCAL_SCHEMA_V15, - LOCAL_SCHEMA_V16, - LOCAL_SCHEMA_V17, - LOCAL_SCHEMA_V18, - LOCAL_SCHEMA_V19, + LOCAL_SCHEMA_V20, SCHEMA_DIGEST, SCHEMA_FINGERPRINT, } from "../src/server/schema-identity" @@ -75,22 +66,21 @@ import { type RawReplayProgress, } from "../src/server/local-store-migrations/legacy-to-current" import { v10ToV11ProductEventsModule } from "../src/server/local-store-migrations/v10-to-v11-product-events" -import { v11ToV12ServiceMapEdgeQuantilesModule } from "../src/server/local-store-migrations/v11-to-v12-service-map-edge-quantiles" import { mkdir, mkdtemp, rm } from "node:fs/promises" import { tmpdir } from "node:os" import { join } from "node:path" describe("current local schema identity", () => { - it("matches the generated v19 revision and keeps the issue-297 identity frozen", () => { - expect(SCHEMA_FINGERPRINT).toBe("778888eceae9d6b7") - expect(SCHEMA_DIGEST).toBe("778888eceae9d6b717004bf2b7cf8b0db84fb0d65c81e2cb72d7cb39a9c3bd74") + it("matches the generated v20 revision and keeps the issue-297 identity frozen", () => { + expect(SCHEMA_FINGERPRINT).toBe("ad8e854c9e2bb021") + expect(SCHEMA_DIGEST).toBe("ad8e854c9e2bb02184ace30e6b6eb483c978626f8a452f54293ee66c358ad1c5") expect(ISSUE_297_TARGET_SCHEMA_PROJECT_REVISION).toBe( "506bc745f7a7eca202ec905a6403a6815e86413faf0cd3cbbf73881023edce91", ) expect(CURRENT_SCHEMA_PROJECT_REVISION).toMatch(/^[0-9a-f]{64}$/) expect(LOCAL_SCHEMA_MANIFEST.objects.length).toBeGreaterThan(60) - expect(CURRENT_LOCAL_SCHEMA.version).toBe(19) - expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V19) + expect(CURRENT_LOCAL_SCHEMA.version).toBe(20) + expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V20) const logs = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === "logs") expect(logs?.columns.some((column) => column.name.startsWith("idx_"))).toBe(false) expect(logs?.indexes).toContain("idx_lower_body") @@ -328,6 +318,7 @@ describe("local migration registry", () => { "local-0016-to-0017-audit-log", "local-0017-to-0018-product-events-from-traces", "local-0018-to-0019-ai-trace-index-usage-conventions", + "local-0019-to-0020-error-events-attribute-fallback", ]) expect(chain[0]?.from.fingerprint).toBe(LEGACY_SCHEMA_FINGERPRINT) expect(chain[0]?.to).toEqual(LOCAL_SCHEMA_V1) @@ -374,7 +365,7 @@ describe("local migration registry", () => { // One past the current tip — bump alongside LOCAL_SCHEMA_VERSION, or this // stops testing the future-store guard and starts testing the // unknown-fingerprint one. - { ...CURRENT_LOCAL_SCHEMA, version: 20, fingerprint: "future", digest: SCHEMA_DIGEST }, + { ...CURRENT_LOCAL_SCHEMA, version: 21, fingerprint: "future", digest: SCHEMA_DIGEST }, CURRENT_LOCAL_SCHEMA, ), ).toThrow(/newer than this build/) @@ -1382,6 +1373,7 @@ describe("v10 -> v11 product events module", () => { "local-0016-to-0017-audit-log", "local-0017-to-0018-product-events-from-traces", "local-0018-to-0019-ai-trace-index-usage-conventions", + "local-0019-to-0020-error-events-attribute-fallback", ]) expect(chain[0]?.to).toEqual(LOCAL_SCHEMA_V11) // The dropped table is declared, and the backfilled ones say what they diff --git a/apps/cli/test/native-local-store-migration.sh b/apps/cli/test/native-local-store-migration.sh index 989f10811..3e450690f 100755 --- a/apps/cli/test/native-local-store-migration.sh +++ b/apps/cli/test/native-local-store-migration.sh @@ -142,7 +142,7 @@ grep -q "local store migrated" "$ROOT/migrate.out" || fail "native migration did # must be bumped in lockstep with LOCAL_SCHEMA_VERSION and the matching # LOCAL_SCHEMA_V.fingerprint in apps/cli/src/server/schema-identity.ts; # leaving it on the previous version is what makes this step fail after a bump. -jq -e '.formatVersion == 2 and .activation == "active" and .schemaVersion == 19 and .schema == "778888eceae9d6b7"' \ +jq -e '.formatVersion == 2 and .activation == "active" and .schemaVersion == 20 and .schema == "ad8e854c9e2bb021"' \ "$ROOT/maple-store-version.json" >/dev/null || fail "native migration wrote the wrong active identity" step "reopening promoted store in a fresh server" diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index beec03206..ce1726a8c 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "46001bd069c07ec8d4e47fd7aa9a5ed09b550f78ee28b93682a3d8242dc67766"; +pub const PROJECT_REVISION: &str = "f87dd560d4607f017adc21f444f7bb1d2f4a4c988c6edbe9b44dab806978c87c"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse diff --git a/docs/error-issue-lifecycle.md b/docs/error-issue-lifecycle.md index 2d05de656..2d5e07ce4 100644 --- a/docs/error-issue-lifecycle.md +++ b/docs/error-issue-lifecycle.md @@ -17,6 +17,12 @@ This is the flow both humans and agents are meant to follow. If you are changing | Investigation | `investigations` | One AI diagnostic run. Zero or more per issue. | | Verification | `error_issue_verifications` | One post-merge "did that actually work?" check. | +The exception comes from the first OTel `exception` event, or the status message when +there is no event. Only spans missing both fall back to `exception.*` span attributes, +then `error.type` / `error.message`, then `Unknown Error`. This keeps existing fingerprints +stable while separating previously unlabelled errors. The SQL in `error_events_mv` +(`packages/domain/src/tinybird/materializations.ts`) owns this precedence. + The distinction that matters: **an incident is a flare-up, an issue is the bug**. An issue can flare up ten times; it gets fixed once. diff --git a/docs/warehouse-rollups.md b/docs/warehouse-rollups.md index b09b7977e..517dd35a5 100644 --- a/docs/warehouse-rollups.md +++ b/docs/warehouse-rollups.md @@ -42,7 +42,8 @@ of a table we already have. 2. **Pre-aggregation for scans.** `*_aggregates_hourly`, `service_overview_*`, `service_operations_*`. Trades write amplification for orders-of-magnitude less read. 3. **Filtered projection.** `error_events` keeps only `StatusCode = 'Error'` and unwraps the - exception event, so error queries never touch the Map columns of the full traces table. + exception event — or, when both the event and status message are absent, the `exception.*` / `error.*` span attributes — + so error queries never touch the Map columns of the full traces table. Storage is not free and the ratio is worse than it looks: `traces` is 110 GB, and its MV descendants total roughly 116 GB. **We store traces more than twice over.** Every new MV on diff --git a/packages/domain/src/clickhouse/migrations/0030_error_events_attribute_fallback.ts b/packages/domain/src/clickhouse/migrations/0030_error_events_attribute_fallback.ts new file mode 100644 index 000000000..0ead27c7f --- /dev/null +++ b/packages/domain/src/clickhouse/migrations/0030_error_events_attribute_fallback.ts @@ -0,0 +1,22 @@ +/** + * Fill missing error details from exception.* / error.* span attributes only + * when both the exception event and StatusMessage are absent. Other spans keep + * their existing fingerprint inputs, including empty fields in an event. + * + * Forward-only: preserve target rows and freeze the emitted DDL below. + * FINGERPRINT_VERSION stays unchanged because this splits only the formerly + * unknown bucket; a version bump would archive unaffected issues (see 0018). + * requiredForIngest is false because the gateway never writes these targets. + */ +export const migration_0030_error_events_attribute_fallback = { + version: 30, + description: + "Recreate the error_events MVs so an exception-less span is labelled from its exception.* / error.* attributes", + requiredForIngest: false, + statements: [ + "DROP VIEW IF EXISTS error_events_mv", + "DROP VIEW IF EXISTS error_events_by_time_mv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n -- Only fill the old Unknown Error bucket. Event values (including\n -- empty fields) and spans with StatusMessage keep every hash input.\n _ei = 0 AND StatusMessage = '' AS _useAttrs,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.type'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '')\n ) AS _exType,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.message'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage)\n ) AS _exMsg,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.stacktrace'],\n if(_useAttrs, SpanAttributes['exception.stacktrace'], '')\n ) AS _exStack,\n if(_useAttrs, _exMsg, StatusMessage) AS _msgText,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(_msgText) AS _isJson,\n _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(_msgText)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'),\n JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'),\n JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'),\n JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'),\n JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'),\n JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'),\n JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n _msgText = '', 'Unknown Error',\n position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0,\n if(\n extract(_msgText, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(_msgText, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel,\n left(_msgText, multiIf(\n position(_msgText, ': ') > 3, toInt64(position(_msgText, ': ')) - 1,\n position(_msgText, ' (') > 3, toInt64(position(_msgText, ' (')) - 1,\n position(_msgText, '\\n') > 3, toInt64(position(_msgText, '\\n')) - 1,\n least(toInt64(length(_msgText)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all hold: 4xx, no exception event, no exception.type\n -- attribute, and no error.type beyond the status code itself (HTTP\n -- semconv sets error.type to the bare status on a non-2xx response,\n -- which carries no exception). 5xx and anything carrying a real\n -- exception still count, and SpanKind is deliberately not consulted —\n -- these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND SpanAttributes['exception.type'] = ''\n AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus))\n )", + "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n -- Only fill the old Unknown Error bucket. Event values (including\n -- empty fields) and spans with StatusMessage keep every hash input.\n _ei = 0 AND StatusMessage = '' AS _useAttrs,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.type'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '')\n ) AS _exType,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.message'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage)\n ) AS _exMsg,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.stacktrace'],\n if(_useAttrs, SpanAttributes['exception.stacktrace'], '')\n ) AS _exStack,\n if(_useAttrs, _exMsg, StatusMessage) AS _msgText,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(_msgText) AS _isJson,\n _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(_msgText)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'),\n JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'),\n JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'),\n JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'),\n JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'),\n JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'),\n JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n _msgText = '', 'Unknown Error',\n position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0,\n if(\n extract(_msgText, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(_msgText, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel,\n left(_msgText, multiIf(\n position(_msgText, ': ') > 3, toInt64(position(_msgText, ': ')) - 1,\n position(_msgText, ' (') > 3, toInt64(position(_msgText, ' (')) - 1,\n position(_msgText, '\\n') > 3, toInt64(position(_msgText, '\\n')) - 1,\n least(toInt64(length(_msgText)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all hold: 4xx, no exception event, no exception.type\n -- attribute, and no error.type beyond the status code itself (HTTP\n -- semconv sets error.type to the bare status on a non-2xx response,\n -- which carries no exception). 5xx and anything carrying a real\n -- exception still count, and SpanKind is deliberately not consulted —\n -- these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND SpanAttributes['exception.type'] = ''\n AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus))\n )", + ], +} as const diff --git a/packages/domain/src/clickhouse/migrations/index.test.ts b/packages/domain/src/clickhouse/migrations/index.test.ts index af578faf9..c41dd15c7 100644 --- a/packages/domain/src/clickhouse/migrations/index.test.ts +++ b/packages/domain/src/clickhouse/migrations/index.test.ts @@ -34,8 +34,7 @@ import { migration_0025_commit_sha_vcs_revision } from "./0025_commit_sha_vcs_re import { migration_0026_ai_trace_index_filter_columns } from "./0026_ai_trace_index_filter_columns" import { migration_0027_audit_log } from "./0027_audit_log" import { migration_0028_product_events_from_traces } from "./0028_product_events_from_traces" -import { migration_0029_ai_trace_index_usage_conventions } from "./0029_ai_trace_index_usage_conventions" -import { migration_0021_product_events } from "./0021_product_events" +import { migration_0030_error_events_attribute_fallback } from "./0030_error_events_attribute_fallback" import { clickHouseSchemaVersion, latestMigrationVersion, migrations } from "./index" const backfills = migration_0004_service_namespace_projections.statements.filter( @@ -52,10 +51,10 @@ describe("ClickHouse migrations", () => { it("keeps migrations ordered by version", () => { expect(migrations.map((m) => m.version)).toEqual([ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, - 28, 29, + 28, 29, 30, ]) - expect(migrations.at(-1)).toBe(migration_0029_ai_trace_index_usage_conventions) - expect(latestMigrationVersion).toBe(29) + expect(migrations.at(-1)).toBe(migration_0030_error_events_attribute_fallback) + expect(latestMigrationVersion).toBe(30) // 0010 and 0014-0020 are read-path only and skipped by the ingest-gating // version; 0021 is not — the gateway writes `session_events`' new identity // columns and `product_events` directly, so a BYO-CH org must apply it @@ -83,6 +82,31 @@ describe("ClickHouse migrations", () => { expect(migration_0026_ai_trace_index_filter_columns.requiredForIngest).toBe(false) expect(migration_0027_audit_log.requiredForIngest).toBe(false) expect(migration_0028_product_events_from_traces.requiredForIngest).toBe(false) + // 0030 only recreates the error-events MVs. + expect(migration_0030_error_events_attribute_fallback.requiredForIngest).toBe(false) + }) + + it("recreates both error-events MVs with the span-attribute exception fallback", () => { + const statements: ReadonlyArray = + migration_0030_error_events_attribute_fallback.statements.filter((stmt) => !isBackfill(stmt)) + const sql = statements.join("\n") + + // An MV's SELECT is frozen at creation, so both views are dropped before + // they are recreated; error_events_by_time_mv shares the projection + // byte-for-byte and must never disagree with error_events_mv on a label. + for (const view of ["error_events_mv", "error_events_by_time_mv"]) { + const dropAt = statements.findIndex((stmt) => stmt === `DROP VIEW IF EXISTS ${view}`) + const createAt = statements.findIndex((stmt) => + stmt.startsWith(`CREATE MATERIALIZED VIEW IF NOT EXISTS ${view} `), + ) + expect(dropAt).toBeGreaterThanOrEqual(0) + expect(createAt).toBeGreaterThan(dropAt) + } + + // Nothing is rewritten: error_events keeps no span attributes to re-derive + // from, and recomputing FingerprintHash would re-bucket every issue. + expect(sql).not.toContain("ALTER TABLE error_events") + expect(migration_0030_error_events_attribute_fallback.statements.some(isBackfill)).toBe(false) }) it("recreates both error-events MVs with the 4xx guard and the widened frame redaction", () => { diff --git a/packages/domain/src/clickhouse/migrations/index.ts b/packages/domain/src/clickhouse/migrations/index.ts index 9e1e8c6f4..528e9fae7 100644 --- a/packages/domain/src/clickhouse/migrations/index.ts +++ b/packages/domain/src/clickhouse/migrations/index.ts @@ -28,6 +28,7 @@ import { migration_0026_ai_trace_index_filter_columns } from "./0026_ai_trace_in import { migration_0027_audit_log } from "./0027_audit_log" import { migration_0028_product_events_from_traces } from "./0028_product_events_from_traces" import { migration_0029_ai_trace_index_usage_conventions } from "./0029_ai_trace_index_usage_conventions" +import { migration_0030_error_events_attribute_fallback } from "./0030_error_events_attribute_fallback" /** * A migration statement is either a raw SQL string (structural DDL) or a @@ -88,6 +89,7 @@ export const migrations: ReadonlyArray = [ migration_0027_audit_log, migration_0028_product_events_from_traces, migration_0029_ai_trace_index_usage_conventions, + migration_0030_error_events_attribute_fallback, ] as const /** Highest migration `version` bundled — i.e. the schema level a fully-applied diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index 0ab45f554..d8153bded 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "46001bd069c07ec8d4e47fd7aa9a5ed09b550f78ee28b93682a3d8242dc67766" as const +export const projectRevision = "f87dd560d4607f017adc21f444f7bb1d2f4a4c988c6edbe9b44dab806978c87c" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS ai_trace_index (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SessionId String,\n VendorId LowCardinality(String),\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n Model LowCardinality(String),\n AgentName LowCardinality(String),\n ToolName LowCardinality(String),\n SpanId String,\n ParentSpanId String,\n Duration UInt64,\n IsError UInt8,\n IsLlmCall UInt8,\n IsToolCall UInt8,\n Tokens Float64,\n Cost Float64,\n ResponseId String\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, TraceId)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", @@ -45,8 +45,8 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS traces (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n TraceState String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n ResourceSchemaUrl String,\n ResourceAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n EventsTimestamp Array(DateTime64(9)),\n EventsName Array(LowCardinality(String)),\n EventsAttributes Array(Map(LowCardinality(String), String)),\n LinksTraceId Array(String),\n LinksSpanId Array(String),\n LinksTraceState Array(String),\n LinksAttributes Array(Map(LowCardinality(String), String)),\n SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0),\n IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0),\n ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)),\n ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)),\n SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)),\n INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp))\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS traces_aggregates_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n StatusCode LowCardinality(String),\n IsEntryPoint UInt8,\n DeploymentEnv LowCardinality(String),\n WeightedCount SimpleAggregateFunction(sum, Float64),\n WeightedDurationSum SimpleAggregateFunction(sum, Float64),\n WeightedErrorCount SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32),\n DurationMin SimpleAggregateFunction(min, UInt64),\n DurationMax SimpleAggregateFunction(max, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv)\nTTL toDate(Hour) + INTERVAL 365 DAY", "CREATE MATERIALIZED VIEW IF NOT EXISTS ai_trace_index_mv TO ai_trace_index AS\nSELECT\n OrgId,\n Timestamp,\n TraceId,\n SpanAttributes['maple_ai.session.id'] AS SessionId,\n SpanAttributes['maple_ai.vendor.id'] AS VendorId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n coalesce(nullIf(SpanAttributes['gen_ai.response.model'], ''), nullIf(SpanAttributes['gen_ai.request.model'], ''), nullIf(SpanAttributes['ai.response.model'], ''), nullIf(SpanAttributes['ai.model.id'], ''), SpanAttributes['llm.model_name']) AS Model,\n coalesce(nullIf(SpanAttributes['gen_ai.agent.name'], ''), SpanAttributes['ai.telemetry.functionId']) AS AgentName,\n coalesce(nullIf(SpanAttributes['gen_ai.tool.name'], ''), nullIf(SpanAttributes['ai.toolCall.name'], ''), SpanAttributes['tool.name']) AS ToolName,\n SpanId,\n ParentSpanId,\n Duration,\n toUInt8(((StatusCode = 'Error' OR SpanAttributes['error.type'] != '') OR SpanAttributes['gen_ai.response.status'] IN ('failed', 'error'))) AS IsError,\n toUInt8((coalesce(nullIf(SpanAttributes['gen_ai.operation.name'], ''), multiIf(SpanAttributes['openinference.span.kind'] = 'LLM', 'chat', SpanAttributes['openinference.span.kind'] = 'TOOL', 'execute_tool', SpanAttributes['openinference.span.kind'] = 'AGENT', 'invoke_agent', SpanAttributes['openinference.span.kind'] = 'EMBEDDING', 'embeddings', SpanAttributes['openinference.span.kind'] = 'RETRIEVER', 'retrieval', '')) IN ('chat', 'generate_content', 'text_completion', 'fetch_response') OR (((coalesce(nullIf(SpanAttributes['gen_ai.operation.name'], ''), multiIf(SpanAttributes['openinference.span.kind'] = 'LLM', 'chat', SpanAttributes['openinference.span.kind'] = 'TOOL', 'execute_tool', SpanAttributes['openinference.span.kind'] = 'AGENT', 'invoke_agent', SpanAttributes['openinference.span.kind'] = 'EMBEDDING', 'embeddings', SpanAttributes['openinference.span.kind'] = 'RETRIEVER', 'retrieval', '')) NOT IN ('chat', 'generate_content', 'text_completion', 'fetch_response', 'embeddings', 'retrieval', 'execute_tool', 'invoke_agent', 'create_agent', 'invoke_workflow', 'plan', 'agent_step') AND NOT ((coalesce(nullIf(SpanAttributes['gen_ai.tool.name'], ''), nullIf(SpanAttributes['ai.toolCall.name'], ''), SpanAttributes['tool.name']) != '' OR lower(SpanName) LIKE '%tool%'))) AND NOT ((lower(SpanName) LIKE '%agent%' OR lower(SpanName) LIKE '%workflow%'))) AND (coalesce(nullIf(SpanAttributes['gen_ai.response.model'], ''), nullIf(SpanAttributes['gen_ai.request.model'], ''), nullIf(SpanAttributes['ai.response.model'], ''), nullIf(SpanAttributes['ai.model.id'], ''), SpanAttributes['llm.model_name']) != '' OR (lower(SpanName) LIKE '%chat%' OR lower(SpanName) LIKE '%completion%'))))) AS IsLlmCall,\n toUInt8((coalesce(nullIf(SpanAttributes['gen_ai.operation.name'], ''), multiIf(SpanAttributes['openinference.span.kind'] = 'LLM', 'chat', SpanAttributes['openinference.span.kind'] = 'TOOL', 'execute_tool', SpanAttributes['openinference.span.kind'] = 'AGENT', 'invoke_agent', SpanAttributes['openinference.span.kind'] = 'EMBEDDING', 'embeddings', SpanAttributes['openinference.span.kind'] = 'RETRIEVER', 'retrieval', '')) IN ('execute_tool') OR (coalesce(nullIf(SpanAttributes['gen_ai.operation.name'], ''), multiIf(SpanAttributes['openinference.span.kind'] = 'LLM', 'chat', SpanAttributes['openinference.span.kind'] = 'TOOL', 'execute_tool', SpanAttributes['openinference.span.kind'] = 'AGENT', 'invoke_agent', SpanAttributes['openinference.span.kind'] = 'EMBEDDING', 'embeddings', SpanAttributes['openinference.span.kind'] = 'RETRIEVER', 'retrieval', '')) NOT IN ('chat', 'generate_content', 'text_completion', 'fetch_response', 'embeddings', 'retrieval', 'execute_tool', 'invoke_agent', 'create_agent', 'invoke_workflow', 'plan', 'agent_step') AND (coalesce(nullIf(SpanAttributes['gen_ai.tool.name'], ''), nullIf(SpanAttributes['ai.toolCall.name'], ''), SpanAttributes['tool.name']) != '' OR lower(SpanName) LIKE '%tool%')))) AS IsToolCall,\n multiIf(SpanAttributes['maple_ai.vendor.id'] IN ('vercel_ai_sdk', 'maple'), greatest(toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.prompt_tokens'], ''), nullIf(SpanAttributes['ai.usage.inputTokens'], ''), nullIf(SpanAttributes['ai.usage.promptTokens'], ''), SpanAttributes['llm.token_count.prompt'])), toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.cache_read.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.input_tokens.cached'], ''), nullIf(SpanAttributes['ai.usage.cachedInputTokens'], ''), nullIf(SpanAttributes['ai.usage.inputTokenDetails.cacheReadTokens'], ''), SpanAttributes['llm.token_count.prompt_details.cache_read'])) + toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.cache_creation.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.cache_write.input_tokens'], ''), SpanAttributes['ai.usage.inputTokenDetails.cacheWriteTokens']))), coalesce(nullIf(SpanAttributes['gen_ai.provider.name'], ''), nullIf(SpanAttributes['gen_ai.system'], ''), nullIf(SpanAttributes['ai.model.provider'], ''), nullIf(SpanAttributes['llm.provider'], ''), SpanAttributes['llm.system']) IN ('openai', 'gcp.gemini', 'gemini', 'gcp.vertex_ai', 'vertex_ai', 'openrouter'), greatest(toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.prompt_tokens'], ''), nullIf(SpanAttributes['ai.usage.inputTokens'], ''), nullIf(SpanAttributes['ai.usage.promptTokens'], ''), SpanAttributes['llm.token_count.prompt'])), toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.cache_read.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.input_tokens.cached'], ''), nullIf(SpanAttributes['ai.usage.cachedInputTokens'], ''), nullIf(SpanAttributes['ai.usage.inputTokenDetails.cacheReadTokens'], ''), SpanAttributes['llm.token_count.prompt_details.cache_read'])) + toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.cache_creation.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.cache_write.input_tokens'], ''), SpanAttributes['ai.usage.inputTokenDetails.cacheWriteTokens']))), coalesce(nullIf(SpanAttributes['gen_ai.provider.name'], ''), nullIf(SpanAttributes['gen_ai.system'], ''), nullIf(SpanAttributes['ai.model.provider'], ''), nullIf(SpanAttributes['llm.provider'], ''), SpanAttributes['llm.system']) IN ('anthropic'), toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.prompt_tokens'], ''), nullIf(SpanAttributes['ai.usage.inputTokens'], ''), nullIf(SpanAttributes['ai.usage.promptTokens'], ''), SpanAttributes['llm.token_count.prompt'])) + toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.cache_read.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.input_tokens.cached'], ''), nullIf(SpanAttributes['ai.usage.cachedInputTokens'], ''), nullIf(SpanAttributes['ai.usage.inputTokenDetails.cacheReadTokens'], ''), SpanAttributes['llm.token_count.prompt_details.cache_read'])) + toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.cache_creation.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.cache_write.input_tokens'], ''), SpanAttributes['ai.usage.inputTokenDetails.cacheWriteTokens'])), greatest(toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.prompt_tokens'], ''), nullIf(SpanAttributes['ai.usage.inputTokens'], ''), nullIf(SpanAttributes['ai.usage.promptTokens'], ''), SpanAttributes['llm.token_count.prompt'])), toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.cache_read.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.input_tokens.cached'], ''), nullIf(SpanAttributes['ai.usage.cachedInputTokens'], ''), nullIf(SpanAttributes['ai.usage.inputTokenDetails.cacheReadTokens'], ''), SpanAttributes['llm.token_count.prompt_details.cache_read'])) + toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.cache_creation.input_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.cache_write.input_tokens'], ''), SpanAttributes['ai.usage.inputTokenDetails.cacheWriteTokens'])))) + multiIf(SpanAttributes['maple_ai.vendor.id'] IN ('vercel_ai_sdk', 'maple'), greatest(toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.output_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.completion_tokens'], ''), nullIf(SpanAttributes['ai.usage.outputTokens'], ''), nullIf(SpanAttributes['ai.usage.completionTokens'], ''), SpanAttributes['llm.token_count.completion'])), toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.reasoning.output_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.output_tokens.reasoning'], ''), nullIf(SpanAttributes['ai.usage.reasoningTokens'], ''), nullIf(SpanAttributes['ai.usage.outputTokenDetails.reasoningTokens'], ''), SpanAttributes['llm.token_count.completion_details.reasoning']))), coalesce(nullIf(SpanAttributes['gen_ai.provider.name'], ''), nullIf(SpanAttributes['gen_ai.system'], ''), nullIf(SpanAttributes['ai.model.provider'], ''), nullIf(SpanAttributes['llm.provider'], ''), SpanAttributes['llm.system']) IN ('anthropic', 'openai', 'openrouter'), greatest(toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.output_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.completion_tokens'], ''), nullIf(SpanAttributes['ai.usage.outputTokens'], ''), nullIf(SpanAttributes['ai.usage.completionTokens'], ''), SpanAttributes['llm.token_count.completion'])), toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.reasoning.output_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.output_tokens.reasoning'], ''), nullIf(SpanAttributes['ai.usage.reasoningTokens'], ''), nullIf(SpanAttributes['ai.usage.outputTokenDetails.reasoningTokens'], ''), SpanAttributes['llm.token_count.completion_details.reasoning']))), coalesce(nullIf(SpanAttributes['gen_ai.provider.name'], ''), nullIf(SpanAttributes['gen_ai.system'], ''), nullIf(SpanAttributes['ai.model.provider'], ''), nullIf(SpanAttributes['llm.provider'], ''), SpanAttributes['llm.system']) IN ('gcp.gemini', 'gemini', 'gcp.vertex_ai', 'vertex_ai'), toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.output_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.completion_tokens'], ''), nullIf(SpanAttributes['ai.usage.outputTokens'], ''), nullIf(SpanAttributes['ai.usage.completionTokens'], ''), SpanAttributes['llm.token_count.completion'])) + toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.reasoning.output_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.output_tokens.reasoning'], ''), nullIf(SpanAttributes['ai.usage.reasoningTokens'], ''), nullIf(SpanAttributes['ai.usage.outputTokenDetails.reasoningTokens'], ''), SpanAttributes['llm.token_count.completion_details.reasoning'])), greatest(toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.output_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.completion_tokens'], ''), nullIf(SpanAttributes['ai.usage.outputTokens'], ''), nullIf(SpanAttributes['ai.usage.completionTokens'], ''), SpanAttributes['llm.token_count.completion'])), toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.reasoning.output_tokens'], ''), nullIf(SpanAttributes['gen_ai.usage.output_tokens.reasoning'], ''), nullIf(SpanAttributes['ai.usage.reasoningTokens'], ''), nullIf(SpanAttributes['ai.usage.outputTokenDetails.reasoningTokens'], ''), SpanAttributes['llm.token_count.completion_details.reasoning'])))) AS Tokens,\n toFloat64OrZero(coalesce(nullIf(SpanAttributes['gen_ai.usage.cost'], ''), nullIf(SpanAttributes['gen_ai.usage.total_cost'], ''), SpanAttributes['llm.cost.total'])) AS Cost,\n coalesce(nullIf(SpanAttributes['gen_ai.response.id'], ''), SpanAttributes['ai.response.id']) AS ResponseId\n FROM traces\n WHERE SpanAttributes['maple_ai.vendor.id'] != ''", - "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", - "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", + "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n -- Only fill the old Unknown Error bucket. Event values (including\n -- empty fields) and spans with StatusMessage keep every hash input.\n _ei = 0 AND StatusMessage = '' AS _useAttrs,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.type'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '')\n ) AS _exType,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.message'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage)\n ) AS _exMsg,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.stacktrace'],\n if(_useAttrs, SpanAttributes['exception.stacktrace'], '')\n ) AS _exStack,\n if(_useAttrs, _exMsg, StatusMessage) AS _msgText,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(_msgText) AS _isJson,\n _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(_msgText)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'),\n JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'),\n JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'),\n JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'),\n JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'),\n JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'),\n JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n _msgText = '', 'Unknown Error',\n position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0,\n if(\n extract(_msgText, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(_msgText, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel,\n left(_msgText, multiIf(\n position(_msgText, ': ') > 3, toInt64(position(_msgText, ': ')) - 1,\n position(_msgText, ' (') > 3, toInt64(position(_msgText, ' (')) - 1,\n position(_msgText, '\\n') > 3, toInt64(position(_msgText, '\\n')) - 1,\n least(toInt64(length(_msgText)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all hold: 4xx, no exception event, no exception.type\n -- attribute, and no error.type beyond the status code itself (HTTP\n -- semconv sets error.type to the bare status on a non-2xx response,\n -- which carries no exception). 5xx and anything carrying a real\n -- exception still count, and SpanKind is deliberately not consulted —\n -- these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND SpanAttributes['exception.type'] = ''\n AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus))\n )", + "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n -- Only fill the old Unknown Error bucket. Event values (including\n -- empty fields) and spans with StatusMessage keep every hash input.\n _ei = 0 AND StatusMessage = '' AS _useAttrs,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.type'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '')\n ) AS _exType,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.message'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage)\n ) AS _exMsg,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.stacktrace'],\n if(_useAttrs, SpanAttributes['exception.stacktrace'], '')\n ) AS _exStack,\n if(_useAttrs, _exMsg, StatusMessage) AS _msgText,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(_msgText) AS _isJson,\n _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(_msgText)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'),\n JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'),\n JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'),\n JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'),\n JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'),\n JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'),\n JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n _msgText = '', 'Unknown Error',\n position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0,\n if(\n extract(_msgText, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(_msgText, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel,\n left(_msgText, multiIf(\n position(_msgText, ': ') > 3, toInt64(position(_msgText, ': ')) - 1,\n position(_msgText, ' (') > 3, toInt64(position(_msgText, ' (')) - 1,\n position(_msgText, '\\n') > 3, toInt64(position(_msgText, '\\n')) - 1,\n least(toInt64(length(_msgText)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all hold: 4xx, no exception event, no exception.type\n -- attribute, and no error.type beyond the status code itself (HTTP\n -- semconv sets error.type to the bare status on a non-2xx response,\n -- which carries no exception). 5xx and anything carrying a real\n -- exception still count, and SpanKind is deliberately not consulted —\n -- these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND SpanAttributes['exception.type'] = ''\n AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus))\n )", "CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS\nSELECT\n OrgId,\n toStartOfMinute(Timestamp) AS Minute,\n FingerprintHash,\n anyLast(ServiceName) AS ServiceName,\n anyLast(ExceptionType) AS ExceptionType,\n anyLast(ExceptionMessage) AS ExceptionMessage,\n anyLast(ErrorLabel) AS ErrorLabel,\n anyLast(TopFrame) AS TopFrame,\n count() AS OccurrenceCount,\n min(Timestamp) AS FirstSeen,\n max(Timestamp) AS LastSeen,\n -- Distinct builds, not a sample: see ServiceVersions on the datasource.\n groupUniqArray(ServiceVersion) AS ServiceVersions\n FROM error_events\n GROUP BY OrgId, Minute, FingerprintHash", "CREATE MATERIALIZED VIEW IF NOT EXISTS identity_links_mv TO identity_links AS\nSELECT\n OrgId,\n VisitorId,\n UserId,\n StartTime AS FirstSeen\n FROM session_replays\n WHERE VisitorId != '' AND UserId != ''", "CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n arrayJoin(mapKeys(LogAttributes)) AS AttributeKey,\n 'log' AS AttributeScope,\n count() AS UsageCount\n FROM logs\n WHERE LogAttributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope", diff --git a/packages/domain/src/generated/tinybird-project-manifest.ts b/packages/domain/src/generated/tinybird-project-manifest.ts index 28bb1790b..6a4a2d83d 100644 --- a/packages/domain/src/generated/tinybird-project-manifest.ts +++ b/packages/domain/src/generated/tinybird-project-manifest.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-tinybird-project-manifest.ts // Do not edit manually. -export const projectRevision = "46001bd069c07ec8d4e47fd7aa9a5ed09b550f78ee28b93682a3d8242dc67766" as const +export const projectRevision = "f87dd560d4607f017adc21f444f7bb1d2f4a4c988c6edbe9b44dab806978c87c" as const export const datasources = [ { @@ -215,12 +215,12 @@ export const pipes = [ { name: "error_events_by_time_mv", content: - "DESCRIPTION >\n Time-ordered copy of error_events_mv's projection, written to error_events_by_time (sorted by OrgId, Timestamp, FingerprintHash) for recent-window error scans.\n\nNODE error_events_by_time_mv_node\nSQL >\n WITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )\n\nTYPE MATERIALIZED\nDATASOURCE error_events_by_time", + "DESCRIPTION >\n Time-ordered copy of error_events_mv's projection, written to error_events_by_time (sorted by OrgId, Timestamp, FingerprintHash) for recent-window error scans.\n\nNODE error_events_by_time_mv_node\nSQL >\n WITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n -- Only fill the old Unknown Error bucket. Event values (including\n -- empty fields) and spans with StatusMessage keep every hash input.\n _ei = 0 AND StatusMessage = '' AS _useAttrs,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.type'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '')\n ) AS _exType,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.message'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage)\n ) AS _exMsg,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.stacktrace'],\n if(_useAttrs, SpanAttributes['exception.stacktrace'], '')\n ) AS _exStack,\n if(_useAttrs, _exMsg, StatusMessage) AS _msgText,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(_msgText) AS _isJson,\n _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(_msgText)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'),\n JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'),\n JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'),\n JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'),\n JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'),\n JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'),\n JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n _msgText = '', 'Unknown Error',\n position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0,\n if(\n extract(_msgText, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(_msgText, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel,\n left(_msgText, multiIf(\n position(_msgText, ': ') > 3, toInt64(position(_msgText, ': ')) - 1,\n position(_msgText, ' (') > 3, toInt64(position(_msgText, ' (')) - 1,\n position(_msgText, '\\n') > 3, toInt64(position(_msgText, '\\n')) - 1,\n least(toInt64(length(_msgText)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all hold: 4xx, no exception event, no exception.type\n -- attribute, and no error.type beyond the status code itself (HTTP\n -- semconv sets error.type to the bare status on a non-2xx response,\n -- which carries no exception). 5xx and anything carrying a real\n -- exception still count, and SpanKind is deliberately not consulted —\n -- these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND SpanAttributes['exception.type'] = ''\n AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus))\n )\n\nTYPE MATERIALIZED\nDATASOURCE error_events_by_time\nDEPLOYMENT_METHOD alter", }, { name: "error_events_mv", content: - "DESCRIPTION >\n Materializes per-occurrence error events from traces. Unwraps the first OTel exception event and computes a cityHash64 FingerprintHash for issue grouping.\n\nNODE error_events_mv_node\nSQL >\n WITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )\n\nTYPE MATERIALIZED\nDATASOURCE error_events", + "DESCRIPTION >\n Materializes per-occurrence error events from traces. Unwraps the first OTel exception event (falling back to exception.* / error.* span attributes) and computes a cityHash64 FingerprintHash for issue grouping.\n\nNODE error_events_mv_node\nSQL >\n WITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n -- Only fill the old Unknown Error bucket. Event values (including\n -- empty fields) and spans with StatusMessage keep every hash input.\n _ei = 0 AND StatusMessage = '' AS _useAttrs,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.type'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '')\n ) AS _exType,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.message'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage)\n ) AS _exMsg,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.stacktrace'],\n if(_useAttrs, SpanAttributes['exception.stacktrace'], '')\n ) AS _exStack,\n if(_useAttrs, _exMsg, StatusMessage) AS _msgText,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(_msgText) AS _isJson,\n _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(_msgText)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'),\n JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'),\n JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'),\n JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'),\n JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'),\n JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'),\n JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n _msgText = '', 'Unknown Error',\n position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0,\n if(\n extract(_msgText, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(_msgText, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel,\n left(_msgText, multiIf(\n position(_msgText, ': ') > 3, toInt64(position(_msgText, ': ')) - 1,\n position(_msgText, ' (') > 3, toInt64(position(_msgText, ' (')) - 1,\n position(_msgText, '\\n') > 3, toInt64(position(_msgText, '\\n')) - 1,\n least(toInt64(length(_msgText)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all hold: 4xx, no exception event, no exception.type\n -- attribute, and no error.type beyond the status code itself (HTTP\n -- semconv sets error.type to the bare status on a non-2xx response,\n -- which carries no exception). 5xx and anything carrying a real\n -- exception still count, and SpanKind is deliberately not consulted —\n -- these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND SpanAttributes['exception.type'] = ''\n AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus))\n )\n\nTYPE MATERIALIZED\nDATASOURCE error_events\nDEPLOYMENT_METHOD alter", }, { name: "error_fingerprints_minutely_mv", diff --git a/packages/domain/src/tinybird/fingerprint.test.ts b/packages/domain/src/tinybird/fingerprint.test.ts index d61dced36..82c8a6aee 100644 --- a/packages/domain/src/tinybird/fingerprint.test.ts +++ b/packages/domain/src/tinybird/fingerprint.test.ts @@ -594,9 +594,9 @@ describe("SQL parity", () => { it("truncates by character in both implementations, not by byte", () => { // ClickHouse `substring` counts bytes while JS `slice` counts UTF-16 units, // so a non-ASCII message would truncate at a different point on each side. - expect(sql).toContain(`substringUTF8(StatusMessage, 1, ${MSG_SCAN_CHARS})`) + expect(sql).toContain(`substringUTF8(_msgText, 1, ${MSG_SCAN_CHARS})`) expect(sql).toContain(`1, ${MSG_SIGNATURE_CHARS}`) - expect(sql).not.toMatch(/substring\(StatusMessage/) + expect(sql).not.toMatch(/substring\((StatusMessage|_msgText)/) }) it("keeps the frame limit in step", () => { diff --git a/packages/domain/src/tinybird/fingerprint.ts b/packages/domain/src/tinybird/fingerprint.ts index 027bedc63..d943738bb 100644 --- a/packages/domain/src/tinybird/fingerprint.ts +++ b/packages/domain/src/tinybird/fingerprint.ts @@ -284,6 +284,7 @@ function messageSignature(statusMessage: string): string { ) } +/** `statusMessage` is the MV's resolved `_msgText`, including attribute fallback. */ export function computeFingerprintInputs(args: { readonly exceptionType: string readonly exceptionStacktrace: string diff --git a/packages/domain/src/tinybird/materializations.ts b/packages/domain/src/tinybird/materializations.ts index 3d44a4ab6..f00158b9d 100644 --- a/packages/domain/src/tinybird/materializations.ts +++ b/packages/domain/src/tinybird/materializations.ts @@ -700,8 +700,9 @@ export const servicePlatformsHourlyMv = defineMaterializedView("service_platform /** * Materialized view populating error_events from traces where StatusCode='Error'. - * Unwraps the first OTel `exception` event and computes a cityHash64 - * FingerprintHash used to group occurrences into Issues. + * Unwraps the first OTel `exception` event. When both the event and StatusMessage + * are absent, reads `exception.*` then `error.*` span attributes. Computes a + * cityHash64 FingerprintHash used to group occurrences into Issues. * * Fingerprint inputs: (OrgId, ServiceName, ExceptionType, top-3 normalized frames, * message signature). @@ -746,9 +747,22 @@ export { errorEventsSelectSql as ERROR_EVENTS_MV_SQL } const errorEventsSelectSql = ` WITH arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, - if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, - if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, - if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + -- Only fill the old Unknown Error bucket. Event values (including + -- empty fields) and spans with StatusMessage keep every hash input. + _ei = 0 AND StatusMessage = '' AS _useAttrs, + if( + _ei > 0, EventsAttributes[_ei]['exception.type'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '') + ) AS _exType, + if( + _ei > 0, EventsAttributes[_ei]['exception.message'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage) + ) AS _exMsg, + if( + _ei > 0, EventsAttributes[_ei]['exception.stacktrace'], + if(_useAttrs, SpanAttributes['exception.stacktrace'], '') + ) AS _exStack, + if(_useAttrs, _exMsg, StatusMessage) AS _msgText, -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old -- rule accepted any line with a colon-digit, which let non-frame lines -- in: Drizzle's \`params: \` line, and the \`Type: message\` @@ -781,8 +795,8 @@ const errorEventsSelectSql = ` if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, arrayStringConcat(_topFrames, '\\n') AS _fpFrames, -- JSON detection for the message signature below. - isValidJSON(StatusMessage) AS _isJson, - _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + isValidJSON(_msgText) AS _isJson, + _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj, -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level -- keys, redact volatile tokens (long hex / numbers) in each raw value, then -- sort by "key=value" so key order & whitespace don't matter. No assumption @@ -792,7 +806,7 @@ const errorEventsSelectSql = ` arraySort( arrayMap( kv -> concat(kv.1, '=', ${chRedactChain("kv.2", JSON_VALUE_REDACTIONS)}), - JSONExtractKeysAndValuesRaw(StatusMessage) + JSONExtractKeysAndValuesRaw(_msgText) ) ), '|' @@ -810,7 +824,7 @@ const errorEventsSelectSql = ` multiIf( _isJsonObj, _jsonSig, substringUTF8( - ${chRedactChain(`substringUTF8(StatusMessage, 1, ${MSG_SCAN_CHARS})`, MSG_TEXT_REDACTIONS)}, + ${chRedactChain(`substringUTF8(_msgText, 1, ${MSG_SCAN_CHARS})`, MSG_TEXT_REDACTIONS)}, 1, ${MSG_SIGNATURE_CHARS} ) ) AS _msgSig, @@ -818,29 +832,29 @@ const errorEventsSelectSql = ` -- many labels may map to one hash). The broad key list here is a DISPLAY -- heuristic only; the fingerprint above makes no key-name assumption. multiIf( - JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), - JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), - JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), - JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), - JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), - JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), - JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'), + JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'), + JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'), + JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'), + JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'), + JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'), + JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'), 'JSON error' ) AS _jsonLabel, multiIf( - StatusMessage = '', 'Unknown Error', - position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + _msgText = '', 'Unknown Error', + position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0, if( - extract(StatusMessage, 'readonly (\\\\w+)') != '', - concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')), + extract(_msgText, 'readonly (\\\\w+)') != '', + concat('Schema parse error: ', extract(_msgText, 'readonly (\\\\w+)')), 'Schema parse error' ), - _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, - left(StatusMessage, multiIf( - position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, - position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, - position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1, - least(toInt64(length(StatusMessage)), 150) + _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel, + left(_msgText, multiIf( + position(_msgText, ': ') > 3, toInt64(position(_msgText, ': ')) - 1, + position(_msgText, ' (') > 3, toInt64(position(_msgText, ' (')) - 1, + position(_msgText, '\\n') > 3, toInt64(position(_msgText, '\\n')) - 1, + least(toInt64(length(_msgText)), 150) )) ) AS _statusLabel, if(_exType != '', _exType, _statusLabel) AS _errorLabel, @@ -874,20 +888,27 @@ const errorEventsSelectSql = ` -- Client-side runtimes (notably the native Cloudflare Workers -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a - -- span only when all three hold: 4xx, no exception event, and no - -- exception type. 5xx and anything carrying an exception still count, - -- and SpanKind is deliberately not consulted — these are Client spans. + -- span only when all hold: 4xx, no exception event, no exception.type + -- attribute, and no error.type beyond the status code itself (HTTP + -- semconv sets error.type to the bare status on a non-2xx response, + -- which carries no exception). 5xx and anything carrying a real + -- exception still count, and SpanKind is deliberately not consulted — + -- these are Client spans. AND NOT ( _httpStatus >= 400 AND _httpStatus < 500 AND _ei = 0 - AND _exType = '' + AND SpanAttributes['exception.type'] = '' + AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus)) ) ` export const errorEventsMv = defineMaterializedView("error_events_mv", { description: - "Materializes per-occurrence error events from traces. Unwraps the first OTel exception event and computes a cityHash64 FingerprintHash for issue grouping.", + "Materializes per-occurrence error events from traces. Unwraps the first OTel exception event (falling back to exception.* / error.* span attributes) and computes a cityHash64 FingerprintHash for issue grouping.", datasource: errorEvents, + // Preserve the target's 90d history; replaying traces would retain only 30d + // and recompute stored fingerprints. Change the SELECT for future inserts. + deploymentMethod: "alter", nodes: [ node({ name: "error_events_mv_node", @@ -908,6 +929,8 @@ export const errorEventsByTimeMv = defineMaterializedView("error_events_by_time_ description: "Time-ordered copy of error_events_mv's projection, written to error_events_by_time (sorted by OrgId, Timestamp, FingerprintHash) for recent-window error scans.", datasource: errorEventsByTime, + // Keep both projections forward-only, with the same retained history. + deploymentMethod: "alter", nodes: [ node({ name: "error_events_by_time_mv_node",