From e55ad1148290840d8e3b59e768a33cc3b79dceb1 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 5 Jun 2026 04:49:02 -0700 Subject: [PATCH 01/21] feat(observability): logging foundations - sanitizer, OTel log bridge, SDK bootstrap (default-off) Co-Authored-By: Claude Opus 4.8 (1M context) --- common/config/build.sbt | 23 +- .../texera/observability/LogSanitizer.scala | 111 +++++ .../texera/observability/OtelInit.scala | 434 ++++++++++++++++++ .../observability/TexeraOtelLogAppender.scala | 140 ++++++ .../observability/LogSanitizerSpec.scala | Bin 0 -> 4667 bytes .../texera/observability/OtelInitSpec.scala | 242 ++++++++++ .../TexeraOtelLogAppenderSpec.scala | 166 +++++++ 7 files changed, 1115 insertions(+), 1 deletion(-) create mode 100644 common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala create mode 100644 common/config/src/main/scala/org/apache/texera/observability/OtelInit.scala create mode 100644 common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala create mode 100644 common/config/src/test/scala/org/apache/texera/observability/LogSanitizerSpec.scala create mode 100644 common/config/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala create mode 100644 common/config/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala diff --git a/common/config/build.sbt b/common/config/build.sbt index bb561de4f3e..fe532ce96b6 100644 --- a/common/config/build.sbt +++ b/common/config/build.sbt @@ -48,7 +48,28 @@ Compile / scalacOptions ++= Seq( // Dependencies ///////////////////////////////////////////////////////////////////////////// +// OpenTelemetry version is pinned here as the single source of truth; all +// services pick it up transitively via dependsOn(Config). Bump deliberately. +val openTelemetryVersion = "1.50.0" + // Core Dependencies libraryDependencies ++= Seq( - "com.typesafe" % "config" % "1.4.6" // For configuration management + "com.typesafe" % "config" % "1.4.6", // For configuration management + "com.typesafe.scala-logging" %% "scala-logging" % "3.9.5", // for LazyLogging in OtelInit + // OpenTelemetry SDK bootstrap (Apache-2.0). We deliberately do NOT use + // sdk-extension-autoconfigure: the security model requires that endpoint + // + resource-attribute filtering run before any exporter is configured. + "io.opentelemetry" % "opentelemetry-api" % openTelemetryVersion, + "io.opentelemetry" % "opentelemetry-sdk" % openTelemetryVersion, + "io.opentelemetry" % "opentelemetry-exporter-otlp" % openTelemetryVersion, + // Logback Classic — needed at compile time to write the OTel log + // appender. Marked `provided` because every service already brings + // Logback in transitively (via Dropwizard / SLF4J), so we don't + // bundle a second copy. + "ch.qos.logback" % "logback-classic" % "1.2.13" % "provided", + // Test-only: in-memory exporter for OtelInitSpec; avoids hitting a real + // collector during unit tests. + "io.opentelemetry" % "opentelemetry-sdk-testing" % openTelemetryVersion % Test, + "ch.qos.logback" % "logback-classic" % "1.2.13" % Test, + "org.scalatest" %% "scalatest" % "3.2.17" % Test ) \ No newline at end of file diff --git a/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala b/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala new file mode 100644 index 00000000000..781d8bd776d --- /dev/null +++ b/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import scala.jdk.CollectionConverters._ + +/** + * Pure functions that sanitize log records before they leave the + * process via the OTel logs bridge. Lives in its own object so the + * security-critical behaviour can be unit-tested without a Logback + * fixture. + * + * Three invariants: + * 1. No control characters in the body (prevents log forging via + * CR/LF injection in user-supplied strings). + * 2. No oversized bodies (a 1 GiB log line must never reach the + * exporter). + * 3. No secrets in plain text (Bearer tokens, password=, AWS keys). + * + * Plus an MDC allowlist so accidental MDC pollution from a downstream + * library cannot leak unintended fields into the exporter. + */ +object LogSanitizer { + + /** Per-record body cap. The OTel SDK and OTLP have higher limits, + * but 16 KiB is plenty for a useful log line and protects the + * collector from a runaway log. */ + val MaxBodyBytes: Int = 16 * 1024 + + /** Suffix appended to truncated bodies. Chosen to be visually + * obvious in a UI but short enough not to dominate the cap. */ + val TruncatedMarker: String = "...[truncated]" + + /** C0 control characters except TAB (0x09). Stripping CR/LF here + * prevents log forging via newline injection in user-supplied + * message bodies. DEL (0x7F) included for the same reason. */ + private val C0ControlRegex = "[\\x00-\\x08\\x0A-\\x1F\\x7F]".r + + /** Secret patterns. Order is significant: most specific first so a + * partial match doesn't shadow a tighter pattern. */ + private val SecretPatterns: Seq[scala.util.matching.Regex] = Seq( + // Authorization: Bearer — bearer token in header form. + """(?i)Bearer\s+[A-Za-z0-9._\-/+=]{8,}""".r, + // password=…, password: … — generic credential keyvalue. + """(?i)password\s*[=:]\s*[^\s,;"']+""".r, + // AWS access key ID (canonical AKIA…16-char format). + """AKIA[0-9A-Z]{16}""".r, + // AWS secret access key, when explicitly labelled. + """(?i)aws_secret_access_key\s*[=:]\s*[A-Za-z0-9/+=]{20,}""".r + ) + + /** MDC keys we will forward to OTel log attributes. Anything else + * is dropped — additions require a code change + reviewer + * acknowledgement of the privacy implications. */ + val AllowedMdcKeys: Set[String] = Set( + "trace_id", + "span_id", + "texera.user.id", + "texera.workflow.id", + "texera.execution.id", + // Computing-unit id identifies the dev process / k8s pod that + // emitted the record. Required for the dashboard's CU-scoped + // log filter — without this key in the allowlist, the OTel + // appender silently strips it and the CU filter matches nothing. + "texera.computing_unit.id", + "texera.project.id", + "texera.operator.id" + ) + + /** Apply all three body-side transformations. Idempotent — running + * sanitize on already-sanitized output is a no-op. */ + def sanitize(body: String): String = { + if (body == null || body.isEmpty) return "" + val stripped = C0ControlRegex.replaceAllIn(body, "") + val scrubbed = SecretPatterns.foldLeft(stripped) { (acc, p) => + p.replaceAllIn(acc, "[REDACTED]") + } + truncate(scrubbed) + } + + /** Truncate to MaxBodyBytes, appending the marker if cut. */ + private def truncate(body: String): String = { + if (body.length <= MaxBodyBytes) body + else body.substring(0, MaxBodyBytes - TruncatedMarker.length) + TruncatedMarker + } + + /** Filter an MDC map to the allowlist. Null-safe. */ + def filterMdc(mdc: java.util.Map[String, String]): Map[String, String] = { + if (mdc == null) return Map.empty + mdc.asScala.iterator + .collect { case (k, v) if k != null && AllowedMdcKeys.contains(k) && v != null => k -> v } + .toMap + } +} diff --git a/common/config/src/main/scala/org/apache/texera/observability/OtelInit.scala b/common/config/src/main/scala/org/apache/texera/observability/OtelInit.scala new file mode 100644 index 00000000000..35e328c49b0 --- /dev/null +++ b/common/config/src/main/scala/org/apache/texera/observability/OtelInit.scala @@ -0,0 +1,434 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import com.typesafe.scalalogging.LazyLogging +import io.opentelemetry.api.{GlobalOpenTelemetry, OpenTelemetry} +import io.opentelemetry.api.common.{AttributeKey, Attributes} +import io.opentelemetry.exporter.otlp.logs.OtlpGrpcLogRecordExporter +import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter +import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.logs.SdkLoggerProvider +import io.opentelemetry.sdk.logs.`export`.{BatchLogRecordProcessor, LogRecordExporter} +import io.opentelemetry.sdk.metrics.SdkMeterProvider +import io.opentelemetry.sdk.metrics.`export`.{MetricExporter, PeriodicMetricReader} +import io.opentelemetry.sdk.resources.Resource +import io.opentelemetry.sdk.trace.SdkTracerProvider +import io.opentelemetry.sdk.trace.`export`.{BatchSpanProcessor, SpanExporter} + +import java.net.URI +import java.time.Duration +import scala.util.{Failure, Success, Try} + +/** + * Bootstraps the OpenTelemetry SDK for a Texera service. + * + * Design notes: + * - Default-disabled. Sets up nothing unless `OTEL_SDK_DISABLED=false`. + * - We deliberately do not use the autoconfigure SPI: the security model + * requires endpoint + resource-attribute filtering to happen BEFORE + * any exporter is constructed. Autoconfigure would parse env vars + * behind our back. + * - Validation is a single pure function so it can be unit-tested + * without spinning the SDK. + * - On any validation failure we log one WARN and return None. We + * do NOT throw — observability is opt-in plumbing; misconfiguration + * must never crash the service. + * - This is the only place in Texera that reads `OTEL_*` environment + * variables. Other modules consume the returned `OpenTelemetry` + * instance directly. + */ +object OtelInit extends LazyLogging { + + /** Resource attribute keys we accept from OTEL_RESOURCE_ATTRIBUTES. + * Resource attrs ride on every record this JVM emits (logs, + * metrics, traces) — so for a per-CU JVM (ComputingUnitMaster / + * ComputingUnitWorker) setting `texera.computing_unit.id=N` at + * boot is enough to tag every record without per-request MDC + * plumbing. Workflow/execution ids vary per task and still need + * MDC at the message boundary, but exposing them in the allowlist + * lets test harnesses + future per-task code populate them via + * the same mechanism. */ + private[observability] val AllowedResourceKeys: Set[String] = Set( + "service.name", + "service.version", + "deployment.environment", + "texera.computing_unit.id", + "texera.workflow.id", + "texera.execution.id" + ) + + /** Endpoint schemes we accept. */ + private[observability] val AllowedSchemes: Set[String] = Set("http", "https", "grpc") + + /** Hosts we accept for the OTLP endpoint by default. */ + private[observability] val DefaultAllowedHosts: Set[String] = Set( + "localhost", + "127.0.0.1", + "::1", + "[::1]" + ) + + /** Default endpoint when SDK is enabled but no endpoint set explicitly. + * Uses 127.0.0.1 (not "localhost") so a natively-run service reaches the + * IPv4-only collector port published by docker-compose — on dual-stack + * hosts "localhost" resolves to ::1 first and the OTLP export silently + * fails. Inside docker the endpoint is overridden to otel-collector:4317. */ + private val DefaultEndpoint = "http://127.0.0.1:4317" + + /** Metric export interval bounds. Values outside this range get + * clamped to the default with a one-shot WARN. The lower bound + * prevents an attacker tipping the exporter into busy-loop mode; + * the upper bound keeps metrics useful for human operators. */ + private[observability] val MinMetricIntervalMs: Long = 1000L + private[observability] val MaxMetricIntervalMs: Long = 10L * 60L * 1000L + private[observability] val DefaultMetricIntervalMs: Long = 60L * 1000L + + // Idempotency guard. The SDK installs global handlers and a shutdown + // hook; calling init() repeatedly must be a no-op after the first call. + @volatile private var initialized: Option[OpenTelemetry] = None + + /** + * Initialize the SDK for the given service name. + * Returns Some(sdk) on success, None on disabled / invalid config. + * + * Side effect when enabled: attaches a [[TexeraOtelLogAppender]] to + * the Logback ROOT logger so application logs are mirrored to the + * OTel collector, with the security guards in [[LogSanitizer]] + * applied to every record. + */ + def init(serviceName: String): Option[OpenTelemetry] = synchronized { + if (initialized.isDefined) return initialized + + val env = (key: String) => Option(System.getenv(key)) + val result = initInternal( + serviceName = serviceName, + envProvider = env, + spanExporterFactory = buildOtlpSpanExporter, + logExporterFactory = endpoint => Some(buildOtlpLogExporter(endpoint)), + metricExporterFactory = endpoint => Some(buildOtlpMetricExporter(endpoint)), + logbackAttacher = LogbackBinder.attach + ) + // Register globally so [[TexeraTracer]] and any other OTel-aware + // code can call ``GlobalOpenTelemetry.getTracer(...)`` without + // threading the SDK through every callsite. set() throws on a + // second call within the same JVM — our outer ``initialized`` + // guard makes that unreachable, but wrap defensively. The test + // path deliberately skips this so multiple isolated SDKs can be + // built within one JVM. + result.foreach { sdk => + Try(GlobalOpenTelemetry.set(sdk)).failed.foreach { t => + logger.warn( + s"GlobalOpenTelemetry already set; using the existing instance: ${t.getMessage}" + ) + } + } + result + } + + /** + * Test-only entry point. Allows the test to inject an env-var map + * and a span exporter so the SDK does not attempt a real network + * connection. The Logback appender is NOT attached in tests — + * appender tests construct it directly with an in-memory log + * exporter. + */ + private[observability] def initForTest( + serviceName: String, + envOverride: Map[String, String], + exporter: SpanExporter, + metricExporter: Option[MetricExporter] = None + ): Option[OpenTelemetry] = synchronized { + initInternal( + serviceName = serviceName, + envProvider = envOverride.get, + spanExporterFactory = _ => exporter, + logExporterFactory = _ => None, + metricExporterFactory = _ => metricExporter, + logbackAttacher = (_, _) => () // no-op in tests + ) + } + + /** Test-only: forget any previously-installed SDK. Does not unregister + * shutdown hooks (the previous SDK is closed instead). */ + private[observability] def resetForTest(): Unit = synchronized { + initialized.foreach { + case sdk: OpenTelemetrySdk => + Try(sdk.getSdkTracerProvider.close()) + Try(sdk.getSdkLoggerProvider.close()) + Try(sdk.getSdkMeterProvider.close()) + case _ => () + } + initialized = None + } + + private def initInternal( + serviceName: String, + envProvider: String => Option[String], + spanExporterFactory: String => SpanExporter, + logExporterFactory: String => Option[LogRecordExporter], + metricExporterFactory: String => Option[MetricExporter], + logbackAttacher: (String, OpenTelemetry) => Unit + ): Option[OpenTelemetry] = { + if (initialized.isDefined) return initialized + + // Default to ENABLED so an `sbt run` of any Texera service emits + // telemetry without per-JVM env-var configuration. Operators who + // need to silence telemetry (CI, embedded-tests, security-locked + // deployments) set OTEL_SDK_DISABLED=true explicitly. If the + // configured endpoint isn't reachable, the OTel SDK's + // BatchProcessor logs a single error and drops records — it + // does NOT crash the host service, so a missing collector at + // dev time is a quiet no-op rather than a startup failure. + val disabled = envProvider("OTEL_SDK_DISABLED").getOrElse("false") + if (disabled.equalsIgnoreCase("true")) { + logger.info("OpenTelemetry SDK disabled (OTEL_SDK_DISABLED=true). No telemetry will be emitted.") + return None + } + + val endpoint = envProvider("OTEL_EXPORTER_OTLP_ENDPOINT").getOrElse(DefaultEndpoint) + val extraAllowed = envProvider("TEXERA_OTEL_ALLOWED_HOSTS") + .map(_.split(',').iterator.map(_.trim.toLowerCase).filter(_.nonEmpty).toSet) + .getOrElse(Set.empty) + val allowedHosts = DefaultAllowedHosts ++ extraAllowed + + validateEndpoint(endpoint, allowedHosts) match { + case Left(reason) => + // One WARN, no further detail (endpoint is not echoed beyond what + // the operator already knows). No spans will be emitted. + logger.warn( + s"OpenTelemetry SDK disabled: invalid OTEL_EXPORTER_OTLP_ENDPOINT — $reason. " + + "Set TEXERA_OTEL_ALLOWED_HOSTS to extend the allowlist." + ) + return None + case Right(_) => // ok + } + + val rawAttrs = envProvider("OTEL_RESOURCE_ATTRIBUTES").getOrElse("") + val resource = buildResource(serviceName, rawAttrs) + + val spanExporter = spanExporterFactory(endpoint) + val tracerProvider = SdkTracerProvider + .builder() + .setResource(resource) + .addSpanProcessor(BatchSpanProcessor.builder(spanExporter).build()) + .build() + + val sdkBuilder = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider) + + // Logger provider is optional — controlled by the factory. Skipped + // in tests so the appender path can be exercised independently. + val loggerProviderOpt = logExporterFactory(endpoint).map { logExporter => + val lp = SdkLoggerProvider + .builder() + .setResource(resource) + .addLogRecordProcessor(BatchLogRecordProcessor.builder(logExporter).build()) + .build() + sdkBuilder.setLoggerProvider(lp) + lp + } + + // Meter provider is optional too. Export interval is clamped to + // [MinMetricIntervalMs, MaxMetricIntervalMs]; an out-of-range + // value gets reset to the default with one WARN — keeps an + // attacker from coaxing the reader into busy-loop mode by + // setting OTEL_METRIC_EXPORT_INTERVAL to a tiny value. + val intervalMs = clampIntervalMs(envProvider("OTEL_METRIC_EXPORT_INTERVAL")) + val meterProviderOpt = metricExporterFactory(endpoint).map { metricExporter => + val reader = PeriodicMetricReader + .builder(metricExporter) + .setInterval(Duration.ofMillis(intervalMs)) + .build() + val mp = SdkMeterProvider + .builder() + .setResource(resource) + .registerMetricReader(reader) + .build() + sdkBuilder.setMeterProvider(mp) + mp + } + + val sdk = sdkBuilder.build() + + // One startup span. Carries only service.name (no env, host, or + // version data beyond the allowlisted resource attrs). + val span = sdk.getTracer("texera.bootstrap").spanBuilder("service.start").startSpan() + Try(span.setAttribute("service.name", serviceName)) + span.end() + + // Wire the Logback appender so subsequent application logs flow to + // the collector with sanitisation applied. Failure here must never + // crash the service — observability is opt-in. + Try(logbackAttacher(serviceName, sdk)).failed.foreach { t => + logger.warn(s"Failed to attach OTel Logback appender (logs not exported): ${t.getMessage}") + } + + // Make sure providers flush on shutdown. We add the hook only after + // the SDK has been fully built so a panic during init doesn't leave + // a dangling hook pointing at a half-constructed provider. + Runtime.getRuntime.addShutdownHook(new Thread(() => { + Try(tracerProvider.close()) + loggerProviderOpt.foreach(lp => Try(lp.close())) + meterProviderOpt.foreach(mp => Try(mp.close())) + () + }, "otel-shutdown")) + + initialized = Some(sdk) + logger.info(s"OpenTelemetry SDK initialized for service '$serviceName' (endpoint=$endpoint).") + initialized + } + + /** + * Validate that the endpoint is parseable, uses an allowlisted scheme, + * and resolves to an allowlisted host. Pure function — safe to test + * without standing up the SDK. + */ + private[observability] def validateEndpoint( + endpoint: String, + allowedHosts: Set[String] + ): Either[String, Unit] = { + Try(URI.create(endpoint)) match { + case Failure(e) => + Left(s"unparseable URI (${e.getClass.getSimpleName})") + case Success(uri) => + val scheme = Option(uri.getScheme).map(_.toLowerCase).getOrElse("") + if (scheme.isEmpty) { + Left("missing scheme") + } else if (!AllowedSchemes.contains(scheme)) { + Left(s"scheme '$scheme' not in allowlist ${AllowedSchemes.toSeq.sorted.mkString("{", ",", "}")}") + } else { + val host = Option(uri.getHost).map(_.toLowerCase).getOrElse("") + if (host.isEmpty) { + Left("missing host") + } else if (!allowedHosts.contains(host)) { + Left(s"host '$host' not in allowlist") + } else { + Right(()) + } + } + } + } + + /** + * Build a Resource from the service name plus the allowlisted subset + * of OTEL_RESOURCE_ATTRIBUTES. Unknown keys are dropped silently; + * service.name from env is ignored in favour of the argument. + */ + private[observability] def buildResource(serviceName: String, rawAttrs: String): Resource = { + val builder = Attributes.builder() + builder.put(AttributeKey.stringKey("service.name"), serviceName) + + parseAttrs(rawAttrs).foreach { + case (key, value) if AllowedResourceKeys.contains(key) && key != "service.name" => + builder.put(AttributeKey.stringKey(key), value) + case _ => // dropped — not in allowlist or overrides service.name + } + + Resource.create(builder.build()) + } + + /** Parse a `k1=v1,k2=v2` string. Malformed entries are skipped. */ + private[observability] def parseAttrs(raw: String): Seq[(String, String)] = { + if (raw == null || raw.isEmpty) return Seq.empty + raw + .split(',') + .iterator + .map(_.trim) + .filter(_.nonEmpty) + .flatMap { entry => + val idx = entry.indexOf('=') + if (idx <= 0 || idx == entry.length - 1) None + else Some(entry.substring(0, idx).trim -> entry.substring(idx + 1).trim) + } + .toSeq + } + + private def buildOtlpSpanExporter(endpoint: String): SpanExporter = + OtlpGrpcSpanExporter.builder().setEndpoint(endpoint).build() + + private def buildOtlpLogExporter(endpoint: String): LogRecordExporter = + OtlpGrpcLogRecordExporter.builder().setEndpoint(endpoint).build() + + private def buildOtlpMetricExporter(endpoint: String): MetricExporter = + OtlpGrpcMetricExporter.builder().setEndpoint(endpoint).build() + + /** + * Parse and clamp OTEL_METRIC_EXPORT_INTERVAL (milliseconds). + * Out-of-range or unparseable input falls back to the default and + * emits a single WARN. Pure-ish — easy to test without standing up + * the meter SDK. + */ + private[observability] def clampIntervalMs(raw: Option[String]): Long = { + raw match { + case None => DefaultMetricIntervalMs + case Some(value) => + Try(value.trim.toLong) match { + case Failure(_) => + logger.warn( + s"OTEL_METRIC_EXPORT_INTERVAL '$value' is not a number; " + + s"using default ${DefaultMetricIntervalMs}ms." + ) + DefaultMetricIntervalMs + case Success(ms) if ms < MinMetricIntervalMs || ms > MaxMetricIntervalMs => + logger.warn( + s"OTEL_METRIC_EXPORT_INTERVAL=${ms}ms out of range " + + s"[${MinMetricIntervalMs}, ${MaxMetricIntervalMs}]; " + + s"using default ${DefaultMetricIntervalMs}ms." + ) + DefaultMetricIntervalMs + case Success(ms) => ms + } + } + } +} + +/** + * Hides the Logback attach step behind a small object so [[OtelInit]] + * doesn't import Logback types directly (keeps the SDK init testable + * without a Logback dependency on the classpath in test runs that + * inject a mock attacher). + */ +private[observability] object LogbackBinder extends LazyLogging { + + /** Attempts to find the Logback ROOT logger, attach a fresh + * [[TexeraOtelLogAppender]] bound to `otel`, and start it. If + * Logback is not the active SLF4J binding (or for any other + * classpath issue), emits one WARN and returns — never throws. + */ + def attach(serviceName: String, otel: OpenTelemetry): Unit = { + val factory = org.slf4j.LoggerFactory.getILoggerFactory + factory match { + case ctx: ch.qos.logback.classic.LoggerContext => + val root = ctx.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME) + val appender = new TexeraOtelLogAppender() + appender.setContext(ctx) + appender.setName(s"texera-otel-$serviceName") + appender.bind(otel) + appender.start() + root.addAppender(appender) + case other => + logger.warn( + s"SLF4J binding is not Logback (${other.getClass.getName}); " + + "OTel log export is not wired. Application logs to stdout/file are unaffected." + ) + } + } +} diff --git a/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala b/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala new file mode 100644 index 00000000000..37d04a303e6 --- /dev/null +++ b/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import ch.qos.logback.classic.Level +import ch.qos.logback.classic.spi.{ILoggingEvent, IThrowableProxy, ThrowableProxyUtil} +import ch.qos.logback.core.UnsynchronizedAppenderBase +import io.opentelemetry.api.OpenTelemetry +import io.opentelemetry.api.common.AttributeKey +import io.opentelemetry.api.logs.{Logger, Severity} +import io.opentelemetry.api.trace.Span +import io.opentelemetry.context.Context + +import java.util.concurrent.TimeUnit + +/** + * Logback appender that forwards every event through [[LogSanitizer]] + * before emitting it as an OTel LogRecord. + * + * Lifecycle: + * - Construct with no args (Logback / programmatic instantiation). + * - Call [[bind]] once with the active [[OpenTelemetry]] instance + * (done by [[OtelInit]] after the SDK is built). Until then, + * [[append]] is a silent no-op — log events keep flowing to + * stdout/file unimpeded. + * - Stopping the appender unbinds; subsequent events drop. + * + * This is intentionally a thin shim. All security-critical logic + * lives in [[LogSanitizer]] so it can be tested without a Logback + * fixture. + */ +class TexeraOtelLogAppender extends UnsynchronizedAppenderBase[ILoggingEvent] { + + // @volatile so a late [[bind]] is visible to appender threads + // without taking a lock on the hot path. + @volatile private var otelLogger: Option[Logger] = None + + def bind(otel: OpenTelemetry): Unit = { + otelLogger = Some(otel.getLogsBridge.get("texera.logback")) + } + + override def stop(): Unit = { + otelLogger = None + super.stop() + } + + override def append(event: ILoggingEvent): Unit = { + otelLogger match { + case None => () // disabled or not yet wired + case Some(logger) => + try { + emit(logger, event) + } catch { + // Logback's addStatus contract: errors from inside an + // appender must not throw out into the calling thread. + case t: Throwable => + addError("OTel log emission failed", t) + } + } + } + + private def emit(logger: Logger, event: ILoggingEvent): Unit = { + // Append the throwable's full stack trace to the body when one is + // attached. Without this, Dropwizard's LoggingExceptionMapper logs + // "Error handling a request: " and the exception itself never + // reaches the observability backend — making 500s impossible to + // diagnose from the dashboard. ThrowableProxyUtil emits a Logback- + // formatted trace that fits inside a single log record. + val baseBody = LogSanitizer.sanitize(event.getFormattedMessage) + val body = Option(event.getThrowableProxy) match { + case Some(proxy) => + // JVM-generated stack frames are trusted (not user input), so + // we skip the C0 strip that would collapse newlines and ruin + // readability. We do still cap the total length implicitly + // via the OTel SDK's per-record body limit, and the body + // stays valid UTF-8 because Logback emits ASCII frame text. + baseBody + "\n" + formatThrowable(proxy) + case None => baseBody + } + val builder = logger + .logRecordBuilder() + .setBody(body) + .setSeverity(severityFromLevel(event.getLevel)) + .setSeverityText(event.getLevel.toString) + .setTimestamp(event.getTimeStamp, TimeUnit.MILLISECONDS) + + // MDC subset: typed AttributeKeys only, so no string injection + // path exists for downstream consumers. + LogSanitizer.filterMdc(event.getMDCPropertyMap).foreach { + case (k, v) => builder.setAttribute(AttributeKey.stringKey(k), v) + } + + builder.setAttribute(AttributeKey.stringKey("logger.name"), event.getLoggerName) + builder.setAttribute(AttributeKey.stringKey("thread.name"), event.getThreadName) + + // Attach the current trace context so the SDK populates trace_id / + // span_id on the LogRecord automatically when a span is active. + val span = Span.current() + if (span.getSpanContext.isValid) { + builder.setContext(Context.current()) + } + + builder.emit() + } + + /** Pretty-print a Logback throwable proxy. Matches what Logback's + * default pattern layout would produce for `%ex` — class name, + * message, full stack frames, then walks the cause chain. */ + private def formatThrowable(proxy: IThrowableProxy): String = + ThrowableProxyUtil.asString(proxy) + + private def severityFromLevel(level: Level): Severity = { + if (level == null) return Severity.UNDEFINED_SEVERITY_NUMBER + level.toInt match { + case Level.TRACE_INT => Severity.TRACE + case Level.DEBUG_INT => Severity.DEBUG + case Level.INFO_INT => Severity.INFO + case Level.WARN_INT => Severity.WARN + case Level.ERROR_INT => Severity.ERROR + case _ => Severity.UNDEFINED_SEVERITY_NUMBER + } + } +} diff --git a/common/config/src/test/scala/org/apache/texera/observability/LogSanitizerSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/LogSanitizerSpec.scala new file mode 100644 index 0000000000000000000000000000000000000000..c5957b65e4b163a89ae7681b69a223339f568d9e GIT binary patch literal 4667 zcmb7I(Qezu675?7{X$;`e3Ao;N>ld+NZ|sqETxKNDX^3#Zc`X5awV~*cG>09vRdS} zUvod~eo44yvq)KH-R+F7eY9%7TuaTbGr7)7Z zI#Z#rkycc1jk+gJyEmbnt`WxYzE` zG>YB+FiP`%a9|nPIo3oeEw#C)DWF6cMr>Nd6e+w|I7is9kuzb!NX=k6S(#`yx1^U! znsgo|us7yVM%~hxGzTutBG#48icl(17i65$j{xG3U--|djsIuT$X%oX3c``wAlx-gHfjepNlQ;R4mgJT>X|s1`JPDBog=(hzUH6#gg;q z(&W+)AC?AutJx_VOiKw=layzlDJvaB;MD~nO(@>(wMNt%d34bl^+u=kzBm41_-;(^ zTQ@hY!MN8M(eQ@a!@*^5+#3$?)TP$omUmwD2A8KKAqNOeJ|>2PfTt1jDZ`4IN*`D& zMV8V;2GKMM5T2T4VkT*(AEZ$%l|-6FlyYKII2l45i^w`jrEK@J*nGTg)@tDTo`Jx2 zNxYSh(g6z3| z=Q@i+|GIcy13OUv*YMtN>o}GHmquGFglBo%U9A?x0$la=Y$Oyi`MWf14kTKs!jvj5 z~|q9 z=pB_~8fnIoggO^UGt(Rh#bt$@;z5v-ON#Xj$x_ney7~j?L=Wf*fr+PY#VXb!q;vA- zGLH2}^HFtMR~_mPUsJy~=0`ZubW-?xAq&JUv^CNv4l=Sk!D*-2 z_Vazbp*s?58H$lKZe8pIlWYQ+NCYpx7`*F$dC}>Av{?L=n1cL2Lm3|nqrj+!kYfR= zL~=j{&TbHMR0a32MPgUfB$dT6O>`K^l!90a6|X8z?`P8Us@9J=%jmmnokM$Mn99IN zkR6z8GT~A_B)#hUoeZG^Lcw$)ajcC1351tu35>-*ao8U0z%_Tf~ZA>6& zs>#1<*JbP8Y1`IhD!?dJ9z?OgT)^X=P;@OGFLb!NSYgs~6CK7|3_5OrUI=q9%`R=2 zl~Q&JQ!=)eCtqpRq`Y3-j{7>}Bazm||6OEp8oT{Cvjp2M$* z4spE42)ym!xF93EcG0n!-J;8tRh-&_r&09!ST7wZoD0wi?q*OMIzsN=4omJj)v|3Y z22YR7({Z=AzOIL=4a~`tXmAm&)r4CWn*MbdJV`{Qm>O;tU1My9(a`9GdnIlbnN3k> zjvs*uav9@NN_C-Z)QI)0kq@E|k)Z3#HYJ$6GakHmD{=u3&!Nbq?lOt4WHB+zy9f=f z@x42pJ>09@kC!Gwt-pk*#LX8s>Ev5-(KOI|8_4SYW>I+}=F5^Z9KX*^xcOdOPxu}^ zML>S(!`X|?8iQF+FMKOzIf{IUirMnS7wH@EAe^%Y6u)M)j<`2!2caXL)7%9H{Q(d4 z4G*Wd0~CSd&pl3VfM7O5P3J(aE%Fsv=}w&gm%?xg?BD9%XB3gQp@TdZ)V?aQI>rSK zUB%6~3n-m0gzh0XT`7qzb#DUSnP1(|%xcT;CclC}hyjw|3k*N`kWK9WAksPB|2)EF z2p4V2fB#a?LoasuNw&?Kv*ob2SLD}F9zQt8AoXRXr*kve>F68p3!F9Mo1C$P%jA^E MPQf|$r`o65-%WVrbpQYW literal 0 HcmV?d00001 diff --git a/common/config/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala new file mode 100644 index 00000000000..5ebd0eb3f4a --- /dev/null +++ b/common/config/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter +import org.scalatest.BeforeAndAfterEach +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import scala.jdk.CollectionConverters._ + +class OtelInitSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { + + override def beforeEach(): Unit = { + OtelInit.resetForTest() + } + + override def afterEach(): Unit = { + OtelInit.resetForTest() + } + + // ----- validateEndpoint: pure function, exhaustive cases ------------- + + "validateEndpoint" should "accept a loopback OTLP gRPC URL" in { + OtelInit.validateEndpoint("http://localhost:4317", OtelInit.DefaultAllowedHosts) shouldBe Right(()) + OtelInit.validateEndpoint("grpc://127.0.0.1:4317", OtelInit.DefaultAllowedHosts) shouldBe Right(()) + OtelInit.validateEndpoint("https://localhost:4318", OtelInit.DefaultAllowedHosts) shouldBe Right(()) + } + + it should "reject file:// schemes (path traversal style attack)" in { + val result = OtelInit.validateEndpoint("file:///etc/passwd", OtelInit.DefaultAllowedHosts) + result.isLeft shouldBe true + result.left.toOption.get should include("scheme") + } + + it should "reject arbitrary remote hosts not in the allowlist" in { + val result = OtelInit.validateEndpoint( + "http://attacker.example.com:4317", + OtelInit.DefaultAllowedHosts + ) + result.isLeft shouldBe true + result.left.toOption.get should include("host") + } + + it should "accept hosts added to the allowlist" in { + val widened = OtelInit.DefaultAllowedHosts + "collector.internal" + OtelInit.validateEndpoint("http://collector.internal:4317", widened) shouldBe Right(()) + } + + it should "reject endpoints with no scheme" in { + val result = OtelInit.validateEndpoint("localhost:4317", OtelInit.DefaultAllowedHosts) + result.isLeft shouldBe true + } + + it should "reject endpoints with no host" in { + val result = OtelInit.validateEndpoint("http:///path", OtelInit.DefaultAllowedHosts) + result.isLeft shouldBe true + } + + it should "reject completely malformed input" in { + val result = OtelInit.validateEndpoint("not a uri at all :: bad", OtelInit.DefaultAllowedHosts) + result.isLeft shouldBe true + } + + // ----- buildResource: only allowlisted keys survive ------------------- + + "buildResource" should "always include the service.name from the argument" in { + val r = OtelInit.buildResource("my-service", "") + Option(r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.name"))) shouldBe Some( + "my-service" + ) + } + + it should "honor allowlisted keys from OTEL_RESOURCE_ATTRIBUTES" in { + val r = OtelInit.buildResource("svc", "service.version=1.2.3,deployment.environment=staging") + Option(r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.version"))) shouldBe Some( + "1.2.3" + ) + Option( + r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("deployment.environment")) + ) shouldBe Some("staging") + } + + it should "silently drop keys not in the allowlist (anti-injection)" in { + val r = OtelInit.buildResource( + "svc", + "service.version=1.0,secret=hunter2,db.password=p4ssw0rd,custom.tag=evil" + ) + val attrs: Map[String, String] = r.getAttributes.asMap.asScala.iterator.map { + case (k, v) => k.getKey -> v.toString + }.toMap + + attrs.keySet should contain only ("service.name", "service.version") + attrs should not contain key("secret") + attrs should not contain key("db.password") + attrs should not contain key("custom.tag") + } + + it should "refuse to let OTEL_RESOURCE_ATTRIBUTES override service.name" in { + val r = OtelInit.buildResource("real-svc", "service.name=spoofed") + Option(r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.name"))) shouldBe Some( + "real-svc" + ) + } + + it should "carry the texera.* resource attrs so a CU JVM auto-tags every emitted record" in { + // Setting the CU id at JVM boot is the only sane place to attach + // it for ComputingUnitMaster / ComputingUnitWorker — those JVMs + // are CU-scoped by deployment, and there is no HTTP request to + // hang an MDC value off. The dashboard's CU filter relies on + // this key being present on every record. + val r = OtelInit.buildResource( + "texera-computing-unit-master", + "texera.computing_unit.id=8,texera.workflow.id=441,texera.execution.id=1234" + ) + val attrs: Map[String, String] = r.getAttributes.asMap.asScala.iterator.map { + case (k, v) => k.getKey -> v.toString + }.toMap + attrs("texera.computing_unit.id") shouldBe "8" + attrs("texera.workflow.id") shouldBe "441" + attrs("texera.execution.id") shouldBe "1234" + } + + it should "ignore malformed pairs without crashing" in { + val r = OtelInit.buildResource("svc", ",,,=,foo,service.version=,=bar,service.version=1.0,") + Option(r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.version"))) shouldBe Some( + "1.0" + ) + } + + it should "handle empty / null input cleanly" in { + OtelInit.parseAttrs("") shouldBe empty + OtelInit.parseAttrs(null) shouldBe empty + } + + // ----- end-to-end init: span emission + disable behaviour ------------- + + "init" should "be a no-op when OTEL_SDK_DISABLED is explicitly set to true" in { + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest("svc", Map("OTEL_SDK_DISABLED" -> "true"), exporter) + result shouldBe None + exporter.getFinishedSpanItems.asScala shouldBe empty + } + + it should "initialize by default (no OTEL_SDK_DISABLED set) so `sbt run` services emit without per-JVM config" in { + // The previous default was `true` (opt-in), which forced every + // service Run Configuration to set OTEL_SDK_DISABLED=false + // explicitly. New default is `false` so a fresh sbt run is + // immediately tagged in the dashboard. Operators who need to + // silence telemetry still set the env var explicitly. + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest( + "svc", + Map( + // OTEL_SDK_DISABLED deliberately omitted — defaults to false. + "OTEL_EXPORTER_OTLP_ENDPOINT" -> "http://localhost:4317" + ), + exporter + ) + result.isDefined shouldBe true + } + + it should "emit a single service.start span when enabled with a valid endpoint" in { + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest( + "my-service", + Map( + "OTEL_SDK_DISABLED" -> "false", + "OTEL_EXPORTER_OTLP_ENDPOINT" -> "http://localhost:4317" + ), + exporter + ) + result.isDefined shouldBe true + + // BatchSpanProcessor is async — flush before reading. + result.get + .asInstanceOf[io.opentelemetry.sdk.OpenTelemetrySdk] + .getSdkTracerProvider + .forceFlush() + .join(2, java.util.concurrent.TimeUnit.SECONDS) + + val spans = exporter.getFinishedSpanItems.asScala + spans should have size 1 + spans.head.getName shouldBe "service.start" + } + + it should "refuse to initialize when the endpoint scheme is file://" in { + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest( + "svc", + Map( + "OTEL_SDK_DISABLED" -> "false", + "OTEL_EXPORTER_OTLP_ENDPOINT" -> "file:///etc/passwd" + ), + exporter + ) + result shouldBe None + exporter.getFinishedSpanItems.asScala shouldBe empty + } + + it should "refuse to initialize when the endpoint host is off-allowlist" in { + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest( + "svc", + Map( + "OTEL_SDK_DISABLED" -> "false", + "OTEL_EXPORTER_OTLP_ENDPOINT" -> "http://attacker.example.com:4317" + ), + exporter + ) + result shouldBe None + exporter.getFinishedSpanItems.asScala shouldBe empty + } + + it should "be idempotent — second init returns the same instance" in { + val exporter = InMemorySpanExporter.create() + val env = Map( + "OTEL_SDK_DISABLED" -> "false", + "OTEL_EXPORTER_OTLP_ENDPOINT" -> "http://localhost:4317" + ) + val first = OtelInit.initForTest("svc", env, exporter) + val second = OtelInit.initForTest("svc", env, exporter) + second shouldBe first + } +} diff --git a/common/config/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala new file mode 100644 index 00000000000..83b9ca9d469 --- /dev/null +++ b/common/config/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import ch.qos.logback.classic.{Level, Logger, LoggerContext} +import ch.qos.logback.classic.spi.LoggingEvent +import io.opentelemetry.api.OpenTelemetry +import io.opentelemetry.api.logs.Severity +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.logs.SdkLoggerProvider +import io.opentelemetry.sdk.logs.`export`.SimpleLogRecordProcessor +import io.opentelemetry.sdk.testing.exporter.InMemoryLogRecordExporter +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.slf4j.LoggerFactory + +import scala.jdk.CollectionConverters._ + +class TexeraOtelLogAppenderSpec extends AnyFlatSpec with Matchers { + + /** Build an OpenTelemetry SDK whose LoggerProvider drains to the + * given in-memory exporter via the synchronous SimpleLogRecordProcessor, + * so tests don't depend on batch timing. */ + private def newFixture(): (OpenTelemetry, InMemoryLogRecordExporter, TexeraOtelLogAppender) = { + val exporter = InMemoryLogRecordExporter.create() + val lp = SdkLoggerProvider + .builder() + .addLogRecordProcessor(SimpleLogRecordProcessor.create(exporter)) + .build() + val sdk = OpenTelemetrySdk.builder().setLoggerProvider(lp).build() + val appender = new TexeraOtelLogAppender() + appender.setContext(LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext]) + appender.bind(sdk) + appender.start() + (sdk, exporter, appender) + } + + private def makeEvent( + message: String, + level: Level = Level.INFO, + mdc: Map[String, String] = Map.empty + ): LoggingEvent = { + val ctx = LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext] + val logger = ctx.getLogger("test.logger").asInstanceOf[Logger] + val ev = new LoggingEvent("fqcn", logger, level, message, null, null) + if (mdc.nonEmpty) ev.setMDCPropertyMap(mdc.asJava) + ev + } + + // ----- positive paths ------------------------------------------------- + + "TexeraOtelLogAppender" should "emit an INFO record with body + severity" in { + val (_, exporter, appender) = newFixture() + appender.doAppend(makeEvent("hello world")) + + val records = exporter.getFinishedLogRecordItems.asScala + records should have size 1 + records.head.getBodyValue.asString shouldBe "hello world" + records.head.getSeverity shouldBe Severity.INFO + records.head.getSeverityText shouldBe "INFO" + } + + it should "map every log level to a distinct OTel severity" in { + val (_, exporter, appender) = newFixture() + Seq(Level.TRACE, Level.DEBUG, Level.INFO, Level.WARN, Level.ERROR).foreach { lvl => + appender.doAppend(makeEvent(s"msg-$lvl", lvl)) + } + val severities = exporter.getFinishedLogRecordItems.asScala.map(_.getSeverity).toSet + severities shouldBe Set(Severity.TRACE, Severity.DEBUG, Severity.INFO, Severity.WARN, Severity.ERROR) + } + + // ----- security: sanitisation happens at the boundary ----------------- + + it should "strip CRLF from a forged log-injection payload before emission" in { + val (_, exporter, appender) = newFixture() + appender.doAppend(makeEvent("hello\r\nFAKE LOG LINE\r\nworld")) + + val body = exporter.getFinishedLogRecordItems.asScala.head.getBodyValue.asString + body shouldBe "helloFAKE LOG LINEworld" + body should not include "\n" + body should not include "\r" + } + + it should "redact Bearer tokens at emission time" in { + val (_, exporter, appender) = newFixture() + appender.doAppend(makeEvent("Authorization: Bearer abc123XYZ.foo")) + + val body = exporter.getFinishedLogRecordItems.asScala.head.getBodyValue.asString + body should include("[REDACTED]") + body should not include "abc123XYZ" + } + + it should "truncate a 1 MiB body to MaxBodyBytes with the marker" in { + val (_, exporter, appender) = newFixture() + val oversize = "x" * (1024 * 1024) + appender.doAppend(makeEvent(oversize)) + + val body = exporter.getFinishedLogRecordItems.asScala.head.getBodyValue.asString + body.length shouldBe LogSanitizer.MaxBodyBytes + body should endWith(LogSanitizer.TruncatedMarker) + } + + // ----- security: MDC allowlist ---------------------------------------- + + it should "forward only allowlisted MDC keys as log attributes" in { + val (_, exporter, appender) = newFixture() + appender.doAppend( + makeEvent( + "msg", + mdc = Map( + "trace_id" -> "abc", + "texera.workflow.id" -> "42", + "secret" -> "should-not-leak", + "password" -> "p4ssw0rd" + ) + ) + ) + + val record = exporter.getFinishedLogRecordItems.asScala.head + val attrs = record.getAttributes.asMap.asScala.iterator.map { + case (k, v) => k.getKey -> v.toString + }.toMap + + attrs.keySet should contain allOf ("trace_id", "texera.workflow.id") + attrs.keySet should not contain ("secret") + attrs.keySet should not contain ("password") + attrs.values should contain noElementsOf Seq("should-not-leak", "p4ssw0rd") + } + + // ----- lifecycle ------------------------------------------------------ + + it should "be a silent no-op when not yet bound to an OpenTelemetry instance" in { + val unbound = new TexeraOtelLogAppender() + unbound.setContext(LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext]) + unbound.start() + // Should not throw, even though no SDK is wired. + noException should be thrownBy unbound.doAppend(makeEvent("hello")) + } + + it should "stop emitting after stop() is called" in { + val (_, exporter, appender) = newFixture() + appender.doAppend(makeEvent("first")) + appender.stop() + appender.doAppend(makeEvent("second")) + + val bodies = exporter.getFinishedLogRecordItems.asScala.map(_.getBodyValue.asString) + bodies should contain only "first" + } +} From df10b8717e7f57087dd03a66962121e448c15632 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 5 Jun 2026 04:49:02 -0700 Subject: [PATCH 02/21] feat(observability): deployment - docker-compose OTel collector + Parca/eBPF profiling Co-Authored-By: Claude Opus 4.8 (1M context) --- bin/observability/otel-collector/config.yaml | 132 +++++++++++ bin/observability/parca/README.md | 98 ++++++++ bin/observability/parca/parca-agent.env | 62 +++++ bin/observability/parca/parca.yaml | 42 ++++ bin/single-node/.env | 34 +++ bin/single-node/docker-compose.yml | 191 +++++++++++++++- bin/single-node/up.sh | 65 ++++++ .../ObservabilityComposeSpec.scala | 216 ++++++++++++++++++ .../observability/ParcaConfigSpec.scala | 129 +++++++++++ 9 files changed, 968 insertions(+), 1 deletion(-) create mode 100644 bin/observability/otel-collector/config.yaml create mode 100644 bin/observability/parca/README.md create mode 100644 bin/observability/parca/parca-agent.env create mode 100644 bin/observability/parca/parca.yaml create mode 100755 bin/single-node/up.sh create mode 100644 common/config/src/test/scala/org/apache/texera/observability/ObservabilityComposeSpec.scala create mode 100644 common/config/src/test/scala/org/apache/texera/observability/ParcaConfigSpec.scala diff --git a/bin/observability/otel-collector/config.yaml b/bin/observability/otel-collector/config.yaml new file mode 100644 index 00000000000..7841e37f2f7 --- /dev/null +++ b/bin/observability/otel-collector/config.yaml @@ -0,0 +1,132 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# OpenTelemetry Collector configuration for the Texera bundled +# observability stack. +# +# Bundled image: otel/opentelemetry-collector-contrib:0.153.0 +# (Apache-2.0). The contrib distribution is needed for the +# prometheusremotewrite exporter that fans metrics to VictoriaMetrics. +# +# Pipeline topology: +# +# Texera services +# │ OTLP/gRPC (4317) or OTLP/HTTP (4318) +# ▼ +# otel-collector +# ├── logs ── OTLP ──────────────────► victorialogs:9428 +# ├── metrics ── prometheusremotewrite ─► victoriametrics:8428 +# └── traces ── OTLP ──────────────────► jaeger:4317 +# +# Network: every receiver and exporter is reachable only on the +# compose bridge network. No host-port mapping for the receivers in +# docker-compose.yml — that keeps OTLP ingest cluster-local, which +# is the security posture the PR plan calls for. +# +# Hostnames below are docker-compose service names — they resolve via +# docker's embedded DNS inside the network. + +receivers: + otlp: + protocols: + grpc: + # 0.0.0.0 binds inside the container only; the compose service + # does NOT publish this port to the host. Reachable only via + # the texera-single-node bridge network. + endpoint: 0.0.0.0:4317 + # 4 MiB cap on a single OTLP message — anyone trying to spam + # the collector with a huge payload gets a clean reject. + max_recv_msg_size_mib: 4 + http: + endpoint: 0.0.0.0:4318 + +processors: + # batch is the upstream-recommended first processor. Caps memory + # per export by batching by either size or timeout, whichever hits + # first. Defaults are sensible for a single-node deployment. + batch: + send_batch_size: 1024 + timeout: 5s + + # memory_limiter prevents the collector OOMing under a sudden burst + # of telemetry. We set a soft limit, not a hard percentage, so the + # numbers are reviewable. + memory_limiter: + check_interval: 1s + limit_mib: 512 + spike_limit_mib: 128 + +exporters: + # VictoriaLogs accepts OTLP for logs natively (verify on every + # version bump). The endpoint path includes /opentelemetry/v1/logs + # per VictoriaLogs's OTLP ingest convention. + otlphttp/victorialogs: + endpoint: http://victorialogs:9428/insert/opentelemetry + compression: gzip + timeout: 10s + # tls is disabled for the compose bridge network. Any deploy + # that opens these ports beyond the bridge must enable TLS here. + tls: + insecure: true + + # VictoriaMetrics consumes Prometheus remote-write natively. The + # prometheusremotewrite exporter is the upstream-recommended way + # to bridge OTel metrics into Prom-ecosystem stores. + prometheusremotewrite: + endpoint: http://victoriametrics:8428/api/v1/write + tls: + insecure: true + # Keep the remote-write batch bounded so a metric burst can't + # become an oversized HTTP request. + remote_write_queue: + queue_size: 1000 + num_consumers: 4 + + # Jaeger v2 accepts OTLP directly — no jaeger exporter type needed. + otlp/jaeger: + endpoint: jaeger:4317 + tls: + insecure: true + +service: + pipelines: + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/victorialogs] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheusremotewrite] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp/jaeger] + + # Collector telemetry binds to loopback inside the container only + # so we can scrape /metrics from a sidecar if needed without + # exposing the collector's own metrics to the host. + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 127.0.0.1 + port: 8888 + logs: + level: info diff --git a/bin/observability/parca/README.md b/bin/observability/parca/README.md new file mode 100644 index 00000000000..27ef602dfb7 --- /dev/null +++ b/bin/observability/parca/README.md @@ -0,0 +1,98 @@ + + +# Parca profiles (PR 5) + +This directory holds configuration for the **profiles** signal in the +Texera observability stack. The compose service definitions that +consume these files land in PR 6; PR 5 ships the configuration only, +so the agent's deploy posture can be reviewed in isolation from the +broader compose changes. + +Components — both Apache-2.0 (see +[`docs/observability/LICENSING.md`](../../../docs/observability/LICENSING.md)): + +| File | Component | Image | +|---|---|---| +| `parca.yaml` | Parca server v0.28.0 | `ghcr.io/parca-dev/parca:v0.28.0` | +| `parca-agent.env` | Parca eBPF agent v0.47.1 | `ghcr.io/parca-dev/parca-agent:v0.47.1` | + +## Deploy posture + +The Parca agent uses eBPF to sample stack traces from running +processes. That puts a few non-negotiable requirements on the host: + +- **Linux only.** eBPF is a Linux kernel feature. macOS and Windows + developers cannot run the agent; the rest of the observability + stack (logs, metrics, traces) works on all platforms. +- **Privileged container.** The agent needs `CAP_SYS_ADMIN`-class + permissions to load eBPF programs and mount the perf-event + facility. The PR 6 compose service will set `privileged: true` + and bind-mount `/sys/kernel/debug`, `/proc`, and `/sys` read-only + into the container. +- **Read-only on host filesystems.** The bind-mounts above are + `ro` — the agent reads kernel state but cannot write to it. No + network exposure outside the cluster: the agent only opens an + outbound connection to the bundled Parca server on + `parca:7070`. + +## Opt-out + +For developers on non-Linux dev machines, or for any deploy that +chooses not to run profiles, set this in the host environment before +`docker compose up`: + +``` +TEXERA_OBSERVABILITY_PROFILES=disabled +``` + +PR 6's compose file gates the `parca-agent` (and optionally the +`parca` server too) on this flag — the rest of the stack continues +to run with `disabled` panels in the UI. + +## What gets profiled + +The agent's default behaviour is to discover and profile every +process on the host. We attach two static labels via +`parca-agent.env`: + +- `deployment=texera` +- `cluster=local` (override per env) + +When the PR 7 Texera query gateway runs Parca queries, it filters on +`deployment=texera` so the dashboard only ever shows Texera-process +profiles, never the operator's other workloads. + +We do **not** label profiles with `workflow.id` / `execution.id`. As +with metrics, those are unbounded identifiers and would blow up +Parca's storage cardinality. Per-execution profile views are reached +by joining on `trace_id` at query time (the Parca query API supports +this). + +## What is not in PR 5 + +- The docker-compose service definitions (PR 6). +- The Angular flame-graph panel that renders pprof data (PR 11 in + the [PR plan](../../../docs/observability/PR-PLAN.md)). +- Kubernetes Helm templates for the agent DaemonSet (deferred to a + later series — single-node compose first). +- TLS between agent and server. The agent dials Parca over plain + gRPC because the bundled deployment binds both to the docker + bridge network. Any deploy that opens those ports to a wider + network must enable TLS at the compose / k8s layer. diff --git a/bin/observability/parca/parca-agent.env b/bin/observability/parca/parca-agent.env new file mode 100644 index 00000000000..5ff164a20da --- /dev/null +++ b/bin/observability/parca/parca-agent.env @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Parca eBPF agent environment. +# +# Bundled image: ghcr.io/parca-dev/parca-agent:v0.47.1 (Apache-2.0). +# +# Consumed by the PR 6 docker-compose service via env_file:. The agent +# itself reads its config as CLI args; we stage the args here so they +# are version-controlled, reviewed, and reused across compose + +# (future) Helm. +# +# Deploy requirements (Linux-only, privileged): +# * The agent uses eBPF and must run on a Linux host kernel. +# * Requires CAP_SYS_ADMIN-class permissions to load eBPF programs. +# The compose service runs the agent container with +# `privileged: true` and mounts /sys/kernel/debug, /proc, /sys +# read-only. macOS / Windows developers must run with +# TEXERA_OBSERVABILITY_PROFILES=disabled (see README.md). +# +# Labels: +# * node: host identifier — set by the compose layer to the docker +# host's hostname. Overridable. +# * metadata-external-labels: static labels attached to every +# profile. We attach service.name so the Parca query layer can +# group profiles by Texera service the same way logs/traces are +# grouped (matches the OTel resource attr). + +PARCA_AGENT_REMOTE_STORE_ADDRESS=parca:7070 +PARCA_AGENT_REMOTE_STORE_INSECURE=true + +# Per-host identification. Compose can override at deploy time: +# `--node=${HOSTNAME:-texera-dev}` +PARCA_AGENT_NODE=texera-dev + +# Static labels attached to every profile this agent emits. +# Must NOT include workflow.id / execution.id — cardinality DoS in +# Parca's storage. Keep to coarse fields only. +PARCA_AGENT_METADATA_EXTERNAL_LABELS=deployment=texera;cluster=local + +# CPU sample rate. 19 Hz is upstream default and a sensible balance +# between overhead and resolution; we keep it explicit so an +# accidental upstream change doesn't silently shift it. +PARCA_AGENT_PROFILING_CPU_SAMPLING_FREQUENCY=19 + +# Log level — info by default; switch to warn in production to keep +# the agent quiet. +PARCA_AGENT_LOG_LEVEL=info diff --git a/bin/observability/parca/parca.yaml b/bin/observability/parca/parca.yaml new file mode 100644 index 00000000000..0af9e88a393 --- /dev/null +++ b/bin/observability/parca/parca.yaml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Parca server configuration. +# +# This is the minimal config the Texera bundled observability stack +# uses. We deliberately keep scrape_configs empty — profiles arrive +# from the Parca eBPF agent (push model) rather than pull-mode scraping. +# Bundled image: ghcr.io/parca-dev/parca:v0.28.0 (Apache-2.0). +# +# Storage is intentionally filesystem-backed for the single-node +# docker-compose deployment. Replace with an object-storage backend +# (S3/GCS/Azure) before any multi-node or production use. + +object_storage: + bucket: + # FILESYSTEM is the simplest backend for single-node development. + # Data lives inside the container at this path; mount a volume + # in docker-compose to persist across container restarts. + type: "FILESYSTEM" + config: + directory: "/var/lib/parca" + +# scrape_configs is intentionally empty: the Texera deployment pushes +# profiles from the parca-agent via OTLP/gRPC. Adding scrape targets +# here would generate significantly more data than agent-based +# profiling per the upstream recommendation. +scrape_configs: [] diff --git a/bin/single-node/.env b/bin/single-node/.env index 54aa2f5b322..4721a1a190f 100644 --- a/bin/single-node/.env +++ b/bin/single-node/.env @@ -97,3 +97,37 @@ LLM_API_KEY=dummy TEXERA_DASHBOARD_SERVICE_ENDPOINT=http://dashboard-service:8080 WORKFLOW_COMPILING_SERVICE_ENDPOINT=http://workflow-compiling-service:9090 WORKFLOW_EXECUTION_SERVICE_ENDPOINT=http://workflow-runtime-coordinator-service:8085 + +# ============================================================================ +# Observability (PR 6) +# ============================================================================ +# Each backend lives behind its own docker-compose profile. By default +# COMPOSE_PROFILES enables every observability profile so `docker compose up` +# brings the whole stack online — there is no separate observability compose +# file or monolithic observability profile. +# +# To disable a signal: +# * Prefer using bin/single-node/up.sh, which translates the +# TEXERA_OBSERVABILITY_* env vars below into the right COMPOSE_PROFILES. +# * Or edit COMPOSE_PROFILES directly here. +# +# Disable env-var conventions (consumed by up.sh): +# TEXERA_OBSERVABILITY_LOGS=disabled drops victorialogs +# TEXERA_OBSERVABILITY_METRICS=disabled drops victoriametrics +# TEXERA_OBSERVABILITY_TRACES=disabled drops jaeger +# TEXERA_OBSERVABILITY_PROFILES=disabled drops parca + parca-agent +# TEXERA_OBSERVABILITY_COLLECTOR=disabled drops the otel-collector (rare) +# +# Parca eBPF agent needs Linux + privileged container; macOS/Windows +# developers must set TEXERA_OBSERVABILITY_PROFILES=disabled. +COMPOSE_PROFILES=observability-collector,observability-logs,observability-metrics,observability-traces,observability-profiles + +# Query endpoints the dashboard gateway calls to read each signal. The +# defaults in observability-gateway.conf are host-local (127.0.0.1), which +# is what a natively-run backend (sbt / IntelliJ) needs. Inside this compose +# the gateway runs in a container, so it must reach the backends by their +# bridge-network service names — these overrides do that. +TEXERA_OBS_LOGS_URL=http://victorialogs:9428 +TEXERA_OBS_METRICS_URL=http://victoriametrics:8428 +TEXERA_OBS_TRACES_URL=http://jaeger:16686 +TEXERA_OBS_PROFILES_URL=http://parca:7070 diff --git a/bin/single-node/docker-compose.yml b/bin/single-node/docker-compose.yml index e26fe8aa957..7930516746e 100644 --- a/bin/single-node/docker-compose.yml +++ b/bin/single-node/docker-compose.yml @@ -487,6 +487,189 @@ services: command: > sh -c 'apk add --no-cache curl jq bash > /dev/null 2>&1 && bash /examples/load-examples.sh' + # ======================================================================== + # Part 5: Observability stack (PR 6). + # + # All six services live in this single compose file (no separate + # observability compose file, no monolithic observability profile). + # Per-signal profiles let an operator drop one without disturbing + # the others — see bin/single-node/up.sh and bin/single-node/.env + # (COMPOSE_PROFILES default includes every observability profile so + # `docker compose up` runs the whole stack). + # + # Network posture: every receiver/HTTP port binds to loopback + # (127.0.0.1) on the host or stays inside the texera-single-node + # bridge network entirely. No 0.0.0.0 host bindings, no Ingress. + # ======================================================================== + + # OpenTelemetry Collector — the single OTLP ingress for all three + # signals. Reads bin/observability/otel-collector/config.yaml. + # Apache-2.0; pinned to the contrib distribution. + otel-collector: + image: otel/opentelemetry-collector-contrib:0.153.0 + container_name: texera-otel-collector + profiles: [observability-collector] + restart: always + user: "10001:10001" + read_only: true + security_opt: + - no-new-privileges:true + volumes: + - ../observability/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml:ro + command: ["--config=/etc/otelcol-contrib/config.yaml"] + # Local dev override: publish OTLP receiver ports on the host + # loopback so a Scala backend running outside docker (sbt / + # IntelliJ) can emit telemetry. In compose-only deploys, services + # talk to otel-collector:4317/:4318 via the bridge network. + ports: + - "127.0.0.1:4317:4317" + - "127.0.0.1:4318:4318" + deploy: + resources: + limits: + memory: 768M + cpus: "1.0" + + # VictoriaLogs — Apache-2.0 log store. LogsQL query API on 9428. + victorialogs: + image: victoriametrics/victoria-logs:v1.50.0 + container_name: texera-victorialogs + profiles: [observability-logs] + restart: always + user: "10002:10002" + read_only: true + security_opt: + - no-new-privileges:true + command: + - "-storageDataPath=/data" + - "-retentionPeriod=30d" + - "-httpListenAddr=:9428" + volumes: + - victorialogs_data:/data + # Loopback-only host binding so an operator can curl the query + # API from the host for ad-hoc debugging without exposing it to + # the network. + ports: + - "127.0.0.1:9428:9428" + deploy: + resources: + limits: + memory: 1G + cpus: "1.0" + + # VictoriaMetrics — Apache-2.0 metrics store. Accepts Prometheus + # remote-write from the collector at /api/v1/write, MetricsQL on + # /api/v1/query. + victoriametrics: + image: victoriametrics/victoria-metrics:v1.144.0 + container_name: texera-victoriametrics + profiles: [observability-metrics] + restart: always + user: "10003:10003" + read_only: true + security_opt: + - no-new-privileges:true + command: + - "-storageDataPath=/data" + - "-retentionPeriod=90d" + - "-httpListenAddr=:8428" + volumes: + - victoriametrics_data:/data + ports: + - "127.0.0.1:8428:8428" + deploy: + resources: + limits: + memory: 1G + cpus: "1.0" + + # Jaeger v2 — Apache-2.0 traces backend + UI. v2 accepts OTLP + # natively. The query API lives at :16686, OTLP ingest at :4317 + # (kept inside the bridge network only — only the collector talks + # to it). + jaeger: + image: jaegertracing/jaeger:2.18.0 + container_name: texera-jaeger + profiles: [observability-traces] + restart: always + security_opt: + - no-new-privileges:true + # In-memory storage for the single-node deployment — restarts + # wipe traces. Any deploy needing trace persistence must swap + # in Cassandra or OpenSearch-backed storage here. + ports: + - "127.0.0.1:16686:16686" + deploy: + resources: + limits: + memory: 1G + cpus: "1.0" + + # Parca server — Apache-2.0 profiles backend. Reads + # bin/observability/parca/parca.yaml (PR 5). FILESYSTEM storage. + parca: + image: ghcr.io/parca-dev/parca:v0.28.0 + container_name: texera-parca + profiles: [observability-profiles] + restart: always + security_opt: + - no-new-privileges:true + volumes: + - ../observability/parca/parca.yaml:/parca.yaml:ro + - parca_data:/var/lib/parca + command: + - "/parca" + - "--config-path=/parca.yaml" + ports: + - "127.0.0.1:7070:7070" + deploy: + resources: + limits: + memory: 1G + cpus: "1.0" + + # Parca eBPF agent — Apache-2.0. Linux-only, privileged. + # See bin/observability/parca/README.md for the full deploy + # posture. The privileged + bind-mount block here is the ONLY + # observability service that requires elevated permissions; the + # surface is documented and reviewed. + parca-agent: + image: ghcr.io/parca-dev/parca-agent:v0.47.1 + container_name: texera-parca-agent + profiles: [observability-profiles] + restart: always + depends_on: + parca: + condition: service_started + env_file: + - ../observability/parca/parca-agent.env + # eBPF requires CAP_SYS_ADMIN-class privileges. macOS / Windows + # developers cannot run this — they should drop the + # observability-profiles profile from COMPOSE_PROFILES. + privileged: true + pid: "host" + # Bind-mount kernel state read-only — the agent reads /proc and + # /sys for stack-trace symbolization but cannot write to either. + volumes: + - /sys/kernel/debug:/sys/kernel/debug:ro + - /proc:/host/proc:ro + - /sys:/host/sys:ro + # parca-agent ships as a distroless image (no /bin/sh), so flags + # are passed directly. env_file values above are kept as the + # source of truth — edit both if you change a value. + command: + - "--remote-store-address=parca:7070" + - "--remote-store-insecure=true" + - "--node=texera-dev" + - "--metadata-external-labels=deployment=texera;cluster=local" + - "--profiling-cpu-sampling-frequency=19" + - "--log-level=info" + deploy: + resources: + limits: + memory: 512M + cpus: "0.5" + networks: default: name: texera-single-node @@ -495,4 +678,10 @@ networks: volumes: minio_data: postgres_data: - workflow_result_data: \ No newline at end of file + workflow_result_data: + # Observability storage. Each backend gets its own named volume so + # an operator can drop one signal's history (e.g. logs) without + # touching the others. + victorialogs_data: + victoriametrics_data: + parca_data: \ No newline at end of file diff --git a/bin/single-node/up.sh b/bin/single-node/up.sh new file mode 100755 index 00000000000..2af7abaf191 --- /dev/null +++ b/bin/single-node/up.sh @@ -0,0 +1,65 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Wrapper around `docker compose up` that honors the per-backend +# observability disable env vars described in .env. Run from +# bin/single-node/. +# +# Usage: +# ./up.sh # bring up everything +# TEXERA_OBSERVABILITY_PROFILES=disabled ./up.sh # no Parca / agent +# ./up.sh -d --remove-orphans # extra args forwarded to compose +# +# Why this exists: docker-compose has no first-class way to say +# "default-on, disable per env var". We translate the TEXERA_OBSERVABILITY_* +# envs into the COMPOSE_PROFILES list, then exec `docker compose`. + +set -eu + +cd "$(dirname "$0")" + +# Start from the full set; drop entries as disable envs are set. +PROFILES="observability-collector observability-logs observability-metrics observability-traces observability-profiles" + +drop_profile() { + # $1 = profile name to remove from PROFILES + PROFILES=$(printf '%s\n' $PROFILES | grep -vx "$1" | tr '\n' ' ') +} + +case "${TEXERA_OBSERVABILITY_LOGS:-enabled}" in + disabled|off|false|0) drop_profile observability-logs ;; +esac +case "${TEXERA_OBSERVABILITY_METRICS:-enabled}" in + disabled|off|false|0) drop_profile observability-metrics ;; +esac +case "${TEXERA_OBSERVABILITY_TRACES:-enabled}" in + disabled|off|false|0) drop_profile observability-traces ;; +esac +case "${TEXERA_OBSERVABILITY_PROFILES:-enabled}" in + disabled|off|false|0) drop_profile observability-profiles ;; +esac +case "${TEXERA_OBSERVABILITY_COLLECTOR:-enabled}" in + disabled|off|false|0) drop_profile observability-collector ;; +esac + +# Comma-separated for COMPOSE_PROFILES. +COMPOSE_PROFILES=$(printf '%s\n' $PROFILES | paste -sd, -) +export COMPOSE_PROFILES + +echo "Bringing up Texera with COMPOSE_PROFILES=${COMPOSE_PROFILES:-}" +exec docker compose up "$@" diff --git a/common/config/src/test/scala/org/apache/texera/observability/ObservabilityComposeSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/ObservabilityComposeSpec.scala new file mode 100644 index 00000000000..5f21df9b407 --- /dev/null +++ b/common/config/src/test/scala/org/apache/texera/observability/ObservabilityComposeSpec.scala @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path, Paths} + +/** + * Smoke tests for the PR 6 docker-compose + collector config. Same + * design as [[ParcaConfigSpec]] — string-level assertions, no YAML + * parser, because the goal is to catch typos and licence-pin drift, + * not to validate the upstream schemas. + */ +class ObservabilityComposeSpec extends AnyFlatSpec with Matchers { + + private def resolveBundled(relative: String): Path = { + var dir = Paths.get("").toAbsolutePath + var hops = 0 + while (hops < 10) { + val candidate = dir.resolve(relative) + if (Files.exists(candidate)) return candidate + val parent = dir.getParent + if (parent == null) return candidate + dir = parent + hops += 1 + } + Paths.get(relative) + } + + private def read(p: Path): String = + new String(Files.readAllBytes(p), StandardCharsets.UTF_8) + + // ----- bundled paths -------------------------------------------------- + + private val compose = resolveBundled("bin/single-node/docker-compose.yml") + private val envFile = resolveBundled("bin/single-node/.env") + private val upScript = resolveBundled("bin/single-node/up.sh") + private val collector = resolveBundled("bin/observability/otel-collector/config.yaml") + + // ----- docker-compose: pinned images --------------------------------- + + "docker-compose.yml" should "exist" in { + Files.exists(compose) shouldBe true + } + + it should "pin the OSS observability image tags from LICENSING.md" in { + val text = read(compose) + text should include("otel/opentelemetry-collector-contrib:0.153.0") + text should include("victoriametrics/victoria-logs:v1.50.0") + text should include("victoriametrics/victoria-metrics:v1.144.0") + text should include("jaegertracing/jaeger:2.18.0") + text should include("ghcr.io/parca-dev/parca:v0.28.0") + text should include("ghcr.io/parca-dev/parca-agent:v0.47.1") + } + + it should "NOT reference the VictoriaMetrics enterprise images" in { + // Tripwire: '-enterprise' images are not Apache-2.0. Texera ships + // OSS only — anyone editing the tags must be reminded of that. + val text = read(compose) + text should not include "-enterprise" + } + + it should "bind every observability host port to loopback (127.0.0.1)" in { + val text = read(compose) + // The four query/UI endpoints we expose to the host MUST use + // the 127.0.0.1: prefix. Any "0.0.0.0:9428" or naked "9428:9428" + // would publish to all interfaces — a misconfig that would + // expose the data store to the network. + Seq("9428", "8428", "16686", "7070").foreach { port => + val pattern = s""""127.0.0.1:$port:$port"""" + withClue(s"port $port should be loopback-only: ") { + text should include(pattern) + } + } + } + + it should "only ever bind the OTel collector OTLP receivers to loopback" in { + // Dev mode publishes 127.0.0.1:4317-4318 so a Scala backend running + // outside docker (sbt / IntelliJ) can emit telemetry. The hard + // requirement is that the receiver is NEVER bound on a non-loopback + // address — otherwise OTLP would be reachable from the LAN. + val text = read(compose) + text should not include "0.0.0.0:4317" + text should not include "0.0.0.0:4318" + // Also reject the bare `"4317:4317"` form which docker treats as + // "bind on all interfaces". Only the explicit loopback form is OK. + text should not include "\"4317:4317\"" + text should not include "\"4318:4318\"" + } + + it should "only mark the parca-agent privileged (eBPF needs CAP_SYS_ADMIN)" in { + // privileged: true is dangerous; we want it on exactly one + // service. If a future contributor copies the agent block as a + // template for another service, this test trips. + val text = read(compose) + val priv = "privileged: true".r.findAllIn(text).length + priv shouldBe 1 + } + + it should "give every observability backend a profile so it can be disabled" in { + val text = read(compose) + Seq( + "observability-collector", + "observability-logs", + "observability-metrics", + "observability-traces", + "observability-profiles" + ).foreach { profile => + text should include(profile) + } + } + + it should "set a memory limit on every observability service" in { + // The new section lives below "Part 5: Observability stack". + val text = read(compose).split("Part 5: Observability stack").last + // Six services, each gets a `deploy.resources.limits.memory:` line. + val memoryLimits = "memory:".r.findAllIn(text).length + memoryLimits should be >= 6 + } + + // ----- .env defaults ------------------------------------------------- + + ".env" should "default COMPOSE_PROFILES to include every observability profile" in { + val text = read(envFile) + text should include("COMPOSE_PROFILES=") + Seq( + "observability-collector", + "observability-logs", + "observability-metrics", + "observability-traces", + "observability-profiles" + ).foreach { profile => + val grepCount = text.split('\n').count(line => + !line.trim.startsWith("#") && line.contains(profile) + ) + withClue(s"$profile should be in default COMPOSE_PROFILES (non-comment): ")( + grepCount should be >= 1 + ) + } + } + + // ----- up.sh ---------------------------------------------------------- + + "up.sh" should "honor each per-signal disable env var" in { + val text = read(upScript) + Seq( + "TEXERA_OBSERVABILITY_LOGS", + "TEXERA_OBSERVABILITY_METRICS", + "TEXERA_OBSERVABILITY_TRACES", + "TEXERA_OBSERVABILITY_PROFILES", + "TEXERA_OBSERVABILITY_COLLECTOR" + ).foreach { v => + text should include(v) + } + } + + // ----- otel-collector config ----------------------------------------- + + "otel-collector/config.yaml" should "exist and declare all three signal pipelines" in { + Files.exists(collector) shouldBe true + val text = read(collector) + // The Service block defines exactly three pipelines. + text should include("logs:") + text should include("metrics:") + text should include("traces:") + } + + it should "route metrics via prometheusremotewrite to VictoriaMetrics" in { + val text = read(collector) + text should include("prometheusremotewrite") + text should include("victoriametrics:8428") + } + + it should "route logs to VictoriaLogs over OTLP HTTP" in { + val text = read(collector) + text should include("otlphttp/victorialogs") + text should include("victorialogs:9428") + } + + it should "route traces to Jaeger over OTLP gRPC" in { + val text = read(collector) + text should include("otlp/jaeger") + text should include("jaeger:4317") + } + + it should "cap incoming OTLP message size (DoS guard)" in { + val text = read(collector) + text should include("max_recv_msg_size_mib") + } + + it should "configure memory_limiter to bound collector memory" in { + val text = read(collector) + text should include("memory_limiter") + text should include("limit_mib") + } +} diff --git a/common/config/src/test/scala/org/apache/texera/observability/ParcaConfigSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/ParcaConfigSpec.scala new file mode 100644 index 00000000000..8e0d162b23f --- /dev/null +++ b/common/config/src/test/scala/org/apache/texera/observability/ParcaConfigSpec.scala @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path, Paths} + +/** + * Smoke test for the bundled Parca configuration files. + * + * Intentionally lightweight: we are guarding against accidental + * deletion / emptying / tag drift, not validating Parca's schema. + * The real config validation happens when the agent starts up + * inside its container — but a unit-level smoke test catches typos + * before a developer pushes them. + */ +class ParcaConfigSpec extends AnyFlatSpec with Matchers { + + // sbt runs tests with the module dir as CWD, but a developer who + // runs `sbt test` from the project root has a different CWD. Walk + // upwards until we find the file, bounded so a missing file fails + // loudly rather than infinite-looping. + private def resolveBundled(relative: String): Path = { + var dir = Paths.get("").toAbsolutePath + var hops = 0 + while (hops < 10) { + val candidate = dir.resolve(relative) + if (Files.exists(candidate)) return candidate + val parent = dir.getParent + if (parent == null) return candidate + dir = parent + hops += 1 + } + Paths.get(relative) // will fail the existence assert below + } + + private val parcaYaml = resolveBundled("bin/observability/parca/parca.yaml") + private val agentEnv = resolveBundled("bin/observability/parca/parca-agent.env") + private val readme = resolveBundled("bin/observability/parca/README.md") + + // ----- parca.yaml ----------------------------------------------------- + + "parca.yaml" should "exist in bin/observability/parca/" in { + Files.exists(parcaYaml) shouldBe true + } + + it should "declare object_storage with the FILESYSTEM bucket type" in { + val text = new String(Files.readAllBytes(parcaYaml), StandardCharsets.UTF_8) + text should include("object_storage") + text should include("FILESYSTEM") + text should include("/var/lib/parca") + } + + it should "include the ASF license header" in { + val text = new String(Files.readAllBytes(parcaYaml), StandardCharsets.UTF_8) + text should include("Apache License, Version 2.0") + } + + // ----- parca-agent.env ------------------------------------------------ + + "parca-agent.env" should "exist and pin the bundled image version reference" in { + Files.exists(agentEnv) shouldBe true + val text = new String(Files.readAllBytes(agentEnv), StandardCharsets.UTF_8) + text should include("v0.47.1") + } + + it should "point the agent at the bundled Parca server hostname" in { + val text = new String(Files.readAllBytes(agentEnv), StandardCharsets.UTF_8) + text should include("PARCA_AGENT_REMOTE_STORE_ADDRESS=parca:7070") + } + + it should "carry the deployment label so the gateway can filter Texera processes" in { + val text = new String(Files.readAllBytes(agentEnv), StandardCharsets.UTF_8) + text should include("deployment=texera") + } + + it should "NOT include high-cardinality labels (workflow.id / execution.id)" in { + // Tripwire: a future contributor might be tempted to add + // workflow.id as a static label. That blows up Parca storage. + // This assertion makes the design intent enforceable. Comments + // are skipped — they're allowed (and required) to explain the + // rule. + val configLines = new String(Files.readAllBytes(agentEnv), StandardCharsets.UTF_8) + .linesIterator + .map(_.trim) + .filter(line => line.nonEmpty && !line.startsWith("#")) + .toSeq + configLines.foreach { line => + line should not include "workflow.id" + line should not include "workflow_id" + line should not include "execution.id" + line should not include "execution_id" + } + } + + // ----- README --------------------------------------------------------- + + "README.md" should "exist and document the opt-out env var" in { + Files.exists(readme) shouldBe true + val text = new String(Files.readAllBytes(readme), StandardCharsets.UTF_8) + text should include("TEXERA_OBSERVABILITY_PROFILES=disabled") + } + + it should "document the Linux-only / privileged-container requirement" in { + val text = new String(Files.readAllBytes(readme), StandardCharsets.UTF_8) + text.toLowerCase should include("linux") + text.toLowerCase should include("privileged") + } +} From 71cfbf64a0133c35da213e43abd7d5efb1472fae Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 5 Jun 2026 04:49:02 -0700 Subject: [PATCH 03/21] feat(observability): backend signal emission - metrics + distributed tracing primitives Co-Authored-By: Claude Opus 4.8 (1M context) --- .../texera/observability/SpanAttrs.scala | 116 ++++++++++ .../texera/observability/TexeraMetrics.scala | 216 ++++++++++++++++++ .../texera/observability/TexeraTracer.scala | 84 +++++++ .../observability/TraceparentValidator.scala | 94 ++++++++ .../texera/observability/SpanAttrsSpec.scala | Bin 0 -> 5448 bytes .../observability/TexeraMetricsSpec.scala | 189 +++++++++++++++ .../TraceparentValidatorSpec.scala | Bin 0 -> 5085 bytes 7 files changed, 699 insertions(+) create mode 100644 common/config/src/main/scala/org/apache/texera/observability/SpanAttrs.scala create mode 100644 common/config/src/main/scala/org/apache/texera/observability/TexeraMetrics.scala create mode 100644 common/config/src/main/scala/org/apache/texera/observability/TexeraTracer.scala create mode 100644 common/config/src/main/scala/org/apache/texera/observability/TraceparentValidator.scala create mode 100644 common/config/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala create mode 100644 common/config/src/test/scala/org/apache/texera/observability/TexeraMetricsSpec.scala create mode 100644 common/config/src/test/scala/org/apache/texera/observability/TraceparentValidatorSpec.scala diff --git a/common/config/src/main/scala/org/apache/texera/observability/SpanAttrs.scala b/common/config/src/main/scala/org/apache/texera/observability/SpanAttrs.scala new file mode 100644 index 00000000000..c66db23f652 --- /dev/null +++ b/common/config/src/main/scala/org/apache/texera/observability/SpanAttrs.scala @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import io.opentelemetry.api.common.AttributeKey +import io.opentelemetry.api.trace.{Span, SpanBuilder} + +/** + * Thin helper for setting span attributes safely. + * + * Three rules: + * 1. Typed setters only — no public escape hatch for arbitrary + * untyped strings to land on a span as untrusted free text. + * 2. Free-text values are CRLF-stripped + capped at + * [[FreeTextMaxLen]] to prevent log/span forging via embedded + * newlines. + * 3. Operator IDs and workflow/execution IDs must match a strict + * character set — otherwise dropped silently (the operator + * identifier should be a stable internal value, not user free + * text). + */ +object SpanAttrs { + + /** Maximum length for free-text span attribute values. */ + val FreeTextMaxLen: Int = 256 + + /** Validates the shape we accept for operator IDs: alnum + `_.-`, + * 1–64 chars. Anything else is dropped (not coerced — we'd rather + * miss a label than leak an unbounded string into a span). */ + private val OperatorIdPattern = "^[A-Za-z0-9_.\\-]{1,64}$".r.pattern + + // ---- Standard Texera correlation labels ------------------------------ + + val WorkflowId: AttributeKey[java.lang.Long] = AttributeKey.longKey("texera.workflow.id") + val ExecutionId: AttributeKey[java.lang.Long] = AttributeKey.longKey("texera.execution.id") + val ProjectId: AttributeKey[java.lang.Long] = AttributeKey.longKey("texera.project.id") + val UserId: AttributeKey[java.lang.Long] = AttributeKey.longKey("texera.user.id") + val OperatorId: AttributeKey[String] = AttributeKey.stringKey("texera.operator.id") + val OperatorName: AttributeKey[String] = AttributeKey.stringKey("texera.operator.name") + val Outcome: AttributeKey[String] = AttributeKey.stringKey("texera.outcome") + + // ---- Typed setters for SpanBuilder (used at span-start time) --------- + + def withWorkflowId(b: SpanBuilder, id: Long): SpanBuilder = + b.setAttribute(WorkflowId, java.lang.Long.valueOf(id)) + + def withExecutionId(b: SpanBuilder, id: Long): SpanBuilder = + b.setAttribute(ExecutionId, java.lang.Long.valueOf(id)) + + def withProjectId(b: SpanBuilder, id: Long): SpanBuilder = + b.setAttribute(ProjectId, java.lang.Long.valueOf(id)) + + def withUserId(b: SpanBuilder, id: Long): SpanBuilder = + b.setAttribute(UserId, java.lang.Long.valueOf(id)) + + /** Sets the operator id only if it passes the strict character + * check; otherwise the attribute is omitted. Returns the same + * builder either way for fluent chaining. */ + def withOperatorId(b: SpanBuilder, id: String): SpanBuilder = { + if (id != null && OperatorIdPattern.matcher(id).matches()) { + b.setAttribute(OperatorId, id) + } + b + } + + /** Sets a free-text label after stripping CRLF and capping length. */ + def withOperatorName(b: SpanBuilder, name: String): SpanBuilder = { + val safe = sanitizeFreeText(name) + if (safe != null) b.setAttribute(OperatorName, safe) else b + } + + // ---- Typed setters for Span (used after a span is active) ------------ + + def setWorkflowId(s: Span, id: Long): Span = s.setAttribute(WorkflowId, java.lang.Long.valueOf(id)) + def setExecutionId(s: Span, id: Long): Span = s.setAttribute(ExecutionId, java.lang.Long.valueOf(id)) + def setOperatorId(s: Span, id: String): Span = { + if (id != null && OperatorIdPattern.matcher(id).matches()) s.setAttribute(OperatorId, id) + else s + } + def setOutcome(s: Span, outcome: String): Span = { + val safe = sanitizeFreeText(outcome) + if (safe != null) s.setAttribute(Outcome, safe) else s + } + + // ---- Pure helpers (exposed for testing) ------------------------------ + + /** + * Strip CR/LF and other C0 control characters from a free-text + * value, then cap at [[FreeTextMaxLen]]. Returns null for + * null/empty input (caller skips the setAttribute call). + */ + def sanitizeFreeText(value: String): String = { + if (value == null || value.isEmpty) return null + val stripped = value.filter(c => c >= 0x20 && c != 0x7F) + if (stripped.isEmpty) null + else if (stripped.length <= FreeTextMaxLen) stripped + else stripped.substring(0, FreeTextMaxLen) + } +} diff --git a/common/config/src/main/scala/org/apache/texera/observability/TexeraMetrics.scala b/common/config/src/main/scala/org/apache/texera/observability/TexeraMetrics.scala new file mode 100644 index 00000000000..c850c055392 --- /dev/null +++ b/common/config/src/main/scala/org/apache/texera/observability/TexeraMetrics.scala @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import com.typesafe.scalalogging.LazyLogging +import io.opentelemetry.api.GlobalOpenTelemetry +import io.opentelemetry.api.common.{AttributeKey, Attributes} +import io.opentelemetry.api.metrics.Meter + +/** + * Strongly-typed façade for Texera-emitted metrics. + * + * Cardinality safety is enforced by the API surface, not by + * documentation: there is no public method that accepts an arbitrary + * string as a label key or value. The only labels that ever land on + * an instrument are the two enums [[Outcome]] and [[WorkflowKind]], + * each restricted to a fixed set. ``workflow.id`` / ``execution.id`` + * are deliberately NOT metric labels — per-execution detail belongs + * in traces and logs, joined on ``trace_id`` at query time. + * + * Histogram bucket bounds are hard-coded constants so they can't be + * coerced by request input. The OTel SDK applies its own default + * attribute-value-length cap to anything that does slip through. + */ +object TexeraMetrics extends LazyLogging { + + /** Outcome enum, the only mutable label on lifecycle counters. */ + sealed abstract class Outcome(val name: String) + object Outcome { + case object Success extends Outcome("success") + case object Failure extends Outcome("failure") + case object Cancelled extends Outcome("cancelled") + } + + /** Workflow kind enum. Distinguishes interactive vs. scheduled + * workflows for the dashboard's basic split — extend deliberately. */ + sealed abstract class WorkflowKind(val name: String) + object WorkflowKind { + case object Interactive extends WorkflowKind("interactive") + case object Scheduled extends WorkflowKind("scheduled") + case object Unknown extends WorkflowKind("unknown") + } + + private val OutcomeKey: AttributeKey[String] = AttributeKey.stringKey("texera.outcome") + private val WorkflowKindKey: AttributeKey[String] = AttributeKey.stringKey("texera.workflow.kind") + + /** Histogram bucket bounds in seconds. Hard-coded — constants so + * request input can't reshape the histogram. Range covers + * fast (<1s) to long (>1h) workflows. */ + private val DurationBuckets: java.util.List[java.lang.Double] = { + val builder = new java.util.ArrayList[java.lang.Double]() + Seq(0.1, 0.5, 1.0, 5.0, 10.0, 30.0, 60.0, 300.0, 600.0, 1800.0, 3600.0) + .foreach(b => builder.add(java.lang.Double.valueOf(b))) + builder + } + + private val InstrumentationScope = "org.apache.texera" + + // Instruments are lazy + memoised. The first call after SDK init + // builds them against the active GlobalOpenTelemetry meter; once + // built they hold the meter instance, so a later GlobalOpenTelemetry + // reset (in tests) wouldn't be visible here — see [[resetForTest]]. + @volatile private var _starts: io.opentelemetry.api.metrics.LongCounter = _ + @volatile private var _completions: io.opentelemetry.api.metrics.LongCounter = _ + @volatile private var _cancellations: io.opentelemetry.api.metrics.LongCounter = _ + // `texera.workflow.active` is an OBSERVABLE gauge, not a manual up/down + // counter. A manual +1/-1 counter leaks whenever a run starts but its + // terminal event never fires (the controller is killed, the process + // restarts mid-run, or the +1 and -1 are split across the engine and web + // tiers and one side is missed) — the gauge then drifts upward forever and + // the dashboard shows phantom "active" executions. An observable gauge + // instead reports the TRUE in-progress count from the live execution + // registry on every collection, so it cannot leak. The count is supplied + // by the host process via [[setActiveExecutionsSupplier]]. + @volatile private var _active: io.opentelemetry.api.metrics.ObservableLongGauge = _ + @volatile private var _duration: io.opentelemetry.api.metrics.DoubleHistogram = _ + + // Supplier of the current in-progress execution count. Defaults to 0 until + // the host process registers the real source (so a process that never + // registers reports a flat 0 rather than a wrong number). Read by the gauge + // callback on every metric collection. + @volatile private var activeExecutionsSupplier: () => Long = () => 0L + + /** Register the authoritative source of "currently active executions". + * The supplier is polled on every metric collection, so the gauge always + * reflects ground truth and can never leak. Called once at process + * startup (e.g. by ComputingUnitMaster). */ + def setActiveExecutionsSupplier(supplier: () => Long): Unit = synchronized { + activeExecutionsSupplier = supplier + ensureBound() + } + + /** Bind instruments to the current global meter. Idempotent — the + * first call wins; later calls are no-ops. Tests can call + * [[bindForTest]] with an explicit Meter, then [[resetForTest]] to + * rebind. */ + def ensureBound(): Unit = synchronized { + if (_starts == null) bind(GlobalOpenTelemetry.getMeter(InstrumentationScope)) + } + + private[observability] def bindForTest(meter: Meter): Unit = synchronized { + bind(meter) + } + + private[observability] def resetForTest(): Unit = synchronized { + _starts = null + _completions = null + _cancellations = null + // The observable gauge registered a collection callback — close it so the + // previous test's meter provider stops being polled after it's discarded. + if (_active != null) _active.close() + _active = null + _duration = null + } + + private def bind(meter: Meter): Unit = { + _starts = meter + .counterBuilder("texera.workflow.starts") + .setDescription("Number of workflow executions started.") + .build() + // Completions carry texera.outcome={success|failure}. Both success + // and self-terminating failures land here so the success/failure-rate + // queries (non-success ÷ all completions) have a denominator that + // means "runs that finished on their own". User-initiated kills are + // NOT completions — see _cancellations. + _completions = meter + .counterBuilder("texera.workflow.completions") + .setDescription("Number of workflow executions that ran to completion (success or failure).") + .build() + // Cancellations (user kills) are tracked separately so they decrement + // the active gauge without polluting the success/failure-rate + // denominator. Deliberately label-free apart from workflow.kind. + _cancellations = meter + .counterBuilder("texera.workflow.cancellations") + .setDescription("Number of workflow executions cancelled/killed before finishing.") + .build() + // Observable gauge: the callback runs on each collection and reports the + // live in-progress count, so the value cannot leak. No per-execution + // labels (cardinality-safe); the dashboard queries sum(texera_workflow_active). + _active = meter + .gaugeBuilder("texera.workflow.active") + .ofLongs() + .setDescription("Number of workflow executions currently in progress (observed from the live registry).") + .buildWithCallback(obs => obs.record(activeExecutionsSupplier())) + _duration = meter + .histogramBuilder("texera.workflow.duration") + .setDescription("End-to-end duration of a workflow execution.") + .setUnit("s") + .setExplicitBucketBoundariesAdvice(DurationBuckets) + .build() + logger.info(s"Texera metric instruments bound to meter scope '$InstrumentationScope'") + } + + // ---- Public emitters — typed, no untyped escape hatch -------------- + + def recordStart(kind: WorkflowKind): Unit = { + ensureBound() + logger.debug(s"metric: workflow started (kind=${kind.name}) — starts +1") + _starts.add(1L, Attributes.of(WorkflowKindKey, kind.name)) + // `active` is no longer mutated here — it is an observable gauge sourced + // from the live execution registry (see setActiveExecutionsSupplier). + } + + def recordCompletion(kind: WorkflowKind, durationSec: Double): Unit = { + ensureBound() + logger.debug( + f"metric: workflow completed successfully (kind=${kind.name}, ${durationSec}%.1fs) — completions +1" + ) + val attrs = Attributes.of(OutcomeKey, Outcome.Success.name, WorkflowKindKey, kind.name) + _completions.add(1L, attrs) + _duration.record(durationSec, attrs) + } + + def recordFailure(kind: WorkflowKind, durationSec: Double): Unit = { + ensureBound() + logger.debug( + f"metric: workflow failed (kind=${kind.name}, ${durationSec}%.1fs) — completions +1 (outcome=failure)" + ) + // A failure is a completion with outcome=failure — it shares the + // completions counter (and duration histogram) with successes so the + // failure-rate query has both numerator and denominator. + val attrs = Attributes.of(OutcomeKey, Outcome.Failure.name, WorkflowKindKey, kind.name) + _completions.add(1L, attrs) + _duration.record(durationSec, attrs) + } + + /** A user-initiated kill/cancel. Bumps a dedicated counter. Deliberately + * NOT recorded as a completion: a cancelled run never finished on its own, + * so it must not drag down the success rate. No duration is recorded for + * the same reason — a killed run's wall-clock time is not a real runtime + * and would skew the duration percentiles. (The active gauge is observed + * from the live registry and needs no decrement here.) */ + def recordCancellation(kind: WorkflowKind): Unit = { + ensureBound() + logger.debug(s"metric: workflow cancelled (kind=${kind.name}) — cancellations +1") + val attrs = Attributes.of(WorkflowKindKey, kind.name) + _cancellations.add(1L, attrs) + } +} diff --git a/common/config/src/main/scala/org/apache/texera/observability/TexeraTracer.scala b/common/config/src/main/scala/org/apache/texera/observability/TexeraTracer.scala new file mode 100644 index 00000000000..8941d97aefa --- /dev/null +++ b/common/config/src/main/scala/org/apache/texera/observability/TexeraTracer.scala @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import io.opentelemetry.api.GlobalOpenTelemetry +import io.opentelemetry.api.trace.{Span, SpanBuilder, StatusCode, Tracer} +import io.opentelemetry.context.{Context, Scope} + +/** + * Thin convenience wrapper around the global OTel tracer. + * + * Two reasons to go through this rather than calling + * ``GlobalOpenTelemetry.getTracer`` directly at every callsite: + * + * 1. Single instrumentation scope name (``org.apache.texera``) — so + * every Texera-produced span shows up under one logical scope in + * the backend, separable from anything emitted by transitive + * libraries. + * 2. One ergonomic ``withSpan`` API that handles exception → status, + * scope cleanup, and span end in a single try/finally. Callers + * don't have to remember the ceremony at every site. + * + * When the SDK is disabled, ``GlobalOpenTelemetry.getTracer`` returns + * a no-op tracer, so calling these methods is safe at any time. + */ +object TexeraTracer { + + private val InstrumentationScope = "org.apache.texera" + + def tracer: Tracer = GlobalOpenTelemetry.getTracer(InstrumentationScope) + + def spanBuilder(name: String): SpanBuilder = tracer.spanBuilder(name) + + /** + * Run ``block`` inside a fresh span; record exceptions, propagate + * the right span status, and ensure the span is ended exactly once. + * + * Use this for synchronous critical sections. For async (Future- + * returning) code paths use ``withAsyncSpan`` so the span doesn't + * close before the async work completes. + */ + def withSpan[T](name: String, configure: SpanBuilder => SpanBuilder = identity)( + block: Span => T + ): T = { + val span = configure(spanBuilder(name)).startSpan() + val scope: Scope = span.makeCurrent() + try { + block(span) + } catch { + case t: Throwable => + span.recordException(t) + span.setStatus(StatusCode.ERROR) + throw t + } finally { + scope.close() + span.end() + } + } + + /** + * Snapshot the current OTel ``Context`` so async callbacks can + * re-attach it via ``Context.makeCurrent`` later. Useful at the + * Scala↔Python boundary where the calling thread is not the + * receiving thread. + */ + def currentContext: Context = Context.current() +} diff --git a/common/config/src/main/scala/org/apache/texera/observability/TraceparentValidator.scala b/common/config/src/main/scala/org/apache/texera/observability/TraceparentValidator.scala new file mode 100644 index 00000000000..9577a9f9438 --- /dev/null +++ b/common/config/src/main/scala/org/apache/texera/observability/TraceparentValidator.scala @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +/** + * Pure validators for W3C Trace Context headers crossing the + * Scala↔Python boundary. The single rule: if the inbound bytes do + * not match the strict regex, we discard the value and start a fresh + * trace. We never echo a rejected value back into a span, log, or + * error message. + * + * Spec reference: https://www.w3.org/TR/trace-context/ + * + * The regexes here intentionally do NOT use any context-sensitive + * grouping or backreferences — keeps the validators safe against + * pathological inputs (ReDoS) and trivially fast. + */ +object TraceparentValidator { + + /** W3C traceparent format: `---`. + * We accept only version `00` (the only published version) with the + * canonical 32-hex / 16-hex / 2-hex layout. All hex lowercase per + * spec — uppercase is invalid. */ + private val TraceparentPattern = + "^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$".r.pattern + + /** Bounded tracestate length. Spec recommends ≤512 chars. We are + * stricter to remove a small DoS surface — an attacker can't send + * a 1 MiB tracestate to balloon downstream context allocations. */ + val MaxTracestateLength: Int = 512 + + /** Validate a traceparent header. Returns the input unchanged on + * success, None on any failure (rejected — caller starts a fresh + * trace). Null and empty are silent failures. */ + def validateTraceparent(header: String): Option[String] = { + if (header == null || header.isEmpty) return None + // Trace-id and parent-id must not be all-zero per spec — an + // all-zero ID is a sentinel for "no value" and MUST be rejected. + if (!TraceparentPattern.matcher(header).matches()) return None + val parts = header.split('-') + val traceId = parts(1) + val spanId = parts(2) + if (isAllZero(traceId) || isAllZero(spanId)) return None + Some(header) + } + + /** Validate a tracestate header. Spec: comma-separated list of + * key=value pairs, ASCII-printable only, total length capped. + * Returns the input unchanged on success, None on rejection. */ + def validateTracestate(header: String): Option[String] = { + if (header == null || header.isEmpty) return None + if (header.length > MaxTracestateLength) return None + if (!isAsciiPrintable(header)) return None + Some(header) + } + + private def isAllZero(s: String): Boolean = { + var i = 0 + while (i < s.length) { + if (s.charAt(i) != '0') return false + i += 1 + } + true + } + + private def isAsciiPrintable(s: String): Boolean = { + var i = 0 + while (i < s.length) { + val c = s.charAt(i) + // Allow printable ASCII range (0x20-0x7E). Tab and CR/LF are + // rejected — a tracestate must not span lines. + if (c < 0x20 || c > 0x7E) return false + i += 1 + } + true + } +} diff --git a/common/config/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala new file mode 100644 index 0000000000000000000000000000000000000000..388ee1d2aaa3a7f102742d7ea6bc083c1e6a00e8 GIT binary patch literal 5448 zcmc&&e{b8y8E*g8m)Hwpz^FhvIY!qO=`uiZEKG2sqg9w(DN;is}>v|?#vX>G77HQL*JQF%NlYKeuXu;qm8lGbOlv1%W^g?3(7Sftb-$L*s7(pS+18bU2bi@TzN7Z zt*l$+CG5>Plw{I6<7&mNS;V?ERTD}l>LwWj`VNXXxl*P1Rh}X|y{zO?(bC)~s~IY#vg_P&VI7>z0FHI;1Ed_=J%~-<>8RI3uCG}U zvfU=$t6OEIFbk*bja=kK?zg=j95fcSH^P*_y($2IZ@0o_*WwaS=IxUtyZ+!}uN;>_ zAb2QLR!sG4g^jjNN~zDAhac8pWvT@1%=wG`t2H+{_U$8NeyWZBBofZIVvCSYQ3JCR z=S;X%7LpwmSNP<#i?QBL@IArz*3jLR*uNkb>x5gGs>sCa=JzgjL8%9i3wW^&MBG;x zvs}N?`a%VL9}|h9LxK0~Zti#PG2xbU_GWZ337f!#Z59aJ6qK$c0K>_CoRiT?6@~f1 z{-8nHMb@=nzp5X9)N%RQ;`8)(*+11C4RVoYDuVF+Aea-d^Awdf*FMl0wj|STT2!Ge zIP6;W*dW&1p(*L+DaEoVSF(S{iG!C%{7GdArluB9ujTE9(wz+6po0|#6!e^;+X!3} z?+brd2=Bh=hjg)wsXRpXYY#xeO6%oVDhj2SXreU|ftkJFT_Z;#=%RMBQknuzN~TS; zjtaR9Z7JZFXN3UgPK0|?2R=p&kIz2uCY7<>IOd`{4zbR!J+Onh#@H*sC0_wriVIk#Mx!aPQ_qb)=0r^0n zeUw3~P0gn9zCaH97q99?f`7fJ@&Zg7N2ZMGb^FxZ^)BY+1B%w$A5A&Dc`8n$zHnYz zFsRgVUmz1OwFg5v&0wo_O)Qn4rDVn{m?e2IQ;vUiV7Jdw zgCEwuLr&MJwon}s$t?DZ#Pm`erji`yAdNcxKXWR?2!G0_qZ0jM17iA6W+k_hPe^Sj z(PhcqDh|3H&UJ^NXJLpw;I=xqns{t+=u&rZ`pL=l##}T+B$GeX-3(FPX^C{}@OnD2PHDT?c6Vkl{07hpVntTw4xc3jr=T3fM@sR-ujfvTfgnucz8^oNRdLXI-{{KvGni;f=|6fV$qDrbsq5mcYRC(ySs+? z$o)fWr1ho0lMd=S#&P8W>-R5R_Ex#Hb(HPZNbJjg{k~Dt=|IrNy|EUEBZg5FY`^cl G@BJ4E#T2pt literal 0 HcmV?d00001 diff --git a/common/config/src/test/scala/org/apache/texera/observability/TexeraMetricsSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/TexeraMetricsSpec.scala new file mode 100644 index 00000000000..10a9eb7651a --- /dev/null +++ b/common/config/src/test/scala/org/apache/texera/observability/TexeraMetricsSpec.scala @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import io.opentelemetry.sdk.metrics.SdkMeterProvider +import io.opentelemetry.sdk.metrics.data.MetricData +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader +import org.scalatest.BeforeAndAfterEach +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import scala.jdk.CollectionConverters._ + +class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { + + private var reader: InMemoryMetricReader = _ + private var provider: SdkMeterProvider = _ + + override def beforeEach(): Unit = { + reader = InMemoryMetricReader.create() + provider = SdkMeterProvider.builder().registerMetricReader(reader).build() + TexeraMetrics.resetForTest() + TexeraMetrics.bindForTest(provider.get("org.apache.texera")) + } + + override def afterEach(): Unit = { + TexeraMetrics.resetForTest() + provider.close() + } + + private def collectAll(): Map[String, MetricData] = { + reader.collectAllMetrics().asScala.map(m => m.getName -> m).toMap + } + + // ----- positive: lifecycle emissions ---------------------------------- + + "TexeraMetrics" should "increment workflow.starts on recordStart" in { + TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) + + val metrics = collectAll() + metrics.keySet should contain("texera.workflow.starts") + + val starts = metrics("texera.workflow.starts").getLongSumData.getPoints.asScala.head + starts.getValue shouldBe 1L + // recordStart no longer mutates `active` — that gauge is observed from the + // live registry (see "report the active-execution count…" below). + } + + it should "report the active-execution count from the registered supplier, never a manual counter" in { + // The whole point of the fix: `active` is an observable gauge sourced from + // ground truth, so it cannot leak. Drive it with a stub supplier and + // confirm the gauge reports exactly what the supplier returns — regardless + // of how many starts/completions were recorded. + @volatile var live = 3L + TexeraMetrics.setActiveExecutionsSupplier(() => live) + TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) + TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) + + val first = collectAll()("texera.workflow.active").getLongGaugeData.getPoints.asScala.head + first.getValue shouldBe 3L + + // It tracks the live source on the next collection — a manual +1/-1 + // counter could never drop like this without an explicit decrement. + live = 0L + val second = collectAll()("texera.workflow.active").getLongGaugeData.getPoints.asScala.head + second.getValue shouldBe 0L + } + + it should "record a completion and duration sample on recordCompletion" in { + TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) + TexeraMetrics.recordCompletion(TexeraMetrics.WorkflowKind.Interactive, durationSec = 2.5) + + val metrics = collectAll() + metrics.keySet should contain allOf ( + "texera.workflow.completions", + "texera.workflow.duration" + ) + + metrics("texera.workflow.completions").getLongSumData.getPoints.asScala.head.getValue shouldBe 1L + + val histogram = metrics("texera.workflow.duration").getHistogramData.getPoints.asScala.head + histogram.getCount shouldBe 1L + histogram.getSum shouldBe 2.5 + } + + it should "record a failure as a non-success completion (so failure-rate queries work)" in { + TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Scheduled) + TexeraMetrics.recordFailure(TexeraMetrics.WorkflowKind.Scheduled, durationSec = 12.0) + + val metrics = collectAll() + // A failure shares the completions counter with successes — the + // failure-rate query divides non-success completions by all + // completions, so failures must live here (not in a separate + // series the query never reads). + val completion = metrics("texera.workflow.completions").getLongSumData.getPoints.asScala.head + completion.getValue shouldBe 1L + completion.getAttributes.asMap.asScala.map { case (k, v) => k.getKey -> v.toString } should contain( + "texera.outcome" -> "failure" + ) + metrics("texera.workflow.duration").getHistogramData.getPoints.asScala.head.getSum shouldBe 12.0 + // The orphan counter the old wiring used must be gone. + metrics.keySet should not contain "texera.workflow.failures" + } + + it should "record a cancellation that is not a completion" in { + TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) + TexeraMetrics.recordCancellation(TexeraMetrics.WorkflowKind.Interactive) + + val metrics = collectAll() + metrics("texera.workflow.cancellations").getLongSumData.getPoints.asScala.head.getValue shouldBe 1L + // A kill is not a completion and records no duration: it must not + // drag down the success rate nor skew the duration percentiles. + metrics.keySet should not contain "texera.workflow.completions" + metrics.keySet should not contain "texera.workflow.duration" + } + + // ----- security: cardinality safety ----------------------------------- + + it should "only emit the texera.outcome and texera.workflow.kind labels" in { + TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) + TexeraMetrics.recordCompletion(TexeraMetrics.WorkflowKind.Interactive, durationSec = 1.0) + + val attrKeys = collectAll().values.flatMap { md => + val pointSet = md.getType.name() match { + case "HISTOGRAM" => md.getHistogramData.getPoints.asScala + case "LONG_GAUGE" => md.getLongGaugeData.getPoints.asScala + case _ => md.getLongSumData.getPoints.asScala + } + pointSet.flatMap(_.getAttributes.asMap.keySet.asScala.map(_.getKey)) + }.toSet + + attrKeys.foreach { key => + Set("texera.outcome", "texera.workflow.kind") should contain(key) + } + attrKeys should not contain "texera.workflow.id" + attrKeys should not contain "texera.execution.id" + } + + it should "expose no public API to attach an arbitrary string label" in { + // The class has only typed emitters whose attribute set is + // hard-coded. This test is intentionally a compile-time check + // disguised as a runtime one — if a future contributor adds an + // untyped public method like recordStart(attrs: Attributes), + // this assertion still passes but the design intent is broken. + // Make the intent explicit: + val methodNames = classOf[TexeraMetrics.type].getDeclaredMethods + .map(_.getName) + .toSet + methodNames should contain allOf ("recordStart", "recordCompletion", "recordFailure") + methodNames should not contain "recordWithAttributes" + } + + // ----- histogram buckets are constants -------------------------------- + + it should "use the hard-coded explicit bucket boundaries for duration" in { + TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) + // Hit a few bucket bounds. + Seq(0.05, 0.6, 7.0, 65.0, 400.0).foreach { d => + TexeraMetrics.recordCompletion(TexeraMetrics.WorkflowKind.Interactive, durationSec = d) + } + + val histogram = collectAll()("texera.workflow.duration").getHistogramData + val point = histogram.getPoints.asScala.head + point.getCount shouldBe 5L + // The point exposes the SDK-configured boundaries; we don't + // assert exact values here (would couple the test to the impl) + // but we do assert there ARE explicit boundaries — anything + // empty would mean the .setExplicitBucketBoundariesAdvice call + // was lost during a refactor. + point.getBoundaries.size should be > 0 + } +} diff --git a/common/config/src/test/scala/org/apache/texera/observability/TraceparentValidatorSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/TraceparentValidatorSpec.scala new file mode 100644 index 0000000000000000000000000000000000000000..d95b61239ba4dfcf2dc87ff4fbbbc19b3d022ab5 GIT binary patch literal 5085 zcmcIo+j84B5bd+RVuw$1V^WgkI+uc|;OxpndalAweJ0$2dDtYq?Sedv#M z7bG3pk`l>vJs#VnfZcO;&n^~j{oH_`;X))_S)PK^;AR|pIZN=@SZ8j@42OYUsFXRO z6*PO}!2#eSH=q@#-P*uh?fiW>Iy(f8azLT+o7@lyL@FqrJgupz!a!Onvt)Cg zh)g6%PfZtW%3-RnxltsQoSV6@lvs-(QxwNsI4>zne%H7~__oz(V5~opA&hnvc*k$J zVUdn4H`go{Qn*#4fdI8Z9pXBhFv%RZF3RvXfr@%+HNd~e`1j_OsBSDhVP^DCZolp~pbGhdh5*nK=18Y- z3PHOawpsS{R?4Ha^h?oK5m}P=b9)y>8BncWCA-u@mk-70Xb3JUwj#=5zX}=-ZEtesi7`cFg_r zT#JO^fAYkET1e@uhtG4j!ZhX!SBuBaNgzA5B#iutf^yi|KoG2Nzn{GlGf6vihG8SH zOorC2u>aXy>x|k0bB?&!hp-5usDvNwjM0x2bD!}3{eLe;Z(hGX zzv}nK=P=_pm>@cE%ZbW@)Jtv>X4g`*$Z^=HPPE>jJkk1R$KA6tI?>Kw;)&MxCtCjz za^&6ZoFqJxLn4#R7z)o}BcRi`65&0up`=S5??3$Uf1k|gY-bj}jfIk{Dc&0_pZcpt z4Lo+TiV|H{JX<=}4NlZ$r}9rFyC2DJ>)3&QDzV?Xk=S8xz0d=+}nvd#zIKWAp zzkn6DdyY_qVGF;z!;{790pDdNUK^Izhh%T-yEC$vYq{*sc)WM z(ajk4HXM5rme;V;IZ78pD?W1yE;|k22{eN^z?)!mBePfR#=rNVE{eYM1Ud&4x?PH^ zAX7Q5rJ$UBw@{*P-5d9ZLueKMXVJ3Oxoea*f<~TnQ!MUjWjG7z6gv9n??OQ9aCWCu zDzh+1rTNGvf7)j6e#FXqXH5jm;q4gDIGkuVYk;{v%SiyAN&#H&Cx8%+N&$FFQFR{q dD?DB Date: Fri, 5 Jun 2026 07:04:36 -0700 Subject: [PATCH 04/21] apply scalafmt to logging foundations (pr1) --- .../texera/observability/LogSanitizer.scala | 60 ++-- .../texera/observability/OtelInit.scala | 260 ++++++++++-------- .../observability/TexeraOtelLogAppender.scala | 35 +-- .../texera/observability/OtelInitSpec.scala | 29 +- .../TexeraOtelLogAppenderSpec.scala | 13 +- 5 files changed, 221 insertions(+), 176 deletions(-) diff --git a/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala b/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala index 781d8bd776d..aa92c558c27 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala @@ -22,39 +22,43 @@ package org.apache.texera.observability import scala.jdk.CollectionConverters._ /** - * Pure functions that sanitize log records before they leave the - * process via the OTel logs bridge. Lives in its own object so the - * security-critical behaviour can be unit-tested without a Logback - * fixture. - * - * Three invariants: - * 1. No control characters in the body (prevents log forging via - * CR/LF injection in user-supplied strings). - * 2. No oversized bodies (a 1 GiB log line must never reach the - * exporter). - * 3. No secrets in plain text (Bearer tokens, password=, AWS keys). - * - * Plus an MDC allowlist so accidental MDC pollution from a downstream - * library cannot leak unintended fields into the exporter. - */ + * Pure functions that sanitize log records before they leave the + * process via the OTel logs bridge. Lives in its own object so the + * security-critical behaviour can be unit-tested without a Logback + * fixture. + * + * Three invariants: + * 1. No control characters in the body (prevents log forging via + * CR/LF injection in user-supplied strings). + * 2. No oversized bodies (a 1 GiB log line must never reach the + * exporter). + * 3. No secrets in plain text (Bearer tokens, password=, AWS keys). + * + * Plus an MDC allowlist so accidental MDC pollution from a downstream + * library cannot leak unintended fields into the exporter. + */ object LogSanitizer { /** Per-record body cap. The OTel SDK and OTLP have higher limits, - * but 16 KiB is plenty for a useful log line and protects the - * collector from a runaway log. */ + * but 16 KiB is plenty for a useful log line and protects the + * collector from a runaway log. + */ val MaxBodyBytes: Int = 16 * 1024 /** Suffix appended to truncated bodies. Chosen to be visually - * obvious in a UI but short enough not to dominate the cap. */ + * obvious in a UI but short enough not to dominate the cap. + */ val TruncatedMarker: String = "...[truncated]" /** C0 control characters except TAB (0x09). Stripping CR/LF here - * prevents log forging via newline injection in user-supplied - * message bodies. DEL (0x7F) included for the same reason. */ + * prevents log forging via newline injection in user-supplied + * message bodies. DEL (0x7F) included for the same reason. + */ private val C0ControlRegex = "[\\x00-\\x08\\x0A-\\x1F\\x7F]".r /** Secret patterns. Order is significant: most specific first so a - * partial match doesn't shadow a tighter pattern. */ + * partial match doesn't shadow a tighter pattern. + */ private val SecretPatterns: Seq[scala.util.matching.Regex] = Seq( // Authorization: Bearer — bearer token in header form. """(?i)Bearer\s+[A-Za-z0-9._\-/+=]{8,}""".r, @@ -67,8 +71,9 @@ object LogSanitizer { ) /** MDC keys we will forward to OTel log attributes. Anything else - * is dropped — additions require a code change + reviewer - * acknowledgement of the privacy implications. */ + * is dropped — additions require a code change + reviewer + * acknowledgement of the privacy implications. + */ val AllowedMdcKeys: Set[String] = Set( "trace_id", "span_id", @@ -85,7 +90,8 @@ object LogSanitizer { ) /** Apply all three body-side transformations. Idempotent — running - * sanitize on already-sanitized output is a no-op. */ + * sanitize on already-sanitized output is a no-op. + */ def sanitize(body: String): String = { if (body == null || body.isEmpty) return "" val stripped = C0ControlRegex.replaceAllIn(body, "") @@ -104,8 +110,8 @@ object LogSanitizer { /** Filter an MDC map to the allowlist. Null-safe. */ def filterMdc(mdc: java.util.Map[String, String]): Map[String, String] = { if (mdc == null) return Map.empty - mdc.asScala.iterator - .collect { case (k, v) if k != null && AllowedMdcKeys.contains(k) && v != null => k -> v } - .toMap + mdc.asScala.iterator.collect { + case (k, v) if k != null && AllowedMdcKeys.contains(k) && v != null => k -> v + }.toMap } } diff --git a/common/config/src/main/scala/org/apache/texera/observability/OtelInit.scala b/common/config/src/main/scala/org/apache/texera/observability/OtelInit.scala index 35e328c49b0..d7c803fc79e 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/OtelInit.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/OtelInit.scala @@ -39,34 +39,35 @@ import java.time.Duration import scala.util.{Failure, Success, Try} /** - * Bootstraps the OpenTelemetry SDK for a Texera service. - * - * Design notes: - * - Default-disabled. Sets up nothing unless `OTEL_SDK_DISABLED=false`. - * - We deliberately do not use the autoconfigure SPI: the security model - * requires endpoint + resource-attribute filtering to happen BEFORE - * any exporter is constructed. Autoconfigure would parse env vars - * behind our back. - * - Validation is a single pure function so it can be unit-tested - * without spinning the SDK. - * - On any validation failure we log one WARN and return None. We - * do NOT throw — observability is opt-in plumbing; misconfiguration - * must never crash the service. - * - This is the only place in Texera that reads `OTEL_*` environment - * variables. Other modules consume the returned `OpenTelemetry` - * instance directly. - */ + * Bootstraps the OpenTelemetry SDK for a Texera service. + * + * Design notes: + * - Default-disabled. Sets up nothing unless `OTEL_SDK_DISABLED=false`. + * - We deliberately do not use the autoconfigure SPI: the security model + * requires endpoint + resource-attribute filtering to happen BEFORE + * any exporter is constructed. Autoconfigure would parse env vars + * behind our back. + * - Validation is a single pure function so it can be unit-tested + * without spinning the SDK. + * - On any validation failure we log one WARN and return None. We + * do NOT throw — observability is opt-in plumbing; misconfiguration + * must never crash the service. + * - This is the only place in Texera that reads `OTEL_*` environment + * variables. Other modules consume the returned `OpenTelemetry` + * instance directly. + */ object OtelInit extends LazyLogging { /** Resource attribute keys we accept from OTEL_RESOURCE_ATTRIBUTES. - * Resource attrs ride on every record this JVM emits (logs, - * metrics, traces) — so for a per-CU JVM (ComputingUnitMaster / - * ComputingUnitWorker) setting `texera.computing_unit.id=N` at - * boot is enough to tag every record without per-request MDC - * plumbing. Workflow/execution ids vary per task and still need - * MDC at the message boundary, but exposing them in the allowlist - * lets test harnesses + future per-task code populate them via - * the same mechanism. */ + * Resource attrs ride on every record this JVM emits (logs, + * metrics, traces) — so for a per-CU JVM (ComputingUnitMaster / + * ComputingUnitWorker) setting `texera.computing_unit.id=N` at + * boot is enough to tag every record without per-request MDC + * plumbing. Workflow/execution ids vary per task and still need + * MDC at the message boundary, but exposing them in the allowlist + * lets test harnesses + future per-task code populate them via + * the same mechanism. + */ private[observability] val AllowedResourceKeys: Set[String] = Set( "service.name", "service.version", @@ -88,16 +89,18 @@ object OtelInit extends LazyLogging { ) /** Default endpoint when SDK is enabled but no endpoint set explicitly. - * Uses 127.0.0.1 (not "localhost") so a natively-run service reaches the - * IPv4-only collector port published by docker-compose — on dual-stack - * hosts "localhost" resolves to ::1 first and the OTLP export silently - * fails. Inside docker the endpoint is overridden to otel-collector:4317. */ + * Uses 127.0.0.1 (not "localhost") so a natively-run service reaches the + * IPv4-only collector port published by docker-compose — on dual-stack + * hosts "localhost" resolves to ::1 first and the OTLP export silently + * fails. Inside docker the endpoint is overridden to otel-collector:4317. + */ private val DefaultEndpoint = "http://127.0.0.1:4317" /** Metric export interval bounds. Values outside this range get - * clamped to the default with a one-shot WARN. The lower bound - * prevents an attacker tipping the exporter into busy-loop mode; - * the upper bound keeps metrics useful for human operators. */ + * clamped to the default with a one-shot WARN. The lower bound + * prevents an attacker tipping the exporter into busy-loop mode; + * the upper bound keeps metrics useful for human operators. + */ private[observability] val MinMetricIntervalMs: Long = 1000L private[observability] val MaxMetricIntervalMs: Long = 10L * 60L * 1000L private[observability] val DefaultMetricIntervalMs: Long = 60L * 1000L @@ -107,78 +110,82 @@ object OtelInit extends LazyLogging { @volatile private var initialized: Option[OpenTelemetry] = None /** - * Initialize the SDK for the given service name. - * Returns Some(sdk) on success, None on disabled / invalid config. - * - * Side effect when enabled: attaches a [[TexeraOtelLogAppender]] to - * the Logback ROOT logger so application logs are mirrored to the - * OTel collector, with the security guards in [[LogSanitizer]] - * applied to every record. - */ - def init(serviceName: String): Option[OpenTelemetry] = synchronized { - if (initialized.isDefined) return initialized - - val env = (key: String) => Option(System.getenv(key)) - val result = initInternal( - serviceName = serviceName, - envProvider = env, - spanExporterFactory = buildOtlpSpanExporter, - logExporterFactory = endpoint => Some(buildOtlpLogExporter(endpoint)), - metricExporterFactory = endpoint => Some(buildOtlpMetricExporter(endpoint)), - logbackAttacher = LogbackBinder.attach - ) - // Register globally so [[TexeraTracer]] and any other OTel-aware - // code can call ``GlobalOpenTelemetry.getTracer(...)`` without - // threading the SDK through every callsite. set() throws on a - // second call within the same JVM — our outer ``initialized`` - // guard makes that unreachable, but wrap defensively. The test - // path deliberately skips this so multiple isolated SDKs can be - // built within one JVM. - result.foreach { sdk => - Try(GlobalOpenTelemetry.set(sdk)).failed.foreach { t => - logger.warn( - s"GlobalOpenTelemetry already set; using the existing instance: ${t.getMessage}" - ) + * Initialize the SDK for the given service name. + * Returns Some(sdk) on success, None on disabled / invalid config. + * + * Side effect when enabled: attaches a [[TexeraOtelLogAppender]] to + * the Logback ROOT logger so application logs are mirrored to the + * OTel collector, with the security guards in [[LogSanitizer]] + * applied to every record. + */ + def init(serviceName: String): Option[OpenTelemetry] = + synchronized { + if (initialized.isDefined) return initialized + + val env = (key: String) => Option(System.getenv(key)) + val result = initInternal( + serviceName = serviceName, + envProvider = env, + spanExporterFactory = buildOtlpSpanExporter, + logExporterFactory = endpoint => Some(buildOtlpLogExporter(endpoint)), + metricExporterFactory = endpoint => Some(buildOtlpMetricExporter(endpoint)), + logbackAttacher = LogbackBinder.attach + ) + // Register globally so [[TexeraTracer]] and any other OTel-aware + // code can call ``GlobalOpenTelemetry.getTracer(...)`` without + // threading the SDK through every callsite. set() throws on a + // second call within the same JVM — our outer ``initialized`` + // guard makes that unreachable, but wrap defensively. The test + // path deliberately skips this so multiple isolated SDKs can be + // built within one JVM. + result.foreach { sdk => + Try(GlobalOpenTelemetry.set(sdk)).failed.foreach { t => + logger.warn( + s"GlobalOpenTelemetry already set; using the existing instance: ${t.getMessage}" + ) + } } + result } - result - } /** - * Test-only entry point. Allows the test to inject an env-var map - * and a span exporter so the SDK does not attempt a real network - * connection. The Logback appender is NOT attached in tests — - * appender tests construct it directly with an in-memory log - * exporter. - */ + * Test-only entry point. Allows the test to inject an env-var map + * and a span exporter so the SDK does not attempt a real network + * connection. The Logback appender is NOT attached in tests — + * appender tests construct it directly with an in-memory log + * exporter. + */ private[observability] def initForTest( serviceName: String, envOverride: Map[String, String], exporter: SpanExporter, metricExporter: Option[MetricExporter] = None - ): Option[OpenTelemetry] = synchronized { - initInternal( - serviceName = serviceName, - envProvider = envOverride.get, - spanExporterFactory = _ => exporter, - logExporterFactory = _ => None, - metricExporterFactory = _ => metricExporter, - logbackAttacher = (_, _) => () // no-op in tests - ) - } + ): Option[OpenTelemetry] = + synchronized { + initInternal( + serviceName = serviceName, + envProvider = envOverride.get, + spanExporterFactory = _ => exporter, + logExporterFactory = _ => None, + metricExporterFactory = _ => metricExporter, + logbackAttacher = (_, _) => () // no-op in tests + ) + } /** Test-only: forget any previously-installed SDK. Does not unregister - * shutdown hooks (the previous SDK is closed instead). */ - private[observability] def resetForTest(): Unit = synchronized { - initialized.foreach { - case sdk: OpenTelemetrySdk => - Try(sdk.getSdkTracerProvider.close()) - Try(sdk.getSdkLoggerProvider.close()) - Try(sdk.getSdkMeterProvider.close()) - case _ => () + * shutdown hooks (the previous SDK is closed instead). + */ + private[observability] def resetForTest(): Unit = + synchronized { + initialized.foreach { + case sdk: OpenTelemetrySdk => + Try(sdk.getSdkTracerProvider.close()) + Try(sdk.getSdkLoggerProvider.close()) + Try(sdk.getSdkMeterProvider.close()) + case _ => () + } + initialized = None } - initialized = None - } private def initInternal( serviceName: String, @@ -200,7 +207,9 @@ object OtelInit extends LazyLogging { // dev time is a quiet no-op rather than a startup failure. val disabled = envProvider("OTEL_SDK_DISABLED").getOrElse("false") if (disabled.equalsIgnoreCase("true")) { - logger.info("OpenTelemetry SDK disabled (OTEL_SDK_DISABLED=true). No telemetry will be emitted.") + logger.info( + "OpenTelemetry SDK disabled (OTEL_SDK_DISABLED=true). No telemetry will be emitted." + ) return None } @@ -284,12 +293,17 @@ object OtelInit extends LazyLogging { // Make sure providers flush on shutdown. We add the hook only after // the SDK has been fully built so a panic during init doesn't leave // a dangling hook pointing at a half-constructed provider. - Runtime.getRuntime.addShutdownHook(new Thread(() => { - Try(tracerProvider.close()) - loggerProviderOpt.foreach(lp => Try(lp.close())) - meterProviderOpt.foreach(mp => Try(mp.close())) - () - }, "otel-shutdown")) + Runtime.getRuntime.addShutdownHook( + new Thread( + () => { + Try(tracerProvider.close()) + loggerProviderOpt.foreach(lp => Try(lp.close())) + meterProviderOpt.foreach(mp => Try(mp.close())) + () + }, + "otel-shutdown" + ) + ) initialized = Some(sdk) logger.info(s"OpenTelemetry SDK initialized for service '$serviceName' (endpoint=$endpoint).") @@ -297,10 +311,10 @@ object OtelInit extends LazyLogging { } /** - * Validate that the endpoint is parseable, uses an allowlisted scheme, - * and resolves to an allowlisted host. Pure function — safe to test - * without standing up the SDK. - */ + * Validate that the endpoint is parseable, uses an allowlisted scheme, + * and resolves to an allowlisted host. Pure function — safe to test + * without standing up the SDK. + */ private[observability] def validateEndpoint( endpoint: String, allowedHosts: Set[String] @@ -313,7 +327,9 @@ object OtelInit extends LazyLogging { if (scheme.isEmpty) { Left("missing scheme") } else if (!AllowedSchemes.contains(scheme)) { - Left(s"scheme '$scheme' not in allowlist ${AllowedSchemes.toSeq.sorted.mkString("{", ",", "}")}") + Left( + s"scheme '$scheme' not in allowlist ${AllowedSchemes.toSeq.sorted.mkString("{", ",", "}")}" + ) } else { val host = Option(uri.getHost).map(_.toLowerCase).getOrElse("") if (host.isEmpty) { @@ -328,10 +344,10 @@ object OtelInit extends LazyLogging { } /** - * Build a Resource from the service name plus the allowlisted subset - * of OTEL_RESOURCE_ATTRIBUTES. Unknown keys are dropped silently; - * service.name from env is ignored in favour of the argument. - */ + * Build a Resource from the service name plus the allowlisted subset + * of OTEL_RESOURCE_ATTRIBUTES. Unknown keys are dropped silently; + * service.name from env is ignored in favour of the argument. + */ private[observability] def buildResource(serviceName: String, rawAttrs: String): Resource = { val builder = Attributes.builder() builder.put(AttributeKey.stringKey("service.name"), serviceName) @@ -371,11 +387,11 @@ object OtelInit extends LazyLogging { OtlpGrpcMetricExporter.builder().setEndpoint(endpoint).build() /** - * Parse and clamp OTEL_METRIC_EXPORT_INTERVAL (milliseconds). - * Out-of-range or unparseable input falls back to the default and - * emits a single WARN. Pure-ish — easy to test without standing up - * the meter SDK. - */ + * Parse and clamp OTEL_METRIC_EXPORT_INTERVAL (milliseconds). + * Out-of-range or unparseable input falls back to the default and + * emits a single WARN. Pure-ish — easy to test without standing up + * the meter SDK. + */ private[observability] def clampIntervalMs(raw: Option[String]): Long = { raw match { case None => DefaultMetricIntervalMs @@ -401,18 +417,18 @@ object OtelInit extends LazyLogging { } /** - * Hides the Logback attach step behind a small object so [[OtelInit]] - * doesn't import Logback types directly (keeps the SDK init testable - * without a Logback dependency on the classpath in test runs that - * inject a mock attacher). - */ + * Hides the Logback attach step behind a small object so [[OtelInit]] + * doesn't import Logback types directly (keeps the SDK init testable + * without a Logback dependency on the classpath in test runs that + * inject a mock attacher). + */ private[observability] object LogbackBinder extends LazyLogging { /** Attempts to find the Logback ROOT logger, attach a fresh - * [[TexeraOtelLogAppender]] bound to `otel`, and start it. If - * Logback is not the active SLF4J binding (or for any other - * classpath issue), emits one WARN and returns — never throws. - */ + * [[TexeraOtelLogAppender]] bound to `otel`, and start it. If + * Logback is not the active SLF4J binding (or for any other + * classpath issue), emits one WARN and returns — never throws. + */ def attach(serviceName: String, otel: OpenTelemetry): Unit = { val factory = org.slf4j.LoggerFactory.getILoggerFactory factory match { diff --git a/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala b/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala index 37d04a303e6..2dbf160bcfb 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala @@ -31,21 +31,21 @@ import io.opentelemetry.context.Context import java.util.concurrent.TimeUnit /** - * Logback appender that forwards every event through [[LogSanitizer]] - * before emitting it as an OTel LogRecord. - * - * Lifecycle: - * - Construct with no args (Logback / programmatic instantiation). - * - Call [[bind]] once with the active [[OpenTelemetry]] instance - * (done by [[OtelInit]] after the SDK is built). Until then, - * [[append]] is a silent no-op — log events keep flowing to - * stdout/file unimpeded. - * - Stopping the appender unbinds; subsequent events drop. - * - * This is intentionally a thin shim. All security-critical logic - * lives in [[LogSanitizer]] so it can be tested without a Logback - * fixture. - */ + * Logback appender that forwards every event through [[LogSanitizer]] + * before emitting it as an OTel LogRecord. + * + * Lifecycle: + * - Construct with no args (Logback / programmatic instantiation). + * - Call [[bind]] once with the active [[OpenTelemetry]] instance + * (done by [[OtelInit]] after the SDK is built). Until then, + * [[append]] is a silent no-op — log events keep flowing to + * stdout/file unimpeded. + * - Stopping the appender unbinds; subsequent events drop. + * + * This is intentionally a thin shim. All security-critical logic + * lives in [[LogSanitizer]] so it can be tested without a Logback + * fixture. + */ class TexeraOtelLogAppender extends UnsynchronizedAppenderBase[ILoggingEvent] { // @volatile so a late [[bind]] is visible to appender threads @@ -121,8 +121,9 @@ class TexeraOtelLogAppender extends UnsynchronizedAppenderBase[ILoggingEvent] { } /** Pretty-print a Logback throwable proxy. Matches what Logback's - * default pattern layout would produce for `%ex` — class name, - * message, full stack frames, then walks the cause chain. */ + * default pattern layout would produce for `%ex` — class name, + * message, full stack frames, then walks the cause chain. + */ private def formatThrowable(proxy: IThrowableProxy): String = ThrowableProxyUtil.asString(proxy) diff --git a/common/config/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala index 5ebd0eb3f4a..45bbac119af 100644 --- a/common/config/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala +++ b/common/config/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala @@ -39,9 +39,16 @@ class OtelInitSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { // ----- validateEndpoint: pure function, exhaustive cases ------------- "validateEndpoint" should "accept a loopback OTLP gRPC URL" in { - OtelInit.validateEndpoint("http://localhost:4317", OtelInit.DefaultAllowedHosts) shouldBe Right(()) - OtelInit.validateEndpoint("grpc://127.0.0.1:4317", OtelInit.DefaultAllowedHosts) shouldBe Right(()) - OtelInit.validateEndpoint("https://localhost:4318", OtelInit.DefaultAllowedHosts) shouldBe Right(()) + OtelInit.validateEndpoint("http://localhost:4317", OtelInit.DefaultAllowedHosts) shouldBe Right( + () + ) + OtelInit.validateEndpoint("grpc://127.0.0.1:4317", OtelInit.DefaultAllowedHosts) shouldBe Right( + () + ) + OtelInit.validateEndpoint( + "https://localhost:4318", + OtelInit.DefaultAllowedHosts + ) shouldBe Right(()) } it should "reject file:// schemes (path traversal style attack)" in { @@ -83,14 +90,18 @@ class OtelInitSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { "buildResource" should "always include the service.name from the argument" in { val r = OtelInit.buildResource("my-service", "") - Option(r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.name"))) shouldBe Some( + Option( + r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.name")) + ) shouldBe Some( "my-service" ) } it should "honor allowlisted keys from OTEL_RESOURCE_ATTRIBUTES" in { val r = OtelInit.buildResource("svc", "service.version=1.2.3,deployment.environment=staging") - Option(r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.version"))) shouldBe Some( + Option( + r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.version")) + ) shouldBe Some( "1.2.3" ) Option( @@ -115,7 +126,9 @@ class OtelInitSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { it should "refuse to let OTEL_RESOURCE_ATTRIBUTES override service.name" in { val r = OtelInit.buildResource("real-svc", "service.name=spoofed") - Option(r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.name"))) shouldBe Some( + Option( + r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.name")) + ) shouldBe Some( "real-svc" ) } @@ -140,7 +153,9 @@ class OtelInitSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { it should "ignore malformed pairs without crashing" in { val r = OtelInit.buildResource("svc", ",,,=,foo,service.version=,=bar,service.version=1.0,") - Option(r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.version"))) shouldBe Some( + Option( + r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.version")) + ) shouldBe Some( "1.0" ) } diff --git a/common/config/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala index 83b9ca9d469..c879d1f9d56 100644 --- a/common/config/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala +++ b/common/config/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala @@ -36,8 +36,9 @@ import scala.jdk.CollectionConverters._ class TexeraOtelLogAppenderSpec extends AnyFlatSpec with Matchers { /** Build an OpenTelemetry SDK whose LoggerProvider drains to the - * given in-memory exporter via the synchronous SimpleLogRecordProcessor, - * so tests don't depend on batch timing. */ + * given in-memory exporter via the synchronous SimpleLogRecordProcessor, + * so tests don't depend on batch timing. + */ private def newFixture(): (OpenTelemetry, InMemoryLogRecordExporter, TexeraOtelLogAppender) = { val exporter = InMemoryLogRecordExporter.create() val lp = SdkLoggerProvider @@ -83,7 +84,13 @@ class TexeraOtelLogAppenderSpec extends AnyFlatSpec with Matchers { appender.doAppend(makeEvent(s"msg-$lvl", lvl)) } val severities = exporter.getFinishedLogRecordItems.asScala.map(_.getSeverity).toSet - severities shouldBe Set(Severity.TRACE, Severity.DEBUG, Severity.INFO, Severity.WARN, Severity.ERROR) + severities shouldBe Set( + Severity.TRACE, + Severity.DEBUG, + Severity.INFO, + Severity.WARN, + Severity.ERROR + ) } // ----- security: sanitisation happens at the boundary ----------------- From 17b1bdc9c1a8f9d711bceaeb476b23d3755f51a4 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 5 Jun 2026 07:16:20 -0700 Subject: [PATCH 05/21] apply scalafmt to metrics and tracing primitives (pr2) --- .../texera/observability/SpanAttrs.scala | 52 ++++---- .../texera/observability/TexeraMetrics.scala | 113 ++++++++++-------- .../texera/observability/TexeraTracer.scala | 56 ++++----- .../observability/TraceparentValidator.scala | 48 ++++---- .../texera/observability/SpanAttrsSpec.scala | Bin 5448 -> 5460 bytes .../observability/TexeraMetricsSpec.scala | 12 +- .../TraceparentValidatorSpec.scala | Bin 5085 -> 5084 bytes 7 files changed, 153 insertions(+), 128 deletions(-) diff --git a/common/config/src/main/scala/org/apache/texera/observability/SpanAttrs.scala b/common/config/src/main/scala/org/apache/texera/observability/SpanAttrs.scala index c66db23f652..a1ac0bbc18b 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/SpanAttrs.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/SpanAttrs.scala @@ -23,27 +23,28 @@ import io.opentelemetry.api.common.AttributeKey import io.opentelemetry.api.trace.{Span, SpanBuilder} /** - * Thin helper for setting span attributes safely. - * - * Three rules: - * 1. Typed setters only — no public escape hatch for arbitrary - * untyped strings to land on a span as untrusted free text. - * 2. Free-text values are CRLF-stripped + capped at - * [[FreeTextMaxLen]] to prevent log/span forging via embedded - * newlines. - * 3. Operator IDs and workflow/execution IDs must match a strict - * character set — otherwise dropped silently (the operator - * identifier should be a stable internal value, not user free - * text). - */ + * Thin helper for setting span attributes safely. + * + * Three rules: + * 1. Typed setters only — no public escape hatch for arbitrary + * untyped strings to land on a span as untrusted free text. + * 2. Free-text values are CRLF-stripped + capped at + * [[FreeTextMaxLen]] to prevent log/span forging via embedded + * newlines. + * 3. Operator IDs and workflow/execution IDs must match a strict + * character set — otherwise dropped silently (the operator + * identifier should be a stable internal value, not user free + * text). + */ object SpanAttrs { /** Maximum length for free-text span attribute values. */ val FreeTextMaxLen: Int = 256 /** Validates the shape we accept for operator IDs: alnum + `_.-`, - * 1–64 chars. Anything else is dropped (not coerced — we'd rather - * miss a label than leak an unbounded string into a span). */ + * 1–64 chars. Anything else is dropped (not coerced — we'd rather + * miss a label than leak an unbounded string into a span). + */ private val OperatorIdPattern = "^[A-Za-z0-9_.\\-]{1,64}$".r.pattern // ---- Standard Texera correlation labels ------------------------------ @@ -71,8 +72,9 @@ object SpanAttrs { b.setAttribute(UserId, java.lang.Long.valueOf(id)) /** Sets the operator id only if it passes the strict character - * check; otherwise the attribute is omitted. Returns the same - * builder either way for fluent chaining. */ + * check; otherwise the attribute is omitted. Returns the same + * builder either way for fluent chaining. + */ def withOperatorId(b: SpanBuilder, id: String): SpanBuilder = { if (id != null && OperatorIdPattern.matcher(id).matches()) { b.setAttribute(OperatorId, id) @@ -88,8 +90,10 @@ object SpanAttrs { // ---- Typed setters for Span (used after a span is active) ------------ - def setWorkflowId(s: Span, id: Long): Span = s.setAttribute(WorkflowId, java.lang.Long.valueOf(id)) - def setExecutionId(s: Span, id: Long): Span = s.setAttribute(ExecutionId, java.lang.Long.valueOf(id)) + def setWorkflowId(s: Span, id: Long): Span = + s.setAttribute(WorkflowId, java.lang.Long.valueOf(id)) + def setExecutionId(s: Span, id: Long): Span = + s.setAttribute(ExecutionId, java.lang.Long.valueOf(id)) def setOperatorId(s: Span, id: String): Span = { if (id != null && OperatorIdPattern.matcher(id).matches()) s.setAttribute(OperatorId, id) else s @@ -102,13 +106,13 @@ object SpanAttrs { // ---- Pure helpers (exposed for testing) ------------------------------ /** - * Strip CR/LF and other C0 control characters from a free-text - * value, then cap at [[FreeTextMaxLen]]. Returns null for - * null/empty input (caller skips the setAttribute call). - */ + * Strip CR/LF and other C0 control characters from a free-text + * value, then cap at [[FreeTextMaxLen]]. Returns null for + * null/empty input (caller skips the setAttribute call). + */ def sanitizeFreeText(value: String): String = { if (value == null || value.isEmpty) return null - val stripped = value.filter(c => c >= 0x20 && c != 0x7F) + val stripped = value.filter(c => c >= 0x20 && c != 0x7f) if (stripped.isEmpty) null else if (stripped.length <= FreeTextMaxLen) stripped else stripped.substring(0, FreeTextMaxLen) diff --git a/common/config/src/main/scala/org/apache/texera/observability/TexeraMetrics.scala b/common/config/src/main/scala/org/apache/texera/observability/TexeraMetrics.scala index c850c055392..ef7ad78aeef 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/TexeraMetrics.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/TexeraMetrics.scala @@ -25,20 +25,20 @@ import io.opentelemetry.api.common.{AttributeKey, Attributes} import io.opentelemetry.api.metrics.Meter /** - * Strongly-typed façade for Texera-emitted metrics. - * - * Cardinality safety is enforced by the API surface, not by - * documentation: there is no public method that accepts an arbitrary - * string as a label key or value. The only labels that ever land on - * an instrument are the two enums [[Outcome]] and [[WorkflowKind]], - * each restricted to a fixed set. ``workflow.id`` / ``execution.id`` - * are deliberately NOT metric labels — per-execution detail belongs - * in traces and logs, joined on ``trace_id`` at query time. - * - * Histogram bucket bounds are hard-coded constants so they can't be - * coerced by request input. The OTel SDK applies its own default - * attribute-value-length cap to anything that does slip through. - */ + * Strongly-typed façade for Texera-emitted metrics. + * + * Cardinality safety is enforced by the API surface, not by + * documentation: there is no public method that accepts an arbitrary + * string as a label key or value. The only labels that ever land on + * an instrument are the two enums [[Outcome]] and [[WorkflowKind]], + * each restricted to a fixed set. ``workflow.id`` / ``execution.id`` + * are deliberately NOT metric labels — per-execution detail belongs + * in traces and logs, joined on ``trace_id`` at query time. + * + * Histogram bucket bounds are hard-coded constants so they can't be + * coerced by request input. The OTel SDK applies its own default + * attribute-value-length cap to anything that does slip through. + */ object TexeraMetrics extends LazyLogging { /** Outcome enum, the only mutable label on lifecycle counters. */ @@ -50,7 +50,8 @@ object TexeraMetrics extends LazyLogging { } /** Workflow kind enum. Distinguishes interactive vs. scheduled - * workflows for the dashboard's basic split — extend deliberately. */ + * workflows for the dashboard's basic split — extend deliberately. + */ sealed abstract class WorkflowKind(val name: String) object WorkflowKind { case object Interactive extends WorkflowKind("interactive") @@ -62,8 +63,9 @@ object TexeraMetrics extends LazyLogging { private val WorkflowKindKey: AttributeKey[String] = AttributeKey.stringKey("texera.workflow.kind") /** Histogram bucket bounds in seconds. Hard-coded — constants so - * request input can't reshape the histogram. Range covers - * fast (<1s) to long (>1h) workflows. */ + * request input can't reshape the histogram. Range covers + * fast (<1s) to long (>1h) workflows. + */ private val DurationBuckets: java.util.List[java.lang.Double] = { val builder = new java.util.ArrayList[java.lang.Double]() Seq(0.1, 0.5, 1.0, 5.0, 10.0, 30.0, 60.0, 300.0, 600.0, 1800.0, 3600.0) @@ -99,36 +101,42 @@ object TexeraMetrics extends LazyLogging { @volatile private var activeExecutionsSupplier: () => Long = () => 0L /** Register the authoritative source of "currently active executions". - * The supplier is polled on every metric collection, so the gauge always - * reflects ground truth and can never leak. Called once at process - * startup (e.g. by ComputingUnitMaster). */ - def setActiveExecutionsSupplier(supplier: () => Long): Unit = synchronized { - activeExecutionsSupplier = supplier - ensureBound() - } + * The supplier is polled on every metric collection, so the gauge always + * reflects ground truth and can never leak. Called once at process + * startup (e.g. by ComputingUnitMaster). + */ + def setActiveExecutionsSupplier(supplier: () => Long): Unit = + synchronized { + activeExecutionsSupplier = supplier + ensureBound() + } /** Bind instruments to the current global meter. Idempotent — the - * first call wins; later calls are no-ops. Tests can call - * [[bindForTest]] with an explicit Meter, then [[resetForTest]] to - * rebind. */ - def ensureBound(): Unit = synchronized { - if (_starts == null) bind(GlobalOpenTelemetry.getMeter(InstrumentationScope)) - } - - private[observability] def bindForTest(meter: Meter): Unit = synchronized { - bind(meter) - } - - private[observability] def resetForTest(): Unit = synchronized { - _starts = null - _completions = null - _cancellations = null - // The observable gauge registered a collection callback — close it so the - // previous test's meter provider stops being polled after it's discarded. - if (_active != null) _active.close() - _active = null - _duration = null - } + * first call wins; later calls are no-ops. Tests can call + * [[bindForTest]] with an explicit Meter, then [[resetForTest]] to + * rebind. + */ + def ensureBound(): Unit = + synchronized { + if (_starts == null) bind(GlobalOpenTelemetry.getMeter(InstrumentationScope)) + } + + private[observability] def bindForTest(meter: Meter): Unit = + synchronized { + bind(meter) + } + + private[observability] def resetForTest(): Unit = + synchronized { + _starts = null + _completions = null + _cancellations = null + // The observable gauge registered a collection callback — close it so the + // previous test's meter provider stops being polled after it's discarded. + if (_active != null) _active.close() + _active = null + _duration = null + } private def bind(meter: Meter): Unit = { _starts = meter @@ -157,7 +165,9 @@ object TexeraMetrics extends LazyLogging { _active = meter .gaugeBuilder("texera.workflow.active") .ofLongs() - .setDescription("Number of workflow executions currently in progress (observed from the live registry).") + .setDescription( + "Number of workflow executions currently in progress (observed from the live registry)." + ) .buildWithCallback(obs => obs.record(activeExecutionsSupplier())) _duration = meter .histogramBuilder("texera.workflow.duration") @@ -202,11 +212,12 @@ object TexeraMetrics extends LazyLogging { } /** A user-initiated kill/cancel. Bumps a dedicated counter. Deliberately - * NOT recorded as a completion: a cancelled run never finished on its own, - * so it must not drag down the success rate. No duration is recorded for - * the same reason — a killed run's wall-clock time is not a real runtime - * and would skew the duration percentiles. (The active gauge is observed - * from the live registry and needs no decrement here.) */ + * NOT recorded as a completion: a cancelled run never finished on its own, + * so it must not drag down the success rate. No duration is recorded for + * the same reason — a killed run's wall-clock time is not a real runtime + * and would skew the duration percentiles. (The active gauge is observed + * from the live registry and needs no decrement here.) + */ def recordCancellation(kind: WorkflowKind): Unit = { ensureBound() logger.debug(s"metric: workflow cancelled (kind=${kind.name}) — cancellations +1") diff --git a/common/config/src/main/scala/org/apache/texera/observability/TexeraTracer.scala b/common/config/src/main/scala/org/apache/texera/observability/TexeraTracer.scala index 8941d97aefa..ae8f71c82e6 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/TexeraTracer.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/TexeraTracer.scala @@ -24,22 +24,22 @@ import io.opentelemetry.api.trace.{Span, SpanBuilder, StatusCode, Tracer} import io.opentelemetry.context.{Context, Scope} /** - * Thin convenience wrapper around the global OTel tracer. - * - * Two reasons to go through this rather than calling - * ``GlobalOpenTelemetry.getTracer`` directly at every callsite: - * - * 1. Single instrumentation scope name (``org.apache.texera``) — so - * every Texera-produced span shows up under one logical scope in - * the backend, separable from anything emitted by transitive - * libraries. - * 2. One ergonomic ``withSpan`` API that handles exception → status, - * scope cleanup, and span end in a single try/finally. Callers - * don't have to remember the ceremony at every site. - * - * When the SDK is disabled, ``GlobalOpenTelemetry.getTracer`` returns - * a no-op tracer, so calling these methods is safe at any time. - */ + * Thin convenience wrapper around the global OTel tracer. + * + * Two reasons to go through this rather than calling + * ``GlobalOpenTelemetry.getTracer`` directly at every callsite: + * + * 1. Single instrumentation scope name (``org.apache.texera``) — so + * every Texera-produced span shows up under one logical scope in + * the backend, separable from anything emitted by transitive + * libraries. + * 2. One ergonomic ``withSpan`` API that handles exception → status, + * scope cleanup, and span end in a single try/finally. Callers + * don't have to remember the ceremony at every site. + * + * When the SDK is disabled, ``GlobalOpenTelemetry.getTracer`` returns + * a no-op tracer, so calling these methods is safe at any time. + */ object TexeraTracer { private val InstrumentationScope = "org.apache.texera" @@ -49,13 +49,13 @@ object TexeraTracer { def spanBuilder(name: String): SpanBuilder = tracer.spanBuilder(name) /** - * Run ``block`` inside a fresh span; record exceptions, propagate - * the right span status, and ensure the span is ended exactly once. - * - * Use this for synchronous critical sections. For async (Future- - * returning) code paths use ``withAsyncSpan`` so the span doesn't - * close before the async work completes. - */ + * Run ``block`` inside a fresh span; record exceptions, propagate + * the right span status, and ensure the span is ended exactly once. + * + * Use this for synchronous critical sections. For async (Future- + * returning) code paths use ``withAsyncSpan`` so the span doesn't + * close before the async work completes. + */ def withSpan[T](name: String, configure: SpanBuilder => SpanBuilder = identity)( block: Span => T ): T = { @@ -75,10 +75,10 @@ object TexeraTracer { } /** - * Snapshot the current OTel ``Context`` so async callbacks can - * re-attach it via ``Context.makeCurrent`` later. Useful at the - * Scala↔Python boundary where the calling thread is not the - * receiving thread. - */ + * Snapshot the current OTel ``Context`` so async callbacks can + * re-attach it via ``Context.makeCurrent`` later. Useful at the + * Scala↔Python boundary where the calling thread is not the + * receiving thread. + */ def currentContext: Context = Context.current() } diff --git a/common/config/src/main/scala/org/apache/texera/observability/TraceparentValidator.scala b/common/config/src/main/scala/org/apache/texera/observability/TraceparentValidator.scala index 9577a9f9438..91868449027 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/TraceparentValidator.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/TraceparentValidator.scala @@ -20,35 +20,38 @@ package org.apache.texera.observability /** - * Pure validators for W3C Trace Context headers crossing the - * Scala↔Python boundary. The single rule: if the inbound bytes do - * not match the strict regex, we discard the value and start a fresh - * trace. We never echo a rejected value back into a span, log, or - * error message. - * - * Spec reference: https://www.w3.org/TR/trace-context/ - * - * The regexes here intentionally do NOT use any context-sensitive - * grouping or backreferences — keeps the validators safe against - * pathological inputs (ReDoS) and trivially fast. - */ + * Pure validators for W3C Trace Context headers crossing the + * Scala↔Python boundary. The single rule: if the inbound bytes do + * not match the strict regex, we discard the value and start a fresh + * trace. We never echo a rejected value back into a span, log, or + * error message. + * + * Spec reference: https://www.w3.org/TR/trace-context/ + * + * The regexes here intentionally do NOT use any context-sensitive + * grouping or backreferences — keeps the validators safe against + * pathological inputs (ReDoS) and trivially fast. + */ object TraceparentValidator { /** W3C traceparent format: `---`. - * We accept only version `00` (the only published version) with the - * canonical 32-hex / 16-hex / 2-hex layout. All hex lowercase per - * spec — uppercase is invalid. */ + * We accept only version `00` (the only published version) with the + * canonical 32-hex / 16-hex / 2-hex layout. All hex lowercase per + * spec — uppercase is invalid. + */ private val TraceparentPattern = "^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$".r.pattern /** Bounded tracestate length. Spec recommends ≤512 chars. We are - * stricter to remove a small DoS surface — an attacker can't send - * a 1 MiB tracestate to balloon downstream context allocations. */ + * stricter to remove a small DoS surface — an attacker can't send + * a 1 MiB tracestate to balloon downstream context allocations. + */ val MaxTracestateLength: Int = 512 /** Validate a traceparent header. Returns the input unchanged on - * success, None on any failure (rejected — caller starts a fresh - * trace). Null and empty are silent failures. */ + * success, None on any failure (rejected — caller starts a fresh + * trace). Null and empty are silent failures. + */ def validateTraceparent(header: String): Option[String] = { if (header == null || header.isEmpty) return None // Trace-id and parent-id must not be all-zero per spec — an @@ -62,8 +65,9 @@ object TraceparentValidator { } /** Validate a tracestate header. Spec: comma-separated list of - * key=value pairs, ASCII-printable only, total length capped. - * Returns the input unchanged on success, None on rejection. */ + * key=value pairs, ASCII-printable only, total length capped. + * Returns the input unchanged on success, None on rejection. + */ def validateTracestate(header: String): Option[String] = { if (header == null || header.isEmpty) return None if (header.length > MaxTracestateLength) return None @@ -86,7 +90,7 @@ object TraceparentValidator { val c = s.charAt(i) // Allow printable ASCII range (0x20-0x7E). Tab and CR/LF are // rejected — a tracestate must not span lines. - if (c < 0x20 || c > 0x7E) return false + if (c < 0x20 || c > 0x7e) return false i += 1 } true diff --git a/common/config/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala index 388ee1d2aaa3a7f102742d7ea6bc083c1e6a00e8..1afca195533073f1d8ff2b0b8c8b4dd524da77e2 100644 GIT binary patch delta 27 gcmX@1bwz7KH4ht?0tigb7q*=Ifm43-S)OVR0DayF-~a#s delta 20 ccmcbjbwX=HHP7Tke*ekE+)|rA@T71609h6XzyJUM diff --git a/common/config/src/test/scala/org/apache/texera/observability/TexeraMetricsSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/TexeraMetricsSpec.scala index 10a9eb7651a..8bed93d6b01 100644 --- a/common/config/src/test/scala/org/apache/texera/observability/TexeraMetricsSpec.scala +++ b/common/config/src/test/scala/org/apache/texera/observability/TexeraMetricsSpec.scala @@ -93,7 +93,9 @@ class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEac "texera.workflow.duration" ) - metrics("texera.workflow.completions").getLongSumData.getPoints.asScala.head.getValue shouldBe 1L + metrics( + "texera.workflow.completions" + ).getLongSumData.getPoints.asScala.head.getValue shouldBe 1L val histogram = metrics("texera.workflow.duration").getHistogramData.getPoints.asScala.head histogram.getCount shouldBe 1L @@ -111,7 +113,9 @@ class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEac // series the query never reads). val completion = metrics("texera.workflow.completions").getLongSumData.getPoints.asScala.head completion.getValue shouldBe 1L - completion.getAttributes.asMap.asScala.map { case (k, v) => k.getKey -> v.toString } should contain( + completion.getAttributes.asMap.asScala.map { + case (k, v) => k.getKey -> v.toString + } should contain( "texera.outcome" -> "failure" ) metrics("texera.workflow.duration").getHistogramData.getPoints.asScala.head.getSum shouldBe 12.0 @@ -124,7 +128,9 @@ class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEac TexeraMetrics.recordCancellation(TexeraMetrics.WorkflowKind.Interactive) val metrics = collectAll() - metrics("texera.workflow.cancellations").getLongSumData.getPoints.asScala.head.getValue shouldBe 1L + metrics( + "texera.workflow.cancellations" + ).getLongSumData.getPoints.asScala.head.getValue shouldBe 1L // A kill is not a completion and records no duration: it must not // drag down the success rate nor skew the duration percentiles. metrics.keySet should not contain "texera.workflow.completions" diff --git a/common/config/src/test/scala/org/apache/texera/observability/TraceparentValidatorSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/TraceparentValidatorSpec.scala index d95b61239ba4dfcf2dc87ff4fbbbc19b3d022ab5..8c4bd670fc1994160c1bb916f1443f08d9d3f20c 100644 GIT binary patch delta 16 Ycmcbsen)*n9`EFdEasb+@tU&%06lXCH2?qr delta 16 Ycmcbkeph`%9`ED{EEb!W@|v>&06l94Gynhq From c89c985c4231eb31e2afdb42995254b5683ea291 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 5 Jun 2026 07:23:58 -0700 Subject: [PATCH 06/21] apply scalafmt to deployment config specs (pr3) --- .../ObservabilityComposeSpec.scala | 15 +++++++-------- .../observability/ParcaConfigSpec.scala | 19 +++++++++---------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/common/config/src/test/scala/org/apache/texera/observability/ObservabilityComposeSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/ObservabilityComposeSpec.scala index 5f21df9b407..af8fe6792bc 100644 --- a/common/config/src/test/scala/org/apache/texera/observability/ObservabilityComposeSpec.scala +++ b/common/config/src/test/scala/org/apache/texera/observability/ObservabilityComposeSpec.scala @@ -26,11 +26,11 @@ import java.nio.charset.StandardCharsets import java.nio.file.{Files, Path, Paths} /** - * Smoke tests for the PR 6 docker-compose + collector config. Same - * design as [[ParcaConfigSpec]] — string-level assertions, no YAML - * parser, because the goal is to catch typos and licence-pin drift, - * not to validate the upstream schemas. - */ + * Smoke tests for the PR 6 docker-compose + collector config. Same + * design as [[ParcaConfigSpec]] — string-level assertions, no YAML + * parser, because the goal is to catch typos and licence-pin drift, + * not to validate the upstream schemas. + */ class ObservabilityComposeSpec extends AnyFlatSpec with Matchers { private def resolveBundled(relative: String): Path = { @@ -150,9 +150,8 @@ class ObservabilityComposeSpec extends AnyFlatSpec with Matchers { "observability-traces", "observability-profiles" ).foreach { profile => - val grepCount = text.split('\n').count(line => - !line.trim.startsWith("#") && line.contains(profile) - ) + val grepCount = + text.split('\n').count(line => !line.trim.startsWith("#") && line.contains(profile)) withClue(s"$profile should be in default COMPOSE_PROFILES (non-comment): ")( grepCount should be >= 1 ) diff --git a/common/config/src/test/scala/org/apache/texera/observability/ParcaConfigSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/ParcaConfigSpec.scala index 8e0d162b23f..1b6a15abcec 100644 --- a/common/config/src/test/scala/org/apache/texera/observability/ParcaConfigSpec.scala +++ b/common/config/src/test/scala/org/apache/texera/observability/ParcaConfigSpec.scala @@ -26,14 +26,14 @@ import java.nio.charset.StandardCharsets import java.nio.file.{Files, Path, Paths} /** - * Smoke test for the bundled Parca configuration files. - * - * Intentionally lightweight: we are guarding against accidental - * deletion / emptying / tag drift, not validating Parca's schema. - * The real config validation happens when the agent starts up - * inside its container — but a unit-level smoke test catches typos - * before a developer pushes them. - */ + * Smoke test for the bundled Parca configuration files. + * + * Intentionally lightweight: we are guarding against accidental + * deletion / emptying / tag drift, not validating Parca's schema. + * The real config validation happens when the agent starts up + * inside its container — but a unit-level smoke test catches typos + * before a developer pushes them. + */ class ParcaConfigSpec extends AnyFlatSpec with Matchers { // sbt runs tests with the module dir as CWD, but a developer who @@ -100,8 +100,7 @@ class ParcaConfigSpec extends AnyFlatSpec with Matchers { // This assertion makes the design intent enforceable. Comments // are skipped — they're allowed (and required) to explain the // rule. - val configLines = new String(Files.readAllBytes(agentEnv), StandardCharsets.UTF_8) - .linesIterator + val configLines = new String(Files.readAllBytes(agentEnv), StandardCharsets.UTF_8).linesIterator .map(_.trim) .filter(line => line.nonEmpty && !line.startsWith("#")) .toSeq From 9e48e0d03ddf20374d4d7fe6fc8d9d3a71172cf9 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 5 Jun 2026 10:54:33 -0700 Subject: [PATCH 07/21] adding licenses for pr1 --- access-control-service/LICENSE-binary | 12 ++++++++++++ amber/LICENSE-binary-java | 12 ++++++++++++ computing-unit-managing-service/LICENSE-binary | 12 ++++++++++++ config-service/LICENSE-binary | 12 ++++++++++++ file-service/LICENSE-binary | 12 ++++++++++++ workflow-compiling-service/LICENSE-binary | 12 ++++++++++++ 6 files changed, 72 insertions(+) diff --git a/access-control-service/LICENSE-binary b/access-control-service/LICENSE-binary index 3abc86ea44c..48cd986319e 100644 --- a/access-control-service/LICENSE-binary +++ b/access-control-service/LICENSE-binary @@ -274,6 +274,18 @@ Scala/Java jars: - io.dropwizard.metrics.metrics-json-4.2.25.jar - io.dropwizard.metrics.metrics-jvm-4.2.25.jar - io.dropwizard.metrics.metrics-logback-4.2.25.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.r2dbc.r2dbc-spi-0.9.0.RELEASE.jar - jakarta.inject.jakarta.inject-api-2.0.1.jar - jakarta.validation.jakarta.validation-api-3.0.2.jar diff --git a/amber/LICENSE-binary-java b/amber/LICENSE-binary-java index fba8dd9cd29..f1911ef6a79 100644 --- a/amber/LICENSE-binary-java +++ b/amber/LICENSE-binary-java @@ -362,6 +362,18 @@ Scala/Java jars: - io.netty.netty-transport-native-unix-common-4.1.96.Final.jar - io.opencensus.opencensus-api-0.31.1.jar - io.opencensus.opencensus-contrib-http-util-0.31.1.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.perfmark.perfmark-api-0.26.0.jar - io.r2dbc.r2dbc-spi-0.9.0.RELEASE.jar - io.reactivex.rxjava3.rxjava-3.1.6.jar diff --git a/computing-unit-managing-service/LICENSE-binary b/computing-unit-managing-service/LICENSE-binary index 37d78afb47c..de8d4699977 100644 --- a/computing-unit-managing-service/LICENSE-binary +++ b/computing-unit-managing-service/LICENSE-binary @@ -369,6 +369,18 @@ Scala/Java jars: - io.netty.netty-tcnative-classes-2.0.61.Final.jar - io.netty.netty-transport-4.1.104.Final.jar - io.netty.netty-transport-native-unix-common-4.1.104.Final.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.perfmark.perfmark-api-0.26.0.jar - io.r2dbc.r2dbc-spi-0.9.0.RELEASE.jar - io.swagger.swagger-annotations-1.6.14.jar diff --git a/config-service/LICENSE-binary b/config-service/LICENSE-binary index 78c3b7878f1..3b5736fa130 100644 --- a/config-service/LICENSE-binary +++ b/config-service/LICENSE-binary @@ -274,6 +274,18 @@ Scala/Java jars: - io.dropwizard.metrics.metrics-json-4.2.25.jar - io.dropwizard.metrics.metrics-jvm-4.2.25.jar - io.dropwizard.metrics.metrics-logback-4.2.25.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.r2dbc.r2dbc-spi-0.9.0.RELEASE.jar - jakarta.inject.jakarta.inject-api-2.0.1.jar - jakarta.validation.jakarta.validation-api-3.0.2.jar diff --git a/file-service/LICENSE-binary b/file-service/LICENSE-binary index 138fd6cad09..393f82e948e 100644 --- a/file-service/LICENSE-binary +++ b/file-service/LICENSE-binary @@ -334,6 +334,18 @@ Scala/Java jars: - io.netty.netty-tcnative-classes-2.0.61.Final.jar - io.netty.netty-transport-4.1.104.Final.jar - io.netty.netty-transport-native-unix-common-4.1.104.Final.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.perfmark.perfmark-api-0.26.0.jar - io.r2dbc.r2dbc-spi-0.9.0.RELEASE.jar - jakarta.inject.jakarta.inject-api-2.0.1.jar diff --git a/workflow-compiling-service/LICENSE-binary b/workflow-compiling-service/LICENSE-binary index ed6a9e1d266..2ad42d6a62f 100644 --- a/workflow-compiling-service/LICENSE-binary +++ b/workflow-compiling-service/LICENSE-binary @@ -336,6 +336,18 @@ Scala/Java jars: - io.netty.netty-tcnative-classes-2.0.61.Final.jar - io.netty.netty-transport-4.1.104.Final.jar - io.netty.netty-transport-native-unix-common-4.1.104.Final.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.perfmark.perfmark-api-0.26.0.jar - io.r2dbc.r2dbc-spi-0.9.0.RELEASE.jar - jakarta.inject.jakarta.inject-api-2.0.1.jar From dca5b7e0bc301b0cfa962038718b0d4f56ba0902 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 5 Jun 2026 11:20:36 -0700 Subject: [PATCH 08/21] adding licenses for pr1 --- access-control-service/LICENSE-binary | 8 ++++++++ config-service/LICENSE-binary | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/access-control-service/LICENSE-binary b/access-control-service/LICENSE-binary index 48cd986319e..4a66e653c11 100644 --- a/access-control-service/LICENSE-binary +++ b/access-control-service/LICENSE-binary @@ -242,6 +242,9 @@ Scala/Java jars: - com.google.guava.listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar - com.google.j2objc.j2objc-annotations-2.8.jar - com.helger.profiler-1.1.1.jar + - com.squareup.okhttp3.okhttp-4.12.0.jar + - com.squareup.okio.okio-3.6.0.jar + - com.squareup.okio.okio-jvm-3.6.0.jar - com.thesamet.scalapb.lenses_2.13-0.11.20.jar - com.thesamet.scalapb.scalapb-json4s_2.13-0.12.0.jar - com.thesamet.scalapb.scalapb-runtime_2.13-0.11.20.jar @@ -304,6 +307,11 @@ Scala/Java jars: - org.hibernate.validator.hibernate-validator-7.0.5.Final.jar - org.javassist.javassist-3.30.2-GA.jar - org.jboss.logging.jboss-logging-3.5.3.Final.jar + - org.jetbrains.annotations-13.0.jar + - org.jetbrains.kotlin.kotlin-stdlib-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-common-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-jdk7-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-jdk8-1.9.10.jar - org.jooq.jooq-3.16.23.jar - org.json4s.json4s-ast_2.13-4.0.1.jar - org.json4s.json4s-jackson-core_2.13-4.0.1.jar diff --git a/config-service/LICENSE-binary b/config-service/LICENSE-binary index 3b5736fa130..3b5a6db4d3e 100644 --- a/config-service/LICENSE-binary +++ b/config-service/LICENSE-binary @@ -242,6 +242,9 @@ Scala/Java jars: - com.google.guava.listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar - com.google.j2objc.j2objc-annotations-2.8.jar - com.helger.profiler-1.1.1.jar + - com.squareup.okhttp3.okhttp-4.12.0.jar + - com.squareup.okio.okio-3.6.0.jar + - com.squareup.okio.okio-jvm-3.6.0.jar - com.thesamet.scalapb.lenses_2.13-0.11.20.jar - com.thesamet.scalapb.scalapb-json4s_2.13-0.12.0.jar - com.thesamet.scalapb.scalapb-runtime_2.13-0.11.20.jar @@ -304,6 +307,11 @@ Scala/Java jars: - org.hibernate.validator.hibernate-validator-7.0.5.Final.jar - org.javassist.javassist-3.30.2-GA.jar - org.jboss.logging.jboss-logging-3.5.3.Final.jar + - org.jetbrains.annotations-13.0.jar + - org.jetbrains.kotlin.kotlin-stdlib-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-common-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-jdk7-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-jdk8-1.9.10.jar - org.jooq.jooq-3.16.23.jar - org.json4s.json4s-ast_2.13-4.0.1.jar - org.json4s.json4s-jackson-core_2.13-4.0.1.jar From 7b8a9e2835b22b4f93ace5d2544df1333006f9a4 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Sun, 14 Jun 2026 03:42:38 -0700 Subject: [PATCH 09/21] fixed the log sanitizer format and allow callers outside sanitize to apply cap --- .../org/apache/texera/observability/LogSanitizer.scala | 6 ++++-- .../texera/observability/TexeraOtelLogAppender.scala | 9 +++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala b/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala index aa92c558c27..cab34fcf58b 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala @@ -101,8 +101,10 @@ object LogSanitizer { truncate(scrubbed) } - /** Truncate to MaxBodyBytes, appending the marker if cut. */ - private def truncate(body: String): String = { + /** Truncate to MaxBodyBytes, appending the marker if cut. Public so + * callers building a body outside `sanitize` can enforce the cap. + */ + def truncate(body: String): String = { if (body.length <= MaxBodyBytes) body else body.substring(0, MaxBodyBytes - TruncatedMarker.length) + TruncatedMarker } diff --git a/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala b/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala index 2dbf160bcfb..f41e88dd569 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala @@ -86,12 +86,9 @@ class TexeraOtelLogAppender extends UnsynchronizedAppenderBase[ILoggingEvent] { val baseBody = LogSanitizer.sanitize(event.getFormattedMessage) val body = Option(event.getThrowableProxy) match { case Some(proxy) => - // JVM-generated stack frames are trusted (not user input), so - // we skip the C0 strip that would collapse newlines and ruin - // readability. We do still cap the total length implicitly - // via the OTel SDK's per-record body limit, and the body - // stays valid UTF-8 because Logback emits ASCII frame text. - baseBody + "\n" + formatThrowable(proxy) + // Trusted JVM frames: skip the C0 strip so newlines survive, + // but still cap length (the OTel SDK does not bound the body). + LogSanitizer.truncate(baseBody + "\n" + formatThrowable(proxy)) case None => baseBody } val builder = logger From 9c45fd531b7638d02fcd9879a508d264484fb76c Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Sun, 14 Jun 2026 03:58:11 -0700 Subject: [PATCH 10/21] fixed incorrect comment in parca-agent.env --- bin/observability/parca/parca-agent.env | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bin/observability/parca/parca-agent.env b/bin/observability/parca/parca-agent.env index 5ff164a20da..2e72ea94b44 100644 --- a/bin/observability/parca/parca-agent.env +++ b/bin/observability/parca/parca-agent.env @@ -36,9 +36,8 @@ # * node: host identifier — set by the compose layer to the docker # host's hostname. Overridable. # * metadata-external-labels: static labels attached to every -# profile. We attach service.name so the Parca query layer can -# group profiles by Texera service the same way logs/traces are -# grouped (matches the OTel resource attr). +# profile (deployment=texera;cluster=local). Coarse fields only; +# no workflow/execution id (cardinality DoS in Parca storage). PARCA_AGENT_REMOTE_STORE_ADDRESS=parca:7070 PARCA_AGENT_REMOTE_STORE_INSECURE=true From 012ee006a1e47472a70f5994b521d7ceb25c1bf9 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Tue, 23 Jun 2026 03:00:48 -0700 Subject: [PATCH 11/21] feat(observability): bootstrap OTel log bridge + framework log caps Call OtelInit.init() in each service main so its logs bridge to the OTel collector under its own service.name; cap noisy framework loggers (pekko/iceberg/hadoop/kafka/jetty/jersey/grpc/ netty/hikari/awssdk) at WARN in each service config. Services: access-control, config, file, computing-unit-managing, workflow-compiling, computing-unit-master, texera-web, amber. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../access-control-service-web-config.yaml | 13 +++++++++++++ .../texera/service/AccessControlService.scala | 2 ++ .../resources/computing-unit-master-config.yml | 12 ++++++++++++ .../texera-compiling-service-web-config.yml | 12 ++++++++++++ amber/src/main/resources/web-config.yml | 12 ++++++++++++ .../apache/texera/web/ComputingUnitMaster.scala | 1 + .../computing-unit-managing-service-config.yaml | 14 +++++++++++++- .../service/ComputingUnitManagingService.scala | 2 ++ .../main/resources/config-service-web-config.yaml | 13 +++++++++++++ .../org/apache/texera/service/ConfigService.scala | 2 ++ .../main/resources/file-service-web-config.yaml | 12 ++++++++++++ .../org/apache/texera/service/FileService.scala | 2 ++ .../workflow-compiling-service-config.yaml | 12 ++++++++++++ .../texera/service/WorkflowCompilingService.scala | 2 ++ 14 files changed, 110 insertions(+), 1 deletion(-) diff --git a/access-control-service/src/main/resources/access-control-service-web-config.yaml b/access-control-service/src/main/resources/access-control-service-web-config.yaml index 8c7895e9858..bd78ecaa82a 100644 --- a/access-control-service/src/main/resources/access-control-service-web-config.yaml +++ b/access-control-service/src/main/resources/access-control-service-web-config.yaml @@ -26,6 +26,19 @@ server: logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + loggers: + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console threshold: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} diff --git a/access-control-service/src/main/scala/org/apache/texera/service/AccessControlService.scala b/access-control-service/src/main/scala/org/apache/texera/service/AccessControlService.scala index f01d06f9417..c1f5114c09f 100644 --- a/access-control-service/src/main/scala/org/apache/texera/service/AccessControlService.scala +++ b/access-control-service/src/main/scala/org/apache/texera/service/AccessControlService.scala @@ -65,6 +65,8 @@ class AccessControlService extends Application[AccessControlServiceConfiguration configuration: AccessControlServiceConfiguration, environment: Environment ): Unit = { + // Bridge this service's logs to the OTel collector under its own service.name. + org.apache.texera.observability.OtelInit.init("access-control-service") // Serve backend at /api environment.jersey.setUrlPattern("/api/*") diff --git a/amber/src/main/resources/computing-unit-master-config.yml b/amber/src/main/resources/computing-unit-master-config.yml index 0dba594b8ae..ee578c3cf90 100644 --- a/amber/src/main/resources/computing-unit-master-config.yml +++ b/amber/src/main/resources/computing-unit-master-config.yml @@ -34,6 +34,18 @@ logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console logFormat: "[%date{ISO8601}] [%level] [%logger] [%thread] - %msg %n" diff --git a/amber/src/main/resources/texera-compiling-service-web-config.yml b/amber/src/main/resources/texera-compiling-service-web-config.yml index ea2c1b9c1e9..c0b6e8aa762 100644 --- a/amber/src/main/resources/texera-compiling-service-web-config.yml +++ b/amber/src/main/resources/texera-compiling-service-web-config.yml @@ -43,6 +43,18 @@ logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console logFormat: "[%date{ISO8601}] [%level] [%logger] [%thread] - %msg %n" diff --git a/amber/src/main/resources/web-config.yml b/amber/src/main/resources/web-config.yml index 9fde1d078e8..9b3c743c89c 100644 --- a/amber/src/main/resources/web-config.yml +++ b/amber/src/main/resources/web-config.yml @@ -43,6 +43,18 @@ logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console logFormat: "[%date{ISO8601}] [%level] [%logger] [%thread] - %msg %n" diff --git a/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala b/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala index d4a061781c9..8d6213e8514 100644 --- a/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala +++ b/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala @@ -139,6 +139,7 @@ class ComputingUnitMaster extends io.dropwizard.Application[Configuration] with } override def run(configuration: Configuration, environment: Environment): Unit = { + org.apache.texera.observability.OtelInit.init("computing-unit-master") ObjectMapperUtils.warmupObjectMapperForOperatorsSerde() SqlServer.initConnection( diff --git a/computing-unit-managing-service/src/main/resources/computing-unit-managing-service-config.yaml b/computing-unit-managing-service/src/main/resources/computing-unit-managing-service-config.yaml index 523b4197989..ea428fcadcb 100644 --- a/computing-unit-managing-service/src/main/resources/computing-unit-managing-service-config.yaml +++ b/computing-unit-managing-service/src/main/resources/computing-unit-managing-service-config.yaml @@ -30,4 +30,16 @@ server: logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: - "com.example": ${TEXERA_SERVICE_LOG_LEVEL:-DEBUG} \ No newline at end of file + "com.example": ${TEXERA_SERVICE_LOG_LEVEL:-DEBUG} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN \ No newline at end of file diff --git a/computing-unit-managing-service/src/main/scala/org/apache/texera/service/ComputingUnitManagingService.scala b/computing-unit-managing-service/src/main/scala/org/apache/texera/service/ComputingUnitManagingService.scala index 0650990264d..dba67a12c3e 100644 --- a/computing-unit-managing-service/src/main/scala/org/apache/texera/service/ComputingUnitManagingService.scala +++ b/computing-unit-managing-service/src/main/scala/org/apache/texera/service/ComputingUnitManagingService.scala @@ -59,6 +59,8 @@ class ComputingUnitManagingService extends Application[ComputingUnitManagingServ configuration: ComputingUnitManagingServiceConfiguration, environment: Environment ): Unit = { + // Bridge this service's logs to the OTel collector under its own service.name. + org.apache.texera.observability.OtelInit.init("computing-unit-managing-service") // Register http resources environment.jersey.setUrlPattern("/api/*") environment.jersey.register(classOf[HealthCheckResource]) diff --git a/config-service/src/main/resources/config-service-web-config.yaml b/config-service/src/main/resources/config-service-web-config.yaml index 4aa67af82e1..8559e1fd507 100644 --- a/config-service/src/main/resources/config-service-web-config.yaml +++ b/config-service/src/main/resources/config-service-web-config.yaml @@ -26,6 +26,19 @@ server: logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + loggers: + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console threshold: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} diff --git a/config-service/src/main/scala/org/apache/texera/service/ConfigService.scala b/config-service/src/main/scala/org/apache/texera/service/ConfigService.scala index e4736cf2511..43fdf5e4840 100644 --- a/config-service/src/main/scala/org/apache/texera/service/ConfigService.scala +++ b/config-service/src/main/scala/org/apache/texera/service/ConfigService.scala @@ -60,6 +60,8 @@ class ConfigService extends Application[ConfigServiceConfiguration] with LazyLog } override def run(configuration: ConfigServiceConfiguration, environment: Environment): Unit = { + // Bridge this service's logs to the OTel collector under its own service.name. + org.apache.texera.observability.OtelInit.init("config-service") // Serve backend at /api environment.jersey.setUrlPattern("/api/*") diff --git a/file-service/src/main/resources/file-service-web-config.yaml b/file-service/src/main/resources/file-service-web-config.yaml index 41f8d1b1748..db5a7ec6645 100644 --- a/file-service/src/main/resources/file-service-web-config.yaml +++ b/file-service/src/main/resources/file-service-web-config.yaml @@ -28,6 +28,18 @@ logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console - type: file diff --git a/file-service/src/main/scala/org/apache/texera/service/FileService.scala b/file-service/src/main/scala/org/apache/texera/service/FileService.scala index 76d78dfef86..d984c3c72b6 100644 --- a/file-service/src/main/scala/org/apache/texera/service/FileService.scala +++ b/file-service/src/main/scala/org/apache/texera/service/FileService.scala @@ -67,6 +67,8 @@ class FileService extends Application[FileServiceConfiguration] with LazyLogging } override def run(configuration: FileServiceConfiguration, environment: Environment): Unit = { + // Bridge this service's logs to the OTel collector under its own service.name. + org.apache.texera.observability.OtelInit.init("file-service") // Serve backend at /api environment.jersey.setUrlPattern("/api/*") SqlServer.initConnection( diff --git a/workflow-compiling-service/src/main/resources/workflow-compiling-service-config.yaml b/workflow-compiling-service/src/main/resources/workflow-compiling-service-config.yaml index 5b9016af1b6..37e413c15b6 100644 --- a/workflow-compiling-service/src/main/resources/workflow-compiling-service-config.yaml +++ b/workflow-compiling-service/src/main/resources/workflow-compiling-service-config.yaml @@ -28,6 +28,18 @@ logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console - type: file diff --git a/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala b/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala index c278b21b4d3..94dcff217cb 100644 --- a/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala +++ b/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala @@ -51,6 +51,8 @@ class WorkflowCompilingService extends Application[WorkflowCompilingServiceConfi configuration: WorkflowCompilingServiceConfiguration, environment: Environment ): Unit = { + // Bridge this service's logs to the OTel collector under its own service.name. + org.apache.texera.observability.OtelInit.init("workflow-compiling-service") ObjectMapperUtils.warmupObjectMapperForOperatorsSerde() // serve backend at /api From 5ee44a567563a302ac31c7cbd6e933c376222939 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Tue, 23 Jun 2026 03:06:58 -0700 Subject: [PATCH 12/21] feat(observability): emit workflow lifecycle metrics + run-level trace span - WorkflowMetricsRecorder: emit workflow lifecycle metrics keyed by execution, driven from the ExecutionStateStore state-transition chokepoint; registered via WorkflowMetricsRecorder.init() in ComputingUnitMaster - WorkflowService: wrap initExecutionService in a run-level TexeraTracer span so setup-path logs carry the trace id Co-Authored-By: Claude Opus 4.8 (1M context) --- .../texera/web/ComputingUnitMaster.scala | 1 + .../WorkflowMetricsRecorder.scala | 107 ++++++++++++++++++ .../texera/web/service/WorkflowService.scala | 24 ++++ .../web/storage/ExecutionStateStore.scala | 4 + 4 files changed, 136 insertions(+) create mode 100644 amber/src/main/scala/org/apache/texera/web/observability/WorkflowMetricsRecorder.scala diff --git a/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala b/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala index 8d6213e8514..97393747ae8 100644 --- a/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala +++ b/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala @@ -140,6 +140,7 @@ class ComputingUnitMaster extends io.dropwizard.Application[Configuration] with override def run(configuration: Configuration, environment: Environment): Unit = { org.apache.texera.observability.OtelInit.init("computing-unit-master") + org.apache.texera.web.observability.WorkflowMetricsRecorder.init() ObjectMapperUtils.warmupObjectMapperForOperatorsSerde() SqlServer.initConnection( diff --git a/amber/src/main/scala/org/apache/texera/web/observability/WorkflowMetricsRecorder.scala b/amber/src/main/scala/org/apache/texera/web/observability/WorkflowMetricsRecorder.scala new file mode 100644 index 00000000000..572aa74e3c2 --- /dev/null +++ b/amber/src/main/scala/org/apache/texera/web/observability/WorkflowMetricsRecorder.scala @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.web.observability + +import com.typesafe.scalalogging.LazyLogging +import org.apache.texera.amber.core.virtualidentity.ExecutionIdentity +import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAggregatedState +import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAggregatedState.{ + COMPLETED, + FAILED, + KILLED, + PAUSED, + PAUSING, + RESUMING, + RUNNING +} +import org.apache.texera.observability.TexeraMetrics +import org.apache.texera.observability.TexeraMetrics.WorkflowKind +import org.apache.texera.web.service.WorkflowService + +import java.util.concurrent.ConcurrentHashMap + +/** + * Drives [[TexeraMetrics]] from amber's execution lifecycle. The metric + * instruments live in common/config; this object is the single place + * that records them, so the lifecycle code only needs one-line calls. + * + * - start / terminal counters and the duration histogram are recorded + * from [[onStart]] and [[onStateChange]]; + * - the always-polled `texera.workflow.active` gauge is sourced from the + * live WorkflowService registry via the supplier registered in [[init]]. + */ +object WorkflowMetricsRecorder extends LazyLogging { + + // Start time + kind per in-flight run, so a terminal transition can emit + // a duration and attribute the outcome. Keyed by the execution identity. + private val inFlight = new ConcurrentHashMap[ExecutionIdentity, (Long, WorkflowKind)]() + + private val ActiveStates: Set[WorkflowAggregatedState] = Set(RUNNING, PAUSING, PAUSED, RESUMING) + private val TerminalStates: Set[WorkflowAggregatedState] = Set(COMPLETED, FAILED, KILLED) + + /** Register the active-executions gauge supplier and bind the instruments. + * Call once at startup, after OtelInit.init. The supplier is polled on + * every metric collection, so it must never throw. + */ + def init(): Unit = { + TexeraMetrics.setActiveExecutionsSupplier(() => + try { + WorkflowService.getAllWorkflowServices.iterator + .flatMap(s => Option(s.executionService.getValue)) + .map(_.executionStateStore.metadataStore.getState.state) + .count(ActiveStates.contains) + .toLong + } catch { + case _: Throwable => 0L + } + ) + TexeraMetrics.ensureBound() + } + + /** Record that a run started. */ + def onStart( + executionId: ExecutionIdentity, + kind: WorkflowKind = WorkflowKind.Interactive + ): Unit = { + inFlight.put(executionId, (System.currentTimeMillis(), kind)) + TexeraMetrics.recordStart(kind) + } + + /** Record terminal counters + duration exactly once, on the first + * transition from a non-terminal into a terminal state. Safe to call on + * every state change; non-terminal and repeat-terminal calls are no-ops. + */ + def onStateChange( + executionId: ExecutionIdentity, + oldState: WorkflowAggregatedState, + newState: WorkflowAggregatedState + ): Unit = { + if (!TerminalStates.contains(newState) || TerminalStates.contains(oldState)) return + val entry = Option(inFlight.remove(executionId)) + val kind = entry.map(_._2).getOrElse(WorkflowKind.Interactive) + val durationSec = entry.map(e => (System.currentTimeMillis() - e._1) / 1000.0).getOrElse(0.0) + newState match { + case COMPLETED => TexeraMetrics.recordCompletion(kind, durationSec) + case FAILED => TexeraMetrics.recordFailure(kind, durationSec) + case KILLED => TexeraMetrics.recordCancellation(kind) + case _ => () + } + } +} diff --git a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala index a241121da20..a4dcd945643 100644 --- a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala +++ b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala @@ -21,6 +21,7 @@ package org.apache.texera.web.service import com.google.protobuf.timestamp.Timestamp import com.typesafe.scalalogging.LazyLogging +import io.opentelemetry.api.trace.Span import io.reactivex.rxjava3.disposables.{CompositeDisposable, Disposable} import io.reactivex.rxjava3.subjects.BehaviorSubject import org.apache.texera.common.config.ApplicationConfig @@ -49,6 +50,7 @@ import org.apache.texera.amber.error.ErrorUtils.{ getStackTraceWithAllCauses } import org.apache.texera.dao.jooq.generated.tables.pojos.User +import org.apache.texera.observability.TexeraTracer import org.apache.texera.service.util.LargeBinaryManager import org.apache.texera.web.model.websocket.event.TexeraWebSocketEvent import org.apache.texera.web.model.websocket.request.WorkflowExecuteRequest @@ -185,6 +187,25 @@ class WorkflowService( userOpt: Option[User], sessionUri: URI ): Unit = { + TexeraTracer.withSpan( + "workflow.execute", + _.setAttribute("texera.workflow.id", workflowId.id.toString) + ) { span => + initExecutionServiceSpanned(req, userOpt, sessionUri, span) + } + } + + /** Body of [[initExecutionService]], run inside the run-level span so + * logs on the setup path carry its trace id. The span covers the + * synchronous setup and the handoff to async execution via + * `executeWorkflow()`; it does not span the full async run. + */ + private def initExecutionServiceSpanned( + req: WorkflowExecuteRequest, + userOpt: Option[User], + sessionUri: URI, + span: Span + ): Unit = { if (executionService.hasValue) { executionService.getValue.unsubscribeAll() @@ -216,6 +237,9 @@ class WorkflowService( convertToJson(req.engineVersion), req.computingUnitId ) + span.setAttribute("texera.execution.id", workflowContext.executionId.id.toString) + // A run has started: record the start counter and stamp its start time. + org.apache.texera.web.observability.WorkflowMetricsRecorder.onStart(workflowContext.executionId) if (ApplicationConfig.faultToleranceLogRootFolder.isDefined) { val writeLocation = ApplicationConfig.faultToleranceLogRootFolder.get.resolve( diff --git a/amber/src/main/scala/org/apache/texera/web/storage/ExecutionStateStore.scala b/amber/src/main/scala/org/apache/texera/web/storage/ExecutionStateStore.scala index 654acbbefd1..bc97b39d185 100644 --- a/amber/src/main/scala/org/apache/texera/web/storage/ExecutionStateStore.scala +++ b/amber/src/main/scala/org/apache/texera/web/storage/ExecutionStateStore.scala @@ -27,6 +27,7 @@ import org.apache.texera.amber.engine.common.executionruntimestate.{ ExecutionMetadataStore, ExecutionStatsStore } +import org.apache.texera.web.observability.WorkflowMetricsRecorder import org.apache.texera.web.service.ExecutionsMetadataPersistService import java.sql.Timestamp @@ -44,6 +45,9 @@ object ExecutionStateStore { execution.setStatus(maptoStatusCode(state)) execution.setLastUpdateTime(new Timestamp(System.currentTimeMillis())) } + // Single chokepoint for every state transition: emit lifecycle metrics + // once on the first transition into a terminal state. + WorkflowMetricsRecorder.onStateChange(metadataStore.executionId, metadataStore.state, state) metadataStore.withState(state) } } From c47a7d0e1b1a7d9cf5a706ed12639215dfedff27 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Tue, 23 Jun 2026 03:08:48 -0700 Subject: [PATCH 13/21] chore(observability): single-node compose wiring for the collector stack - bin/observability/docker-compose.yml: collector + parca-agent stack - bin/single-node/docker-compose.yml: mount the otel-collector and parca configs and run the parca-agent sidecar Co-Authored-By: Claude Opus 4.8 (1M context) --- bin/observability/docker-compose.yml | 39 ++++++++++++++++++++++++++++ bin/single-node/docker-compose.yml | 6 ++--- 2 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 bin/observability/docker-compose.yml diff --git a/bin/observability/docker-compose.yml b/bin/observability/docker-compose.yml new file mode 100644 index 00000000000..adce75a4a40 --- /dev/null +++ b/bin/observability/docker-compose.yml @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Dedicated entry point for the observability stack, so it can be started like +# the other dev targets (e.g. file-service's own compose) instead of reaching +# into the single-node deployment file. +# +# It does NOT redefine the services: it `include`s the single-node compose, the +# single source of truth for every container definition. The observability +# services there are gated behind compose profiles, so they are selected via +# COMPOSE_PROFILES + the service list (see start_app.sh `up_obs`). The other +# single-node services are not profile-gated, so they are simply not named and +# stay down. +# +# Relative volume paths inside the included file (e.g. the collector/parca +# configs under ../observability/) are resolved relative to the included file's +# own directory, so they keep working unchanged. +# +# The project name matches single-node so this manages the SAME containers +# whether obs is brought up here or as part of the full single-node deployment +# (the containers use fixed container_names, so a different project would clash). +name: texera-single-node + +include: + - ../single-node/docker-compose.yml diff --git a/bin/single-node/docker-compose.yml b/bin/single-node/docker-compose.yml index e796d1baf4c..370744ea7ed 100644 --- a/bin/single-node/docker-compose.yml +++ b/bin/single-node/docker-compose.yml @@ -515,7 +515,7 @@ services: security_opt: - no-new-privileges:true volumes: - - ../observability/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml:ro + - ../observability/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml:ro,z command: ["--config=/etc/otelcol-contrib/config.yaml"] # Local dev override: publish OTLP receiver ports on the host # loopback so a Scala backend running outside docker (sbt / @@ -615,7 +615,7 @@ services: security_opt: - no-new-privileges:true volumes: - - ../observability/parca/parca.yaml:/parca.yaml:ro + - ../observability/parca/parca.yaml:/parca.yaml:ro,z - parca_data:/var/lib/parca command: - "/parca" @@ -634,7 +634,7 @@ services: # observability service that requires elevated permissions; the # surface is documented and reviewed. parca-agent: - image: ghcr.io/parca-dev/parca-agent:v0.47.1 + image: ghcr.io/parca-dev/parca-agent:v0.48.0 container_name: texera-parca-agent profiles: [observability-profiles] restart: always From 1c57ed4b87be62a2aa037ae60b93d38483802fa8 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 26 Jun 2026 16:56:08 -0700 Subject: [PATCH 14/21] added license binaries to appropriate sections --- notebook-migration-service/LICENSE-binary | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/notebook-migration-service/LICENSE-binary b/notebook-migration-service/LICENSE-binary index 3abc86ea44c..4a66e653c11 100644 --- a/notebook-migration-service/LICENSE-binary +++ b/notebook-migration-service/LICENSE-binary @@ -242,6 +242,9 @@ Scala/Java jars: - com.google.guava.listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar - com.google.j2objc.j2objc-annotations-2.8.jar - com.helger.profiler-1.1.1.jar + - com.squareup.okhttp3.okhttp-4.12.0.jar + - com.squareup.okio.okio-3.6.0.jar + - com.squareup.okio.okio-jvm-3.6.0.jar - com.thesamet.scalapb.lenses_2.13-0.11.20.jar - com.thesamet.scalapb.scalapb-json4s_2.13-0.12.0.jar - com.thesamet.scalapb.scalapb-runtime_2.13-0.11.20.jar @@ -274,6 +277,18 @@ Scala/Java jars: - io.dropwizard.metrics.metrics-json-4.2.25.jar - io.dropwizard.metrics.metrics-jvm-4.2.25.jar - io.dropwizard.metrics.metrics-logback-4.2.25.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.r2dbc.r2dbc-spi-0.9.0.RELEASE.jar - jakarta.inject.jakarta.inject-api-2.0.1.jar - jakarta.validation.jakarta.validation-api-3.0.2.jar @@ -292,6 +307,11 @@ Scala/Java jars: - org.hibernate.validator.hibernate-validator-7.0.5.Final.jar - org.javassist.javassist-3.30.2-GA.jar - org.jboss.logging.jboss-logging-3.5.3.Final.jar + - org.jetbrains.annotations-13.0.jar + - org.jetbrains.kotlin.kotlin-stdlib-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-common-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-jdk7-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-jdk8-1.9.10.jar - org.jooq.jooq-3.16.23.jar - org.json4s.json4s-ast_2.13-4.0.1.jar - org.json4s.json4s-jackson-core_2.13-4.0.1.jar From d927cff7994363d6f7f7f502b1cdcae56cf372fe Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 26 Jun 2026 17:04:02 -0700 Subject: [PATCH 15/21] changed parac-agent to compatible version --- bin/single-node/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/single-node/docker-compose.yml b/bin/single-node/docker-compose.yml index 370744ea7ed..b1d827511a6 100644 --- a/bin/single-node/docker-compose.yml +++ b/bin/single-node/docker-compose.yml @@ -634,7 +634,7 @@ services: # observability service that requires elevated permissions; the # surface is documented and reviewed. parca-agent: - image: ghcr.io/parca-dev/parca-agent:v0.48.0 + image: ghcr.io/parca-dev/parca-agent:v0.47.1 container_name: texera-parca-agent profiles: [observability-profiles] restart: always From 2f467ace1902a5dde3f7c2eb0308dbf8e9e18cc6 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 26 Jun 2026 20:45:17 -0700 Subject: [PATCH 16/21] fixed spec submitted as bytes --- .../observability/LogSanitizerSpec.scala | Bin 4667 -> 4687 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/common/config/src/test/scala/org/apache/texera/observability/LogSanitizerSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/LogSanitizerSpec.scala index c5957b65e4b163a89ae7681b69a223339f568d9e..ba6ebc37b80d9fcc5a8c6fa7cf284c58e18b5a68 100644 GIT binary patch delta 69 zcmdn3a$aS_0#@yqQUf6H3-tjr%$;0)VoE|Boj^Rpq{Ot6)FKemJS`BO{>)S~*7)SSs5SY$RoXKi3&e$=pRa~8)_ FP5`qJ5^?|l From 8deddfdb0def592c070a41672e0b8e6c90730afa Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Sat, 27 Jun 2026 11:18:16 -0700 Subject: [PATCH 17/21] fix(comments): remove excess details and old information reduced the comments to include less design details and information not needed in the codebase. --- .../texera/observability/LogSanitizer.scala | 60 ++------ .../texera/observability/OtelInit.scala | 136 +++++------------- .../observability/TexeraOtelLogAppender.scala | 46 ++---- .../texera/observability/OtelInitSpec.scala | 14 +- 4 files changed, 66 insertions(+), 190 deletions(-) diff --git a/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala b/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala index cab34fcf58b..f73fcdf88bc 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala @@ -22,76 +22,46 @@ package org.apache.texera.observability import scala.jdk.CollectionConverters._ /** - * Pure functions that sanitize log records before they leave the - * process via the OTel logs bridge. Lives in its own object so the - * security-critical behaviour can be unit-tested without a Logback - * fixture. - * - * Three invariants: - * 1. No control characters in the body (prevents log forging via - * CR/LF injection in user-supplied strings). - * 2. No oversized bodies (a 1 GiB log line must never reach the - * exporter). - * 3. No secrets in plain text (Bearer tokens, password=, AWS keys). - * - * Plus an MDC allowlist so accidental MDC pollution from a downstream - * library cannot leak unintended fields into the exporter. + * Pure functions that sanitize log bodies and MDC before export: + * strip control characters, redact secrets, cap body size, and + * filter MDC to an allowlist. */ object LogSanitizer { - /** Per-record body cap. The OTel SDK and OTLP have higher limits, - * but 16 KiB is plenty for a useful log line and protects the - * collector from a runaway log. - */ + /** Per-record body byte cap. */ val MaxBodyBytes: Int = 16 * 1024 - /** Suffix appended to truncated bodies. Chosen to be visually - * obvious in a UI but short enough not to dominate the cap. - */ + /** Suffix appended to truncated bodies. */ val TruncatedMarker: String = "...[truncated]" - /** C0 control characters except TAB (0x09). Stripping CR/LF here - * prevents log forging via newline injection in user-supplied - * message bodies. DEL (0x7F) included for the same reason. - */ + /** C0 control characters except TAB (0x09), plus DEL (0x7F). */ private val C0ControlRegex = "[\\x00-\\x08\\x0A-\\x1F\\x7F]".r - /** Secret patterns. Order is significant: most specific first so a - * partial match doesn't shadow a tighter pattern. - */ + /** Secret patterns, redacted from bodies. Most specific first. */ private val SecretPatterns: Seq[scala.util.matching.Regex] = Seq( - // Authorization: Bearer — bearer token in header form. + // Bearer token """(?i)Bearer\s+[A-Za-z0-9._\-/+=]{8,}""".r, - // password=…, password: … — generic credential keyvalue. + // password=... or password: ... """(?i)password\s*[=:]\s*[^\s,;"']+""".r, - // AWS access key ID (canonical AKIA…16-char format). + // AWS access key ID """AKIA[0-9A-Z]{16}""".r, - // AWS secret access key, when explicitly labelled. + // labelled AWS secret access key """(?i)aws_secret_access_key\s*[=:]\s*[A-Za-z0-9/+=]{20,}""".r ) - /** MDC keys we will forward to OTel log attributes. Anything else - * is dropped — additions require a code change + reviewer - * acknowledgement of the privacy implications. - */ + /** MDC keys forwarded to OTel log attributes; others are dropped. */ val AllowedMdcKeys: Set[String] = Set( "trace_id", "span_id", "texera.user.id", "texera.workflow.id", "texera.execution.id", - // Computing-unit id identifies the dev process / k8s pod that - // emitted the record. Required for the dashboard's CU-scoped - // log filter — without this key in the allowlist, the OTel - // appender silently strips it and the CU filter matches nothing. "texera.computing_unit.id", "texera.project.id", "texera.operator.id" ) - /** Apply all three body-side transformations. Idempotent — running - * sanitize on already-sanitized output is a no-op. - */ + /** Strip control chars, redact secrets, then truncate. Idempotent. */ def sanitize(body: String): String = { if (body == null || body.isEmpty) return "" val stripped = C0ControlRegex.replaceAllIn(body, "") @@ -101,9 +71,7 @@ object LogSanitizer { truncate(scrubbed) } - /** Truncate to MaxBodyBytes, appending the marker if cut. Public so - * callers building a body outside `sanitize` can enforce the cap. - */ + /** Truncate to MaxBodyBytes, appending the marker if cut. */ def truncate(body: String): String = { if (body.length <= MaxBodyBytes) body else body.substring(0, MaxBodyBytes - TruncatedMarker.length) + TruncatedMarker diff --git a/common/config/src/main/scala/org/apache/texera/observability/OtelInit.scala b/common/config/src/main/scala/org/apache/texera/observability/OtelInit.scala index d7c803fc79e..5302362bc3a 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/OtelInit.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/OtelInit.scala @@ -41,32 +41,15 @@ import scala.util.{Failure, Success, Try} /** * Bootstraps the OpenTelemetry SDK for a Texera service. * - * Design notes: - * - Default-disabled. Sets up nothing unless `OTEL_SDK_DISABLED=false`. - * - We deliberately do not use the autoconfigure SPI: the security model - * requires endpoint + resource-attribute filtering to happen BEFORE - * any exporter is constructed. Autoconfigure would parse env vars - * behind our back. - * - Validation is a single pure function so it can be unit-tested - * without spinning the SDK. - * - On any validation failure we log one WARN and return None. We - * do NOT throw — observability is opt-in plumbing; misconfiguration - * must never crash the service. - * - This is the only place in Texera that reads `OTEL_*` environment - * variables. Other modules consume the returned `OpenTelemetry` - * instance directly. + * Enabled by default; set OTEL_SDK_DISABLED=true to turn it off. Reads + * OTEL_* env vars, validates the endpoint against an allowlist, builds + * tracer/log/metric providers, and attaches a Logback appender. + * Returns None when disabled or misconfigured; never throws. */ object OtelInit extends LazyLogging { - /** Resource attribute keys we accept from OTEL_RESOURCE_ATTRIBUTES. - * Resource attrs ride on every record this JVM emits (logs, - * metrics, traces) — so for a per-CU JVM (ComputingUnitMaster / - * ComputingUnitWorker) setting `texera.computing_unit.id=N` at - * boot is enough to tag every record without per-request MDC - * plumbing. Workflow/execution ids vary per task and still need - * MDC at the message boundary, but exposing them in the allowlist - * lets test harnesses + future per-task code populate them via - * the same mechanism. + /** Resource attribute keys accepted from OTEL_RESOURCE_ATTRIBUTES; + * applied to every record this JVM emits. Others are dropped. */ private[observability] val AllowedResourceKeys: Set[String] = Set( "service.name", @@ -88,35 +71,25 @@ object OtelInit extends LazyLogging { "[::1]" ) - /** Default endpoint when SDK is enabled but no endpoint set explicitly. - * Uses 127.0.0.1 (not "localhost") so a natively-run service reaches the - * IPv4-only collector port published by docker-compose — on dual-stack - * hosts "localhost" resolves to ::1 first and the OTLP export silently - * fails. Inside docker the endpoint is overridden to otel-collector:4317. + /** Default endpoint. 127.0.0.1 (not "localhost") to force IPv4 so a + * natively-run service reaches the collector on dual-stack hosts. */ private val DefaultEndpoint = "http://127.0.0.1:4317" - /** Metric export interval bounds. Values outside this range get - * clamped to the default with a one-shot WARN. The lower bound - * prevents an attacker tipping the exporter into busy-loop mode; - * the upper bound keeps metrics useful for human operators. + /** Metric export interval bounds; out-of-range values clamp to the + * default (see clampIntervalMs). */ private[observability] val MinMetricIntervalMs: Long = 1000L private[observability] val MaxMetricIntervalMs: Long = 10L * 60L * 1000L private[observability] val DefaultMetricIntervalMs: Long = 60L * 1000L - // Idempotency guard. The SDK installs global handlers and a shutdown - // hook; calling init() repeatedly must be a no-op after the first call. + // Idempotency guard: init() is a no-op after the first call. @volatile private var initialized: Option[OpenTelemetry] = None /** - * Initialize the SDK for the given service name. - * Returns Some(sdk) on success, None on disabled / invalid config. - * - * Side effect when enabled: attaches a [[TexeraOtelLogAppender]] to - * the Logback ROOT logger so application logs are mirrored to the - * OTel collector, with the security guards in [[LogSanitizer]] - * applied to every record. + * Initialize the SDK for the given service name. Returns Some on + * success, None when disabled or misconfigured. When enabled, also + * attaches a [[TexeraOtelLogAppender]] to the Logback ROOT logger. */ def init(serviceName: String): Option[OpenTelemetry] = synchronized { @@ -131,13 +104,9 @@ object OtelInit extends LazyLogging { metricExporterFactory = endpoint => Some(buildOtlpMetricExporter(endpoint)), logbackAttacher = LogbackBinder.attach ) - // Register globally so [[TexeraTracer]] and any other OTel-aware - // code can call ``GlobalOpenTelemetry.getTracer(...)`` without - // threading the SDK through every callsite. set() throws on a - // second call within the same JVM — our outer ``initialized`` - // guard makes that unreachable, but wrap defensively. The test - // path deliberately skips this so multiple isolated SDKs can be - // built within one JVM. + // Register globally so OTel-aware code can use GlobalOpenTelemetry + // without threading the SDK through callsites. set() throws on a + // second call; wrap defensively. result.foreach { sdk => Try(GlobalOpenTelemetry.set(sdk)).failed.foreach { t => logger.warn( @@ -149,11 +118,8 @@ object OtelInit extends LazyLogging { } /** - * Test-only entry point. Allows the test to inject an env-var map - * and a span exporter so the SDK does not attempt a real network - * connection. The Logback appender is NOT attached in tests — - * appender tests construct it directly with an in-memory log - * exporter. + * Test-only entry point: injects an env-var map and exporters so the + * SDK makes no network connection. Does not attach the Logback appender. */ private[observability] def initForTest( serviceName: String, @@ -197,14 +163,8 @@ object OtelInit extends LazyLogging { ): Option[OpenTelemetry] = { if (initialized.isDefined) return initialized - // Default to ENABLED so an `sbt run` of any Texera service emits - // telemetry without per-JVM env-var configuration. Operators who - // need to silence telemetry (CI, embedded-tests, security-locked - // deployments) set OTEL_SDK_DISABLED=true explicitly. If the - // configured endpoint isn't reachable, the OTel SDK's - // BatchProcessor logs a single error and drops records — it - // does NOT crash the host service, so a missing collector at - // dev time is a quiet no-op rather than a startup failure. + // Enabled by default; OTEL_SDK_DISABLED=true opts out. An + // unreachable endpoint drops records without crashing the service. val disabled = envProvider("OTEL_SDK_DISABLED").getOrElse("false") if (disabled.equalsIgnoreCase("true")) { logger.info( @@ -221,8 +181,7 @@ object OtelInit extends LazyLogging { validateEndpoint(endpoint, allowedHosts) match { case Left(reason) => - // One WARN, no further detail (endpoint is not echoed beyond what - // the operator already knows). No spans will be emitted. + // One WARN; no telemetry is emitted. logger.warn( s"OpenTelemetry SDK disabled: invalid OTEL_EXPORTER_OTLP_ENDPOINT — $reason. " + "Set TEXERA_OTEL_ALLOWED_HOSTS to extend the allowlist." @@ -243,8 +202,7 @@ object OtelInit extends LazyLogging { val sdkBuilder = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider) - // Logger provider is optional — controlled by the factory. Skipped - // in tests so the appender path can be exercised independently. + // Logger provider is optional; the factory returns None in tests. val loggerProviderOpt = logExporterFactory(endpoint).map { logExporter => val lp = SdkLoggerProvider .builder() @@ -255,11 +213,7 @@ object OtelInit extends LazyLogging { lp } - // Meter provider is optional too. Export interval is clamped to - // [MinMetricIntervalMs, MaxMetricIntervalMs]; an out-of-range - // value gets reset to the default with one WARN — keeps an - // attacker from coaxing the reader into busy-loop mode by - // setting OTEL_METRIC_EXPORT_INTERVAL to a tiny value. + // Meter provider is optional too; export interval is clamped. val intervalMs = clampIntervalMs(envProvider("OTEL_METRIC_EXPORT_INTERVAL")) val meterProviderOpt = metricExporterFactory(endpoint).map { metricExporter => val reader = PeriodicMetricReader @@ -277,22 +231,17 @@ object OtelInit extends LazyLogging { val sdk = sdkBuilder.build() - // One startup span. Carries only service.name (no env, host, or - // version data beyond the allowlisted resource attrs). + // One startup span carrying only service.name. val span = sdk.getTracer("texera.bootstrap").spanBuilder("service.start").startSpan() Try(span.setAttribute("service.name", serviceName)) span.end() - // Wire the Logback appender so subsequent application logs flow to - // the collector with sanitisation applied. Failure here must never - // crash the service — observability is opt-in. + // Wire the Logback appender; failure here must not crash the service. Try(logbackAttacher(serviceName, sdk)).failed.foreach { t => logger.warn(s"Failed to attach OTel Logback appender (logs not exported): ${t.getMessage}") } - // Make sure providers flush on shutdown. We add the hook only after - // the SDK has been fully built so a panic during init doesn't leave - // a dangling hook pointing at a half-constructed provider. + // Flush providers on shutdown. Added after the SDK is fully built. Runtime.getRuntime.addShutdownHook( new Thread( () => { @@ -311,9 +260,8 @@ object OtelInit extends LazyLogging { } /** - * Validate that the endpoint is parseable, uses an allowlisted scheme, - * and resolves to an allowlisted host. Pure function — safe to test - * without standing up the SDK. + * Validate the endpoint is parseable and uses an allowlisted scheme + * and host. Pure function. */ private[observability] def validateEndpoint( endpoint: String, @@ -344,9 +292,8 @@ object OtelInit extends LazyLogging { } /** - * Build a Resource from the service name plus the allowlisted subset - * of OTEL_RESOURCE_ATTRIBUTES. Unknown keys are dropped silently; - * service.name from env is ignored in favour of the argument. + * Build a Resource from the service name and the allowlisted subset + * of OTEL_RESOURCE_ATTRIBUTES. service.name from env is ignored. */ private[observability] def buildResource(serviceName: String, rawAttrs: String): Resource = { val builder = Attributes.builder() @@ -355,7 +302,7 @@ object OtelInit extends LazyLogging { parseAttrs(rawAttrs).foreach { case (key, value) if AllowedResourceKeys.contains(key) && key != "service.name" => builder.put(AttributeKey.stringKey(key), value) - case _ => // dropped — not in allowlist or overrides service.name + case _ => // not in allowlist, or overrides service.name } Resource.create(builder.build()) @@ -387,10 +334,8 @@ object OtelInit extends LazyLogging { OtlpGrpcMetricExporter.builder().setEndpoint(endpoint).build() /** - * Parse and clamp OTEL_METRIC_EXPORT_INTERVAL (milliseconds). - * Out-of-range or unparseable input falls back to the default and - * emits a single WARN. Pure-ish — easy to test without standing up - * the meter SDK. + * Parse and clamp OTEL_METRIC_EXPORT_INTERVAL (ms). Out-of-range or + * unparseable input falls back to the default with one WARN. */ private[observability] def clampIntervalMs(raw: Option[String]): Long = { raw match { @@ -417,17 +362,14 @@ object OtelInit extends LazyLogging { } /** - * Hides the Logback attach step behind a small object so [[OtelInit]] - * doesn't import Logback types directly (keeps the SDK init testable - * without a Logback dependency on the classpath in test runs that - * inject a mock attacher). + * Isolates the Logback attach step so [[OtelInit]] does not import + * Logback types directly, keeping SDK init testable with a mock attacher. */ private[observability] object LogbackBinder extends LazyLogging { - /** Attempts to find the Logback ROOT logger, attach a fresh - * [[TexeraOtelLogAppender]] bound to `otel`, and start it. If - * Logback is not the active SLF4J binding (or for any other - * classpath issue), emits one WARN and returns — never throws. + /** Attach a [[TexeraOtelLogAppender]] bound to `otel` to the Logback + * ROOT logger. Emits one WARN and returns if Logback is not the + * active SLF4J binding. */ def attach(serviceName: String, otel: OpenTelemetry): Unit = { val factory = org.slf4j.LoggerFactory.getILoggerFactory diff --git a/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala b/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala index f41e88dd569..5105031ad92 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala @@ -31,25 +31,13 @@ import io.opentelemetry.context.Context import java.util.concurrent.TimeUnit /** - * Logback appender that forwards every event through [[LogSanitizer]] - * before emitting it as an OTel LogRecord. - * - * Lifecycle: - * - Construct with no args (Logback / programmatic instantiation). - * - Call [[bind]] once with the active [[OpenTelemetry]] instance - * (done by [[OtelInit]] after the SDK is built). Until then, - * [[append]] is a silent no-op — log events keep flowing to - * stdout/file unimpeded. - * - Stopping the appender unbinds; subsequent events drop. - * - * This is intentionally a thin shim. All security-critical logic - * lives in [[LogSanitizer]] so it can be tested without a Logback - * fixture. + * Logback appender that sanitizes each event via [[LogSanitizer]] and + * emits it as an OTel LogRecord. [[append]] is a no-op until [[bind]] + * is called and after [[stop]]. */ class TexeraOtelLogAppender extends UnsynchronizedAppenderBase[ILoggingEvent] { - // @volatile so a late [[bind]] is visible to appender threads - // without taking a lock on the hot path. + // @volatile so a late bind() is visible to appender threads. @volatile private var otelLogger: Option[Logger] = None def bind(otel: OpenTelemetry): Unit = { @@ -63,13 +51,12 @@ class TexeraOtelLogAppender extends UnsynchronizedAppenderBase[ILoggingEvent] { override def append(event: ILoggingEvent): Unit = { otelLogger match { - case None => () // disabled or not yet wired + case None => () // not bound case Some(logger) => try { emit(logger, event) } catch { - // Logback's addStatus contract: errors from inside an - // appender must not throw out into the calling thread. + // An appender must not throw into the calling thread. case t: Throwable => addError("OTel log emission failed", t) } @@ -77,17 +64,11 @@ class TexeraOtelLogAppender extends UnsynchronizedAppenderBase[ILoggingEvent] { } private def emit(logger: Logger, event: ILoggingEvent): Unit = { - // Append the throwable's full stack trace to the body when one is - // attached. Without this, Dropwizard's LoggingExceptionMapper logs - // "Error handling a request: " and the exception itself never - // reaches the observability backend — making 500s impossible to - // diagnose from the dashboard. ThrowableProxyUtil emits a Logback- - // formatted trace that fits inside a single log record. + // Append the stack trace to the body when a throwable is attached. val baseBody = LogSanitizer.sanitize(event.getFormattedMessage) val body = Option(event.getThrowableProxy) match { case Some(proxy) => - // Trusted JVM frames: skip the C0 strip so newlines survive, - // but still cap length (the OTel SDK does not bound the body). + // Skip the C0 strip so trace newlines survive, but still cap. LogSanitizer.truncate(baseBody + "\n" + formatThrowable(proxy)) case None => baseBody } @@ -98,8 +79,7 @@ class TexeraOtelLogAppender extends UnsynchronizedAppenderBase[ILoggingEvent] { .setSeverityText(event.getLevel.toString) .setTimestamp(event.getTimeStamp, TimeUnit.MILLISECONDS) - // MDC subset: typed AttributeKeys only, so no string injection - // path exists for downstream consumers. + // Allowlisted MDC keys as typed attributes. LogSanitizer.filterMdc(event.getMDCPropertyMap).foreach { case (k, v) => builder.setAttribute(AttributeKey.stringKey(k), v) } @@ -107,8 +87,7 @@ class TexeraOtelLogAppender extends UnsynchronizedAppenderBase[ILoggingEvent] { builder.setAttribute(AttributeKey.stringKey("logger.name"), event.getLoggerName) builder.setAttribute(AttributeKey.stringKey("thread.name"), event.getThreadName) - // Attach the current trace context so the SDK populates trace_id / - // span_id on the LogRecord automatically when a span is active. + // Attach trace context so the SDK sets trace_id / span_id. val span = Span.current() if (span.getSpanContext.isValid) { builder.setContext(Context.current()) @@ -117,10 +96,7 @@ class TexeraOtelLogAppender extends UnsynchronizedAppenderBase[ILoggingEvent] { builder.emit() } - /** Pretty-print a Logback throwable proxy. Matches what Logback's - * default pattern layout would produce for `%ex` — class name, - * message, full stack frames, then walks the cause chain. - */ + /** Format a throwable proxy as a Logback-style stack trace. */ private def formatThrowable(proxy: IThrowableProxy): String = ThrowableProxyUtil.asString(proxy) diff --git a/common/config/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala index 45bbac119af..4124c16e7f4 100644 --- a/common/config/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala +++ b/common/config/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala @@ -134,11 +134,6 @@ class OtelInitSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { } it should "carry the texera.* resource attrs so a CU JVM auto-tags every emitted record" in { - // Setting the CU id at JVM boot is the only sane place to attach - // it for ComputingUnitMaster / ComputingUnitWorker — those JVMs - // are CU-scoped by deployment, and there is no HTTP request to - // hang an MDC value off. The dashboard's CU filter relies on - // this key being present on every record. val r = OtelInit.buildResource( "texera-computing-unit-master", "texera.computing_unit.id=8,texera.workflow.id=441,texera.execution.id=1234" @@ -175,16 +170,11 @@ class OtelInitSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { } it should "initialize by default (no OTEL_SDK_DISABLED set) so `sbt run` services emit without per-JVM config" in { - // The previous default was `true` (opt-in), which forced every - // service Run Configuration to set OTEL_SDK_DISABLED=false - // explicitly. New default is `false` so a fresh sbt run is - // immediately tagged in the dashboard. Operators who need to - // silence telemetry still set the env var explicitly. val exporter = InMemorySpanExporter.create() val result = OtelInit.initForTest( "svc", Map( - // OTEL_SDK_DISABLED deliberately omitted — defaults to false. + // OTEL_SDK_DISABLED omitted; defaults to false. "OTEL_EXPORTER_OTLP_ENDPOINT" -> "http://localhost:4317" ), exporter @@ -204,7 +194,7 @@ class OtelInitSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { ) result.isDefined shouldBe true - // BatchSpanProcessor is async — flush before reading. + // BatchSpanProcessor is async; flush before reading. result.get .asInstanceOf[io.opentelemetry.sdk.OpenTelemetrySdk] .getSdkTracerProvider From 3d7ae02879641771b8b8acc7347aed41328f5f75 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Sat, 11 Jul 2026 15:17:37 -0700 Subject: [PATCH 18/21] wiring up observability into the helm and docker install --- bin/k8s/Chart.lock | 21 ++++ bin/k8s/install.sh | 56 +++++++++ bin/k8s/templates/base/_helpers.tpl | 20 ++++ .../access-control-service-deployment.yaml | 1 + .../config-service-deployment.yaml | 1 + .../file-service/file-service-deployment.yaml | 3 +- .../templates/base/observability/jaeger.yaml | 81 +++++++++++++ .../otel-collector-configmap.yaml | 108 +++++++++++++++++ .../base/observability/otel-collector.yaml | 95 +++++++++++++++ .../base/observability/victorialogs.yaml | 111 +++++++++++++++++ .../base/observability/victoriametrics.yaml | 112 ++++++++++++++++++ .../base/webserver/webserver-deployment.yaml | 3 +- ...workflow-compiling-service-deployment.yaml | 3 +- ...low-computing-unit-manager-deployment.yaml | 3 +- bin/k8s/values.yaml | 62 ++++++++++ bin/single-node/.env | 11 ++ 16 files changed, 687 insertions(+), 4 deletions(-) create mode 100644 bin/k8s/Chart.lock create mode 100755 bin/k8s/install.sh create mode 100644 bin/k8s/templates/base/observability/jaeger.yaml create mode 100644 bin/k8s/templates/base/observability/otel-collector-configmap.yaml create mode 100644 bin/k8s/templates/base/observability/otel-collector.yaml create mode 100644 bin/k8s/templates/base/observability/victorialogs.yaml create mode 100644 bin/k8s/templates/base/observability/victoriametrics.yaml diff --git a/bin/k8s/Chart.lock b/bin/k8s/Chart.lock new file mode 100644 index 00000000000..af90dd0b29b --- /dev/null +++ b/bin/k8s/Chart.lock @@ -0,0 +1,21 @@ +dependencies: +- name: postgresql + repository: https://charts.bitnami.com/bitnami + version: 16.5.6 +- name: minio + repository: https://charts.bitnami.com/bitnami + version: 15.0.7 +- name: lakefs + repository: https://charts.lakefs.io + version: 1.8.1 +- name: gateway-helm + repository: oci://docker.io/envoyproxy + version: 1.6.3 +- name: lakekeeper + repository: https://lakekeeper.github.io/lakekeeper-charts/ + version: 0.9.0 +- name: metrics-server + repository: https://kubernetes-sigs.github.io/metrics-server/ + version: 3.12.2 +digest: sha256:031184824e3bd3e03883a3debe64459f462610e4f5bd7c147f03fd05333b169e +generated: "2026-07-11T11:09:44.800500493-07:00" diff --git a/bin/k8s/install.sh b/bin/k8s/install.sh new file mode 100755 index 00000000000..d1ed9b55e6e --- /dev/null +++ b/bin/k8s/install.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# One-command install/upgrade of the Texera Helm chart. +# +# `helm install` on its own fails on a fresh checkout because the subchart +# dependencies (postgresql, minio, lakefs, envoy-gateway, lakekeeper, +# metrics-server) are declared in Chart.yaml but not vendored into charts/. +# This wrapper fetches them first (idempotent), then upgrades-or-installs. +# +# Usage (run from anywhere): +# bin/k8s/install.sh # app only +# bin/k8s/install.sh --set observability.enabled=true # app + observability +# RELEASE=texera NAMESPACE=texera bin/k8s/install.sh --set observability.enabled=true +# +# Any extra args are forwarded verbatim to `helm upgrade --install`, so +# --set / -f values.override.yaml / --namespace etc. all work. + +set -euo pipefail + +CHART_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RELEASE="${RELEASE:-texera}" +NAMESPACE="${NAMESPACE:-texera}" + +command -v helm >/dev/null 2>&1 || { echo "helm not found on PATH" >&2; exit 1; } + +# Fetch/refresh subcharts into charts/. `dependency build` uses Chart.lock when +# present (reproducible); it falls back to `dependency update` if there is no +# lock yet. Both are safe to re-run. +echo "==> Resolving chart dependencies" +if [ -f "${CHART_DIR}/Chart.lock" ]; then + helm dependency build "${CHART_DIR}" +else + helm dependency update "${CHART_DIR}" +fi + +echo "==> helm upgrade --install ${RELEASE} (namespace: ${NAMESPACE})" +exec helm upgrade --install "${RELEASE}" "${CHART_DIR}" \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + "$@" diff --git a/bin/k8s/templates/base/_helpers.tpl b/bin/k8s/templates/base/_helpers.tpl index e044b7285a8..e90e98cabee 100644 --- a/bin/k8s/templates/base/_helpers.tpl +++ b/bin/k8s/templates/base/_helpers.tpl @@ -54,3 +54,23 @@ services fall back to the in-cluster MinIO Service and its auto-generated {{- define "texera.s3.secretAccessKeyKey" -}} {{- if .Values.storage.s3.endpoint -}}secret-access-key{{- else -}}root-password{{- end -}} {{- end -}} + +{{/* +Observability emission env for a Scala service pod. + +Renders OTEL_EXPORTER_OTLP_ENDPOINT (pointing at the in-cluster OTel +Collector Service) plus TEXERA_OTEL_ALLOWED_HOSTS. OtelInit validates the +endpoint host against an allowlist whose default is localhost only, so the +collector's Service name must be added explicitly or the SDK rejects it and +emits nothing. Renders nothing unless both the observability stack and the +collector are enabled, so the default install is unchanged. Include inside a +container's `env:` list, e.g. `{{- include "texera.observability.env" . | nindent 12 }}`. +*/}} +{{- define "texera.observability.env" -}} +{{- if and .Values.observability.enabled .Values.observability.collector.enabled }} +- name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://{{ .Release.Name }}-otel-collector:{{ .Values.observability.collector.grpcPort }}" +- name: TEXERA_OTEL_ALLOWED_HOSTS + value: "{{ .Release.Name }}-otel-collector" +{{- end }} +{{- end -}} diff --git a/bin/k8s/templates/base/access-control-service/access-control-service-deployment.yaml b/bin/k8s/templates/base/access-control-service/access-control-service-deployment.yaml index 99713e70713..4056d637638 100644 --- a/bin/k8s/templates/base/access-control-service/access-control-service-deployment.yaml +++ b/bin/k8s/templates/base/access-control-service/access-control-service-deployment.yaml @@ -64,6 +64,7 @@ spec: - name: {{ .name }} value: "{{ .value }}" {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} livenessProbe: httpGet: path: /api/healthcheck diff --git a/bin/k8s/templates/base/config-service/config-service-deployment.yaml b/bin/k8s/templates/base/config-service/config-service-deployment.yaml index f0748785c3a..3817be8a68a 100644 --- a/bin/k8s/templates/base/config-service/config-service-deployment.yaml +++ b/bin/k8s/templates/base/config-service/config-service-deployment.yaml @@ -51,6 +51,7 @@ spec: - name: {{ .name }} value: "{{ .value }}" {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} livenessProbe: httpGet: path: /api/healthcheck diff --git a/bin/k8s/templates/base/file-service/file-service-deployment.yaml b/bin/k8s/templates/base/file-service/file-service-deployment.yaml index 6a9190bc6cf..37568ef2254 100644 --- a/bin/k8s/templates/base/file-service/file-service-deployment.yaml +++ b/bin/k8s/templates/base/file-service/file-service-deployment.yaml @@ -79,4 +79,5 @@ spec: {{- range .Values.texeraEnvVars }} - name: {{ .name }} value: "{{ .value }}" - {{- end }} \ No newline at end of file + {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} \ No newline at end of file diff --git a/bin/k8s/templates/base/observability/jaeger.yaml b/bin/k8s/templates/base/observability/jaeger.yaml new file mode 100644 index 00000000000..2821a977736 --- /dev/null +++ b/bin/k8s/templates/base/observability/jaeger.yaml @@ -0,0 +1,81 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.traces.enabled }} +# Jaeger v2: traces backend + UI, k8s counterpart of the docker-compose +# jaeger service. v2 accepts OTLP natively (the collector exports here on +# otlpPort). In-memory storage: a restart wipes traces, same as the +# single-node compose. Swap in a persistent storage backend for +# multi-node / production use. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-jaeger + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-jaeger +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-jaeger + template: + metadata: + labels: + app: {{ .Release.Name }}-jaeger + spec: + securityContext: + runAsNonRoot: true + containers: + - name: jaeger + image: {{ .Values.observability.traces.image }} + imagePullPolicy: {{ .Values.texeraImages.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + ports: + - name: query + containerPort: {{ .Values.observability.traces.queryPort }} + - name: otlp-grpc + containerPort: {{ .Values.observability.traces.otlpPort }} + resources: + {{- toYaml .Values.observability.traces.resources | nindent 12 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-jaeger + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-jaeger +spec: + type: ClusterIP + selector: + app: {{ .Release.Name }}-jaeger + ports: + - name: query + protocol: TCP + port: {{ .Values.observability.traces.queryPort }} + targetPort: {{ .Values.observability.traces.queryPort }} + - name: otlp-grpc + protocol: TCP + port: {{ .Values.observability.traces.otlpPort }} + targetPort: {{ .Values.observability.traces.otlpPort }} +{{- end }} diff --git a/bin/k8s/templates/base/observability/otel-collector-configmap.yaml b/bin/k8s/templates/base/observability/otel-collector-configmap.yaml new file mode 100644 index 00000000000..4a12dc8a791 --- /dev/null +++ b/bin/k8s/templates/base/observability/otel-collector-configmap.yaml @@ -0,0 +1,108 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.collector.enabled }} +# OpenTelemetry Collector config, k8s counterpart of +# bin/observability/otel-collector/config.yaml. Same pipeline topology; +# exporters point at the in-cluster backend Services instead of the +# docker-compose service names. Only the pipelines for enabled signals +# are rendered, so disabling a signal drops its exporter cleanly. +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-otel-collector-config + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-otel-collector +data: + config.yaml: | + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:{{ .Values.observability.collector.grpcPort }} + max_recv_msg_size_mib: 4 + http: + endpoint: 0.0.0.0:{{ .Values.observability.collector.httpPort }} + + processors: + batch: + send_batch_size: 1024 + timeout: 5s + memory_limiter: + check_interval: 1s + limit_mib: 512 + spike_limit_mib: 128 + + exporters: + {{- if .Values.observability.logs.enabled }} + otlphttp/victorialogs: + endpoint: http://{{ .Release.Name }}-victorialogs:{{ .Values.observability.logs.port }}/insert/opentelemetry + compression: gzip + timeout: 10s + tls: + insecure: true + {{- end }} + {{- if .Values.observability.metrics.enabled }} + prometheusremotewrite: + endpoint: http://{{ .Release.Name }}-victoriametrics:{{ .Values.observability.metrics.port }}/api/v1/write + tls: + insecure: true + remote_write_queue: + queue_size: 1000 + num_consumers: 4 + {{- end }} + {{- if .Values.observability.traces.enabled }} + otlp/jaeger: + endpoint: {{ .Release.Name }}-jaeger:{{ .Values.observability.traces.otlpPort }} + tls: + insecure: true + {{- end }} + + service: + pipelines: + {{- if .Values.observability.logs.enabled }} + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/victorialogs] + {{- end }} + {{- if .Values.observability.metrics.enabled }} + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheusremotewrite] + {{- end }} + {{- if .Values.observability.traces.enabled }} + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp/jaeger] + {{- end }} + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 127.0.0.1 + port: 8888 + logs: + level: info +{{- end }} diff --git a/bin/k8s/templates/base/observability/otel-collector.yaml b/bin/k8s/templates/base/observability/otel-collector.yaml new file mode 100644 index 00000000000..89c537317d8 --- /dev/null +++ b/bin/k8s/templates/base/observability/otel-collector.yaml @@ -0,0 +1,95 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.collector.enabled }} +# OpenTelemetry Collector: the single OTLP ingress for all signals, +# k8s counterpart of the docker-compose otel-collector service. The +# receiver Ports are exposed only on the ClusterIP Service (no Ingress), +# mirroring the loopback-only posture of the compose deployment. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-otel-collector + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-otel-collector +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-otel-collector + template: + metadata: + labels: + app: {{ .Release.Name }}-otel-collector + annotations: + # Roll the pod when the rendered config changes. + checksum/config: {{ include (print $.Template.BasePath "/base/observability/otel-collector-configmap.yaml") . | sha256sum }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 10001 + fsGroup: 10001 + containers: + - name: otel-collector + image: {{ .Values.observability.collector.image }} + imagePullPolicy: {{ .Values.texeraImages.pullPolicy }} + args: ["--config=/etc/otelcol-contrib/config.yaml"] + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + ports: + - name: otlp-grpc + containerPort: {{ .Values.observability.collector.grpcPort }} + - name: otlp-http + containerPort: {{ .Values.observability.collector.httpPort }} + volumeMounts: + - name: config + mountPath: /etc/otelcol-contrib + readOnly: true + resources: + {{- toYaml .Values.observability.collector.resources | nindent 12 }} + volumes: + - name: config + configMap: + name: {{ .Release.Name }}-otel-collector-config +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-otel-collector + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-otel-collector +spec: + type: ClusterIP + selector: + app: {{ .Release.Name }}-otel-collector + ports: + - name: otlp-grpc + protocol: TCP + port: {{ .Values.observability.collector.grpcPort }} + targetPort: {{ .Values.observability.collector.grpcPort }} + - name: otlp-http + protocol: TCP + port: {{ .Values.observability.collector.httpPort }} + targetPort: {{ .Values.observability.collector.httpPort }} +{{- end }} diff --git a/bin/k8s/templates/base/observability/victorialogs.yaml b/bin/k8s/templates/base/observability/victorialogs.yaml new file mode 100644 index 00000000000..6896ffd7b2e --- /dev/null +++ b/bin/k8s/templates/base/observability/victorialogs.yaml @@ -0,0 +1,111 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.logs.enabled }} +# VictoriaLogs: log store, k8s counterpart of the docker-compose +# victorialogs service. OTLP ingest + LogsQL query API on one port. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Release.Name }}-victorialogs-data + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-victorialogs +spec: + accessModes: ["ReadWriteOnce"] + {{- with .Values.observability.logs.storageClass }} + storageClassName: {{ . }} + {{- end }} + resources: + requests: + storage: {{ .Values.observability.logs.storage }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-victorialogs + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-victorialogs +spec: + replicas: 1 + # Single writer bound to one RWO volume; never run two at once. + strategy: + type: Recreate + selector: + matchLabels: + app: {{ .Release.Name }}-victorialogs + template: + metadata: + labels: + app: {{ .Release.Name }}-victorialogs + spec: + securityContext: + runAsNonRoot: true + runAsUser: 10002 + fsGroup: 10002 + containers: + - name: victorialogs + image: {{ .Values.observability.logs.image }} + imagePullPolicy: {{ .Values.texeraImages.pullPolicy }} + args: + - "-storageDataPath=/data" + - "-retentionPeriod={{ .Values.observability.logs.retentionPeriod }}" + - "-httpListenAddr=:{{ .Values.observability.logs.port }}" + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + ports: + - name: http + containerPort: {{ .Values.observability.logs.port }} + volumeMounts: + - name: data + mountPath: /data + livenessProbe: + httpGet: + path: /health + port: {{ .Values.observability.logs.port }} + initialDelaySeconds: 10 + periodSeconds: 15 + resources: + {{- toYaml .Values.observability.logs.resources | nindent 12 }} + volumes: + - name: data + persistentVolumeClaim: + claimName: {{ .Release.Name }}-victorialogs-data +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-victorialogs + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-victorialogs +spec: + type: ClusterIP + selector: + app: {{ .Release.Name }}-victorialogs + ports: + - name: http + protocol: TCP + port: {{ .Values.observability.logs.port }} + targetPort: {{ .Values.observability.logs.port }} +{{- end }} diff --git a/bin/k8s/templates/base/observability/victoriametrics.yaml b/bin/k8s/templates/base/observability/victoriametrics.yaml new file mode 100644 index 00000000000..9e793de5a80 --- /dev/null +++ b/bin/k8s/templates/base/observability/victoriametrics.yaml @@ -0,0 +1,112 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.metrics.enabled }} +# VictoriaMetrics: metrics store, k8s counterpart of the docker-compose +# victoriametrics service. Accepts Prometheus remote-write from the +# collector; MetricsQL query API on the same port. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Release.Name }}-victoriametrics-data + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-victoriametrics +spec: + accessModes: ["ReadWriteOnce"] + {{- with .Values.observability.metrics.storageClass }} + storageClassName: {{ . }} + {{- end }} + resources: + requests: + storage: {{ .Values.observability.metrics.storage }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-victoriametrics + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-victoriametrics +spec: + replicas: 1 + # Single writer bound to one RWO volume; never run two at once. + strategy: + type: Recreate + selector: + matchLabels: + app: {{ .Release.Name }}-victoriametrics + template: + metadata: + labels: + app: {{ .Release.Name }}-victoriametrics + spec: + securityContext: + runAsNonRoot: true + runAsUser: 10003 + fsGroup: 10003 + containers: + - name: victoriametrics + image: {{ .Values.observability.metrics.image }} + imagePullPolicy: {{ .Values.texeraImages.pullPolicy }} + args: + - "-storageDataPath=/data" + - "-retentionPeriod={{ .Values.observability.metrics.retentionPeriod }}" + - "-httpListenAddr=:{{ .Values.observability.metrics.port }}" + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + ports: + - name: http + containerPort: {{ .Values.observability.metrics.port }} + volumeMounts: + - name: data + mountPath: /data + livenessProbe: + httpGet: + path: /health + port: {{ .Values.observability.metrics.port }} + initialDelaySeconds: 10 + periodSeconds: 15 + resources: + {{- toYaml .Values.observability.metrics.resources | nindent 12 }} + volumes: + - name: data + persistentVolumeClaim: + claimName: {{ .Release.Name }}-victoriametrics-data +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-victoriametrics + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-victoriametrics +spec: + type: ClusterIP + selector: + app: {{ .Release.Name }}-victoriametrics + ports: + - name: http + protocol: TCP + port: {{ .Values.observability.metrics.port }} + targetPort: {{ .Values.observability.metrics.port }} +{{- end }} diff --git a/bin/k8s/templates/base/webserver/webserver-deployment.yaml b/bin/k8s/templates/base/webserver/webserver-deployment.yaml index 983c6269947..ca0fe4a39af 100644 --- a/bin/k8s/templates/base/webserver/webserver-deployment.yaml +++ b/bin/k8s/templates/base/webserver/webserver-deployment.yaml @@ -74,4 +74,5 @@ spec: {{- range .Values.texeraEnvVars }} - name: {{ .name }} value: "{{ .value }}" - {{- end }} \ No newline at end of file + {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} \ No newline at end of file diff --git a/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml b/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml index 50a0a04e1b7..b12bb68266b 100644 --- a/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml +++ b/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml @@ -66,4 +66,5 @@ spec: {{- range .Values.texeraEnvVars }} - name: {{ .name }} value: "{{ .value }}" - {{- end }} \ No newline at end of file + {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} \ No newline at end of file diff --git a/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml b/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml index a9118450412..2057444e5e9 100644 --- a/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml +++ b/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml @@ -123,4 +123,5 @@ spec: {{- range .Values.texeraEnvVars }} - name: {{ .name }} value: "{{ .value }}" - {{- end }} \ No newline at end of file + {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} \ No newline at end of file diff --git a/bin/k8s/values.yaml b/bin/k8s/values.yaml index 64642b517a9..3b4c3ac2592 100644 --- a/bin/k8s/values.yaml +++ b/bin/k8s/values.yaml @@ -408,3 +408,65 @@ envoy-gateway: extensionApis: enableBackend: true enableEnvoyPatchPolicy: true + +# Observability stack (hand-rolled, mirrors bin/observability + the +# single-node docker-compose profiles). Everything is gated behind +# `enabled` (default off) so the standard install is unchanged. When +# enabled, the Scala services emit OTLP to the in-cluster collector, +# which fans logs/metrics/traces to the backends below. All Services are +# ClusterIP only (no Ingress), matching the loopback-only posture of the +# compose deployment. Profiling (Parca) is intentionally not included: it +# needs a privileged eBPF DaemonSet and is a separate change. +observability: + # Master switch for the whole stack. + enabled: false + + # OpenTelemetry Collector: the single OTLP ingress. Required whenever + # any signal below is enabled (it is what the services emit to). + collector: + enabled: true + image: otel/opentelemetry-collector-contrib:0.153.0 + grpcPort: 4317 + httpPort: 4318 + resources: + limits: + memory: 768Mi + cpu: "1" + + # VictoriaLogs: log store (OTLP ingest + LogsQL). + logs: + enabled: true + image: victoriametrics/victoria-logs:v1.50.0 + port: 9428 + retentionPeriod: 30d + storage: 10Gi + storageClass: local-path + resources: + limits: + memory: 1Gi + cpu: "1" + + # VictoriaMetrics: metrics store (Prometheus remote-write + MetricsQL). + metrics: + enabled: true + image: victoriametrics/victoria-metrics:v1.144.0 + port: 8428 + retentionPeriod: 90d + storage: 10Gi + storageClass: local-path + resources: + limits: + memory: 1Gi + cpu: "1" + + # Jaeger v2: traces backend + UI. In-memory storage (restart wipes + # traces); swap in persistent storage for production. + traces: + enabled: true + image: jaegertracing/jaeger:2.18.0 + queryPort: 16686 + otlpPort: 4317 + resources: + limits: + memory: 1Gi + cpu: "1" diff --git a/bin/single-node/.env b/bin/single-node/.env index 8f282ec8fc4..be9b2f88aa1 100644 --- a/bin/single-node/.env +++ b/bin/single-node/.env @@ -130,3 +130,14 @@ TEXERA_OBS_LOGS_URL=http://victorialogs:9428 TEXERA_OBS_METRICS_URL=http://victoriametrics:8428 TEXERA_OBS_TRACES_URL=http://jaeger:16686 TEXERA_OBS_PROFILES_URL=http://parca:7070 + +# Emission side: the Scala services (via OtelInit) push OTLP to the bundled +# collector over the bridge network. OtelInit validates the endpoint host +# against an allowlist (localhost only by default), so the collector's compose +# service name has to be allowlisted explicitly, or the SDK rejects it and +# emits nothing. These are read by every service that loads this .env; only the +# OtelInit-enabled Scala services act on them. When the observability-collector +# profile is disabled the collector is absent and OtelInit fails quietly +# (telemetry dropped, one warning, service unaffected). +OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 +TEXERA_OTEL_ALLOWED_HOSTS=otel-collector From 82dbb217bdafdb947ca4e0dba9c9c3c755b48359 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Wed, 15 Jul 2026 23:24:11 -0700 Subject: [PATCH 19/21] feat(observability): address #5376 review - direct OTel API + WorkflowMetrics rename Follow the OTel Java demo patterns instead of custom wrappers (zuozhiw review): - WorkflowService: start the run-level span with the standard OTel API, name it WorkflowService.initExecutionService, and drop the initExecutionServiceSpanned split so no span is passed as an argument. The real execution failure is now recorded onto the span from errorHandler, where it is actually caught. - TexeraTracer: drop the withSpan wrapper, keeping only the tracer accessor (single instrumentation scope) and currentContext. - SpanAttrs: drop the awkward with*/set* setter helpers, keep the standard label keys and sanitizeFreeText; callers set attributes via the OTel API. - Rename TexeraMetrics to WorkflowMetrics, document that this facade is only for the workflow-execution cluster, and add an example of calling the OTel meter API directly at a call site. Update SpanAttrsSpec and WorkflowMetricsSpec to match. --- .../WorkflowMetricsRecorder.scala | 18 +- .../texera/web/service/WorkflowService.scala | 255 +++++++++--------- .../texera/observability/SpanAttrs.scala | 70 +---- .../texera/observability/TexeraTracer.scala | 63 ++--- ...eraMetrics.scala => WorkflowMetrics.scala} | 24 +- .../texera/observability/SpanAttrsSpec.scala | Bin 5460 -> 3870 bytes ...csSpec.scala => WorkflowMetricsSpec.scala} | 40 +-- 7 files changed, 207 insertions(+), 263 deletions(-) rename common/config/src/main/scala/org/apache/texera/observability/{TexeraMetrics.scala => WorkflowMetrics.scala} (90%) rename common/config/src/test/scala/org/apache/texera/observability/{TexeraMetricsSpec.scala => WorkflowMetricsSpec.scala} (82%) diff --git a/amber/src/main/scala/org/apache/texera/web/observability/WorkflowMetricsRecorder.scala b/amber/src/main/scala/org/apache/texera/web/observability/WorkflowMetricsRecorder.scala index 572aa74e3c2..0f9ab2401aa 100644 --- a/amber/src/main/scala/org/apache/texera/web/observability/WorkflowMetricsRecorder.scala +++ b/amber/src/main/scala/org/apache/texera/web/observability/WorkflowMetricsRecorder.scala @@ -31,14 +31,14 @@ import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAg RESUMING, RUNNING } -import org.apache.texera.observability.TexeraMetrics -import org.apache.texera.observability.TexeraMetrics.WorkflowKind +import org.apache.texera.observability.WorkflowMetrics +import org.apache.texera.observability.WorkflowMetrics.WorkflowKind import org.apache.texera.web.service.WorkflowService import java.util.concurrent.ConcurrentHashMap /** - * Drives [[TexeraMetrics]] from amber's execution lifecycle. The metric + * Drives [[WorkflowMetrics]] from amber's execution lifecycle. The metric * instruments live in common/config; this object is the single place * that records them, so the lifecycle code only needs one-line calls. * @@ -61,7 +61,7 @@ object WorkflowMetricsRecorder extends LazyLogging { * every metric collection, so it must never throw. */ def init(): Unit = { - TexeraMetrics.setActiveExecutionsSupplier(() => + WorkflowMetrics.setActiveExecutionsSupplier(() => try { WorkflowService.getAllWorkflowServices.iterator .flatMap(s => Option(s.executionService.getValue)) @@ -72,7 +72,7 @@ object WorkflowMetricsRecorder extends LazyLogging { case _: Throwable => 0L } ) - TexeraMetrics.ensureBound() + WorkflowMetrics.ensureBound() } /** Record that a run started. */ @@ -81,7 +81,7 @@ object WorkflowMetricsRecorder extends LazyLogging { kind: WorkflowKind = WorkflowKind.Interactive ): Unit = { inFlight.put(executionId, (System.currentTimeMillis(), kind)) - TexeraMetrics.recordStart(kind) + WorkflowMetrics.recordStart(kind) } /** Record terminal counters + duration exactly once, on the first @@ -98,9 +98,9 @@ object WorkflowMetricsRecorder extends LazyLogging { val kind = entry.map(_._2).getOrElse(WorkflowKind.Interactive) val durationSec = entry.map(e => (System.currentTimeMillis() - e._1) / 1000.0).getOrElse(0.0) newState match { - case COMPLETED => TexeraMetrics.recordCompletion(kind, durationSec) - case FAILED => TexeraMetrics.recordFailure(kind, durationSec) - case KILLED => TexeraMetrics.recordCancellation(kind) + case COMPLETED => WorkflowMetrics.recordCompletion(kind, durationSec) + case FAILED => WorkflowMetrics.recordFailure(kind, durationSec) + case KILLED => WorkflowMetrics.recordCancellation(kind) case _ => () } } diff --git a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala index 3d3f79d10cd..bf591ef282c 100644 --- a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala +++ b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala @@ -21,7 +21,7 @@ package org.apache.texera.web.service import com.google.protobuf.timestamp.Timestamp import com.typesafe.scalalogging.LazyLogging -import io.opentelemetry.api.trace.Span +import io.opentelemetry.api.trace.StatusCode import io.reactivex.rxjava3.disposables.{CompositeDisposable, Disposable} import io.reactivex.rxjava3.subjects.BehaviorSubject import org.apache.texera.common.config.ApplicationConfig @@ -180,148 +180,157 @@ class WorkflowService( new WorkflowContext(workflowId = workflowId, cuid = Some(computingUnitId)) } + /** Sets up and launches a workflow execution inside a run-level span so + * setup-path logs carry its trace id. The span covers the synchronous + * setup and the handoff to async execution via `executeWorkflow()`; it + * does not span the full async run. The real execution failure is + * recorded onto the current span from `errorHandler`. + */ def initExecutionService( req: WorkflowExecuteRequest, userOpt: Option[User], sessionUri: URI ): Unit = { - TexeraTracer.withSpan( - "workflow.execute", - _.setAttribute("texera.workflow.id", workflowId.id.toString) - ) { span => - initExecutionServiceSpanned(req, userOpt, sessionUri, span) - } - } - - /** Body of [[initExecutionService]], run inside the run-level span so - * logs on the setup path carry its trace id. The span covers the - * synchronous setup and the handoff to async execution via - * `executeWorkflow()`; it does not span the full async run. - */ - private def initExecutionServiceSpanned( - req: WorkflowExecuteRequest, - userOpt: Option[User], - sessionUri: URI, - span: Span - ): Unit = { + val span = TexeraTracer.tracer + .spanBuilder("WorkflowService.initExecutionService") + .setAttribute("texera.workflow.id", workflowId.id.toString) + .startSpan() + val scope = span.makeCurrent() + try { - if (executionService.hasValue) { - executionService.getValue.unsubscribeAll() - } + if (executionService.hasValue) { + executionService.getValue.unsubscribeAll() + } - val (uidOpt, userEmailOpt) = userOpt.map(user => (user.getUid, user.getEmail)).unzip + val (uidOpt, userEmailOpt) = userOpt.map(user => (user.getUid, user.getEmail)).unzip - // uid is NOT NULL in the DB; fail early here rather than letting the insert fail downstream. - val uid = uidOpt.getOrElse( - throw new IllegalArgumentException( - "Cannot start execution: a user id (uid) is required but none was provided." + // uid is NOT NULL in the DB; fail early here rather than letting the insert fail downstream. + val uid = uidOpt.getOrElse( + throw new IllegalArgumentException( + "Cannot start execution: a user id (uid) is required but none was provided." + ) ) - ) - - val workflowContext: WorkflowContext = createWorkflowContext() - var controllerConf = ControllerConfig.default - - // clean up results from previous run - val previousExecutionId = - WorkflowExecutionService.getLatestExecutionId(workflowId, req.computingUnitId) - previousExecutionId.foreach(eid => { - clearExecutionResources(eid) - }) // TODO: change this behavior after enabling cache. - - workflowContext.executionId = ExecutionsMetadataPersistService.insertNewExecution( - workflowContext.workflowId, - uid, - req.executionName, - convertToJson(req.engineVersion), - req.computingUnitId - ) - span.setAttribute("texera.execution.id", workflowContext.executionId.id.toString) - // A run has started: record the start counter and stamp its start time. - org.apache.texera.web.observability.WorkflowMetricsRecorder.onStart(workflowContext.executionId) - if (ApplicationConfig.faultToleranceLogRootFolder.isDefined) { - val writeLocation = ApplicationConfig.faultToleranceLogRootFolder.get.resolve( - s"${workflowContext.workflowId}/${workflowContext.executionId}/" + val workflowContext: WorkflowContext = createWorkflowContext() + var controllerConf = ControllerConfig.default + + // clean up results from previous run + val previousExecutionId = + WorkflowExecutionService.getLatestExecutionId(workflowId, req.computingUnitId) + previousExecutionId.foreach(eid => { + clearExecutionResources(eid) + }) // TODO: change this behavior after enabling cache. + + workflowContext.executionId = ExecutionsMetadataPersistService.insertNewExecution( + workflowContext.workflowId, + uid, + req.executionName, + convertToJson(req.engineVersion), + req.computingUnitId ) - ExecutionsMetadataPersistService.tryUpdateExistingExecution(workflowContext.executionId) { - execution => execution.setLogLocation(writeLocation.toString) + span.setAttribute("texera.execution.id", workflowContext.executionId.id.toString) + // A run has started: record the start counter and stamp its start time. + org.apache.texera.web.observability.WorkflowMetricsRecorder + .onStart(workflowContext.executionId) + + if (ApplicationConfig.faultToleranceLogRootFolder.isDefined) { + val writeLocation = ApplicationConfig.faultToleranceLogRootFolder.get.resolve( + s"${workflowContext.workflowId}/${workflowContext.executionId}/" + ) + ExecutionsMetadataPersistService.tryUpdateExistingExecution(workflowContext.executionId) { + execution => execution.setLogLocation(writeLocation.toString) + } + controllerConf = controllerConf.copy(faultToleranceConfOpt = + Some(FaultToleranceConfig(writeTo = writeLocation)) + ) } - controllerConf = controllerConf.copy(faultToleranceConfOpt = - Some(FaultToleranceConfig(writeTo = writeLocation)) - ) - } - if (req.replayFromExecution.isDefined) { - val replayInfo = req.replayFromExecution.get - ExecutionsMetadataPersistService - .tryGetExistingExecution(ExecutionIdentity(replayInfo.eid)) - .foreach { execution => - val readLocation = new URI(execution.getLogLocation) - controllerConf = controllerConf.copy(stateRestoreConfOpt = - Some( - StateRestoreConfig( - readFrom = readLocation, - replayDestination = EmbeddedControlMessageIdentity(replayInfo.interaction) + if (req.replayFromExecution.isDefined) { + val replayInfo = req.replayFromExecution.get + ExecutionsMetadataPersistService + .tryGetExistingExecution(ExecutionIdentity(replayInfo.eid)) + .foreach { execution => + val readLocation = new URI(execution.getLogLocation) + controllerConf = controllerConf.copy(stateRestoreConfOpt = + Some( + StateRestoreConfig( + readFrom = readLocation, + replayDestination = EmbeddedControlMessageIdentity(replayInfo.interaction) + ) ) ) - ) - } - } + } + } - val executionStateStore = new ExecutionStateStore() - // assign execution id to find the execution from DB in case the constructor fails. - executionStateStore.metadataStore.updateState(state => - state.withExecutionId(workflowContext.executionId) - ) - val errorHandler: Throwable => Unit = { t => - { - val fromActorOpt = t match { - case ex: WorkflowRuntimeException => - ex.relatedWorkerId - case other => - None - } - val (operatorId, workerId) = getOperatorFromActorIdOpt(fromActorOpt) - logger.error("error during execution", t) - executionStateStore.statsStore.updateState(stats => - stats.withEndTimeStamp(System.currentTimeMillis()) - ) - executionStateStore.metadataStore.updateState { metadataStore => - updateWorkflowState(FAILED, metadataStore).addFatalErrors( - WorkflowFatalError( - EXECUTION_FAILURE, - Timestamp(Instant.now), - t.toString, - getStackTraceWithAllCauses(t), - operatorId, - workerId - ) + val executionStateStore = new ExecutionStateStore() + // assign execution id to find the execution from DB in case the constructor fails. + executionStateStore.metadataStore.updateState(state => + state.withExecutionId(workflowContext.executionId) + ) + val errorHandler: Throwable => Unit = { t => + { + val fromActorOpt = t match { + case ex: WorkflowRuntimeException => + ex.relatedWorkerId + case other => + None + } + val (operatorId, workerId) = getOperatorFromActorIdOpt(fromActorOpt) + logger.error("error during execution", t) + // Record the real execution failure on the run-level span. Handled + // here rather than in initExecutionService's catch because this is + // where the failure is actually caught (it does not propagate up). + span.recordException(t) + span.setStatus(StatusCode.ERROR) + executionStateStore.statsStore.updateState(stats => + stats.withEndTimeStamp(System.currentTimeMillis()) ) + executionStateStore.metadataStore.updateState { metadataStore => + updateWorkflowState(FAILED, metadataStore).addFatalErrors( + WorkflowFatalError( + EXECUTION_FAILURE, + Timestamp(Instant.now), + t.toString, + getStackTraceWithAllCauses(t), + operatorId, + workerId + ) + ) + } } } - } - // WorkflowExecutionService construction does no external work and cannot - // throw; it registers its error/state diff handler up front. Once published - // via `executionService.onNext`, any failure in `executeWorkflow()` is - // recorded by `errorHandler` into the metadata store, whose handler emits a - // WorkflowErrorEvent that `connectToExecution` forwards. - try { - val execution = new WorkflowExecutionService( - controllerConf, - workflowContext, - resultService, - req, - executionStateStore, - errorHandler, - userEmailOpt, - sessionUri - ) - lifeCycleManager.registerCleanUpOnStateChange(executionStateStore) - executionService.onNext(execution) - execution.executeWorkflow() + // WorkflowExecutionService construction does no external work and cannot + // throw; it registers its error/state diff handler up front. Once published + // via `executionService.onNext`, any failure in `executeWorkflow()` is + // recorded by `errorHandler` into the metadata store, whose handler emits a + // WorkflowErrorEvent that `connectToExecution` forwards. + try { + val execution = new WorkflowExecutionService( + controllerConf, + workflowContext, + resultService, + req, + executionStateStore, + errorHandler, + userEmailOpt, + sessionUri + ) + lifeCycleManager.registerCleanUpOnStateChange(executionStateStore) + executionService.onNext(execution) + execution.executeWorkflow() + } catch { + case e: Throwable => errorHandler(e) + } + } catch { - case e: Throwable => errorHandler(e) + case t: Throwable => + // Synchronous setup failure (before the run's own errorHandler is wired). + span.recordException(t) + span.setStatus(StatusCode.ERROR) + throw t + } finally { + scope.close() + span.end() } - } def convertToJson(frontendVersion: String): String = { diff --git a/common/config/src/main/scala/org/apache/texera/observability/SpanAttrs.scala b/common/config/src/main/scala/org/apache/texera/observability/SpanAttrs.scala index a1ac0bbc18b..c818f69bc16 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/SpanAttrs.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/SpanAttrs.scala @@ -20,33 +20,21 @@ package org.apache.texera.observability import io.opentelemetry.api.common.AttributeKey -import io.opentelemetry.api.trace.{Span, SpanBuilder} /** - * Thin helper for setting span attributes safely. + * Standard Texera span-attribute keys plus a free-text sanitizer. * - * Three rules: - * 1. Typed setters only — no public escape hatch for arbitrary - * untyped strings to land on a span as untrusted free text. - * 2. Free-text values are CRLF-stripped + capped at - * [[FreeTextMaxLen]] to prevent log/span forging via embedded - * newlines. - * 3. Operator IDs and workflow/execution IDs must match a strict - * character set — otherwise dropped silently (the operator - * identifier should be a stable internal value, not user free - * text). + * These are the shared label keys so every callsite tags spans with the + * same names. Set them with the standard OTel API at the callsite, e.g. + * ``spanBuilder.setAttribute(SpanAttrs.WorkflowId, id)`` or + * ``span.setAttribute(SpanAttrs.WorkflowId, id)``. Run any free-text value + * through [[sanitizeFreeText]] first to strip CRLF and cap its length. */ object SpanAttrs { /** Maximum length for free-text span attribute values. */ val FreeTextMaxLen: Int = 256 - /** Validates the shape we accept for operator IDs: alnum + `_.-`, - * 1–64 chars. Anything else is dropped (not coerced — we'd rather - * miss a label than leak an unbounded string into a span). - */ - private val OperatorIdPattern = "^[A-Za-z0-9_.\\-]{1,64}$".r.pattern - // ---- Standard Texera correlation labels ------------------------------ val WorkflowId: AttributeKey[java.lang.Long] = AttributeKey.longKey("texera.workflow.id") @@ -57,52 +45,6 @@ object SpanAttrs { val OperatorName: AttributeKey[String] = AttributeKey.stringKey("texera.operator.name") val Outcome: AttributeKey[String] = AttributeKey.stringKey("texera.outcome") - // ---- Typed setters for SpanBuilder (used at span-start time) --------- - - def withWorkflowId(b: SpanBuilder, id: Long): SpanBuilder = - b.setAttribute(WorkflowId, java.lang.Long.valueOf(id)) - - def withExecutionId(b: SpanBuilder, id: Long): SpanBuilder = - b.setAttribute(ExecutionId, java.lang.Long.valueOf(id)) - - def withProjectId(b: SpanBuilder, id: Long): SpanBuilder = - b.setAttribute(ProjectId, java.lang.Long.valueOf(id)) - - def withUserId(b: SpanBuilder, id: Long): SpanBuilder = - b.setAttribute(UserId, java.lang.Long.valueOf(id)) - - /** Sets the operator id only if it passes the strict character - * check; otherwise the attribute is omitted. Returns the same - * builder either way for fluent chaining. - */ - def withOperatorId(b: SpanBuilder, id: String): SpanBuilder = { - if (id != null && OperatorIdPattern.matcher(id).matches()) { - b.setAttribute(OperatorId, id) - } - b - } - - /** Sets a free-text label after stripping CRLF and capping length. */ - def withOperatorName(b: SpanBuilder, name: String): SpanBuilder = { - val safe = sanitizeFreeText(name) - if (safe != null) b.setAttribute(OperatorName, safe) else b - } - - // ---- Typed setters for Span (used after a span is active) ------------ - - def setWorkflowId(s: Span, id: Long): Span = - s.setAttribute(WorkflowId, java.lang.Long.valueOf(id)) - def setExecutionId(s: Span, id: Long): Span = - s.setAttribute(ExecutionId, java.lang.Long.valueOf(id)) - def setOperatorId(s: Span, id: String): Span = { - if (id != null && OperatorIdPattern.matcher(id).matches()) s.setAttribute(OperatorId, id) - else s - } - def setOutcome(s: Span, outcome: String): Span = { - val safe = sanitizeFreeText(outcome) - if (safe != null) s.setAttribute(Outcome, safe) else s - } - // ---- Pure helpers (exposed for testing) ------------------------------ /** diff --git a/common/config/src/main/scala/org/apache/texera/observability/TexeraTracer.scala b/common/config/src/main/scala/org/apache/texera/observability/TexeraTracer.scala index ae8f71c82e6..38e1c36b620 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/TexeraTracer.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/TexeraTracer.scala @@ -20,25 +20,30 @@ package org.apache.texera.observability import io.opentelemetry.api.GlobalOpenTelemetry -import io.opentelemetry.api.trace.{Span, SpanBuilder, StatusCode, Tracer} -import io.opentelemetry.context.{Context, Scope} +import io.opentelemetry.api.trace.Tracer +import io.opentelemetry.context.Context /** - * Thin convenience wrapper around the global OTel tracer. + * Accessor for the Texera OTel tracer. * - * Two reasons to go through this rather than calling - * ``GlobalOpenTelemetry.getTracer`` directly at every callsite: + * The only thing this adds over calling ``GlobalOpenTelemetry.getTracer`` + * directly is a single instrumentation scope name (``org.apache.texera``), + * so every Texera-produced span shows up under one logical scope in the + * backend, separable from anything emitted by transitive libraries. * - * 1. Single instrumentation scope name (``org.apache.texera``) — so - * every Texera-produced span shows up under one logical scope in - * the backend, separable from anything emitted by transitive - * libraries. - * 2. One ergonomic ``withSpan`` API that handles exception → status, - * scope cleanup, and span end in a single try/finally. Callers - * don't have to remember the ceremony at every site. + * Start and end spans with the standard OTel API at the callsite (see the + * OpenTelemetry Java demo for the recommended pattern): * - * When the SDK is disabled, ``GlobalOpenTelemetry.getTracer`` returns - * a no-op tracer, so calling these methods is safe at any time. + * {{{ + * val span = TexeraTracer.tracer.spanBuilder("MyClass.myMethod").startSpan() + * val scope = span.makeCurrent() + * try { ... } catch { + * case t: Throwable => span.recordException(t); span.setStatus(ERROR); throw t + * } finally { scope.close(); span.end() } + * }}} + * + * When the SDK is disabled, ``GlobalOpenTelemetry.getTracer`` returns a + * no-op tracer, so calling this is safe at any time. */ object TexeraTracer { @@ -46,38 +51,10 @@ object TexeraTracer { def tracer: Tracer = GlobalOpenTelemetry.getTracer(InstrumentationScope) - def spanBuilder(name: String): SpanBuilder = tracer.spanBuilder(name) - - /** - * Run ``block`` inside a fresh span; record exceptions, propagate - * the right span status, and ensure the span is ended exactly once. - * - * Use this for synchronous critical sections. For async (Future- - * returning) code paths use ``withAsyncSpan`` so the span doesn't - * close before the async work completes. - */ - def withSpan[T](name: String, configure: SpanBuilder => SpanBuilder = identity)( - block: Span => T - ): T = { - val span = configure(spanBuilder(name)).startSpan() - val scope: Scope = span.makeCurrent() - try { - block(span) - } catch { - case t: Throwable => - span.recordException(t) - span.setStatus(StatusCode.ERROR) - throw t - } finally { - scope.close() - span.end() - } - } - /** * Snapshot the current OTel ``Context`` so async callbacks can * re-attach it via ``Context.makeCurrent`` later. Useful at the - * Scala↔Python boundary where the calling thread is not the + * Scala to Python boundary where the calling thread is not the * receiving thread. */ def currentContext: Context = Context.current() diff --git a/common/config/src/main/scala/org/apache/texera/observability/TexeraMetrics.scala b/common/config/src/main/scala/org/apache/texera/observability/WorkflowMetrics.scala similarity index 90% rename from common/config/src/main/scala/org/apache/texera/observability/TexeraMetrics.scala rename to common/config/src/main/scala/org/apache/texera/observability/WorkflowMetrics.scala index ef7ad78aeef..1035e8a52ec 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/TexeraMetrics.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/WorkflowMetrics.scala @@ -25,21 +25,37 @@ import io.opentelemetry.api.common.{AttributeKey, Attributes} import io.opentelemetry.api.metrics.Meter /** - * Strongly-typed façade for Texera-emitted metrics. + * Strongly-typed facade for the workflow-execution metrics cluster. * - * Cardinality safety is enforced by the API surface, not by + * This facade pattern is deliberate here because these instruments are a + * small, fixed, correlated set with strict cardinality rules, and it is + * worth centralizing that control in one place. It is NOT the default + * pattern for metrics in general: most call sites should just call the OTel + * meter API directly next to the business logic, e.g. + * + * {{{ + * private val meter = GlobalOpenTelemetry.getMeter("org.apache.texera") + * private val requests = meter.counterBuilder("myfeature.requests").build() + * // in the handler: + * requests.add(1, Attributes.of(AttributeKey.stringKey("route"), route)) + * }}} + * + * Only reach for a facade like this one when the metrics are complex or + * need centralized, standardized control. Do not copy it by default. + * + * Cardinality safety here is enforced by the API surface, not by * documentation: there is no public method that accepts an arbitrary * string as a label key or value. The only labels that ever land on * an instrument are the two enums [[Outcome]] and [[WorkflowKind]], * each restricted to a fixed set. ``workflow.id`` / ``execution.id`` - * are deliberately NOT metric labels — per-execution detail belongs + * are deliberately NOT metric labels: per-execution detail belongs * in traces and logs, joined on ``trace_id`` at query time. * * Histogram bucket bounds are hard-coded constants so they can't be * coerced by request input. The OTel SDK applies its own default * attribute-value-length cap to anything that does slip through. */ -object TexeraMetrics extends LazyLogging { +object WorkflowMetrics extends LazyLogging { /** Outcome enum, the only mutable label on lifecycle counters. */ sealed abstract class Outcome(val name: String) diff --git a/common/config/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala index 1afca195533073f1d8ff2b0b8c8b4dd524da77e2..af98d327b6de267ae6bbefae18d05a52d97410c5 100644 GIT binary patch delta 357 zcmcbjHBWBCL=J=ElEl1}#G(|1oW!Km9EI%E%3>fhFGZm&Gf|-=BUPawCowZm!9N5j z=@{UtF!>;dh&MD6;?xp_qB;Y9eY9(z7z!ZLYi+6LhB2!B12Qbf)wlcme|!5mGW z^RPHBI~CJ*B1weQCoJ1R36^an7-Qlib&DbwI}5o$JON!TCrYab9XLPhKdbf`_PYMEt+iLFD-?(D#HgA7ewC%5afU5UK71i!$6Y=^F^)$D4Bb7au=;)Pi zLEEb@rz)xJW%g7lYtM%9K;xN#_Rhl60@hcm^PuaA_xo-WxHvuW`2@$a1lqpF8FOPD>-0KFwhzHAq!W2zR)lU1n|k1vD`JUfMg4uK gRyvn0k`zf&cDJKO##a^ReqDv1uiku#^PjH#1{~k0!~g&Q diff --git a/common/config/src/test/scala/org/apache/texera/observability/TexeraMetricsSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/WorkflowMetricsSpec.scala similarity index 82% rename from common/config/src/test/scala/org/apache/texera/observability/TexeraMetricsSpec.scala rename to common/config/src/test/scala/org/apache/texera/observability/WorkflowMetricsSpec.scala index 8bed93d6b01..17c9c4c74dd 100644 --- a/common/config/src/test/scala/org/apache/texera/observability/TexeraMetricsSpec.scala +++ b/common/config/src/test/scala/org/apache/texera/observability/WorkflowMetricsSpec.scala @@ -28,7 +28,7 @@ import org.scalatest.matchers.should.Matchers import scala.jdk.CollectionConverters._ -class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { +class WorkflowMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { private var reader: InMemoryMetricReader = _ private var provider: SdkMeterProvider = _ @@ -36,12 +36,12 @@ class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEac override def beforeEach(): Unit = { reader = InMemoryMetricReader.create() provider = SdkMeterProvider.builder().registerMetricReader(reader).build() - TexeraMetrics.resetForTest() - TexeraMetrics.bindForTest(provider.get("org.apache.texera")) + WorkflowMetrics.resetForTest() + WorkflowMetrics.bindForTest(provider.get("org.apache.texera")) } override def afterEach(): Unit = { - TexeraMetrics.resetForTest() + WorkflowMetrics.resetForTest() provider.close() } @@ -51,8 +51,8 @@ class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEac // ----- positive: lifecycle emissions ---------------------------------- - "TexeraMetrics" should "increment workflow.starts on recordStart" in { - TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) + "WorkflowMetrics" should "increment workflow.starts on recordStart" in { + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) val metrics = collectAll() metrics.keySet should contain("texera.workflow.starts") @@ -69,9 +69,9 @@ class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEac // confirm the gauge reports exactly what the supplier returns — regardless // of how many starts/completions were recorded. @volatile var live = 3L - TexeraMetrics.setActiveExecutionsSupplier(() => live) - TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) - TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) + WorkflowMetrics.setActiveExecutionsSupplier(() => live) + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) val first = collectAll()("texera.workflow.active").getLongGaugeData.getPoints.asScala.head first.getValue shouldBe 3L @@ -84,8 +84,8 @@ class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEac } it should "record a completion and duration sample on recordCompletion" in { - TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) - TexeraMetrics.recordCompletion(TexeraMetrics.WorkflowKind.Interactive, durationSec = 2.5) + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + WorkflowMetrics.recordCompletion(WorkflowMetrics.WorkflowKind.Interactive, durationSec = 2.5) val metrics = collectAll() metrics.keySet should contain allOf ( @@ -103,8 +103,8 @@ class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEac } it should "record a failure as a non-success completion (so failure-rate queries work)" in { - TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Scheduled) - TexeraMetrics.recordFailure(TexeraMetrics.WorkflowKind.Scheduled, durationSec = 12.0) + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Scheduled) + WorkflowMetrics.recordFailure(WorkflowMetrics.WorkflowKind.Scheduled, durationSec = 12.0) val metrics = collectAll() // A failure shares the completions counter with successes — the @@ -124,8 +124,8 @@ class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEac } it should "record a cancellation that is not a completion" in { - TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) - TexeraMetrics.recordCancellation(TexeraMetrics.WorkflowKind.Interactive) + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + WorkflowMetrics.recordCancellation(WorkflowMetrics.WorkflowKind.Interactive) val metrics = collectAll() metrics( @@ -140,8 +140,8 @@ class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEac // ----- security: cardinality safety ----------------------------------- it should "only emit the texera.outcome and texera.workflow.kind labels" in { - TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) - TexeraMetrics.recordCompletion(TexeraMetrics.WorkflowKind.Interactive, durationSec = 1.0) + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + WorkflowMetrics.recordCompletion(WorkflowMetrics.WorkflowKind.Interactive, durationSec = 1.0) val attrKeys = collectAll().values.flatMap { md => val pointSet = md.getType.name() match { @@ -166,7 +166,7 @@ class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEac // untyped public method like recordStart(attrs: Attributes), // this assertion still passes but the design intent is broken. // Make the intent explicit: - val methodNames = classOf[TexeraMetrics.type].getDeclaredMethods + val methodNames = classOf[WorkflowMetrics.type].getDeclaredMethods .map(_.getName) .toSet methodNames should contain allOf ("recordStart", "recordCompletion", "recordFailure") @@ -176,10 +176,10 @@ class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEac // ----- histogram buckets are constants -------------------------------- it should "use the hard-coded explicit bucket boundaries for duration" in { - TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) // Hit a few bucket bounds. Seq(0.05, 0.6, 7.0, 65.0, 400.0).foreach { d => - TexeraMetrics.recordCompletion(TexeraMetrics.WorkflowKind.Interactive, durationSec = d) + WorkflowMetrics.recordCompletion(WorkflowMetrics.WorkflowKind.Interactive, durationSec = d) } val histogram = collectAll()("texera.workflow.duration").getHistogramData From 30b841c1b84709713e7a0c051522d72c1c1ecd81 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Tue, 4 Aug 2026 12:30:02 -0700 Subject: [PATCH 20/21] use SpanAttrs keys and stop recording on ended span in WorkflowService --- .../texera/web/service/WorkflowService.scala | 27 ++++++++++-------- .../texera/observability/SpanAttrsSpec.scala | Bin 3870 -> 4084 bytes 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala index b37d9d821b1..2e1fee4c8ba 100644 --- a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala +++ b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala @@ -50,7 +50,7 @@ import org.apache.texera.amber.error.ErrorUtils.{ getStackTraceWithAllCauses } import org.apache.texera.dao.jooq.generated.tables.pojos.User -import org.apache.texera.observability.TexeraTracer +import org.apache.texera.observability.{SpanAttrs, TexeraTracer} import org.apache.texera.service.util.LargeBinaryManager import org.apache.texera.web.model.websocket.event.TexeraWebSocketEvent import org.apache.texera.web.model.websocket.request.WorkflowExecuteRequest @@ -181,10 +181,12 @@ class WorkflowService( } /** Sets up and launches a workflow execution inside a run-level span so - * setup-path logs carry its trace id. The span covers the synchronous - * setup and the handoff to async execution via `executeWorkflow()`; it - * does not span the full async run. The real execution failure is - * recorded onto the current span from `errorHandler`. + * setup-path logs carry its trace id. The span covers only the synchronous + * setup and the handoff to async execution via `executeWorkflow()`; it does + * not span the full async run. Only synchronous setup failures are recorded + * on the span (in the catch below); async execution failures arrive via + * `errorHandler` after the span has ended and are surfaced through the + * metadata store instead. */ def initExecutionService( req: WorkflowExecuteRequest, @@ -193,7 +195,7 @@ class WorkflowService( ): Unit = { val span = TexeraTracer.tracer .spanBuilder("WorkflowService.initExecutionService") - .setAttribute("texera.workflow.id", workflowId.id.toString) + .setAttribute(SpanAttrs.WorkflowId, Long.box(workflowId.id)) .startSpan() val scope = span.makeCurrent() try { @@ -228,7 +230,7 @@ class WorkflowService( convertToJson(req.engineVersion), req.computingUnitId ) - span.setAttribute("texera.execution.id", workflowContext.executionId.id.toString) + span.setAttribute(SpanAttrs.ExecutionId, Long.box(workflowContext.executionId.id)) // A run has started: record the start counter and stamp its start time. org.apache.texera.web.observability.WorkflowMetricsRecorder .onStart(workflowContext.executionId) @@ -276,11 +278,12 @@ class WorkflowService( } val (operatorId, workerId) = getOperatorFromActorIdOpt(fromActorOpt) logger.error("error during execution", t) - // Record the real execution failure on the run-level span. Handled - // here rather than in initExecutionService's catch because this is - // where the failure is actually caught (it does not propagate up). - span.recordException(t) - span.setStatus(StatusCode.ERROR) + // Do NOT touch `span` here: this handler is passed into + // WorkflowExecutionService and invoked asynchronously (runtime, + // websocket, startWorkflow callbacks) after initExecutionService has + // returned and the setup span has already ended, so recording onto it + // would be a silent no-op. The failure is surfaced via the metadata + // store below; setup-span errors are recorded in the catch block. executionStateStore.statsStore.updateState(stats => stats.withEndTimeStamp(System.currentTimeMillis()) ) diff --git a/common/config/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala index af98d327b6de267ae6bbefae18d05a52d97410c5..5553a36e997809029167db2558a2c36b0c8f6243 100644 GIT binary patch delta 180 zcmZ{cu?@m75Jd|lilm}QOHPq!C_DruOmR*QV&(9G?T~~u+_M1^5?fI-07lgO?n&?c z+wxsLuKCsk@U8URKt;n)3Kj{Df delta 12 Tcmew&KTmE$75C- Date: Wed, 12 Aug 2026 14:01:16 -0700 Subject: [PATCH 21/21] feat(observability): logging foundations (bootstrap, bridge, sanitizer) --- .github/workflows/build.yml | 2 +- access-control-service/LICENSE-binary | 20 + .../access-control-service-web-config.yaml | 13 + .../texera/service/AccessControlService.scala | 2 + amber/LICENSE-binary-java | 12 + .../computing-unit-master-config.yml | 12 + .../texera-compiling-service-web-config.yml | 12 + amber/src/main/resources/web-config.yml | 12 + .../texera/web/ComputingUnitMaster.scala | 1 + bin/k8s/values.yaml | 15 + bin/single-node/.env | 12 + build.sbt | 18 +- .../src/main/resources/observability.conf | 45 ++ .../common/config/EnvironmentalVariable.scala | 7 + .../common/config/ObservabilityConfig.scala | 38 ++ .../config/ObservabilityConfigSpec.scala | 65 +++ common/observability/build.sbt | 73 ++++ .../texera/observability/LogSanitizer.scala | 122 ++++++ .../texera/observability/OtelInit.scala | 403 ++++++++++++++++++ .../observability/TexeraOtelLogAppender.scala | 127 ++++++ .../observability/LogSanitizerSpec.scala | 140 ++++++ .../texera/observability/OtelInitSpec.scala | 263 ++++++++++++ .../TexeraOtelLogAppenderSpec.scala | 214 ++++++++++ .../LICENSE-binary | 12 + ...omputing-unit-managing-service-config.yaml | 14 +- .../ComputingUnitManagingService.scala | 2 + config-service/LICENSE-binary | 20 + .../resources/config-service-web-config.yaml | 13 + .../apache/texera/service/ConfigService.scala | 2 + file-service/LICENSE-binary | 12 + .../resources/file-service-web-config.yaml | 12 + .../apache/texera/service/FileService.scala | 2 + workflow-compiling-service/LICENSE-binary | 12 + .../workflow-compiling-service-config.yaml | 12 + .../service/WorkflowCompilingService.scala | 2 + 35 files changed, 1735 insertions(+), 8 deletions(-) create mode 100644 common/config/src/main/resources/observability.conf create mode 100644 common/config/src/main/scala/org/apache/texera/common/config/ObservabilityConfig.scala create mode 100644 common/config/src/test/scala/org/apache/texera/common/config/ObservabilityConfigSpec.scala create mode 100644 common/observability/build.sbt create mode 100644 common/observability/src/main/scala/org/apache/texera/observability/LogSanitizer.scala create mode 100644 common/observability/src/main/scala/org/apache/texera/observability/OtelInit.scala create mode 100644 common/observability/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala create mode 100644 common/observability/src/test/scala/org/apache/texera/observability/LogSanitizerSpec.scala create mode 100644 common/observability/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala create mode 100644 common/observability/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c28457b1986..d826606b310 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -307,7 +307,7 @@ jobs: BACKPORT_TARGET_BRANCH: ${{ inputs.backport_target_branch }} run: | # Backport builds filter by the checked-out build.sbt. - want=(DAO Auth Config Resource Util PyBuilder WorkflowCore + want=(DAO Auth Config Observability Resource Util PyBuilder WorkflowCore WorkflowOperator WorkflowCompiler WorkflowExecutionService) tasks=() if [ -n "${BACKPORT_TARGET_BRANCH}" ]; then diff --git a/access-control-service/LICENSE-binary b/access-control-service/LICENSE-binary index 78f1df46a94..a4ceb120c84 100644 --- a/access-control-service/LICENSE-binary +++ b/access-control-service/LICENSE-binary @@ -242,6 +242,9 @@ Scala/Java jars: - com.google.guava.listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar - com.google.j2objc.j2objc-annotations-2.8.jar - com.helger.profiler-1.1.1.jar + - com.squareup.okhttp3.okhttp-4.12.0.jar + - com.squareup.okio.okio-3.6.0.jar + - com.squareup.okio.okio-jvm-3.6.0.jar - com.thesamet.scalapb.lenses_2.13-0.11.20.jar - com.thesamet.scalapb.scalapb-json4s_2.13-0.12.0.jar - com.thesamet.scalapb.scalapb-runtime_2.13-0.11.20.jar @@ -274,6 +277,18 @@ Scala/Java jars: - io.dropwizard.metrics.metrics-json-4.2.25.jar - io.dropwizard.metrics.metrics-jvm-4.2.25.jar - io.dropwizard.metrics.metrics-logback-4.2.25.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.r2dbc.r2dbc-spi-1.0.0.RELEASE.jar - jakarta.inject.jakarta.inject-api-2.0.1.jar - jakarta.validation.jakarta.validation-api-3.0.2.jar @@ -292,6 +307,11 @@ Scala/Java jars: - org.hibernate.validator.hibernate-validator-7.0.5.Final.jar - org.javassist.javassist-3.30.2-GA.jar - org.jboss.logging.jboss-logging-3.5.3.Final.jar + - org.jetbrains.annotations-13.0.jar + - org.jetbrains.kotlin.kotlin-stdlib-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-common-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-jdk7-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-jdk8-1.9.10.jar - org.jooq.jooq-3.19.36.jar - org.json4s.json4s-ast_2.13-4.0.1.jar - org.json4s.json4s-jackson-core_2.13-4.0.1.jar diff --git a/access-control-service/src/main/resources/access-control-service-web-config.yaml b/access-control-service/src/main/resources/access-control-service-web-config.yaml index 8c7895e9858..bd78ecaa82a 100644 --- a/access-control-service/src/main/resources/access-control-service-web-config.yaml +++ b/access-control-service/src/main/resources/access-control-service-web-config.yaml @@ -26,6 +26,19 @@ server: logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + loggers: + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console threshold: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} diff --git a/access-control-service/src/main/scala/org/apache/texera/service/AccessControlService.scala b/access-control-service/src/main/scala/org/apache/texera/service/AccessControlService.scala index 1f50c86c9f5..a7942d34b53 100644 --- a/access-control-service/src/main/scala/org/apache/texera/service/AccessControlService.scala +++ b/access-control-service/src/main/scala/org/apache/texera/service/AccessControlService.scala @@ -58,6 +58,8 @@ class AccessControlService extends Application[AccessControlServiceConfiguration configuration: AccessControlServiceConfiguration, environment: Environment ): Unit = { + // Bridge this service's logs to the OTel collector under its own service.name. + org.apache.texera.observability.OtelInit.init("access-control-service") // Serve backend at /api environment.jersey.setUrlPattern("/api/*") diff --git a/amber/LICENSE-binary-java b/amber/LICENSE-binary-java index 90d27b4447c..424194d8e32 100644 --- a/amber/LICENSE-binary-java +++ b/amber/LICENSE-binary-java @@ -363,6 +363,18 @@ Scala/Java jars: - org.jspecify.jspecify-1.0.0.jar - io.opencensus.opencensus-api-0.31.1.jar - io.opencensus.opencensus-contrib-http-util-0.31.1.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.perfmark.perfmark-api-0.27.0.jar - io.r2dbc.r2dbc-spi-1.0.0.RELEASE.jar - io.reactivex.rxjava3.rxjava-3.1.12.jar diff --git a/amber/src/main/resources/computing-unit-master-config.yml b/amber/src/main/resources/computing-unit-master-config.yml index 0dba594b8ae..ee578c3cf90 100644 --- a/amber/src/main/resources/computing-unit-master-config.yml +++ b/amber/src/main/resources/computing-unit-master-config.yml @@ -34,6 +34,18 @@ logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console logFormat: "[%date{ISO8601}] [%level] [%logger] [%thread] - %msg %n" diff --git a/amber/src/main/resources/texera-compiling-service-web-config.yml b/amber/src/main/resources/texera-compiling-service-web-config.yml index ea2c1b9c1e9..c0b6e8aa762 100644 --- a/amber/src/main/resources/texera-compiling-service-web-config.yml +++ b/amber/src/main/resources/texera-compiling-service-web-config.yml @@ -43,6 +43,18 @@ logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console logFormat: "[%date{ISO8601}] [%level] [%logger] [%thread] - %msg %n" diff --git a/amber/src/main/resources/web-config.yml b/amber/src/main/resources/web-config.yml index 9fde1d078e8..9b3c743c89c 100644 --- a/amber/src/main/resources/web-config.yml +++ b/amber/src/main/resources/web-config.yml @@ -43,6 +43,18 @@ logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console logFormat: "[%date{ISO8601}] [%level] [%logger] [%thread] - %msg %n" diff --git a/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala b/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala index 6616ff47d1e..b31a906c5d0 100644 --- a/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala +++ b/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala @@ -139,6 +139,7 @@ class ComputingUnitMaster extends io.dropwizard.Application[Configuration] with } override def run(configuration: Configuration, environment: Environment): Unit = { + org.apache.texera.observability.OtelInit.init("computing-unit-master") ObjectMapperUtils.warmupObjectMapperForOperatorsSerde() SqlServer.initConnection( diff --git a/bin/k8s/values.yaml b/bin/k8s/values.yaml index 4651bae6947..8085b975eb3 100644 --- a/bin/k8s/values.yaml +++ b/bin/k8s/values.yaml @@ -358,6 +358,21 @@ texeraEnvVars: - name: AUTH_JWT_SECRET # Development-only default (256-bit HS256 secret). Production environments MUST override this with a different, securely generated secret. value: "a7f3c8e9b14d2e6f5a0b9c3d8e1f4a6b2c5d7e9f0a3b6c8d1e4f7a9b2c5d8e1f" + # OpenTelemetry (observability). Disabled by default; set OTEL_SDK_DISABLED to + # "false" and point the endpoint at a reachable OTLP collector (http/https only) to enable. + - name: OTEL_SDK_DISABLED + value: "true" + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://127.0.0.1:4317" + # Comma-separated resource attributes (k1=v1,k2=v2); service.name is set per service. + - name: OTEL_RESOURCE_ATTRIBUTES + value: "" + # Comma-separated extra hosts allowed for the OTLP endpoint (loopback allowed by default). + - name: TEXERA_OTEL_ALLOWED_HOSTS + value: "" + # Metric export interval in ms. + - name: OTEL_METRIC_EXPORT_INTERVAL + value: "30000" yWebsocketServer: name: y-websocket-server diff --git a/bin/single-node/.env b/bin/single-node/.env index 555e14db7df..cf5552c2224 100644 --- a/bin/single-node/.env +++ b/bin/single-node/.env @@ -96,3 +96,15 @@ LLM_ENDPOINT=http://nginx:8080 TEXERA_DASHBOARD_SERVICE_ENDPOINT=http://dashboard-service:8080 WORKFLOW_COMPILING_SERVICE_ENDPOINT=http://workflow-compiling-service:9090 WORKFLOW_EXECUTION_SERVICE_ENDPOINT=http://workflow-runtime-coordinator-service:8085 + +# OpenTelemetry (observability). Disabled by default; the SDK stays inert until +# OTEL_SDK_DISABLED=false. Point the endpoint at a reachable OTLP collector +# (http/https only) before enabling. +OTEL_SDK_DISABLED=true +OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4317 +# Comma-separated resource attributes (k1=v1,k2=v2); service.name is set per service. +OTEL_RESOURCE_ATTRIBUTES= +# Comma-separated extra hosts allowed for the OTLP endpoint (loopback allowed by default). +TEXERA_OTEL_ALLOWED_HOSTS= +# Metric export interval in ms. +OTEL_METRIC_EXPORT_INTERVAL=30000 diff --git a/build.sbt b/build.sbt index 3779413e041..64504508e98 100644 --- a/build.sbt +++ b/build.sbt @@ -122,6 +122,11 @@ ThisBuild / excludeDependencies += ExclusionRule("log4j", "log4j") lazy val Util = (project in file("common/util")).settings(commonModuleSettings) lazy val DAO = (project in file("common/dao")).settings(commonModuleSettings) lazy val Config = (project in file("common/config")).settings(commonModuleSettings) +// OpenTelemetry bootstrap (OtelInit, log appender, sanitizer) shared by every +// service entry point; pins the OTel dependency versions in one place. Depends +// on Config to read OTEL_* settings from observability.conf. +lazy val Observability = + (project in file("common/observability")).settings(commonModuleSettings).dependsOn(Config) lazy val Resource = (project in file("common/resource")).settings(commonModuleSettings) lazy val Auth = (project in file("common/auth")) .settings(commonModuleSettings) @@ -129,7 +134,7 @@ lazy val Auth = (project in file("common/auth")) .dependsOn(DAO, Config) .dependsOn(DAO % "test->test") // reuse MockTexeraDB embedded Postgres in tests lazy val ConfigService = (project in file("config-service")) - .dependsOn(Auth, Config, DAO, Resource) + .dependsOn(Auth, Config, DAO, Resource, Observability) .dependsOn(DAO % "test->test") // reuse MockTexeraDB embedded Postgres in tests .settings(commonModuleSettings) .settings( @@ -139,7 +144,7 @@ lazy val ConfigService = (project in file("config-service")) ) ) lazy val AccessControlService = (project in file("access-control-service")) - .dependsOn(Auth, Config, DAO, Resource) + .dependsOn(Auth, Config, DAO, Resource, Observability) .settings(commonModuleSettings) .settings( dependencyOverrides ++= Seq( @@ -163,7 +168,7 @@ lazy val WorkflowCore = (project in file("common/workflow-core")) .configs(Test) .dependsOn(DAO % "test->test") // test scope dependency lazy val ComputingUnitManagingService = (project in file("computing-unit-managing-service")) - .dependsOn(WorkflowCore, Auth, Config, Resource) + .dependsOn(WorkflowCore, Auth, Config, Resource, Observability) .configs(Test) .dependsOn(DAO % "test->test") // reuse MockTexeraDB embedded Postgres in tests .settings(commonModuleSettings) @@ -210,7 +215,7 @@ lazy val ComputingUnitManagingService = (project in file("computing-unit-managin ) lazy val FileService = (project in file("file-service")) .settings(commonModuleSettings) - .dependsOn(WorkflowCore, Auth, Config, Resource, Util) + .dependsOn(WorkflowCore, Auth, Config, Resource, Util, Observability) .configs(Test) .dependsOn(DAO % "test->test") // test scope dependency .settings( @@ -238,7 +243,7 @@ lazy val WorkflowCompiler = (project in file("common/workflow-compiler")) .configs(Test) .dependsOn(WorkflowOperator) lazy val WorkflowCompilingService = (project in file("workflow-compiling-service")) - .dependsOn(WorkflowCompiler, Auth, Config, Resource) + .dependsOn(WorkflowCompiler, Auth, Config, Resource, Observability) .settings(commonModuleSettings) .settings( dependencyOverrides ++= Seq( @@ -250,7 +255,7 @@ lazy val WorkflowCompilingService = (project in file("workflow-compiling-service ) lazy val WorkflowExecutionService = (project in file("amber")) - .dependsOn(WorkflowCompiler, Auth, Config) + .dependsOn(WorkflowCompiler, Auth, Config, Observability) .settings(commonModuleSettings) .settings( dependencyOverrides ++= Seq( @@ -285,6 +290,7 @@ lazy val TexeraProject = (project in file(".")) // common libraries Auth, Config, + Observability, Resource, Util, DAO, diff --git a/common/config/src/main/resources/observability.conf b/common/config/src/main/resources/observability.conf new file mode 100644 index 00000000000..2bf7a4e10d6 --- /dev/null +++ b/common/config/src/main/resources/observability.conf @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# OpenTelemetry SDK bootstrap settings, consumed by OtelInit. Each value has a +# safe default and an optional environment override, so an operator can find and +# tune every knob here instead of in code. +observability { + # Master switch. Disabled by default: no exporters start and no telemetry is + # emitted until this is set to false (i.e. OTEL_SDK_DISABLED=false). + sdk-disabled = "true" + sdk-disabled = ${?OTEL_SDK_DISABLED} + + # OTLP collector endpoint. http/https only; loopback-only host allowlist by + # default (extend via allowed-hosts below). + endpoint = "http://127.0.0.1:4317" + endpoint = ${?OTEL_EXPORTER_OTLP_ENDPOINT} + + # Comma-separated resource attributes (k1=v1,k2=v2). service.name is set by + # the service and cannot be overridden here. + resource-attributes = "" + resource-attributes = ${?OTEL_RESOURCE_ATTRIBUTES} + + # Comma-separated extra hosts added to the endpoint allowlist. + allowed-hosts = "" + allowed-hosts = ${?TEXERA_OTEL_ALLOWED_HOSTS} + + # Metric export interval in milliseconds; out-of-range values fall back to + # the default. + metric-export-interval-ms = "30000" + metric-export-interval-ms = ${?OTEL_METRIC_EXPORT_INTERVAL} +} diff --git a/common/config/src/main/scala/org/apache/texera/common/config/EnvironmentalVariable.scala b/common/config/src/main/scala/org/apache/texera/common/config/EnvironmentalVariable.scala index a335ddeff6c..2876770df57 100644 --- a/common/config/src/main/scala/org/apache/texera/common/config/EnvironmentalVariable.scala +++ b/common/config/src/main/scala/org/apache/texera/common/config/EnvironmentalVariable.scala @@ -45,6 +45,13 @@ object EnvironmentalVariable { val ENV_USER_JWT_TOKEN = "USER_JWT_TOKEN" val ENV_AUTH_JWT_SECRET = "AUTH_JWT_SECRET" + // OpenTelemetry observability (see observability.conf) + val ENV_OTEL_SDK_DISABLED = "OTEL_SDK_DISABLED" + val ENV_OTEL_EXPORTER_OTLP_ENDPOINT = "OTEL_EXPORTER_OTLP_ENDPOINT" + val ENV_OTEL_RESOURCE_ATTRIBUTES = "OTEL_RESOURCE_ATTRIBUTES" + val ENV_OTEL_METRIC_EXPORT_INTERVAL = "OTEL_METRIC_EXPORT_INTERVAL" + val ENV_TEXERA_OTEL_ALLOWED_HOSTS = "TEXERA_OTEL_ALLOWED_HOSTS" + // JDBC val ENV_JDBC_URL = "STORAGE_JDBC_URL" val ENV_JDBC_USERNAME = "STORAGE_JDBC_USERNAME" diff --git a/common/config/src/main/scala/org/apache/texera/common/config/ObservabilityConfig.scala b/common/config/src/main/scala/org/apache/texera/common/config/ObservabilityConfig.scala new file mode 100644 index 00000000000..29d121dbd4d --- /dev/null +++ b/common/config/src/main/scala/org/apache/texera/common/config/ObservabilityConfig.scala @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.texera.common.config + +import com.typesafe.config.{Config, ConfigFactory} + +/** + * Typed view over observability.conf. Each field carries the HOCON default + * already merged with its OTEL_* environment override, so OtelInit reads its + * settings from one place instead of calling System.getenv directly. Values + * are kept as strings and interpreted by OtelInit, which tolerates malformed + * input without throwing. + */ +object ObservabilityConfig { + private val conf: Config = ConfigFactory.parseResources("observability.conf").resolve() + + val sdkDisabled: String = conf.getString("observability.sdk-disabled") + val endpoint: String = conf.getString("observability.endpoint") + val resourceAttributes: String = conf.getString("observability.resource-attributes") + val allowedHosts: String = conf.getString("observability.allowed-hosts") + val metricExportIntervalMs: String = conf.getString("observability.metric-export-interval-ms") +} diff --git a/common/config/src/test/scala/org/apache/texera/common/config/ObservabilityConfigSpec.scala b/common/config/src/test/scala/org/apache/texera/common/config/ObservabilityConfigSpec.scala new file mode 100644 index 00000000000..659485bbc9f --- /dev/null +++ b/common/config/src/test/scala/org/apache/texera/common/config/ObservabilityConfigSpec.scala @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.common.config + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** + * Spec for [[ObservabilityConfig]]. Reading each value forces resolution from + * observability.conf, so a renamed key surfaces here as a ConfigException. + * Exact-value assertions are guarded on the OTEL_* override being unset. + */ +class ObservabilityConfigSpec extends AnyFlatSpec with Matchers { + + // `${?VAR}` in HOCON can be satisfied by an OS env var or a JVM system property. + private def isOverridden(name: String): Boolean = + sys.env.contains(name) || sys.props.contains(name) + + "ObservabilityConfig" should "default to disabled per issue #5367" in { + if (!isOverridden(EnvironmentalVariable.ENV_OTEL_SDK_DISABLED)) { + ObservabilityConfig.sdkDisabled shouldBe "true" + } else { + ObservabilityConfig.sdkDisabled should not be empty + } + } + + it should "default to a loopback OTLP endpoint" in { + if (!isOverridden(EnvironmentalVariable.ENV_OTEL_EXPORTER_OTLP_ENDPOINT)) { + ObservabilityConfig.endpoint shouldBe "http://127.0.0.1:4317" + } else { + ObservabilityConfig.endpoint should not be empty + } + } + + it should "default the metric export interval to 30s" in { + if (!isOverridden(EnvironmentalVariable.ENV_OTEL_METRIC_EXPORT_INTERVAL)) { + ObservabilityConfig.metricExportIntervalMs shouldBe "30000" + } else { + ObservabilityConfig.metricExportIntervalMs should not be empty + } + } + + it should "resolve resource-attributes and allowed-hosts without error" in { + // Empty by default; the point is that the keys exist and resolve. + noException should be thrownBy ObservabilityConfig.resourceAttributes + noException should be thrownBy ObservabilityConfig.allowedHosts + } +} diff --git a/common/observability/build.sbt b/common/observability/build.sbt new file mode 100644 index 00000000000..ac33d09e417 --- /dev/null +++ b/common/observability/build.sbt @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import scala.collection.Seq + +name := "observability" + + +enablePlugins(JavaAppPackaging) + +// Enable semanticdb for Scalafix +ThisBuild / semanticdbEnabled := true +ThisBuild / semanticdbVersion := scalafixSemanticdb.revision + +// Manage dependency conflicts by always using the latest revision +ThisBuild / conflictManager := ConflictManager.latestRevision + +// Restrict parallel execution of tests to avoid conflicts +Global / concurrentRestrictions += Tags.limit(Tags.Test, 1) + +///////////////////////////////////////////////////////////////////////////// +// Compiler Options +///////////////////////////////////////////////////////////////////////////// + +// Scala compiler options +Compile / scalacOptions ++= Seq( + "-Xelide-below", "WARNING", // Turn on optimizations with "WARNING" as the threshold + "-feature", // Check feature warnings + "-deprecation", // Check deprecation warnings + "-Ywarn-unused:imports" // Check for unused imports +) + +///////////////////////////////////////////////////////////////////////////// +// Dependencies +///////////////////////////////////////////////////////////////////////////// + +// OpenTelemetry version is pinned here as the single source of truth; every +// service picks it up via dependsOn(Observability). Bump deliberately. +val openTelemetryVersion = "1.50.0" + +libraryDependencies ++= Seq( + "com.typesafe.scala-logging" %% "scala-logging" % "3.9.5", // for LazyLogging in OtelInit + // OpenTelemetry SDK bootstrap (Apache-2.0). We deliberately do NOT use + // sdk-extension-autoconfigure: the security model requires that endpoint + // + resource-attribute filtering run before any exporter is configured. + "io.opentelemetry" % "opentelemetry-api" % openTelemetryVersion, + "io.opentelemetry" % "opentelemetry-sdk" % openTelemetryVersion, + "io.opentelemetry" % "opentelemetry-exporter-otlp" % openTelemetryVersion, + // Logback Classic is needed at compile time to write the OTel log + // appender. Marked `provided` because every service already brings + // Logback in transitively (via Dropwizard / SLF4J), so we don't + // bundle a second copy. + "ch.qos.logback" % "logback-classic" % "1.2.13" % "provided", + // Test-only: in-memory exporter for OtelInitSpec; avoids hitting a real + // collector during unit tests. + "io.opentelemetry" % "opentelemetry-sdk-testing" % openTelemetryVersion % Test, + "ch.qos.logback" % "logback-classic" % "1.2.13" % Test, + "org.scalatest" %% "scalatest" % "3.2.15" % Test // ScalaTest (for unit tests) +) diff --git a/common/observability/src/main/scala/org/apache/texera/observability/LogSanitizer.scala b/common/observability/src/main/scala/org/apache/texera/observability/LogSanitizer.scala new file mode 100644 index 00000000000..ffb35cebb8e --- /dev/null +++ b/common/observability/src/main/scala/org/apache/texera/observability/LogSanitizer.scala @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import scala.jdk.CollectionConverters._ + +/** + * Pure functions that sanitize log bodies and MDC before export: + * strip control characters, redact secrets, cap body size, and + * filter MDC down by dropping denied keys. + */ +object LogSanitizer { + + /** Per-record body length cap, in chars. */ + val MaxBodyBytes: Int = 16 * 1024 + + /** Suffix appended to truncated bodies. */ + val TruncatedMarker: String = "...[truncated]" + + /** C0 control characters except TAB (0x09), plus DEL (0x7F). */ + private val C0ControlRegex = "[\\x00-\\x08\\x0A-\\x1F\\x7F]".r + + /** Secret patterns, redacted from bodies. Most specific first. */ + private val SecretPatterns: Seq[scala.util.matching.Regex] = Seq( + // Bearer token + """(?i)Bearer\s+[A-Za-z0-9._\-/+=]{8,}""".r, + // password=... or password: ... + """(?i)password\s*[=:]\s*[^\s,;"']+""".r, + // AWS access key ID + """AKIA[0-9A-Z]{16}""".r, + // labelled AWS secret access key + """(?i)aws_secret_access_key\s*[=:]\s*[A-Za-z0-9/+=]{20,}""".r + ) + + /** MDC keys never forwarded to OTel log attributes. Default-allow: any key + * our instrumentation sets is exported, so adding a new correlation field + * needs no edit here. Only the noisy keys Pekko's SLF4J bridge injects are + * dropped, since they are redundant with the log body and would bloat every + * exported record. Values that pass through are still run through + * [[sanitize]], so secret-shaped content is redacted regardless of key; a + * key whose name looks credential-bearing (see [[isSecretKey]]) has its + * value redacted wholesale. + */ + val DeniedMdcKeys: Set[String] = Set( + "sourceThread", + "pekkoSource", + "pekkoAddress", + "pekkoTimestamp", + "sourceActorSystem" + ) + + /** Substrings marking an MDC key as credential-bearing. A matching key has + * its value redacted whole, since the value alone (e.g. a bare password) + * need not match any [[SecretPatterns]] regex to be a secret. + */ + private val SecretKeySubstrings: Seq[String] = + Seq( + "password", + "passwd", + "pwd", + "secret", + "token", + "apikey", + "api_key", + "authorization", + "credential" + ) + + private def isSecretKey(key: String): Boolean = { + val k = key.toLowerCase + SecretKeySubstrings.exists(k.contains) + } + + /** Strip C0 control characters (except TAB) and DEL. Null-safe. */ + def stripControlChars(body: String): String = + if (body == null) "" else C0ControlRegex.replaceAllIn(body, "") + + /** Redact secret-shaped substrings. Null-safe, idempotent. */ + def redactSecrets(body: String): String = + if (body == null) "" + else SecretPatterns.foldLeft(body)((acc, p) => p.replaceAllIn(acc, "[REDACTED]")) + + /** Strip control chars, redact secrets, then truncate. Idempotent. */ + def sanitize(body: String): String = { + if (body == null || body.isEmpty) return "" + truncate(redactSecrets(stripControlChars(body))) + } + + /** Truncate to MaxBodyBytes, appending the marker if cut. */ + def truncate(body: String): String = { + if (body.length <= MaxBodyBytes) body + else body.substring(0, MaxBodyBytes - TruncatedMarker.length) + TruncatedMarker + } + + /** Drop denied MDC keys, sanitize the surviving values. A key whose name is + * credential-bearing has its value redacted whole. Null-safe. + */ + def filterMdc(mdc: java.util.Map[String, String]): Map[String, String] = { + if (mdc == null) return Map.empty + mdc.asScala.iterator.collect { + case (k, v) if k != null && v != null && !DeniedMdcKeys.contains(k) => + if (isSecretKey(k)) k -> "[REDACTED]" else k -> sanitize(v) + }.toMap + } +} diff --git a/common/observability/src/main/scala/org/apache/texera/observability/OtelInit.scala b/common/observability/src/main/scala/org/apache/texera/observability/OtelInit.scala new file mode 100644 index 00000000000..50390986773 --- /dev/null +++ b/common/observability/src/main/scala/org/apache/texera/observability/OtelInit.scala @@ -0,0 +1,403 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import com.typesafe.scalalogging.LazyLogging +import org.apache.texera.common.config.{EnvironmentalVariable, ObservabilityConfig} +import io.opentelemetry.api.{GlobalOpenTelemetry, OpenTelemetry} +import io.opentelemetry.api.common.{AttributeKey, Attributes} +import io.opentelemetry.exporter.otlp.logs.OtlpGrpcLogRecordExporter +import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter +import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.logs.SdkLoggerProvider +import io.opentelemetry.sdk.logs.`export`.{BatchLogRecordProcessor, LogRecordExporter} +import io.opentelemetry.sdk.metrics.SdkMeterProvider +import io.opentelemetry.sdk.metrics.`export`.{MetricExporter, PeriodicMetricReader} +import io.opentelemetry.sdk.resources.Resource +import io.opentelemetry.sdk.trace.SdkTracerProvider +import io.opentelemetry.sdk.trace.`export`.{BatchSpanProcessor, SpanExporter} + +import java.net.URI +import java.time.Duration +import scala.util.{Failure, Success, Try} + +/** + * Bootstraps the OpenTelemetry SDK for a Texera service. + * + * Disabled by default; set OTEL_SDK_DISABLED=false to enable it. Reads its + * settings from observability.conf (each defaulted, each OTEL_*-overridable), + * validates the endpoint against an allowlist, builds tracer/log/metric + * providers, and attaches a Logback appender. Returns None when disabled or + * misconfigured; never throws. + */ +object OtelInit extends LazyLogging { + + /** Endpoint schemes we accept. OTLP-over-gRPC uses http/https endpoints; + * the exporter rejects a `grpc://` scheme outright, so it is not allowed. + */ + private[observability] val AllowedSchemes: Set[String] = Set("http", "https") + + /** Hosts we accept for the OTLP endpoint by default. */ + private[observability] val DefaultAllowedHosts: Set[String] = Set( + "localhost", + "127.0.0.1", + "[::1]" + ) + + /** Default endpoint. 127.0.0.1 (not "localhost") to force IPv4 so a + * natively-run service reaches the collector on dual-stack hosts. + */ + private val DefaultEndpoint = "http://127.0.0.1:4317" + + /** Metric export interval bounds; out-of-range values fall back to the + * default (see clampIntervalMs). + */ + private[observability] val MinMetricIntervalMs: Long = 1000L + private[observability] val MaxMetricIntervalMs: Long = 10L * 60L * 1000L + private[observability] val DefaultMetricIntervalMs: Long = 30L * 1000L + + // Idempotency guard: init() is a no-op after the first call. + @volatile private var initialized: Option[OpenTelemetry] = None + + /** + * Initialize the SDK for the given service name. Returns Some on + * success, None when disabled or misconfigured. When enabled, also + * attaches a [[TexeraOtelLogAppender]] to the Logback ROOT logger. + */ + def init(serviceName: String): Option[OpenTelemetry] = + synchronized { + if (initialized.isDefined) return initialized + + // Source the OTEL_* settings from observability.conf (HOCON defaults + // already merged with any env override); fall back to the raw environment + // for anything else. + val env = (key: String) => + key match { + case EnvironmentalVariable.ENV_OTEL_SDK_DISABLED => Some(ObservabilityConfig.sdkDisabled) + case EnvironmentalVariable.ENV_OTEL_EXPORTER_OTLP_ENDPOINT => + Some(ObservabilityConfig.endpoint) + case EnvironmentalVariable.ENV_OTEL_RESOURCE_ATTRIBUTES => + Some(ObservabilityConfig.resourceAttributes) + case EnvironmentalVariable.ENV_TEXERA_OTEL_ALLOWED_HOSTS => + Some(ObservabilityConfig.allowedHosts) + case EnvironmentalVariable.ENV_OTEL_METRIC_EXPORT_INTERVAL => + Some(ObservabilityConfig.metricExportIntervalMs) + case other => Option(System.getenv(other)) + } + val result = initInternal( + serviceName = serviceName, + envProvider = env, + spanExporterFactory = buildOtlpSpanExporter, + logExporterFactory = endpoint => Some(buildOtlpLogExporter(endpoint)), + metricExporterFactory = endpoint => Some(buildOtlpMetricExporter(endpoint)), + logbackAttacher = LogbackBinder.attach + ) + // Register globally so OTel-aware code can use GlobalOpenTelemetry + // without threading the SDK through callsites. set() throws on a + // second call; wrap defensively. + result.foreach { sdk => + Try(GlobalOpenTelemetry.set(sdk)).failed.foreach { t => + logger.warn( + s"GlobalOpenTelemetry already set; using the existing instance: ${t.getMessage}" + ) + } + } + result + } + + /** + * Test-only entry point: injects an env-var map and exporters so the + * SDK makes no network connection. Does not attach the Logback appender. + */ + private[observability] def initForTest( + serviceName: String, + envOverride: Map[String, String], + exporter: SpanExporter, + metricExporter: Option[MetricExporter] = None + ): Option[OpenTelemetry] = + synchronized { + initInternal( + serviceName = serviceName, + envProvider = envOverride.get, + spanExporterFactory = _ => exporter, + logExporterFactory = _ => None, + metricExporterFactory = _ => metricExporter, + logbackAttacher = (_, _) => () // no-op in tests + ) + } + + /** Test-only: forget any previously-installed SDK. Does not unregister + * shutdown hooks (the previous SDK is closed instead). + */ + private[observability] def resetForTest(): Unit = + synchronized { + initialized.foreach { + case sdk: OpenTelemetrySdk => + Try(sdk.getSdkTracerProvider.close()) + Try(sdk.getSdkLoggerProvider.close()) + Try(sdk.getSdkMeterProvider.close()) + case _ => () + } + initialized = None + } + + private def initInternal( + serviceName: String, + envProvider: String => Option[String], + spanExporterFactory: String => SpanExporter, + logExporterFactory: String => Option[LogRecordExporter], + metricExporterFactory: String => Option[MetricExporter], + logbackAttacher: (String, OpenTelemetry) => Unit + ): Option[OpenTelemetry] = { + if (initialized.isDefined) return initialized + + // Disabled by default (issue #5367): stay inert unless OTEL_SDK_DISABLED is + // explicitly false. An unreachable endpoint drops records without crashing. + val disabled = envProvider(EnvironmentalVariable.ENV_OTEL_SDK_DISABLED).getOrElse("true") + if (!disabled.equalsIgnoreCase("false")) { + logger.info( + "OpenTelemetry SDK disabled (OTEL_SDK_DISABLED not false). No telemetry will be emitted." + ) + return None + } + + val endpoint = + envProvider(EnvironmentalVariable.ENV_OTEL_EXPORTER_OTLP_ENDPOINT).getOrElse(DefaultEndpoint) + val extraAllowed = envProvider(EnvironmentalVariable.ENV_TEXERA_OTEL_ALLOWED_HOSTS) + .map(_.split(',').iterator.map(_.trim.toLowerCase).filter(_.nonEmpty).toSet) + .getOrElse(Set.empty) + val allowedHosts = DefaultAllowedHosts ++ extraAllowed + + validateEndpoint(endpoint, allowedHosts) match { + case Left(reason) => + // One WARN; no telemetry is emitted. + logger.warn( + s"OpenTelemetry SDK disabled: invalid OTEL_EXPORTER_OTLP_ENDPOINT — $reason. " + + "Set TEXERA_OTEL_ALLOWED_HOSTS to extend the allowlist." + ) + return None + case Right(_) => // ok + } + + val rawAttrs = envProvider(EnvironmentalVariable.ENV_OTEL_RESOURCE_ATTRIBUTES).getOrElse("") + val resource = buildResource(serviceName, rawAttrs) + + val spanExporter = spanExporterFactory(endpoint) + val tracerProvider = SdkTracerProvider + .builder() + .setResource(resource) + .addSpanProcessor(BatchSpanProcessor.builder(spanExporter).build()) + .build() + + val sdkBuilder = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider) + + // Logger provider is optional; the factory returns None in tests. + val loggerProviderOpt = logExporterFactory(endpoint).map { logExporter => + val lp = SdkLoggerProvider + .builder() + .setResource(resource) + .addLogRecordProcessor(BatchLogRecordProcessor.builder(logExporter).build()) + .build() + sdkBuilder.setLoggerProvider(lp) + lp + } + + // Meter provider is optional too; interval falls back to the default + // when out of range. + val intervalMs = + clampIntervalMs(envProvider(EnvironmentalVariable.ENV_OTEL_METRIC_EXPORT_INTERVAL)) + val meterProviderOpt = metricExporterFactory(endpoint).map { metricExporter => + val reader = PeriodicMetricReader + .builder(metricExporter) + .setInterval(Duration.ofMillis(intervalMs)) + .build() + val mp = SdkMeterProvider + .builder() + .setResource(resource) + .registerMetricReader(reader) + .build() + sdkBuilder.setMeterProvider(mp) + mp + } + + val sdk = sdkBuilder.build() + + // One startup span carrying only service.name. + val span = sdk.getTracer("texera.bootstrap").spanBuilder("service.start").startSpan() + Try(span.setAttribute("service.name", serviceName)) + span.end() + + // Wire the Logback appender; failure here must not crash the service. + Try(logbackAttacher(serviceName, sdk)).failed.foreach { t => + logger.warn(s"Failed to attach OTel Logback appender (logs not exported): ${t.getMessage}") + } + + // Flush providers on shutdown. Added after the SDK is fully built. + Runtime.getRuntime.addShutdownHook( + new Thread( + () => { + Try(tracerProvider.close()) + loggerProviderOpt.foreach(lp => Try(lp.close())) + meterProviderOpt.foreach(mp => Try(mp.close())) + () + }, + "otel-shutdown" + ) + ) + + initialized = Some(sdk) + logger.info(s"OpenTelemetry SDK initialized for service '$serviceName' (endpoint=$endpoint).") + initialized + } + + /** + * Validate the endpoint is parseable and uses an allowlisted scheme + * and host. Pure function. + */ + private[observability] def validateEndpoint( + endpoint: String, + allowedHosts: Set[String] + ): Either[String, Unit] = { + Try(URI.create(endpoint)) match { + case Failure(e) => + Left(s"unparseable URI (${e.getClass.getSimpleName})") + case Success(uri) => + val scheme = Option(uri.getScheme).map(_.toLowerCase).getOrElse("") + if (scheme.isEmpty) { + Left("missing scheme") + } else if (!AllowedSchemes.contains(scheme)) { + Left( + s"scheme '$scheme' not in allowlist ${AllowedSchemes.toSeq.sorted.mkString("{", ",", "}")}" + ) + } else { + val host = Option(uri.getHost).map(_.toLowerCase).getOrElse("") + if (host.isEmpty) { + Left("missing host") + } else if (!allowedHosts.contains(host)) { + Left(s"host '$host' not in allowlist") + } else { + Right(()) + } + } + } + } + + /** + * Build a Resource from the service name and OTEL_RESOURCE_ATTRIBUTES. + * Every parsed attribute is applied so new resource fields need no edit + * here; the one exception is service.name, which the argument controls + * and env cannot override. + */ + private[observability] def buildResource(serviceName: String, rawAttrs: String): Resource = { + val builder = Attributes.builder() + builder.put(AttributeKey.stringKey("service.name"), serviceName) + + parseAttrs(rawAttrs).foreach { + case (key, value) if key != "service.name" => + builder.put(AttributeKey.stringKey(key), value) + case _ => // env cannot override service.name + } + + Resource.create(builder.build()) + } + + /** Parse a `k1=v1,k2=v2` string. Malformed entries are skipped. */ + private[observability] def parseAttrs(raw: String): Seq[(String, String)] = { + if (raw == null || raw.isEmpty) return Seq.empty + raw + .split(',') + .iterator + .map(_.trim) + .filter(_.nonEmpty) + .flatMap { entry => + val idx = entry.indexOf('=') + if (idx <= 0 || idx == entry.length - 1) None + else Some(entry.substring(0, idx).trim -> entry.substring(idx + 1).trim) + } + .toSeq + } + + private def buildOtlpSpanExporter(endpoint: String): SpanExporter = + OtlpGrpcSpanExporter.builder().setEndpoint(endpoint).build() + + private def buildOtlpLogExporter(endpoint: String): LogRecordExporter = + OtlpGrpcLogRecordExporter.builder().setEndpoint(endpoint).build() + + private def buildOtlpMetricExporter(endpoint: String): MetricExporter = + OtlpGrpcMetricExporter.builder().setEndpoint(endpoint).build() + + /** + * Parse and clamp OTEL_METRIC_EXPORT_INTERVAL (ms). Out-of-range or + * unparseable input falls back to the default with one WARN. + */ + private[observability] def clampIntervalMs(raw: Option[String]): Long = { + raw match { + case None => DefaultMetricIntervalMs + case Some(value) => + Try(value.trim.toLong) match { + case Failure(_) => + logger.warn( + s"OTEL_METRIC_EXPORT_INTERVAL '$value' is not a number; " + + s"using default ${DefaultMetricIntervalMs}ms." + ) + DefaultMetricIntervalMs + case Success(ms) if ms < MinMetricIntervalMs || ms > MaxMetricIntervalMs => + logger.warn( + s"OTEL_METRIC_EXPORT_INTERVAL=${ms}ms out of range " + + s"[${MinMetricIntervalMs}, ${MaxMetricIntervalMs}]; " + + s"using default ${DefaultMetricIntervalMs}ms." + ) + DefaultMetricIntervalMs + case Success(ms) => ms + } + } + } +} + +/** + * Isolates the Logback attach step so [[OtelInit]] does not import + * Logback types directly, keeping SDK init testable with a mock attacher. + */ +private[observability] object LogbackBinder extends LazyLogging { + + /** Attach a [[TexeraOtelLogAppender]] bound to `otel` to the Logback + * ROOT logger. Emits one WARN and returns if Logback is not the + * active SLF4J binding. + */ + def attach(serviceName: String, otel: OpenTelemetry): Unit = { + val factory = org.slf4j.LoggerFactory.getILoggerFactory + factory match { + case ctx: ch.qos.logback.classic.LoggerContext => + val root = ctx.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME) + val appender = new TexeraOtelLogAppender() + appender.setContext(ctx) + appender.setName(s"texera-otel-$serviceName") + appender.bind(otel) + appender.start() + root.addAppender(appender) + case other => + logger.warn( + s"SLF4J binding is not Logback (${other.getClass.getName}); " + + "OTel log export is not wired. Application logs to stdout/file are unaffected." + ) + } + } +} diff --git a/common/observability/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala b/common/observability/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala new file mode 100644 index 00000000000..306236d3318 --- /dev/null +++ b/common/observability/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import ch.qos.logback.classic.Level +import ch.qos.logback.classic.spi.{ILoggingEvent, IThrowableProxy, ThrowableProxyUtil} +import ch.qos.logback.core.UnsynchronizedAppenderBase +import io.opentelemetry.api.OpenTelemetry +import io.opentelemetry.api.common.AttributeKey +import io.opentelemetry.api.logs.{Logger, Severity} +import io.opentelemetry.api.trace.Span +import io.opentelemetry.context.Context + +import java.util.concurrent.TimeUnit + +/** + * Logback appender that sanitizes each event via [[LogSanitizer]] and + * emits it as an OTel LogRecord. [[append]] is a no-op until [[bind]] + * is called and after [[stop]]. + * + * This is internal plumbing, not the developer logging API. Code logs + * through the normal SLF4J / scala-logging interface and adds correlation + * ids via MDC; [[OtelInit.init]] attaches this appender to the ROOT logger + * so those records also reach OTel: + * + * {{{ + * class Foo extends LazyLogging { + * MDC.put("workflowId", id) // forwarded as an OTel log attribute + * try logger.info("started") // body + severity + trace context + * finally MDC.remove("workflowId") + * } + * }}} + */ +class TexeraOtelLogAppender extends UnsynchronizedAppenderBase[ILoggingEvent] { + + // @volatile so a late bind() is visible to appender threads. + @volatile private var otelLogger: Option[Logger] = None + + def bind(otel: OpenTelemetry): Unit = { + otelLogger = Some(otel.getLogsBridge.get("texera.logback")) + } + + override def stop(): Unit = { + otelLogger = None + super.stop() + } + + override def append(event: ILoggingEvent): Unit = { + otelLogger match { + case None => () // not bound + case Some(logger) => + try { + emit(logger, event) + } catch { + // An appender must not throw into the calling thread. + case t: Throwable => + addError("OTel log emission failed", t) + } + } + } + + private def emit(logger: Logger, event: ILoggingEvent): Unit = { + // Control-strip the message only (trace newlines must survive), then append + // the stack trace, redact secrets across the whole body, and cap length. + val message = LogSanitizer.stripControlChars(event.getFormattedMessage) + val combined = Option(event.getThrowableProxy) match { + case Some(proxy) => message + "\n" + formatThrowable(proxy) + case None => message + } + val body = LogSanitizer.truncate(LogSanitizer.redactSecrets(combined)) + val builder = logger + .logRecordBuilder() + .setBody(body) + .setSeverity(severityFromLevel(event.getLevel)) + .setSeverityText(event.getLevel.toString) + .setTimestamp(event.getTimeStamp, TimeUnit.MILLISECONDS) + + // Surviving (deny-list filtered) MDC keys as typed attributes. + LogSanitizer.filterMdc(event.getMDCPropertyMap).foreach { + case (k, v) => builder.setAttribute(AttributeKey.stringKey(k), v) + } + + builder.setAttribute(AttributeKey.stringKey("logger.name"), event.getLoggerName) + builder.setAttribute(AttributeKey.stringKey("thread.name"), event.getThreadName) + + // Attach trace context so the SDK sets trace_id / span_id. + val span = Span.current() + if (span.getSpanContext.isValid) { + builder.setContext(Context.current()) + } + + builder.emit() + } + + /** Format a throwable proxy as a Logback-style stack trace. */ + private def formatThrowable(proxy: IThrowableProxy): String = + ThrowableProxyUtil.asString(proxy) + + private def severityFromLevel(level: Level): Severity = { + if (level == null) return Severity.UNDEFINED_SEVERITY_NUMBER + level.toInt match { + case Level.TRACE_INT => Severity.TRACE + case Level.DEBUG_INT => Severity.DEBUG + case Level.INFO_INT => Severity.INFO + case Level.WARN_INT => Severity.WARN + case Level.ERROR_INT => Severity.ERROR + case _ => Severity.UNDEFINED_SEVERITY_NUMBER + } + } +} diff --git a/common/observability/src/test/scala/org/apache/texera/observability/LogSanitizerSpec.scala b/common/observability/src/test/scala/org/apache/texera/observability/LogSanitizerSpec.scala new file mode 100644 index 00000000000..022b52aa68a --- /dev/null +++ b/common/observability/src/test/scala/org/apache/texera/observability/LogSanitizerSpec.scala @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import scala.jdk.CollectionConverters._ + +class LogSanitizerSpec extends AnyFlatSpec with Matchers { + + // ----- sanitize: control characters ---------------------------------- + + "sanitize" should "strip CR/LF so a user-supplied message cannot forge a new log line" in { + val crlfPayload = "hello\r\nFAKE LOG LINE\r\nworld" + LogSanitizer.sanitize(crlfPayload) shouldBe "helloFAKE LOG LINEworld" + } + + it should "strip other C0 control characters but preserve TAB" in { + val payload = "before\u0000NUL\u0007BEL\tTAB\u001bafter\u007fdel" + LogSanitizer.sanitize(payload) shouldBe "beforeNULBEL\tTABafterdel" + } + + it should "handle empty / null bodies cleanly" in { + LogSanitizer.sanitize("") shouldBe "" + LogSanitizer.sanitize(null) shouldBe "" + } + + // ----- sanitize: secret scrubbing ------------------------------------ + + it should "redact Bearer tokens regardless of case" in { + LogSanitizer.sanitize("Authorization: Bearer abc123XYZ.foo") should include("[REDACTED]") + LogSanitizer.sanitize("Authorization: Bearer abc123XYZ.foo") should not include "abc123XYZ" + LogSanitizer.sanitize("auth = bearer eyJhbGci.tok") should include("[REDACTED]") + } + + it should "redact password=... key/value forms" in { + val out = LogSanitizer.sanitize("connecting: user=alice password=hunter2 host=db") + out should include("[REDACTED]") + out should not include "hunter2" + // surrounding context preserved + out should include("user=alice") + out should include("host=db") + } + + it should "redact AWS access key IDs" in { + val out = LogSanitizer.sanitize("found key AKIAIOSFODNN7EXAMPLE in env") + out should include("[REDACTED]") + out should not include "AKIAIOSFODNN7EXAMPLE" + } + + it should "redact AWS secret access keys when explicitly labelled" in { + val out = LogSanitizer.sanitize( + "aws_secret_access_key=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY some text" + ) + out should include("[REDACTED]") + out should not include "wJalrXUtnFEMI" + } + + it should "leave already-redacted content alone (idempotent)" in { + val once = LogSanitizer.sanitize("Authorization: Bearer abc12345.deadbeef") + val twice = LogSanitizer.sanitize(once) + twice shouldBe once + } + + // ----- sanitize: size cap -------------------------------------------- + + it should "truncate bodies larger than MaxBodyBytes and append the marker" in { + val oversize = "a" * (LogSanitizer.MaxBodyBytes * 4) // ~64 KiB + val out = LogSanitizer.sanitize(oversize) + out.length shouldBe LogSanitizer.MaxBodyBytes + out should endWith(LogSanitizer.TruncatedMarker) + } + + it should "leave bodies at or below the cap unchanged in length" in { + val rightAtCap = "x" * LogSanitizer.MaxBodyBytes + LogSanitizer.sanitize(rightAtCap).length shouldBe LogSanitizer.MaxBodyBytes + } + + // ----- filterMdc ----------------------------------------------------- + + "filterMdc" should "drop denied Pekko keys and pass every other key through" in { + val mdc = Map( + "trace_id" -> "abc", + "span_id" -> "def", + "texera.workflow.id" -> "42", + "app.new.tag" -> "kept", + "sourceThread" -> "dispatcher-3", + "pekkoSource" -> "akka://sys/user/actor" + ).asJava + val out = LogSanitizer.filterMdc(mdc) + out.keySet shouldBe Set("trace_id", "span_id", "texera.workflow.id", "app.new.tag") + } + + it should "scrub secret-shaped values on keys that pass through" in { + val mdc = Map("note" -> "password=p4ssw0rd").asJava + LogSanitizer.filterMdc(mdc)("note") should not include "p4ssw0rd" + } + + it should "redact the value of a credential-named key even when the value itself is benign" in { + val mdc = Map( + "password" -> "hunter2", + "user.api_key" -> "abcdef", + "authToken" -> "xyz", + "trace_id" -> "keep-me" + ).asJava + val out = LogSanitizer.filterMdc(mdc) + out("password") shouldBe "[REDACTED]" + out("user.api_key") shouldBe "[REDACTED]" + out("authToken") shouldBe "[REDACTED]" + out("trace_id") shouldBe "keep-me" + } + + it should "tolerate null map and null values" in { + LogSanitizer.filterMdc(null) shouldBe empty + + val javaMap = new java.util.HashMap[String, String]() + javaMap.put("trace_id", null) + javaMap.put("texera.user.id", "7") + val out = LogSanitizer.filterMdc(javaMap) + out shouldBe Map("texera.user.id" -> "7") + } +} diff --git a/common/observability/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala b/common/observability/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala new file mode 100644 index 00000000000..6b8c9bcb437 --- /dev/null +++ b/common/observability/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala @@ -0,0 +1,263 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter +import org.scalatest.BeforeAndAfterEach +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import scala.jdk.CollectionConverters._ + +class OtelInitSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { + + override def beforeEach(): Unit = { + OtelInit.resetForTest() + } + + override def afterEach(): Unit = { + OtelInit.resetForTest() + } + + // ----- validateEndpoint: pure function, exhaustive cases ------------- + + "validateEndpoint" should "accept loopback OTLP http(s) URLs" in { + OtelInit.validateEndpoint("http://localhost:4317", OtelInit.DefaultAllowedHosts) shouldBe Right( + () + ) + OtelInit.validateEndpoint("http://127.0.0.1:4317", OtelInit.DefaultAllowedHosts) shouldBe Right( + () + ) + OtelInit.validateEndpoint( + "https://localhost:4318", + OtelInit.DefaultAllowedHosts + ) shouldBe Right(()) + } + + it should "reject a grpc:// endpoint (the OTLP exporter accepts only http/https)" in { + val result = OtelInit.validateEndpoint("grpc://127.0.0.1:4317", OtelInit.DefaultAllowedHosts) + result.isLeft shouldBe true + result.left.toOption.get should include("scheme") + } + + it should "reject file:// schemes (path traversal style attack)" in { + val result = OtelInit.validateEndpoint("file:///etc/passwd", OtelInit.DefaultAllowedHosts) + result.isLeft shouldBe true + result.left.toOption.get should include("scheme") + } + + it should "reject arbitrary remote hosts not in the allowlist" in { + val result = OtelInit.validateEndpoint( + "http://attacker.example.com:4317", + OtelInit.DefaultAllowedHosts + ) + result.isLeft shouldBe true + result.left.toOption.get should include("host") + } + + it should "accept hosts added to the allowlist" in { + val widened = OtelInit.DefaultAllowedHosts + "collector.internal" + OtelInit.validateEndpoint("http://collector.internal:4317", widened) shouldBe Right(()) + } + + it should "reject endpoints with no scheme" in { + val result = OtelInit.validateEndpoint("localhost:4317", OtelInit.DefaultAllowedHosts) + result.isLeft shouldBe true + } + + it should "reject endpoints with no host" in { + val result = OtelInit.validateEndpoint("http:///path", OtelInit.DefaultAllowedHosts) + result.isLeft shouldBe true + } + + it should "reject completely malformed input" in { + val result = OtelInit.validateEndpoint("not a uri at all :: bad", OtelInit.DefaultAllowedHosts) + result.isLeft shouldBe true + } + + // ----- buildResource: passthrough with service.name protected --------- + + "buildResource" should "always include the service.name from the argument" in { + val r = OtelInit.buildResource("my-service", "") + Option( + r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.name")) + ) shouldBe Some( + "my-service" + ) + } + + it should "honor keys from OTEL_RESOURCE_ATTRIBUTES" in { + val r = OtelInit.buildResource("svc", "service.version=1.2.3,deployment.environment=staging") + Option( + r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.version")) + ) shouldBe Some( + "1.2.3" + ) + Option( + r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("deployment.environment")) + ) shouldBe Some("staging") + } + + it should "pass custom keys through so new resource fields need no code edit" in { + val r = OtelInit.buildResource( + "svc", + "service.version=1.0,custom.tag=team-a,texera.region.id=us-west" + ) + val attrs: Map[String, String] = r.getAttributes.asMap.asScala.iterator.map { + case (k, v) => k.getKey -> v.toString + }.toMap + + attrs("service.version") shouldBe "1.0" + attrs("custom.tag") shouldBe "team-a" + attrs("texera.region.id") shouldBe "us-west" + } + + it should "refuse to let OTEL_RESOURCE_ATTRIBUTES override service.name" in { + val r = OtelInit.buildResource("real-svc", "service.name=spoofed") + Option( + r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.name")) + ) shouldBe Some( + "real-svc" + ) + } + + it should "carry the texera.* resource attrs so a CU JVM auto-tags every emitted record" in { + val r = OtelInit.buildResource( + "texera-computing-unit-master", + "texera.computing_unit.id=8,texera.workflow.id=441,texera.execution.id=1234" + ) + val attrs: Map[String, String] = r.getAttributes.asMap.asScala.iterator.map { + case (k, v) => k.getKey -> v.toString + }.toMap + attrs("texera.computing_unit.id") shouldBe "8" + attrs("texera.workflow.id") shouldBe "441" + attrs("texera.execution.id") shouldBe "1234" + } + + it should "ignore malformed pairs without crashing" in { + val r = OtelInit.buildResource("svc", ",,,=,foo,service.version=,=bar,service.version=1.0,") + Option( + r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.version")) + ) shouldBe Some( + "1.0" + ) + } + + it should "handle empty / null input cleanly" in { + OtelInit.parseAttrs("") shouldBe empty + OtelInit.parseAttrs(null) shouldBe empty + } + + // ----- end-to-end init: span emission + disable behaviour ------------- + + "init" should "be a no-op when OTEL_SDK_DISABLED is explicitly set to true" in { + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest("svc", Map("OTEL_SDK_DISABLED" -> "true"), exporter) + result shouldBe None + exporter.getFinishedSpanItems.asScala shouldBe empty + } + + it should "stay inert by default when OTEL_SDK_DISABLED is unset (issue #5367)" in { + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest( + "svc", + Map( + // OTEL_SDK_DISABLED omitted; the SDK stays disabled by default. + "OTEL_EXPORTER_OTLP_ENDPOINT" -> "http://localhost:4317" + ), + exporter + ) + result shouldBe None + exporter.getFinishedSpanItems.asScala shouldBe empty + } + + it should "stay inert for any OTEL_SDK_DISABLED value other than an explicit false" in { + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest( + "svc", + Map("OTEL_SDK_DISABLED" -> "", "OTEL_EXPORTER_OTLP_ENDPOINT" -> "http://localhost:4317"), + exporter + ) + result shouldBe None + } + + it should "emit a single service.start span when enabled with a valid endpoint" in { + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest( + "my-service", + Map( + "OTEL_SDK_DISABLED" -> "false", + "OTEL_EXPORTER_OTLP_ENDPOINT" -> "http://localhost:4317" + ), + exporter + ) + result.isDefined shouldBe true + + // BatchSpanProcessor is async; flush before reading. + result.get + .asInstanceOf[io.opentelemetry.sdk.OpenTelemetrySdk] + .getSdkTracerProvider + .forceFlush() + .join(2, java.util.concurrent.TimeUnit.SECONDS) + + val spans = exporter.getFinishedSpanItems.asScala + spans should have size 1 + spans.head.getName shouldBe "service.start" + } + + it should "refuse to initialize when the endpoint scheme is file://" in { + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest( + "svc", + Map( + "OTEL_SDK_DISABLED" -> "false", + "OTEL_EXPORTER_OTLP_ENDPOINT" -> "file:///etc/passwd" + ), + exporter + ) + result shouldBe None + exporter.getFinishedSpanItems.asScala shouldBe empty + } + + it should "refuse to initialize when the endpoint host is off-allowlist" in { + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest( + "svc", + Map( + "OTEL_SDK_DISABLED" -> "false", + "OTEL_EXPORTER_OTLP_ENDPOINT" -> "http://attacker.example.com:4317" + ), + exporter + ) + result shouldBe None + exporter.getFinishedSpanItems.asScala shouldBe empty + } + + it should "be idempotent — second init returns the same instance" in { + val exporter = InMemorySpanExporter.create() + val env = Map( + "OTEL_SDK_DISABLED" -> "false", + "OTEL_EXPORTER_OTLP_ENDPOINT" -> "http://localhost:4317" + ) + val first = OtelInit.initForTest("svc", env, exporter) + val second = OtelInit.initForTest("svc", env, exporter) + second shouldBe first + } +} diff --git a/common/observability/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala b/common/observability/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala new file mode 100644 index 00000000000..21407f13035 --- /dev/null +++ b/common/observability/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import ch.qos.logback.classic.{Level, Logger, LoggerContext} +import ch.qos.logback.classic.spi.LoggingEvent +import io.opentelemetry.api.OpenTelemetry +import io.opentelemetry.api.logs.Severity +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.logs.SdkLoggerProvider +import io.opentelemetry.sdk.logs.`export`.SimpleLogRecordProcessor +import io.opentelemetry.sdk.testing.exporter.InMemoryLogRecordExporter +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.slf4j.LoggerFactory + +import scala.jdk.CollectionConverters._ + +class TexeraOtelLogAppenderSpec extends AnyFlatSpec with Matchers { + + /** Build an OpenTelemetry SDK whose LoggerProvider drains to the + * given in-memory exporter via the synchronous SimpleLogRecordProcessor, + * so tests don't depend on batch timing. + */ + private def newFixture(): (OpenTelemetry, InMemoryLogRecordExporter, TexeraOtelLogAppender) = { + val exporter = InMemoryLogRecordExporter.create() + val lp = SdkLoggerProvider + .builder() + .addLogRecordProcessor(SimpleLogRecordProcessor.create(exporter)) + .build() + val sdk = OpenTelemetrySdk.builder().setLoggerProvider(lp).build() + val appender = new TexeraOtelLogAppender() + appender.setContext(LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext]) + appender.bind(sdk) + appender.start() + (sdk, exporter, appender) + } + + private def makeEvent( + message: String, + level: Level = Level.INFO, + mdc: Map[String, String] = Map.empty + ): LoggingEvent = { + val ctx = LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext] + val logger = ctx.getLogger("test.logger").asInstanceOf[Logger] + val ev = new LoggingEvent("fqcn", logger, level, message, null, null) + if (mdc.nonEmpty) ev.setMDCPropertyMap(mdc.asJava) + ev + } + + // ----- positive paths ------------------------------------------------- + + "TexeraOtelLogAppender" should "emit an INFO record with body + severity" in { + val (_, exporter, appender) = newFixture() + appender.doAppend(makeEvent("hello world")) + + val records = exporter.getFinishedLogRecordItems.asScala + records should have size 1 + records.head.getBodyValue.asString shouldBe "hello world" + records.head.getSeverity shouldBe Severity.INFO + records.head.getSeverityText shouldBe "INFO" + } + + it should "map every log level to a distinct OTel severity" in { + val (_, exporter, appender) = newFixture() + Seq(Level.TRACE, Level.DEBUG, Level.INFO, Level.WARN, Level.ERROR).foreach { lvl => + appender.doAppend(makeEvent(s"msg-$lvl", lvl)) + } + val severities = exporter.getFinishedLogRecordItems.asScala.map(_.getSeverity).toSet + severities shouldBe Set( + Severity.TRACE, + Severity.DEBUG, + Severity.INFO, + Severity.WARN, + Severity.ERROR + ) + } + + // ----- security: sanitisation happens at the boundary ----------------- + + it should "strip CRLF from a forged log-injection payload before emission" in { + val (_, exporter, appender) = newFixture() + appender.doAppend(makeEvent("hello\r\nFAKE LOG LINE\r\nworld")) + + val body = exporter.getFinishedLogRecordItems.asScala.head.getBodyValue.asString + body shouldBe "helloFAKE LOG LINEworld" + body should not include "\n" + body should not include "\r" + } + + it should "redact Bearer tokens at emission time" in { + val (_, exporter, appender) = newFixture() + appender.doAppend(makeEvent("Authorization: Bearer abc123XYZ.foo")) + + val body = exporter.getFinishedLogRecordItems.asScala.head.getBodyValue.asString + body should include("[REDACTED]") + body should not include "abc123XYZ" + } + + it should "redact secrets inside an attached stack trace, not just the message" in { + val (_, exporter, appender) = newFixture() + val ctx = LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext] + val logger = ctx.getLogger("test.logger").asInstanceOf[Logger] + val boom = new RuntimeException("db connect failed for password=hunter2") + val ev = new LoggingEvent("fqcn", logger, Level.ERROR, "operation failed", boom, null) + appender.doAppend(ev) + + val body = exporter.getFinishedLogRecordItems.asScala.head.getBodyValue.asString + body should include("[REDACTED]") + body should not include "hunter2" + // The stack trace's newlines are preserved (only the message is C0-stripped). + body should include("\n") + } + + it should "truncate a 1 MiB body to MaxBodyBytes with the marker" in { + val (_, exporter, appender) = newFixture() + val oversize = "x" * (1024 * 1024) + appender.doAppend(makeEvent(oversize)) + + val body = exporter.getFinishedLogRecordItems.asScala.head.getBodyValue.asString + body.length shouldBe LogSanitizer.MaxBodyBytes + body should endWith(LogSanitizer.TruncatedMarker) + } + + // ----- security: MDC deny-list ---------------------------------------- + + private def attrsOf(record: io.opentelemetry.sdk.logs.data.LogRecordData): Map[String, String] = + record.getAttributes.asMap.asScala.iterator.map { case (k, v) => k.getKey -> v.toString }.toMap + + it should "forward arbitrary correlation MDC keys (deny-list, not allow-list)" in { + val (_, exporter, appender) = newFixture() + appender.doAppend( + makeEvent( + "msg", + mdc = Map( + "trace_id" -> "abc", + "texera.workflow.id" -> "42", + "some.new.key" -> "kept" + ) + ) + ) + + val attrs = attrsOf(exporter.getFinishedLogRecordItems.asScala.head) + attrs.keySet should contain allOf ("trace_id", "texera.workflow.id", "some.new.key") + attrs("some.new.key") shouldBe "kept" + } + + it should "drop the noisy Pekko bridge MDC keys" in { + val (_, exporter, appender) = newFixture() + val denied = LogSanitizer.DeniedMdcKeys.iterator.map(_ -> "noise").toMap + appender.doAppend(makeEvent("msg", mdc = denied + ("trace_id" -> "abc"))) + + val attrs = attrsOf(exporter.getFinishedLogRecordItems.asScala.head) + attrs.keySet should contain("trace_id") + attrs.keySet should contain noElementsOf LogSanitizer.DeniedMdcKeys + } + + it should "redact a secret-shaped MDC value while keeping its key" in { + val (_, exporter, appender) = newFixture() + appender.doAppend(makeEvent("msg", mdc = Map("authz" -> "Bearer abc123XYZ.foo"))) + + val attrs = attrsOf(exporter.getFinishedLogRecordItems.asScala.head) + attrs.keySet should contain("authz") + attrs("authz") should include("[REDACTED]") + attrs("authz") should not include "abc123XYZ" + } + + it should "redact the value of a credential-named MDC key even when the value looks benign" in { + val (_, exporter, appender) = newFixture() + appender.doAppend(makeEvent("msg", mdc = Map("password" -> "p4ssw0rd", "api_key" -> "plain"))) + + val attrs = attrsOf(exporter.getFinishedLogRecordItems.asScala.head) + attrs("password") shouldBe "[REDACTED]" + attrs("api_key") shouldBe "[REDACTED]" + attrs.values should contain noElementsOf Seq("p4ssw0rd", "plain") + } + + // ----- lifecycle ------------------------------------------------------ + + it should "be a silent no-op when not yet bound to an OpenTelemetry instance" in { + val unbound = new TexeraOtelLogAppender() + unbound.setContext(LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext]) + unbound.start() + // Should not throw, even though no SDK is wired. + noException should be thrownBy unbound.doAppend(makeEvent("hello")) + } + + it should "stop emitting after stop() is called" in { + val (_, exporter, appender) = newFixture() + appender.doAppend(makeEvent("first")) + appender.stop() + appender.doAppend(makeEvent("second")) + + val bodies = exporter.getFinishedLogRecordItems.asScala.map(_.getBodyValue.asString) + bodies should contain only "first" + } +} diff --git a/computing-unit-managing-service/LICENSE-binary b/computing-unit-managing-service/LICENSE-binary index 0b8e4c1286b..bf057b8c40e 100644 --- a/computing-unit-managing-service/LICENSE-binary +++ b/computing-unit-managing-service/LICENSE-binary @@ -369,6 +369,18 @@ Scala/Java jars: - io.netty.netty-transport-native-epoll-4.2.15.Final.jar - io.netty.netty-transport-native-unix-common-4.2.15.Final.jar - org.jspecify.jspecify-1.0.0.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.perfmark.perfmark-api-0.27.0.jar - io.r2dbc.r2dbc-spi-1.0.0.RELEASE.jar - io.swagger.swagger-annotations-1.6.14.jar diff --git a/computing-unit-managing-service/src/main/resources/computing-unit-managing-service-config.yaml b/computing-unit-managing-service/src/main/resources/computing-unit-managing-service-config.yaml index 523b4197989..ea428fcadcb 100644 --- a/computing-unit-managing-service/src/main/resources/computing-unit-managing-service-config.yaml +++ b/computing-unit-managing-service/src/main/resources/computing-unit-managing-service-config.yaml @@ -30,4 +30,16 @@ server: logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: - "com.example": ${TEXERA_SERVICE_LOG_LEVEL:-DEBUG} \ No newline at end of file + "com.example": ${TEXERA_SERVICE_LOG_LEVEL:-DEBUG} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN \ No newline at end of file diff --git a/computing-unit-managing-service/src/main/scala/org/apache/texera/service/ComputingUnitManagingService.scala b/computing-unit-managing-service/src/main/scala/org/apache/texera/service/ComputingUnitManagingService.scala index f0dffc89e11..cd43b888997 100644 --- a/computing-unit-managing-service/src/main/scala/org/apache/texera/service/ComputingUnitManagingService.scala +++ b/computing-unit-managing-service/src/main/scala/org/apache/texera/service/ComputingUnitManagingService.scala @@ -53,6 +53,8 @@ class ComputingUnitManagingService extends Application[ComputingUnitManagingServ configuration: ComputingUnitManagingServiceConfiguration, environment: Environment ): Unit = { + // Bridge this service's logs to the OTel collector under its own service.name. + org.apache.texera.observability.OtelInit.init("computing-unit-managing-service") // Register http resources environment.jersey.setUrlPattern("/api/*") environment.jersey.register(classOf[HealthCheckResource]) diff --git a/config-service/LICENSE-binary b/config-service/LICENSE-binary index 93348da932e..5f93f2b6780 100644 --- a/config-service/LICENSE-binary +++ b/config-service/LICENSE-binary @@ -242,6 +242,9 @@ Scala/Java jars: - com.google.guava.listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar - com.google.j2objc.j2objc-annotations-2.8.jar - com.helger.profiler-1.1.1.jar + - com.squareup.okhttp3.okhttp-4.12.0.jar + - com.squareup.okio.okio-3.6.0.jar + - com.squareup.okio.okio-jvm-3.6.0.jar - com.thesamet.scalapb.lenses_2.13-0.11.20.jar - com.thesamet.scalapb.scalapb-json4s_2.13-0.12.0.jar - com.thesamet.scalapb.scalapb-runtime_2.13-0.11.20.jar @@ -274,6 +277,18 @@ Scala/Java jars: - io.dropwizard.metrics.metrics-json-4.2.25.jar - io.dropwizard.metrics.metrics-jvm-4.2.25.jar - io.dropwizard.metrics.metrics-logback-4.2.25.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.r2dbc.r2dbc-spi-1.0.0.RELEASE.jar - jakarta.inject.jakarta.inject-api-2.0.1.jar - jakarta.validation.jakarta.validation-api-3.0.2.jar @@ -292,6 +307,11 @@ Scala/Java jars: - org.hibernate.validator.hibernate-validator-7.0.5.Final.jar - org.javassist.javassist-3.30.2-GA.jar - org.jboss.logging.jboss-logging-3.5.3.Final.jar + - org.jetbrains.annotations-13.0.jar + - org.jetbrains.kotlin.kotlin-stdlib-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-common-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-jdk7-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-jdk8-1.9.10.jar - org.jooq.jooq-3.19.36.jar - org.json4s.json4s-ast_2.13-4.0.1.jar - org.json4s.json4s-jackson-core_2.13-4.0.1.jar diff --git a/config-service/src/main/resources/config-service-web-config.yaml b/config-service/src/main/resources/config-service-web-config.yaml index 4aa67af82e1..8559e1fd507 100644 --- a/config-service/src/main/resources/config-service-web-config.yaml +++ b/config-service/src/main/resources/config-service-web-config.yaml @@ -26,6 +26,19 @@ server: logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + loggers: + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console threshold: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} diff --git a/config-service/src/main/scala/org/apache/texera/service/ConfigService.scala b/config-service/src/main/scala/org/apache/texera/service/ConfigService.scala index a7e9b61d994..df7335d21f7 100644 --- a/config-service/src/main/scala/org/apache/texera/service/ConfigService.scala +++ b/config-service/src/main/scala/org/apache/texera/service/ConfigService.scala @@ -53,6 +53,8 @@ class ConfigService extends Application[ConfigServiceConfiguration] with LazyLog } override def run(configuration: ConfigServiceConfiguration, environment: Environment): Unit = { + // Bridge this service's logs to the OTel collector under its own service.name. + org.apache.texera.observability.OtelInit.init("config-service") // Serve backend at /api environment.jersey.setUrlPattern("/api/*") diff --git a/file-service/LICENSE-binary b/file-service/LICENSE-binary index b84e5912d67..681bf5444a9 100644 --- a/file-service/LICENSE-binary +++ b/file-service/LICENSE-binary @@ -334,6 +334,18 @@ Scala/Java jars: - io.netty.netty-transport-native-epoll-4.2.15.Final.jar - io.netty.netty-transport-native-unix-common-4.2.15.Final.jar - org.jspecify.jspecify-1.0.0.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.perfmark.perfmark-api-0.27.0.jar - io.r2dbc.r2dbc-spi-1.0.0.RELEASE.jar - jakarta.inject.jakarta.inject-api-2.0.1.jar diff --git a/file-service/src/main/resources/file-service-web-config.yaml b/file-service/src/main/resources/file-service-web-config.yaml index 41f8d1b1748..db5a7ec6645 100644 --- a/file-service/src/main/resources/file-service-web-config.yaml +++ b/file-service/src/main/resources/file-service-web-config.yaml @@ -28,6 +28,18 @@ logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console - type: file diff --git a/file-service/src/main/scala/org/apache/texera/service/FileService.scala b/file-service/src/main/scala/org/apache/texera/service/FileService.scala index 1bb29f5dab3..e0e40573659 100644 --- a/file-service/src/main/scala/org/apache/texera/service/FileService.scala +++ b/file-service/src/main/scala/org/apache/texera/service/FileService.scala @@ -62,6 +62,8 @@ class FileService extends Application[FileServiceConfiguration] with LazyLogging } override def run(configuration: FileServiceConfiguration, environment: Environment): Unit = { + // Bridge this service's logs to the OTel collector under its own service.name. + org.apache.texera.observability.OtelInit.init("file-service") // Serve backend at /api environment.jersey.setUrlPattern("/api/*") SqlServer.initConnection( diff --git a/workflow-compiling-service/LICENSE-binary b/workflow-compiling-service/LICENSE-binary index 96c723694ae..90befd23b9a 100644 --- a/workflow-compiling-service/LICENSE-binary +++ b/workflow-compiling-service/LICENSE-binary @@ -336,6 +336,18 @@ Scala/Java jars: - io.netty.netty-transport-native-epoll-4.2.15.Final.jar - io.netty.netty-transport-native-unix-common-4.2.15.Final.jar - org.jspecify.jspecify-1.0.0.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.perfmark.perfmark-api-0.27.0.jar - io.r2dbc.r2dbc-spi-1.0.0.RELEASE.jar - jakarta.inject.jakarta.inject-api-2.0.1.jar diff --git a/workflow-compiling-service/src/main/resources/workflow-compiling-service-config.yaml b/workflow-compiling-service/src/main/resources/workflow-compiling-service-config.yaml index 5b9016af1b6..37e413c15b6 100644 --- a/workflow-compiling-service/src/main/resources/workflow-compiling-service-config.yaml +++ b/workflow-compiling-service/src/main/resources/workflow-compiling-service-config.yaml @@ -28,6 +28,18 @@ logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console - type: file diff --git a/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala b/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala index a69ef545246..9938694d452 100644 --- a/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala +++ b/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala @@ -49,6 +49,8 @@ class WorkflowCompilingService extends Application[WorkflowCompilingServiceConfi configuration: WorkflowCompilingServiceConfiguration, environment: Environment ): Unit = { + // Bridge this service's logs to the OTel collector under its own service.name. + org.apache.texera.observability.OtelInit.init("workflow-compiling-service") ObjectMapperUtils.warmupObjectMapperForOperatorsSerde() // serve backend at /api