From e55ad1148290840d8e3b59e768a33cc3b79dceb1 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 5 Jun 2026 04:49:02 -0700 Subject: [PATCH 01/26] feat(observability): logging foundations - sanitizer, OTel log bridge, SDK bootstrap (default-off) Co-Authored-By: Claude Opus 4.8 (1M context) --- common/config/build.sbt | 23 +- .../texera/observability/LogSanitizer.scala | 111 +++++ .../texera/observability/OtelInit.scala | 434 ++++++++++++++++++ .../observability/TexeraOtelLogAppender.scala | 140 ++++++ .../observability/LogSanitizerSpec.scala | Bin 0 -> 4667 bytes .../texera/observability/OtelInitSpec.scala | 242 ++++++++++ .../TexeraOtelLogAppenderSpec.scala | 166 +++++++ 7 files changed, 1115 insertions(+), 1 deletion(-) create mode 100644 common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala create mode 100644 common/config/src/main/scala/org/apache/texera/observability/OtelInit.scala create mode 100644 common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala create mode 100644 common/config/src/test/scala/org/apache/texera/observability/LogSanitizerSpec.scala create mode 100644 common/config/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala create mode 100644 common/config/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala diff --git a/common/config/build.sbt b/common/config/build.sbt index bb561de4f3e..fe532ce96b6 100644 --- a/common/config/build.sbt +++ b/common/config/build.sbt @@ -48,7 +48,28 @@ Compile / scalacOptions ++= Seq( // Dependencies ///////////////////////////////////////////////////////////////////////////// +// OpenTelemetry version is pinned here as the single source of truth; all +// services pick it up transitively via dependsOn(Config). Bump deliberately. +val openTelemetryVersion = "1.50.0" + // Core Dependencies libraryDependencies ++= Seq( - "com.typesafe" % "config" % "1.4.6" // For configuration management + "com.typesafe" % "config" % "1.4.6", // For configuration management + "com.typesafe.scala-logging" %% "scala-logging" % "3.9.5", // for LazyLogging in OtelInit + // OpenTelemetry SDK bootstrap (Apache-2.0). We deliberately do NOT use + // sdk-extension-autoconfigure: the security model requires that endpoint + // + resource-attribute filtering run before any exporter is configured. + "io.opentelemetry" % "opentelemetry-api" % openTelemetryVersion, + "io.opentelemetry" % "opentelemetry-sdk" % openTelemetryVersion, + "io.opentelemetry" % "opentelemetry-exporter-otlp" % openTelemetryVersion, + // Logback Classic — needed at compile time to write the OTel log + // appender. Marked `provided` because every service already brings + // Logback in transitively (via Dropwizard / SLF4J), so we don't + // bundle a second copy. + "ch.qos.logback" % "logback-classic" % "1.2.13" % "provided", + // Test-only: in-memory exporter for OtelInitSpec; avoids hitting a real + // collector during unit tests. + "io.opentelemetry" % "opentelemetry-sdk-testing" % openTelemetryVersion % Test, + "ch.qos.logback" % "logback-classic" % "1.2.13" % Test, + "org.scalatest" %% "scalatest" % "3.2.17" % Test ) \ No newline at end of file diff --git a/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala b/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala new file mode 100644 index 00000000000..781d8bd776d --- /dev/null +++ b/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import scala.jdk.CollectionConverters._ + +/** + * Pure functions that sanitize log records before they leave the + * process via the OTel logs bridge. Lives in its own object so the + * security-critical behaviour can be unit-tested without a Logback + * fixture. + * + * Three invariants: + * 1. No control characters in the body (prevents log forging via + * CR/LF injection in user-supplied strings). + * 2. No oversized bodies (a 1 GiB log line must never reach the + * exporter). + * 3. No secrets in plain text (Bearer tokens, password=, AWS keys). + * + * Plus an MDC allowlist so accidental MDC pollution from a downstream + * library cannot leak unintended fields into the exporter. + */ +object LogSanitizer { + + /** Per-record body cap. The OTel SDK and OTLP have higher limits, + * but 16 KiB is plenty for a useful log line and protects the + * collector from a runaway log. */ + val MaxBodyBytes: Int = 16 * 1024 + + /** Suffix appended to truncated bodies. Chosen to be visually + * obvious in a UI but short enough not to dominate the cap. */ + val TruncatedMarker: String = "...[truncated]" + + /** C0 control characters except TAB (0x09). Stripping CR/LF here + * prevents log forging via newline injection in user-supplied + * message bodies. DEL (0x7F) included for the same reason. */ + private val C0ControlRegex = "[\\x00-\\x08\\x0A-\\x1F\\x7F]".r + + /** Secret patterns. Order is significant: most specific first so a + * partial match doesn't shadow a tighter pattern. */ + private val SecretPatterns: Seq[scala.util.matching.Regex] = Seq( + // Authorization: Bearer — bearer token in header form. + """(?i)Bearer\s+[A-Za-z0-9._\-/+=]{8,}""".r, + // password=…, password: … — generic credential keyvalue. + """(?i)password\s*[=:]\s*[^\s,;"']+""".r, + // AWS access key ID (canonical AKIA…16-char format). + """AKIA[0-9A-Z]{16}""".r, + // AWS secret access key, when explicitly labelled. + """(?i)aws_secret_access_key\s*[=:]\s*[A-Za-z0-9/+=]{20,}""".r + ) + + /** MDC keys we will forward to OTel log attributes. Anything else + * is dropped — additions require a code change + reviewer + * acknowledgement of the privacy implications. */ + val AllowedMdcKeys: Set[String] = Set( + "trace_id", + "span_id", + "texera.user.id", + "texera.workflow.id", + "texera.execution.id", + // Computing-unit id identifies the dev process / k8s pod that + // emitted the record. Required for the dashboard's CU-scoped + // log filter — without this key in the allowlist, the OTel + // appender silently strips it and the CU filter matches nothing. + "texera.computing_unit.id", + "texera.project.id", + "texera.operator.id" + ) + + /** Apply all three body-side transformations. Idempotent — running + * sanitize on already-sanitized output is a no-op. */ + def sanitize(body: String): String = { + if (body == null || body.isEmpty) return "" + val stripped = C0ControlRegex.replaceAllIn(body, "") + val scrubbed = SecretPatterns.foldLeft(stripped) { (acc, p) => + p.replaceAllIn(acc, "[REDACTED]") + } + truncate(scrubbed) + } + + /** Truncate to MaxBodyBytes, appending the marker if cut. */ + private def truncate(body: String): String = { + if (body.length <= MaxBodyBytes) body + else body.substring(0, MaxBodyBytes - TruncatedMarker.length) + TruncatedMarker + } + + /** Filter an MDC map to the allowlist. Null-safe. */ + def filterMdc(mdc: java.util.Map[String, String]): Map[String, String] = { + if (mdc == null) return Map.empty + mdc.asScala.iterator + .collect { case (k, v) if k != null && AllowedMdcKeys.contains(k) && v != null => k -> v } + .toMap + } +} diff --git a/common/config/src/main/scala/org/apache/texera/observability/OtelInit.scala b/common/config/src/main/scala/org/apache/texera/observability/OtelInit.scala new file mode 100644 index 00000000000..35e328c49b0 --- /dev/null +++ b/common/config/src/main/scala/org/apache/texera/observability/OtelInit.scala @@ -0,0 +1,434 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import com.typesafe.scalalogging.LazyLogging +import io.opentelemetry.api.{GlobalOpenTelemetry, OpenTelemetry} +import io.opentelemetry.api.common.{AttributeKey, Attributes} +import io.opentelemetry.exporter.otlp.logs.OtlpGrpcLogRecordExporter +import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter +import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.logs.SdkLoggerProvider +import io.opentelemetry.sdk.logs.`export`.{BatchLogRecordProcessor, LogRecordExporter} +import io.opentelemetry.sdk.metrics.SdkMeterProvider +import io.opentelemetry.sdk.metrics.`export`.{MetricExporter, PeriodicMetricReader} +import io.opentelemetry.sdk.resources.Resource +import io.opentelemetry.sdk.trace.SdkTracerProvider +import io.opentelemetry.sdk.trace.`export`.{BatchSpanProcessor, SpanExporter} + +import java.net.URI +import java.time.Duration +import scala.util.{Failure, Success, Try} + +/** + * Bootstraps the OpenTelemetry SDK for a Texera service. + * + * Design notes: + * - Default-disabled. Sets up nothing unless `OTEL_SDK_DISABLED=false`. + * - We deliberately do not use the autoconfigure SPI: the security model + * requires endpoint + resource-attribute filtering to happen BEFORE + * any exporter is constructed. Autoconfigure would parse env vars + * behind our back. + * - Validation is a single pure function so it can be unit-tested + * without spinning the SDK. + * - On any validation failure we log one WARN and return None. We + * do NOT throw — observability is opt-in plumbing; misconfiguration + * must never crash the service. + * - This is the only place in Texera that reads `OTEL_*` environment + * variables. Other modules consume the returned `OpenTelemetry` + * instance directly. + */ +object OtelInit extends LazyLogging { + + /** Resource attribute keys we accept from OTEL_RESOURCE_ATTRIBUTES. + * Resource attrs ride on every record this JVM emits (logs, + * metrics, traces) — so for a per-CU JVM (ComputingUnitMaster / + * ComputingUnitWorker) setting `texera.computing_unit.id=N` at + * boot is enough to tag every record without per-request MDC + * plumbing. Workflow/execution ids vary per task and still need + * MDC at the message boundary, but exposing them in the allowlist + * lets test harnesses + future per-task code populate them via + * the same mechanism. */ + private[observability] val AllowedResourceKeys: Set[String] = Set( + "service.name", + "service.version", + "deployment.environment", + "texera.computing_unit.id", + "texera.workflow.id", + "texera.execution.id" + ) + + /** Endpoint schemes we accept. */ + private[observability] val AllowedSchemes: Set[String] = Set("http", "https", "grpc") + + /** Hosts we accept for the OTLP endpoint by default. */ + private[observability] val DefaultAllowedHosts: Set[String] = Set( + "localhost", + "127.0.0.1", + "::1", + "[::1]" + ) + + /** Default endpoint when SDK is enabled but no endpoint set explicitly. + * Uses 127.0.0.1 (not "localhost") so a natively-run service reaches the + * IPv4-only collector port published by docker-compose — on dual-stack + * hosts "localhost" resolves to ::1 first and the OTLP export silently + * fails. Inside docker the endpoint is overridden to otel-collector:4317. */ + private val DefaultEndpoint = "http://127.0.0.1:4317" + + /** Metric export interval bounds. Values outside this range get + * clamped to the default with a one-shot WARN. The lower bound + * prevents an attacker tipping the exporter into busy-loop mode; + * the upper bound keeps metrics useful for human operators. */ + private[observability] val MinMetricIntervalMs: Long = 1000L + private[observability] val MaxMetricIntervalMs: Long = 10L * 60L * 1000L + private[observability] val DefaultMetricIntervalMs: Long = 60L * 1000L + + // Idempotency guard. The SDK installs global handlers and a shutdown + // hook; calling init() repeatedly must be a no-op after the first call. + @volatile private var initialized: Option[OpenTelemetry] = None + + /** + * Initialize the SDK for the given service name. + * Returns Some(sdk) on success, None on disabled / invalid config. + * + * Side effect when enabled: attaches a [[TexeraOtelLogAppender]] to + * the Logback ROOT logger so application logs are mirrored to the + * OTel collector, with the security guards in [[LogSanitizer]] + * applied to every record. + */ + def init(serviceName: String): Option[OpenTelemetry] = synchronized { + if (initialized.isDefined) return initialized + + val env = (key: String) => Option(System.getenv(key)) + val result = initInternal( + serviceName = serviceName, + envProvider = env, + spanExporterFactory = buildOtlpSpanExporter, + logExporterFactory = endpoint => Some(buildOtlpLogExporter(endpoint)), + metricExporterFactory = endpoint => Some(buildOtlpMetricExporter(endpoint)), + logbackAttacher = LogbackBinder.attach + ) + // Register globally so [[TexeraTracer]] and any other OTel-aware + // code can call ``GlobalOpenTelemetry.getTracer(...)`` without + // threading the SDK through every callsite. set() throws on a + // second call within the same JVM — our outer ``initialized`` + // guard makes that unreachable, but wrap defensively. The test + // path deliberately skips this so multiple isolated SDKs can be + // built within one JVM. + result.foreach { sdk => + Try(GlobalOpenTelemetry.set(sdk)).failed.foreach { t => + logger.warn( + s"GlobalOpenTelemetry already set; using the existing instance: ${t.getMessage}" + ) + } + } + result + } + + /** + * Test-only entry point. Allows the test to inject an env-var map + * and a span exporter so the SDK does not attempt a real network + * connection. The Logback appender is NOT attached in tests — + * appender tests construct it directly with an in-memory log + * exporter. + */ + private[observability] def initForTest( + serviceName: String, + envOverride: Map[String, String], + exporter: SpanExporter, + metricExporter: Option[MetricExporter] = None + ): Option[OpenTelemetry] = synchronized { + initInternal( + serviceName = serviceName, + envProvider = envOverride.get, + spanExporterFactory = _ => exporter, + logExporterFactory = _ => None, + metricExporterFactory = _ => metricExporter, + logbackAttacher = (_, _) => () // no-op in tests + ) + } + + /** Test-only: forget any previously-installed SDK. Does not unregister + * shutdown hooks (the previous SDK is closed instead). */ + private[observability] def resetForTest(): Unit = synchronized { + initialized.foreach { + case sdk: OpenTelemetrySdk => + Try(sdk.getSdkTracerProvider.close()) + Try(sdk.getSdkLoggerProvider.close()) + Try(sdk.getSdkMeterProvider.close()) + case _ => () + } + initialized = None + } + + private def initInternal( + serviceName: String, + envProvider: String => Option[String], + spanExporterFactory: String => SpanExporter, + logExporterFactory: String => Option[LogRecordExporter], + metricExporterFactory: String => Option[MetricExporter], + logbackAttacher: (String, OpenTelemetry) => Unit + ): Option[OpenTelemetry] = { + if (initialized.isDefined) return initialized + + // Default to ENABLED so an `sbt run` of any Texera service emits + // telemetry without per-JVM env-var configuration. Operators who + // need to silence telemetry (CI, embedded-tests, security-locked + // deployments) set OTEL_SDK_DISABLED=true explicitly. If the + // configured endpoint isn't reachable, the OTel SDK's + // BatchProcessor logs a single error and drops records — it + // does NOT crash the host service, so a missing collector at + // dev time is a quiet no-op rather than a startup failure. + val disabled = envProvider("OTEL_SDK_DISABLED").getOrElse("false") + if (disabled.equalsIgnoreCase("true")) { + logger.info("OpenTelemetry SDK disabled (OTEL_SDK_DISABLED=true). No telemetry will be emitted.") + return None + } + + val endpoint = envProvider("OTEL_EXPORTER_OTLP_ENDPOINT").getOrElse(DefaultEndpoint) + val extraAllowed = envProvider("TEXERA_OTEL_ALLOWED_HOSTS") + .map(_.split(',').iterator.map(_.trim.toLowerCase).filter(_.nonEmpty).toSet) + .getOrElse(Set.empty) + val allowedHosts = DefaultAllowedHosts ++ extraAllowed + + validateEndpoint(endpoint, allowedHosts) match { + case Left(reason) => + // One WARN, no further detail (endpoint is not echoed beyond what + // the operator already knows). No spans will be emitted. + logger.warn( + s"OpenTelemetry SDK disabled: invalid OTEL_EXPORTER_OTLP_ENDPOINT — $reason. " + + "Set TEXERA_OTEL_ALLOWED_HOSTS to extend the allowlist." + ) + return None + case Right(_) => // ok + } + + val rawAttrs = envProvider("OTEL_RESOURCE_ATTRIBUTES").getOrElse("") + val resource = buildResource(serviceName, rawAttrs) + + val spanExporter = spanExporterFactory(endpoint) + val tracerProvider = SdkTracerProvider + .builder() + .setResource(resource) + .addSpanProcessor(BatchSpanProcessor.builder(spanExporter).build()) + .build() + + val sdkBuilder = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider) + + // Logger provider is optional — controlled by the factory. Skipped + // in tests so the appender path can be exercised independently. + val loggerProviderOpt = logExporterFactory(endpoint).map { logExporter => + val lp = SdkLoggerProvider + .builder() + .setResource(resource) + .addLogRecordProcessor(BatchLogRecordProcessor.builder(logExporter).build()) + .build() + sdkBuilder.setLoggerProvider(lp) + lp + } + + // Meter provider is optional too. Export interval is clamped to + // [MinMetricIntervalMs, MaxMetricIntervalMs]; an out-of-range + // value gets reset to the default with one WARN — keeps an + // attacker from coaxing the reader into busy-loop mode by + // setting OTEL_METRIC_EXPORT_INTERVAL to a tiny value. + val intervalMs = clampIntervalMs(envProvider("OTEL_METRIC_EXPORT_INTERVAL")) + val meterProviderOpt = metricExporterFactory(endpoint).map { metricExporter => + val reader = PeriodicMetricReader + .builder(metricExporter) + .setInterval(Duration.ofMillis(intervalMs)) + .build() + val mp = SdkMeterProvider + .builder() + .setResource(resource) + .registerMetricReader(reader) + .build() + sdkBuilder.setMeterProvider(mp) + mp + } + + val sdk = sdkBuilder.build() + + // One startup span. Carries only service.name (no env, host, or + // version data beyond the allowlisted resource attrs). + val span = sdk.getTracer("texera.bootstrap").spanBuilder("service.start").startSpan() + Try(span.setAttribute("service.name", serviceName)) + span.end() + + // Wire the Logback appender so subsequent application logs flow to + // the collector with sanitisation applied. Failure here must never + // crash the service — observability is opt-in. + Try(logbackAttacher(serviceName, sdk)).failed.foreach { t => + logger.warn(s"Failed to attach OTel Logback appender (logs not exported): ${t.getMessage}") + } + + // Make sure providers flush on shutdown. We add the hook only after + // the SDK has been fully built so a panic during init doesn't leave + // a dangling hook pointing at a half-constructed provider. + Runtime.getRuntime.addShutdownHook(new Thread(() => { + Try(tracerProvider.close()) + loggerProviderOpt.foreach(lp => Try(lp.close())) + meterProviderOpt.foreach(mp => Try(mp.close())) + () + }, "otel-shutdown")) + + initialized = Some(sdk) + logger.info(s"OpenTelemetry SDK initialized for service '$serviceName' (endpoint=$endpoint).") + initialized + } + + /** + * Validate that the endpoint is parseable, uses an allowlisted scheme, + * and resolves to an allowlisted host. Pure function — safe to test + * without standing up the SDK. + */ + private[observability] def validateEndpoint( + endpoint: String, + allowedHosts: Set[String] + ): Either[String, Unit] = { + Try(URI.create(endpoint)) match { + case Failure(e) => + Left(s"unparseable URI (${e.getClass.getSimpleName})") + case Success(uri) => + val scheme = Option(uri.getScheme).map(_.toLowerCase).getOrElse("") + if (scheme.isEmpty) { + Left("missing scheme") + } else if (!AllowedSchemes.contains(scheme)) { + Left(s"scheme '$scheme' not in allowlist ${AllowedSchemes.toSeq.sorted.mkString("{", ",", "}")}") + } else { + val host = Option(uri.getHost).map(_.toLowerCase).getOrElse("") + if (host.isEmpty) { + Left("missing host") + } else if (!allowedHosts.contains(host)) { + Left(s"host '$host' not in allowlist") + } else { + Right(()) + } + } + } + } + + /** + * Build a Resource from the service name plus the allowlisted subset + * of OTEL_RESOURCE_ATTRIBUTES. Unknown keys are dropped silently; + * service.name from env is ignored in favour of the argument. + */ + private[observability] def buildResource(serviceName: String, rawAttrs: String): Resource = { + val builder = Attributes.builder() + builder.put(AttributeKey.stringKey("service.name"), serviceName) + + parseAttrs(rawAttrs).foreach { + case (key, value) if AllowedResourceKeys.contains(key) && key != "service.name" => + builder.put(AttributeKey.stringKey(key), value) + case _ => // dropped — not in allowlist or overrides service.name + } + + Resource.create(builder.build()) + } + + /** Parse a `k1=v1,k2=v2` string. Malformed entries are skipped. */ + private[observability] def parseAttrs(raw: String): Seq[(String, String)] = { + if (raw == null || raw.isEmpty) return Seq.empty + raw + .split(',') + .iterator + .map(_.trim) + .filter(_.nonEmpty) + .flatMap { entry => + val idx = entry.indexOf('=') + if (idx <= 0 || idx == entry.length - 1) None + else Some(entry.substring(0, idx).trim -> entry.substring(idx + 1).trim) + } + .toSeq + } + + private def buildOtlpSpanExporter(endpoint: String): SpanExporter = + OtlpGrpcSpanExporter.builder().setEndpoint(endpoint).build() + + private def buildOtlpLogExporter(endpoint: String): LogRecordExporter = + OtlpGrpcLogRecordExporter.builder().setEndpoint(endpoint).build() + + private def buildOtlpMetricExporter(endpoint: String): MetricExporter = + OtlpGrpcMetricExporter.builder().setEndpoint(endpoint).build() + + /** + * Parse and clamp OTEL_METRIC_EXPORT_INTERVAL (milliseconds). + * Out-of-range or unparseable input falls back to the default and + * emits a single WARN. Pure-ish — easy to test without standing up + * the meter SDK. + */ + private[observability] def clampIntervalMs(raw: Option[String]): Long = { + raw match { + case None => DefaultMetricIntervalMs + case Some(value) => + Try(value.trim.toLong) match { + case Failure(_) => + logger.warn( + s"OTEL_METRIC_EXPORT_INTERVAL '$value' is not a number; " + + s"using default ${DefaultMetricIntervalMs}ms." + ) + DefaultMetricIntervalMs + case Success(ms) if ms < MinMetricIntervalMs || ms > MaxMetricIntervalMs => + logger.warn( + s"OTEL_METRIC_EXPORT_INTERVAL=${ms}ms out of range " + + s"[${MinMetricIntervalMs}, ${MaxMetricIntervalMs}]; " + + s"using default ${DefaultMetricIntervalMs}ms." + ) + DefaultMetricIntervalMs + case Success(ms) => ms + } + } + } +} + +/** + * Hides the Logback attach step behind a small object so [[OtelInit]] + * doesn't import Logback types directly (keeps the SDK init testable + * without a Logback dependency on the classpath in test runs that + * inject a mock attacher). + */ +private[observability] object LogbackBinder extends LazyLogging { + + /** Attempts to find the Logback ROOT logger, attach a fresh + * [[TexeraOtelLogAppender]] bound to `otel`, and start it. If + * Logback is not the active SLF4J binding (or for any other + * classpath issue), emits one WARN and returns — never throws. + */ + def attach(serviceName: String, otel: OpenTelemetry): Unit = { + val factory = org.slf4j.LoggerFactory.getILoggerFactory + factory match { + case ctx: ch.qos.logback.classic.LoggerContext => + val root = ctx.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME) + val appender = new TexeraOtelLogAppender() + appender.setContext(ctx) + appender.setName(s"texera-otel-$serviceName") + appender.bind(otel) + appender.start() + root.addAppender(appender) + case other => + logger.warn( + s"SLF4J binding is not Logback (${other.getClass.getName}); " + + "OTel log export is not wired. Application logs to stdout/file are unaffected." + ) + } + } +} diff --git a/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala b/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala new file mode 100644 index 00000000000..37d04a303e6 --- /dev/null +++ b/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import ch.qos.logback.classic.Level +import ch.qos.logback.classic.spi.{ILoggingEvent, IThrowableProxy, ThrowableProxyUtil} +import ch.qos.logback.core.UnsynchronizedAppenderBase +import io.opentelemetry.api.OpenTelemetry +import io.opentelemetry.api.common.AttributeKey +import io.opentelemetry.api.logs.{Logger, Severity} +import io.opentelemetry.api.trace.Span +import io.opentelemetry.context.Context + +import java.util.concurrent.TimeUnit + +/** + * Logback appender that forwards every event through [[LogSanitizer]] + * before emitting it as an OTel LogRecord. + * + * Lifecycle: + * - Construct with no args (Logback / programmatic instantiation). + * - Call [[bind]] once with the active [[OpenTelemetry]] instance + * (done by [[OtelInit]] after the SDK is built). Until then, + * [[append]] is a silent no-op — log events keep flowing to + * stdout/file unimpeded. + * - Stopping the appender unbinds; subsequent events drop. + * + * This is intentionally a thin shim. All security-critical logic + * lives in [[LogSanitizer]] so it can be tested without a Logback + * fixture. + */ +class TexeraOtelLogAppender extends UnsynchronizedAppenderBase[ILoggingEvent] { + + // @volatile so a late [[bind]] is visible to appender threads + // without taking a lock on the hot path. + @volatile private var otelLogger: Option[Logger] = None + + def bind(otel: OpenTelemetry): Unit = { + otelLogger = Some(otel.getLogsBridge.get("texera.logback")) + } + + override def stop(): Unit = { + otelLogger = None + super.stop() + } + + override def append(event: ILoggingEvent): Unit = { + otelLogger match { + case None => () // disabled or not yet wired + case Some(logger) => + try { + emit(logger, event) + } catch { + // Logback's addStatus contract: errors from inside an + // appender must not throw out into the calling thread. + case t: Throwable => + addError("OTel log emission failed", t) + } + } + } + + private def emit(logger: Logger, event: ILoggingEvent): Unit = { + // Append the throwable's full stack trace to the body when one is + // attached. Without this, Dropwizard's LoggingExceptionMapper logs + // "Error handling a request: " and the exception itself never + // reaches the observability backend — making 500s impossible to + // diagnose from the dashboard. ThrowableProxyUtil emits a Logback- + // formatted trace that fits inside a single log record. + val baseBody = LogSanitizer.sanitize(event.getFormattedMessage) + val body = Option(event.getThrowableProxy) match { + case Some(proxy) => + // JVM-generated stack frames are trusted (not user input), so + // we skip the C0 strip that would collapse newlines and ruin + // readability. We do still cap the total length implicitly + // via the OTel SDK's per-record body limit, and the body + // stays valid UTF-8 because Logback emits ASCII frame text. + baseBody + "\n" + formatThrowable(proxy) + case None => baseBody + } + val builder = logger + .logRecordBuilder() + .setBody(body) + .setSeverity(severityFromLevel(event.getLevel)) + .setSeverityText(event.getLevel.toString) + .setTimestamp(event.getTimeStamp, TimeUnit.MILLISECONDS) + + // MDC subset: typed AttributeKeys only, so no string injection + // path exists for downstream consumers. + LogSanitizer.filterMdc(event.getMDCPropertyMap).foreach { + case (k, v) => builder.setAttribute(AttributeKey.stringKey(k), v) + } + + builder.setAttribute(AttributeKey.stringKey("logger.name"), event.getLoggerName) + builder.setAttribute(AttributeKey.stringKey("thread.name"), event.getThreadName) + + // Attach the current trace context so the SDK populates trace_id / + // span_id on the LogRecord automatically when a span is active. + val span = Span.current() + if (span.getSpanContext.isValid) { + builder.setContext(Context.current()) + } + + builder.emit() + } + + /** Pretty-print a Logback throwable proxy. Matches what Logback's + * default pattern layout would produce for `%ex` — class name, + * message, full stack frames, then walks the cause chain. */ + private def formatThrowable(proxy: IThrowableProxy): String = + ThrowableProxyUtil.asString(proxy) + + private def severityFromLevel(level: Level): Severity = { + if (level == null) return Severity.UNDEFINED_SEVERITY_NUMBER + level.toInt match { + case Level.TRACE_INT => Severity.TRACE + case Level.DEBUG_INT => Severity.DEBUG + case Level.INFO_INT => Severity.INFO + case Level.WARN_INT => Severity.WARN + case Level.ERROR_INT => Severity.ERROR + case _ => Severity.UNDEFINED_SEVERITY_NUMBER + } + } +} diff --git a/common/config/src/test/scala/org/apache/texera/observability/LogSanitizerSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/LogSanitizerSpec.scala new file mode 100644 index 0000000000000000000000000000000000000000..c5957b65e4b163a89ae7681b69a223339f568d9e GIT binary patch literal 4667 zcmb7I(Qezu675?7{X$;`e3Ao;N>ld+NZ|sqETxKNDX^3#Zc`X5awV~*cG>09vRdS} zUvod~eo44yvq)KH-R+F7eY9%7TuaTbGr7)7Z zI#Z#rkycc1jk+gJyEmbnt`WxYzE` zG>YB+FiP`%a9|nPIo3oeEw#C)DWF6cMr>Nd6e+w|I7is9kuzb!NX=k6S(#`yx1^U! znsgo|us7yVM%~hxGzTutBG#48icl(17i65$j{xG3U--|djsIuT$X%oX3c``wAlx-gHfjepNlQ;R4mgJT>X|s1`JPDBog=(hzUH6#gg;q z(&W+)AC?AutJx_VOiKw=layzlDJvaB;MD~nO(@>(wMNt%d34bl^+u=kzBm41_-;(^ zTQ@hY!MN8M(eQ@a!@*^5+#3$?)TP$omUmwD2A8KKAqNOeJ|>2PfTt1jDZ`4IN*`D& zMV8V;2GKMM5T2T4VkT*(AEZ$%l|-6FlyYKII2l45i^w`jrEK@J*nGTg)@tDTo`Jx2 zNxYSh(g6z3| z=Q@i+|GIcy13OUv*YMtN>o}GHmquGFglBo%U9A?x0$la=Y$Oyi`MWf14kTKs!jvj5 z~|q9 z=pB_~8fnIoggO^UGt(Rh#bt$@;z5v-ON#Xj$x_ney7~j?L=Wf*fr+PY#VXb!q;vA- zGLH2}^HFtMR~_mPUsJy~=0`ZubW-?xAq&JUv^CNv4l=Sk!D*-2 z_Vazbp*s?58H$lKZe8pIlWYQ+NCYpx7`*F$dC}>Av{?L=n1cL2Lm3|nqrj+!kYfR= zL~=j{&TbHMR0a32MPgUfB$dT6O>`K^l!90a6|X8z?`P8Us@9J=%jmmnokM$Mn99IN zkR6z8GT~A_B)#hUoeZG^Lcw$)ajcC1351tu35>-*ao8U0z%_Tf~ZA>6& zs>#1<*JbP8Y1`IhD!?dJ9z?OgT)^X=P;@OGFLb!NSYgs~6CK7|3_5OrUI=q9%`R=2 zl~Q&JQ!=)eCtqpRq`Y3-j{7>}Bazm||6OEp8oT{Cvjp2M$* z4spE42)ym!xF93EcG0n!-J;8tRh-&_r&09!ST7wZoD0wi?q*OMIzsN=4omJj)v|3Y z22YR7({Z=AzOIL=4a~`tXmAm&)r4CWn*MbdJV`{Qm>O;tU1My9(a`9GdnIlbnN3k> zjvs*uav9@NN_C-Z)QI)0kq@E|k)Z3#HYJ$6GakHmD{=u3&!Nbq?lOt4WHB+zy9f=f z@x42pJ>09@kC!Gwt-pk*#LX8s>Ev5-(KOI|8_4SYW>I+}=F5^Z9KX*^xcOdOPxu}^ zML>S(!`X|?8iQF+FMKOzIf{IUirMnS7wH@EAe^%Y6u)M)j<`2!2caXL)7%9H{Q(d4 z4G*Wd0~CSd&pl3VfM7O5P3J(aE%Fsv=}w&gm%?xg?BD9%XB3gQp@TdZ)V?aQI>rSK zUB%6~3n-m0gzh0XT`7qzb#DUSnP1(|%xcT;CclC}hyjw|3k*N`kWK9WAksPB|2)EF z2p4V2fB#a?LoasuNw&?Kv*ob2SLD}F9zQt8AoXRXr*kve>F68p3!F9Mo1C$P%jA^E MPQf|$r`o65-%WVrbpQYW literal 0 HcmV?d00001 diff --git a/common/config/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala new file mode 100644 index 00000000000..5ebd0eb3f4a --- /dev/null +++ b/common/config/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter +import org.scalatest.BeforeAndAfterEach +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import scala.jdk.CollectionConverters._ + +class OtelInitSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { + + override def beforeEach(): Unit = { + OtelInit.resetForTest() + } + + override def afterEach(): Unit = { + OtelInit.resetForTest() + } + + // ----- validateEndpoint: pure function, exhaustive cases ------------- + + "validateEndpoint" should "accept a loopback OTLP gRPC URL" in { + OtelInit.validateEndpoint("http://localhost:4317", OtelInit.DefaultAllowedHosts) shouldBe Right(()) + OtelInit.validateEndpoint("grpc://127.0.0.1:4317", OtelInit.DefaultAllowedHosts) shouldBe Right(()) + OtelInit.validateEndpoint("https://localhost:4318", OtelInit.DefaultAllowedHosts) shouldBe Right(()) + } + + it should "reject file:// schemes (path traversal style attack)" in { + val result = OtelInit.validateEndpoint("file:///etc/passwd", OtelInit.DefaultAllowedHosts) + result.isLeft shouldBe true + result.left.toOption.get should include("scheme") + } + + it should "reject arbitrary remote hosts not in the allowlist" in { + val result = OtelInit.validateEndpoint( + "http://attacker.example.com:4317", + OtelInit.DefaultAllowedHosts + ) + result.isLeft shouldBe true + result.left.toOption.get should include("host") + } + + it should "accept hosts added to the allowlist" in { + val widened = OtelInit.DefaultAllowedHosts + "collector.internal" + OtelInit.validateEndpoint("http://collector.internal:4317", widened) shouldBe Right(()) + } + + it should "reject endpoints with no scheme" in { + val result = OtelInit.validateEndpoint("localhost:4317", OtelInit.DefaultAllowedHosts) + result.isLeft shouldBe true + } + + it should "reject endpoints with no host" in { + val result = OtelInit.validateEndpoint("http:///path", OtelInit.DefaultAllowedHosts) + result.isLeft shouldBe true + } + + it should "reject completely malformed input" in { + val result = OtelInit.validateEndpoint("not a uri at all :: bad", OtelInit.DefaultAllowedHosts) + result.isLeft shouldBe true + } + + // ----- buildResource: only allowlisted keys survive ------------------- + + "buildResource" should "always include the service.name from the argument" in { + val r = OtelInit.buildResource("my-service", "") + Option(r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.name"))) shouldBe Some( + "my-service" + ) + } + + it should "honor allowlisted keys from OTEL_RESOURCE_ATTRIBUTES" in { + val r = OtelInit.buildResource("svc", "service.version=1.2.3,deployment.environment=staging") + Option(r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.version"))) shouldBe Some( + "1.2.3" + ) + Option( + r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("deployment.environment")) + ) shouldBe Some("staging") + } + + it should "silently drop keys not in the allowlist (anti-injection)" in { + val r = OtelInit.buildResource( + "svc", + "service.version=1.0,secret=hunter2,db.password=p4ssw0rd,custom.tag=evil" + ) + val attrs: Map[String, String] = r.getAttributes.asMap.asScala.iterator.map { + case (k, v) => k.getKey -> v.toString + }.toMap + + attrs.keySet should contain only ("service.name", "service.version") + attrs should not contain key("secret") + attrs should not contain key("db.password") + attrs should not contain key("custom.tag") + } + + it should "refuse to let OTEL_RESOURCE_ATTRIBUTES override service.name" in { + val r = OtelInit.buildResource("real-svc", "service.name=spoofed") + Option(r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.name"))) shouldBe Some( + "real-svc" + ) + } + + it should "carry the texera.* resource attrs so a CU JVM auto-tags every emitted record" in { + // Setting the CU id at JVM boot is the only sane place to attach + // it for ComputingUnitMaster / ComputingUnitWorker — those JVMs + // are CU-scoped by deployment, and there is no HTTP request to + // hang an MDC value off. The dashboard's CU filter relies on + // this key being present on every record. + val r = OtelInit.buildResource( + "texera-computing-unit-master", + "texera.computing_unit.id=8,texera.workflow.id=441,texera.execution.id=1234" + ) + val attrs: Map[String, String] = r.getAttributes.asMap.asScala.iterator.map { + case (k, v) => k.getKey -> v.toString + }.toMap + attrs("texera.computing_unit.id") shouldBe "8" + attrs("texera.workflow.id") shouldBe "441" + attrs("texera.execution.id") shouldBe "1234" + } + + it should "ignore malformed pairs without crashing" in { + val r = OtelInit.buildResource("svc", ",,,=,foo,service.version=,=bar,service.version=1.0,") + Option(r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.version"))) shouldBe Some( + "1.0" + ) + } + + it should "handle empty / null input cleanly" in { + OtelInit.parseAttrs("") shouldBe empty + OtelInit.parseAttrs(null) shouldBe empty + } + + // ----- end-to-end init: span emission + disable behaviour ------------- + + "init" should "be a no-op when OTEL_SDK_DISABLED is explicitly set to true" in { + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest("svc", Map("OTEL_SDK_DISABLED" -> "true"), exporter) + result shouldBe None + exporter.getFinishedSpanItems.asScala shouldBe empty + } + + it should "initialize by default (no OTEL_SDK_DISABLED set) so `sbt run` services emit without per-JVM config" in { + // The previous default was `true` (opt-in), which forced every + // service Run Configuration to set OTEL_SDK_DISABLED=false + // explicitly. New default is `false` so a fresh sbt run is + // immediately tagged in the dashboard. Operators who need to + // silence telemetry still set the env var explicitly. + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest( + "svc", + Map( + // OTEL_SDK_DISABLED deliberately omitted — defaults to false. + "OTEL_EXPORTER_OTLP_ENDPOINT" -> "http://localhost:4317" + ), + exporter + ) + result.isDefined shouldBe true + } + + it should "emit a single service.start span when enabled with a valid endpoint" in { + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest( + "my-service", + Map( + "OTEL_SDK_DISABLED" -> "false", + "OTEL_EXPORTER_OTLP_ENDPOINT" -> "http://localhost:4317" + ), + exporter + ) + result.isDefined shouldBe true + + // BatchSpanProcessor is async — flush before reading. + result.get + .asInstanceOf[io.opentelemetry.sdk.OpenTelemetrySdk] + .getSdkTracerProvider + .forceFlush() + .join(2, java.util.concurrent.TimeUnit.SECONDS) + + val spans = exporter.getFinishedSpanItems.asScala + spans should have size 1 + spans.head.getName shouldBe "service.start" + } + + it should "refuse to initialize when the endpoint scheme is file://" in { + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest( + "svc", + Map( + "OTEL_SDK_DISABLED" -> "false", + "OTEL_EXPORTER_OTLP_ENDPOINT" -> "file:///etc/passwd" + ), + exporter + ) + result shouldBe None + exporter.getFinishedSpanItems.asScala shouldBe empty + } + + it should "refuse to initialize when the endpoint host is off-allowlist" in { + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest( + "svc", + Map( + "OTEL_SDK_DISABLED" -> "false", + "OTEL_EXPORTER_OTLP_ENDPOINT" -> "http://attacker.example.com:4317" + ), + exporter + ) + result shouldBe None + exporter.getFinishedSpanItems.asScala shouldBe empty + } + + it should "be idempotent — second init returns the same instance" in { + val exporter = InMemorySpanExporter.create() + val env = Map( + "OTEL_SDK_DISABLED" -> "false", + "OTEL_EXPORTER_OTLP_ENDPOINT" -> "http://localhost:4317" + ) + val first = OtelInit.initForTest("svc", env, exporter) + val second = OtelInit.initForTest("svc", env, exporter) + second shouldBe first + } +} diff --git a/common/config/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala new file mode 100644 index 00000000000..83b9ca9d469 --- /dev/null +++ b/common/config/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import ch.qos.logback.classic.{Level, Logger, LoggerContext} +import ch.qos.logback.classic.spi.LoggingEvent +import io.opentelemetry.api.OpenTelemetry +import io.opentelemetry.api.logs.Severity +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.logs.SdkLoggerProvider +import io.opentelemetry.sdk.logs.`export`.SimpleLogRecordProcessor +import io.opentelemetry.sdk.testing.exporter.InMemoryLogRecordExporter +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.slf4j.LoggerFactory + +import scala.jdk.CollectionConverters._ + +class TexeraOtelLogAppenderSpec extends AnyFlatSpec with Matchers { + + /** Build an OpenTelemetry SDK whose LoggerProvider drains to the + * given in-memory exporter via the synchronous SimpleLogRecordProcessor, + * so tests don't depend on batch timing. */ + private def newFixture(): (OpenTelemetry, InMemoryLogRecordExporter, TexeraOtelLogAppender) = { + val exporter = InMemoryLogRecordExporter.create() + val lp = SdkLoggerProvider + .builder() + .addLogRecordProcessor(SimpleLogRecordProcessor.create(exporter)) + .build() + val sdk = OpenTelemetrySdk.builder().setLoggerProvider(lp).build() + val appender = new TexeraOtelLogAppender() + appender.setContext(LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext]) + appender.bind(sdk) + appender.start() + (sdk, exporter, appender) + } + + private def makeEvent( + message: String, + level: Level = Level.INFO, + mdc: Map[String, String] = Map.empty + ): LoggingEvent = { + val ctx = LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext] + val logger = ctx.getLogger("test.logger").asInstanceOf[Logger] + val ev = new LoggingEvent("fqcn", logger, level, message, null, null) + if (mdc.nonEmpty) ev.setMDCPropertyMap(mdc.asJava) + ev + } + + // ----- positive paths ------------------------------------------------- + + "TexeraOtelLogAppender" should "emit an INFO record with body + severity" in { + val (_, exporter, appender) = newFixture() + appender.doAppend(makeEvent("hello world")) + + val records = exporter.getFinishedLogRecordItems.asScala + records should have size 1 + records.head.getBodyValue.asString shouldBe "hello world" + records.head.getSeverity shouldBe Severity.INFO + records.head.getSeverityText shouldBe "INFO" + } + + it should "map every log level to a distinct OTel severity" in { + val (_, exporter, appender) = newFixture() + Seq(Level.TRACE, Level.DEBUG, Level.INFO, Level.WARN, Level.ERROR).foreach { lvl => + appender.doAppend(makeEvent(s"msg-$lvl", lvl)) + } + val severities = exporter.getFinishedLogRecordItems.asScala.map(_.getSeverity).toSet + severities shouldBe Set(Severity.TRACE, Severity.DEBUG, Severity.INFO, Severity.WARN, Severity.ERROR) + } + + // ----- security: sanitisation happens at the boundary ----------------- + + it should "strip CRLF from a forged log-injection payload before emission" in { + val (_, exporter, appender) = newFixture() + appender.doAppend(makeEvent("hello\r\nFAKE LOG LINE\r\nworld")) + + val body = exporter.getFinishedLogRecordItems.asScala.head.getBodyValue.asString + body shouldBe "helloFAKE LOG LINEworld" + body should not include "\n" + body should not include "\r" + } + + it should "redact Bearer tokens at emission time" in { + val (_, exporter, appender) = newFixture() + appender.doAppend(makeEvent("Authorization: Bearer abc123XYZ.foo")) + + val body = exporter.getFinishedLogRecordItems.asScala.head.getBodyValue.asString + body should include("[REDACTED]") + body should not include "abc123XYZ" + } + + it should "truncate a 1 MiB body to MaxBodyBytes with the marker" in { + val (_, exporter, appender) = newFixture() + val oversize = "x" * (1024 * 1024) + appender.doAppend(makeEvent(oversize)) + + val body = exporter.getFinishedLogRecordItems.asScala.head.getBodyValue.asString + body.length shouldBe LogSanitizer.MaxBodyBytes + body should endWith(LogSanitizer.TruncatedMarker) + } + + // ----- security: MDC allowlist ---------------------------------------- + + it should "forward only allowlisted MDC keys as log attributes" in { + val (_, exporter, appender) = newFixture() + appender.doAppend( + makeEvent( + "msg", + mdc = Map( + "trace_id" -> "abc", + "texera.workflow.id" -> "42", + "secret" -> "should-not-leak", + "password" -> "p4ssw0rd" + ) + ) + ) + + val record = exporter.getFinishedLogRecordItems.asScala.head + val attrs = record.getAttributes.asMap.asScala.iterator.map { + case (k, v) => k.getKey -> v.toString + }.toMap + + attrs.keySet should contain allOf ("trace_id", "texera.workflow.id") + attrs.keySet should not contain ("secret") + attrs.keySet should not contain ("password") + attrs.values should contain noElementsOf Seq("should-not-leak", "p4ssw0rd") + } + + // ----- lifecycle ------------------------------------------------------ + + it should "be a silent no-op when not yet bound to an OpenTelemetry instance" in { + val unbound = new TexeraOtelLogAppender() + unbound.setContext(LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext]) + unbound.start() + // Should not throw, even though no SDK is wired. + noException should be thrownBy unbound.doAppend(makeEvent("hello")) + } + + it should "stop emitting after stop() is called" in { + val (_, exporter, appender) = newFixture() + appender.doAppend(makeEvent("first")) + appender.stop() + appender.doAppend(makeEvent("second")) + + val bodies = exporter.getFinishedLogRecordItems.asScala.map(_.getBodyValue.asString) + bodies should contain only "first" + } +} From de8a98cfa3542dd02014d43393162a590ba0433e Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 5 Jun 2026 04:49:02 -0700 Subject: [PATCH 02/26] feat(observability): query gateway core + dashboard shell (dtos, tenant scope, health, routing) Co-Authored-By: Claude Opus 4.8 (1M context) --- .../texera/web/TexeraWebApplication.scala | 39 +- .../RequestContextMdcFilter.scala | 164 ++++++++ .../observability/UserContextMdcFilter.scala | 82 ++++ .../observability/gateway/AuditLogger.scala | 90 +++++ .../observability/gateway/BackendClient.scala | 158 ++++++++ .../gateway/GatewayContext.scala | 66 +++ .../gateway/ObservabilityResources.scala | 129 ++++++ .../observability/gateway/RateLimiter.scala | 97 +++++ .../observability/gateway/ScopeResolver.scala | 135 +++++++ .../web/observability/gateway/builders.scala | 62 +++ .../web/observability/gateway/dtos.scala | 375 ++++++++++++++++++ .../RequestContextMdcFilterSpec.scala | 272 +++++++++++++ .../UserContextMdcFilterSpec.scala | 139 +++++++ .../gateway/BackendClientSpec.scala | 171 ++++++++ .../gateway/DtoValidationSpec.scala | 146 +++++++ .../gateway/RateLimiterSpec.scala | 76 ++++ .../gateway/ScopeResolverSpec.scala | 70 ++++ .../main/resources/observability-gateway.conf | 50 +++ .../config/ObservabilityGatewayConfig.scala | 47 +++ frontend/src/app/app-routing.constant.ts | 1 + frontend/src/app/app-routing.module.ts | 5 + .../blob-error-http-interceptor.service.ts | 8 +- .../component/dashboard.component.html | 11 + .../component/dashboard.component.ts | 2 + .../observability.component.html | 48 +++ .../observability.component.scss | 61 +++ .../observability.component.spec.ts | 103 +++++ .../observability/observability.component.ts | 116 ++++++ .../observability.service.spec.ts | 228 +++++++++++ .../observability/observability.service.ts | 174 ++++++++ .../user/observability/observability.types.ts | 197 +++++++++ .../traces-pivot.service.spec.ts | 51 +++ .../observability/traces-pivot.service.ts | 49 +++ 33 files changed, 3419 insertions(+), 3 deletions(-) create mode 100644 amber/src/main/scala/org/apache/texera/web/observability/RequestContextMdcFilter.scala create mode 100644 amber/src/main/scala/org/apache/texera/web/observability/UserContextMdcFilter.scala create mode 100644 amber/src/main/scala/org/apache/texera/web/observability/gateway/AuditLogger.scala create mode 100644 amber/src/main/scala/org/apache/texera/web/observability/gateway/BackendClient.scala create mode 100644 amber/src/main/scala/org/apache/texera/web/observability/gateway/GatewayContext.scala create mode 100644 amber/src/main/scala/org/apache/texera/web/observability/gateway/ObservabilityResources.scala create mode 100644 amber/src/main/scala/org/apache/texera/web/observability/gateway/RateLimiter.scala create mode 100644 amber/src/main/scala/org/apache/texera/web/observability/gateway/ScopeResolver.scala create mode 100644 amber/src/main/scala/org/apache/texera/web/observability/gateway/builders.scala create mode 100644 amber/src/main/scala/org/apache/texera/web/observability/gateway/dtos.scala create mode 100644 amber/src/test/scala/org/apache/texera/web/observability/RequestContextMdcFilterSpec.scala create mode 100644 amber/src/test/scala/org/apache/texera/web/observability/UserContextMdcFilterSpec.scala create mode 100644 amber/src/test/scala/org/apache/texera/web/observability/gateway/BackendClientSpec.scala create mode 100644 amber/src/test/scala/org/apache/texera/web/observability/gateway/DtoValidationSpec.scala create mode 100644 amber/src/test/scala/org/apache/texera/web/observability/gateway/RateLimiterSpec.scala create mode 100644 amber/src/test/scala/org/apache/texera/web/observability/gateway/ScopeResolverSpec.scala create mode 100644 common/config/src/main/resources/observability-gateway.conf create mode 100644 common/config/src/main/scala/org/apache/texera/config/ObservabilityGatewayConfig.scala create mode 100644 frontend/src/app/dashboard/component/user/observability/observability.component.html create mode 100644 frontend/src/app/dashboard/component/user/observability/observability.component.scss create mode 100644 frontend/src/app/dashboard/component/user/observability/observability.component.spec.ts create mode 100644 frontend/src/app/dashboard/component/user/observability/observability.component.ts create mode 100644 frontend/src/app/dashboard/service/user/observability/observability.service.spec.ts create mode 100644 frontend/src/app/dashboard/service/user/observability/observability.service.ts create mode 100644 frontend/src/app/dashboard/service/user/observability/observability.types.ts create mode 100644 frontend/src/app/dashboard/service/user/observability/traces-pivot.service.spec.ts create mode 100644 frontend/src/app/dashboard/service/user/observability/traces-pivot.service.ts diff --git a/amber/src/main/scala/org/apache/texera/web/TexeraWebApplication.scala b/amber/src/main/scala/org/apache/texera/web/TexeraWebApplication.scala index 5438eea4d0f..4a185f76e52 100644 --- a/amber/src/main/scala/org/apache/texera/web/TexeraWebApplication.scala +++ b/amber/src/main/scala/org/apache/texera/web/TexeraWebApplication.scala @@ -31,7 +31,12 @@ import org.apache.texera.amber.engine.common.Utils import org.apache.texera.amber.util.ObjectMapperUtils import org.apache.texera.auth.SessionUser import org.apache.texera.dao.SqlServer +import org.apache.texera.observability.OtelInit import org.apache.texera.web.auth.JwtAuth.setupJwtAuth +import org.apache.texera.web.observability.gateway.{ + GatewayContext, + ObservabilityHealthResource +} import org.apache.texera.web.resource._ import org.apache.texera.web.resource.auth.{AuthResource, GoogleAuthResource} import org.apache.texera.web.resource.dashboard.DashboardResource @@ -59,7 +64,7 @@ import org.glassfish.jersey.server.filter.RolesAllowedDynamicFeature import java.time.Duration -object TexeraWebApplication { +object TexeraWebApplication extends LazyLogging { def main(args: Array[String]): Unit = { @@ -67,6 +72,7 @@ object TexeraWebApplication { // Currently in kubernetes, multiple pods calling this function can result into thread competition // discardUncommittedChangesOfAllDatasets() + logger.info("Starting TexeraWebApplication") // start web server new TexeraWebApplication().run( "server", @@ -101,11 +107,15 @@ class TexeraWebApplication } override def run(configuration: TexeraWebConfiguration, environment: Environment): Unit = { + logger.info("Initializing TexeraWebApplication web server") + OtelInit.init("texera-web") + ObjectMapperUtils.warmupObjectMapperForOperatorsSerde() // serve backend at /api environment.jersey.setUrlPattern("/api/*") + logger.debug(s"Connecting to database at ${StorageConfig.jdbcUrl}") SqlServer.initConnection( StorageConfig.jdbcUrl, StorageConfig.jdbcUsername, @@ -141,6 +151,11 @@ class TexeraWebApplication ) environment.jersey.register(classOf[RolesAllowedDynamicFeature]) + // Tag every log record emitted while handling a request with the + // authenticated user's id (MDC key `texera.user.id`). Registered + // here so it runs AFTER the JWT auth filter set up by setupJwtAuth. + environment.jersey.register(classOf[org.apache.texera.web.observability.UserContextMdcFilter]) + environment.jersey.register(classOf[AuthResource]) environment.jersey.register(classOf[GoogleAuthResource]) environment.jersey.register(classOf[UserConfigResource]) @@ -162,8 +177,28 @@ class TexeraWebApplication environment.jersey.register(classOf[AIAssistantResource]) environment.jersey.register(classOf[HuggingFaceModelResource]) + // Observability gateway. A single GatewayContext is shared by all + // resources so the rate-limiter buckets and the backend HTTP clients + // are reused across requests. The health resource lands here in + // gateway-core; the per-signal resources are registered by their + // respective PRs (obs/pr10..13). + logger.debug("Registering observability gateway resources") + val obsCtx = GatewayContext.default() + environment.jersey.register(new ObservabilityHealthResource(obsCtx)) + AuthResource.createAdminUser() + // Tag every log record emitted during request handling with the + // workflow / execution / computing-unit id when one is visible in + // the URL or headers. Required so the observability dashboard's + // CU/workflow filters match live records (the JVM otherwise emits + // them without any per-request context). + environment.getApplicationContext.addFilter( + new FilterHolder(new org.apache.texera.web.observability.RequestContextMdcFilter()), + "/*", + java.util.EnumSet.allOf(classOf[javax.servlet.DispatcherType]) + ) + // Route request logs through SLF4J, controlled by TEXERA_SERVICE_LOG_LEVEL. // TODO: replace with RequestLoggingFilter.register() from common/auth once Dropwizard is upgraded to 4.x val requestLogger = org.slf4j.LoggerFactory.getLogger("org.eclipse.jetty.server.RequestLog") @@ -189,5 +224,7 @@ class TexeraWebApplication "/*", java.util.EnumSet.allOf(classOf[javax.servlet.DispatcherType]) ) + + logger.info("TexeraWebApplication web server initialized and ready") } } diff --git a/amber/src/main/scala/org/apache/texera/web/observability/RequestContextMdcFilter.scala b/amber/src/main/scala/org/apache/texera/web/observability/RequestContextMdcFilter.scala new file mode 100644 index 00000000000..054360c1c44 --- /dev/null +++ b/amber/src/main/scala/org/apache/texera/web/observability/RequestContextMdcFilter.scala @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.web.observability + +import org.slf4j.MDC + +import javax.servlet._ +import javax.servlet.http.HttpServletRequest + +/** + * Servlet filter that pushes request-scoped IDs into the SLF4J MDC + * so every log record emitted while handling a request carries + * `texera.workflow.id`, `texera.execution.id`, and + * `texera.computing_unit.id` when those are visible in the URL or + * request headers. + * + * Without this, the OTel log appender bridges every Logback event + * with no per-request context, so the dashboard's CU/workflow + * filters match nothing live — the only records carrying those keys + * are seed data pushed via the VictoriaLogs ingest API. + * + * Sources of IDs, in priority order: + * 1. HTTP headers `X-Texera-Workflow-Id`, `X-Texera-Execution-Id`, + * `X-Texera-Computing-Unit-Id` — set explicitly by the + * Angular client when known. + * 2. URL path segments matching `/workflow/`, + * `/execution/`, `/computing-unit/` (also the + * `/{wid}` / `/{cuid}` patterns used by some resources). + * + * MDC is ALWAYS cleared in a `finally` block so a thread reused for + * a different request doesn't leak the previous request's labels. + * + * Keys here must stay in sync with [[LogSanitizer.AllowedMdcKeys]] — + * any new key added here must be allowlisted there or the OTel + * appender will silently drop it. + */ +class RequestContextMdcFilter extends Filter { + + override def init(filterConfig: FilterConfig): Unit = () + override def destroy(): Unit = () + + override def doFilter( + request: ServletRequest, + response: ServletResponse, + chain: FilterChain + ): Unit = { + val pushed = scala.collection.mutable.ArrayBuffer.empty[String] + request match { + case http: HttpServletRequest => + // Header overrides come first — an explicit client header is + // the most precise signal we have. + applyHeader(http, "X-Texera-Workflow-Id", "texera.workflow.id", pushed) + applyHeader(http, "X-Texera-Execution-Id", "texera.execution.id", pushed) + applyHeader(http, "X-Texera-Computing-Unit-Id", "texera.computing_unit.id", pushed) + + // URL extraction — only fill keys that weren't already set + // by headers. + val path = Option(http.getRequestURI).getOrElse("") + applyPath(path, RequestContextMdcFilter.WorkflowPattern, "texera.workflow.id", pushed) + applyPath(path, RequestContextMdcFilter.ExecutionPattern, "texera.execution.id", pushed) + applyPath(path, RequestContextMdcFilter.CuPattern, "texera.computing_unit.id", pushed) + + // Query-parameter extraction — the WebSocket upgrade URL is + // `/wsapi/workflow-websocket?wid=&cuid=` and Texera + // also accepts `?cuid=N` on the REST PVE endpoints. Filling + // the MDC here means every record emitted during the WS + // handshake AND the long-running session that follows + // carries the ids the client supplied. + applyParam(http, "wid", "texera.workflow.id", pushed) + applyParam(http, "eid", "texera.execution.id", pushed) + applyParam(http, "cuid", "texera.computing_unit.id", pushed) + case _ => + // Non-HTTP request (websocket upgrades fall through here on + // some Servlet versions). No MDC context to add. + } + try { + chain.doFilter(request, response) + } finally { + // Defence in depth: clear only what THIS filter set so we don't + // step on MDC populated by other middleware. + pushed.foreach(MDC.remove) + } + } + + private def applyHeader( + req: HttpServletRequest, + headerName: String, + mdcKey: String, + pushed: scala.collection.mutable.ArrayBuffer[String] + ): Unit = { + val raw = Option(req.getHeader(headerName)).map(_.trim).filter(_.nonEmpty) + raw.foreach { value => + // Cheap allowlist: only digits get through. Prevents log + // forging via a CRLF in the header value. + if (value.forall(_.isDigit) && value.length <= 19) { + MDC.put(mdcKey, value) + pushed += mdcKey + } + } + } + + private def applyPath( + path: String, + pattern: scala.util.matching.Regex, + mdcKey: String, + pushed: scala.collection.mutable.ArrayBuffer[String] + ): Unit = { + if (MDC.get(mdcKey) == null) { + pattern.findFirstMatchIn(path).foreach { m => + val v = m.group(1) + MDC.put(mdcKey, v) + pushed += mdcKey + } + } + } + + /** Read a query parameter, validate it's a positive integer, push + * to MDC. Same shape as applyHeader: numeric-only allowlist and a + * length cap so a forged value can't inject newlines into the + * rendered log line. */ + private def applyParam( + req: HttpServletRequest, + paramName: String, + mdcKey: String, + pushed: scala.collection.mutable.ArrayBuffer[String] + ): Unit = { + if (MDC.get(mdcKey) == null) { + val raw = Option(req.getParameter(paramName)).map(_.trim).filter(_.nonEmpty) + raw.foreach { value => + if (value.forall(_.isDigit) && value.length <= 19) { + MDC.put(mdcKey, value) + pushed += mdcKey + } + } + } + } +} + +object RequestContextMdcFilter { + // The URL patterns cover the existing JAX-RS @Path templates that + // embed numeric IDs. `(\d+)` is the only allowlisted shape because + // every Texera id is a positive integer; non-digit segments don't + // need MDC propagation. + val WorkflowPattern: scala.util.matching.Regex = """/workflow/(\d+)""".r + val ExecutionPattern: scala.util.matching.Regex = """/execution/(\d+)""".r + val CuPattern: scala.util.matching.Regex = """/computing-unit/(\d+)""".r +} diff --git a/amber/src/main/scala/org/apache/texera/web/observability/UserContextMdcFilter.scala b/amber/src/main/scala/org/apache/texera/web/observability/UserContextMdcFilter.scala new file mode 100644 index 00000000000..4c0dd3df468 --- /dev/null +++ b/amber/src/main/scala/org/apache/texera/web/observability/UserContextMdcFilter.scala @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.web.observability + +import org.apache.texera.auth.SessionUser +import org.slf4j.MDC + +import javax.annotation.Priority +import javax.ws.rs.container.{ + ContainerRequestContext, + ContainerRequestFilter, + ContainerResponseContext, + ContainerResponseFilter +} +import javax.ws.rs.ext.Provider +import javax.ws.rs.Priorities + +/** + * Jersey filter that pushes the authenticated user's id into the + * SLF4J MDC so every log record emitted while handling the request + * carries `texera.user.id`. + * + * Why Jersey and not a Servlet filter: + * - Authentication is wired as a Jersey `ContainerRequestFilter` + * (see [[org.apache.texera.auth.JwtAuthFilter]]) — by the time + * Servlet filters run, `SecurityContext.getUserPrincipal` is + * still null. A Jersey filter ordered after auth picks it up. + * - Priorities.AUTHENTICATION + 100 places us in the + * AUTHORIZATION bucket which is guaranteed to run AFTER auth. + * + * Symmetric request/response interfaces let us pair set + clear + * without leaking MDC across threads in the worker pool. + */ +@Provider +@Priority(Priorities.AUTHORIZATION) +class UserContextMdcFilter extends ContainerRequestFilter with ContainerResponseFilter { + + override def filter(requestContext: ContainerRequestContext): Unit = { + val secCtx = requestContext.getSecurityContext + if (secCtx != null) { + secCtx.getUserPrincipal match { + case user: SessionUser => + val uid = user.getUid + if (uid != null) MDC.put(UserContextMdcFilter.UserIdKey, uid.toString) + case _ => // anonymous request, or auth chose a different Principal type + } + } + } + + override def filter( + requestContext: ContainerRequestContext, + responseContext: ContainerResponseContext + ): Unit = { + // Defence in depth: clear only the key we own. The Servlet-layer + // filter clears its own keys via the same pattern. + MDC.remove(UserContextMdcFilter.UserIdKey) + } +} + +object UserContextMdcFilter { + /** MDC key — must stay in sync with + * [[org.apache.texera.observability.LogSanitizer.AllowedMdcKeys]] + * or the OTel log appender drops it before emit. */ + val UserIdKey: String = "texera.user.id" +} diff --git a/amber/src/main/scala/org/apache/texera/web/observability/gateway/AuditLogger.scala b/amber/src/main/scala/org/apache/texera/web/observability/gateway/AuditLogger.scala new file mode 100644 index 00000000000..5a0daa9ee82 --- /dev/null +++ b/amber/src/main/scala/org/apache/texera/web/observability/gateway/AuditLogger.scala @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.web.observability.gateway + +import org.apache.texera.observability.LogSanitizer +import org.slf4j.{Logger, LoggerFactory} + +/** + * Per-query audit log for the gateway. + * + * Lives on its own SLF4J logger name so operators can route audit + * events to a dedicated appender (file/syslog/SIEM) separate from + * application logs. The logback.xml in each service can selectively + * disable propagation to the root appender if audit events should + * NOT flow into the OTel collector. + * + * The audit line is one JSON-shaped string per query. Free-text + * fields go through [[LogSanitizer.sanitize]] first so secret + * patterns and CRLF injections cannot land in audit storage. + */ +object AuditLogger { + + private val log: Logger = LoggerFactory.getLogger("texera.audit.observability") + + case class Entry( + userId: Long, + remoteIp: String, + endpoint: String, + signal: String, + scope: GatewayScope, + query: String, + fromMs: Long, + toMs: Long, + hits: Long + ) + + def record(entry: Entry): Unit = { + if (!log.isInfoEnabled) return + val safeQuery = LogSanitizer.sanitize(entry.query) + val msg = new StringBuilder + msg.append("{") + msg.append(s""""ts":${System.currentTimeMillis()}""") + msg.append(s""","user":${entry.userId}""") + msg.append(s""","ip":"${escapeJson(entry.remoteIp)}"""") + msg.append(s""","endpoint":"${escapeJson(entry.endpoint)}"""") + msg.append(s""","signal":"${escapeJson(entry.signal)}"""") + msg.append(s""","allowed_workflows":${entry.scope.allowedWorkflowIds.size}""") + msg.append(s""","allowed_projects":${entry.scope.allowedProjectIds.size}""") + msg.append(s""","query":"${escapeJson(safeQuery)}"""") + msg.append(s""","from":${entry.fromMs}""") + msg.append(s""","to":${entry.toMs}""") + msg.append(s""","hits":${entry.hits}""") + msg.append("}") + log.info(msg.toString) + } + + private def escapeJson(s: String): String = { + if (s == null) return "" + val out = new StringBuilder(s.length + 8) + var i = 0 + while (i < s.length) { + val c = s.charAt(i) + c match { + case '"' => out.append("\\\"") + case '\\' => out.append("\\\\") + case _ if c < 0x20 || c == 0x7F => // control chars stripped (defense in depth) + case _ => out.append(c) + } + i += 1 + } + out.toString + } +} diff --git a/amber/src/main/scala/org/apache/texera/web/observability/gateway/BackendClient.scala b/amber/src/main/scala/org/apache/texera/web/observability/gateway/BackendClient.scala new file mode 100644 index 00000000000..3de66f350a4 --- /dev/null +++ b/amber/src/main/scala/org/apache/texera/web/observability/gateway/BackendClient.scala @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.web.observability.gateway + +import com.typesafe.scalalogging.LazyLogging +import org.apache.texera.web.observability.gateway.dtos.{ + GatewayError, + MaxResponseBytes +} + +import java.net.URI +import java.net.http.HttpResponse.BodyHandlers +import java.net.http.{HttpClient, HttpRequest, HttpResponse} +import java.nio.charset.StandardCharsets +import java.time.Duration +import scala.util.{Failure, Success, Try} + +/** + * Thin HTTP wrapper used by the per-backend clients. We deliberately + * use the JDK 11 HttpClient (no new dependency) and keep the surface + * minimal: one ``get`` / one ``post`` taking a typed body, with + * VictoriaLogs/Metrics tenancy headers injected at the boundary. + * + * Two security rails enforced here: + * 1. Hard response-byte cap. We read the body into a buffer that + * stops at [[dtos.MaxResponseBytes]] — an attacker / runaway + * backend cannot make us swallow a 1 GiB body. + * 2. Per-request scope: the GatewayScope arrives validated. Tenant + * isolation against VictoriaLogs / VictoriaMetrics is enforced + * by the LogsQL / MetricsQL stream filters that the builders + * derive from `scope.allowedWorkflowIds` — NOT by header + * multi-tenancy. We previously sent an `AccountID: ` + * header for defence-in-depth, but the OTel collector does not + * set AccountID at ingest (every record lands in tenant 0), so + * that header caused every authenticated query to return zero + * results. TODO(observability/multi-tenant): wire per-user + * AccountID through the collector + ingest pipeline, then + * re-introduce the header here. + * + * Secret redaction is intentionally NOT done at this layer: it runs + * per-field inside the parsers (parseLogs / parseTraces), so a single + * oversized value can't truncate and corrupt the whole JSON response. + */ +class BackendClient( + baseUrl: String, + timeoutMs: Long = 5000L +) extends LazyLogging { + + private val http: HttpClient = HttpClient + .newBuilder() + .connectTimeout(Duration.ofMillis(timeoutMs)) + .followRedirects(HttpClient.Redirect.NEVER) // backends shouldn't redirect; if they do, treat as error + .build() + + /** GET that returns a (status, body) tuple or a typed error. The + * body is decoded as UTF-8 and is truncated at MaxResponseBytes + * with a [[GatewayError.ResponseTooLarge]] surfaced. */ + def get(path: String, scope: GatewayScope, signal: String): Either[GatewayError, BackendResponse] = { + val uri = URI.create(baseUrl + path) + val req = HttpRequest + .newBuilder(uri) + .timeout(Duration.ofMillis(timeoutMs)) + .header("Accept", "application/json") + // Tenancy via VL/VM multi-tenant headers is disabled — see the + // class comment. Project header is still useful as a logging + // breadcrumb on the backend access logs and costs us nothing. + .header("ProjectID", scope.allowedProjectIds.headOption.map(_.toString).getOrElse("0")) + .GET() + .build() + send(req, signal) + } + + /** POST a typed body. ``contentType`` is the only place we accept + * an arbitrary string — but it's a CONST passed by the caller, + * never from request input. */ + def post( + path: String, + body: Array[Byte], + contentType: String, + scope: GatewayScope, + signal: String + ): Either[GatewayError, BackendResponse] = { + val uri = URI.create(baseUrl + path) + val req = HttpRequest + .newBuilder(uri) + .timeout(Duration.ofMillis(timeoutMs)) + .header("Accept", "application/json") + .header("Content-Type", contentType) + // See class comment for why we no longer send AccountID. + .header("ProjectID", scope.allowedProjectIds.headOption.map(_.toString).getOrElse("0")) + .POST(HttpRequest.BodyPublishers.ofByteArray(body)) + .build() + send(req, signal) + } + + private def send(req: HttpRequest, signal: String): Either[GatewayError, BackendResponse] = { + logger.debug(s"[$signal] sending ${req.method()} ${req.uri()} (timeout ${timeoutMs}ms)") + val startNanos = System.nanoTime() + Try(http.send(req, BodyHandlers.ofByteArray())) match { + case Failure(e) => + val elapsedMs = (System.nanoTime() - startNanos) / 1000000L + // This is the line that explains the dashboard's "Unreachable" + // badge: connection refused, DNS failure, or timeout against the + // backend's base URL. Logged at WARN with the cause so operators + // don't have to attach a debugger to find a misconfigured host. + logger.warn( + s"[$signal] backend unreachable at $baseUrl after ${elapsedMs}ms " + + s"(${req.method()} ${req.uri()}): ${e.getClass.getSimpleName}: ${e.getMessage}" + ) + Left(GatewayError.BackendUnreachable(signal)) + case Success(resp: HttpResponse[Array[Byte]]) => + val elapsedMs = (System.nanoTime() - startNanos) / 1000000L + val raw = resp.body() + if (raw == null) { + logger.debug(s"[$signal] ${resp.statusCode()} from $baseUrl in ${elapsedMs}ms (empty body)") + Right(BackendResponse(resp.statusCode(), "")) + } else if (raw.length.toLong > MaxResponseBytes) { + logger.warn( + s"[$signal] response from $baseUrl exceeds the ${MaxResponseBytes}-byte cap " + + s"(${raw.length} bytes in ${elapsedMs}ms) — rejecting to protect the gateway" + ) + Left(GatewayError.ResponseTooLarge) + } else { + logger.debug( + s"[$signal] ${resp.statusCode()} from $baseUrl in ${elapsedMs}ms (${raw.length} bytes)" + ) + Right(BackendResponse(resp.statusCode(), new String(raw, StandardCharsets.UTF_8))) + } + } + } +} + +/** Wrapped backend response — status + body. Secret redaction is NOT + * applied here: it happens per-field inside the parsers (parseLogs / + * parseTraces sanitize individual message/attribute values). A + * whole-body LogSanitizer.sanitize pass is unsafe on these JSON + * payloads — its 16 KiB cap truncates large responses mid-value and + * corrupts the JSON. */ +case class BackendResponse(status: Int, body: String) { + def isOk: Boolean = status >= 200 && status < 300 +} diff --git a/amber/src/main/scala/org/apache/texera/web/observability/gateway/GatewayContext.scala b/amber/src/main/scala/org/apache/texera/web/observability/gateway/GatewayContext.scala new file mode 100644 index 00000000000..b38f0ed0476 --- /dev/null +++ b/amber/src/main/scala/org/apache/texera/web/observability/gateway/GatewayContext.scala @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.web.observability.gateway + +import org.apache.texera.config.ObservabilityGatewayConfig + +/** + * Single bag of collaborators every observability resource needs. + * Centralises wiring so the resource constructors stay short and so + * tests can override one collaborator (e.g. stub the scope resolver) + * without touching the rest. + */ +case class GatewayContext( + scopeResolver: ScopeResolver, + perUserLimiter: RateLimiter, + perIpLimiter: RateLimiter, + logsClient: BackendClient, + metricsClient: BackendClient, + tracesClient: BackendClient, + profilesClient: BackendClient, + // Parca's API is gRPC-only — the BackendClient (HTTP/1.1 JSON) can't + // reach it. We keep the BackendClient field above for symmetry and + // for the reachability check, but the actual query path uses + // [[ParcaClient]] which talks gRPC-Web. Both pull from the same config + // key so operators have one URL to set. + profilesBaseUrl: String +) + +object GatewayContext { + + /** Build the production context. The backend query URLs come from + * [[ObservabilityGatewayConfig]] (`observability-gateway.conf`), which + * defaults to the host-local stack and is overridden inside docker via + * the TEXERA_OBS_*_URL env vars set in bin/single-node/.env. A natively + * run backend therefore reaches the loopback-published backends with no + * extra configuration. */ + def default(): GatewayContext = { + GatewayContext( + scopeResolver = new ScopeResolver.Jooq(), + perUserLimiter = RateLimiter.defaultPerUser(), + perIpLimiter = RateLimiter.defaultPerIp(), + logsClient = new BackendClient(ObservabilityGatewayConfig.logsUrl), + metricsClient = new BackendClient(ObservabilityGatewayConfig.metricsUrl), + tracesClient = new BackendClient(ObservabilityGatewayConfig.tracesUrl), + profilesClient = new BackendClient(ObservabilityGatewayConfig.profilesUrl), + profilesBaseUrl = ObservabilityGatewayConfig.profilesUrl + ) + } +} diff --git a/amber/src/main/scala/org/apache/texera/web/observability/gateway/ObservabilityResources.scala b/amber/src/main/scala/org/apache/texera/web/observability/gateway/ObservabilityResources.scala new file mode 100644 index 00000000000..12ab7618eec --- /dev/null +++ b/amber/src/main/scala/org/apache/texera/web/observability/gateway/ObservabilityResources.scala @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.web.observability.gateway + +import com.typesafe.scalalogging.LazyLogging +import io.dropwizard.auth.Auth +import javax.annotation.security.RolesAllowed +import javax.ws.rs._ +import javax.ws.rs.core.{Context, MediaType, Response} +import javax.servlet.http.HttpServletRequest +import org.apache.texera.auth.SessionUser +import org.apache.texera.web.observability.gateway.dtos._ + +/** + * Dropwizard resources for the observability gateway. + * + * Every endpoint runs the same five-step skeleton (see DESIGN.md): + * 1. Auth — handled by @Auth + @RolesAllowed. + * 2. Rate limit — token bucket per user, per IP. + * 3. Scope — resolved from the SessionUser via ScopeResolver. + * 4. Validate — typed DTO validators reject anything out of range. + * 5. Query — typed builders only, then BackendClient with the + * AccountID/ProjectID headers. Response goes through redaction + * before reaching JSON. + * + * The resources are intentionally small — most of the security logic + * lives in the dtos / builders / scope objects so it's unit-testable + * without a Dropwizard fixture. + */ + +/** Tiny helper that turns a [[GatewayError]] into a Dropwizard + * Response. Kept here so every resource uses the same shape. */ +private[gateway] object Respond extends LazyLogging { + def err(e: GatewayError): Response = { + // One breadcrumb per rejected request, at a level keyed to severity: + // 5xx — the gateway or a backend misbehaved → WARN (operator should look) + // 4xx — the caller was rejected as designed (rate limit, forbidden, + // bad input) → DEBUG (expected, only useful when tracing a + // specific user's failing request). + val line = s"observability request rejected: ${e.code} (HTTP ${e.status}) — ${e.message}" + if (e.status >= 500) logger.warn(line) else logger.debug(line) + Response + .status(e.status) + .entity(Map("code" -> e.code, "message" -> e.message)) + .`type`(MediaType.APPLICATION_JSON) + .build() + } + + def json(value: Any): Response = + Response.ok(value).`type`(MediaType.APPLICATION_JSON).build() +} + +/** Shared pre-flight: rate limit + scope resolution. */ +private[gateway] object Preflight extends LazyLogging { + def run( + ctx: GatewayContext, + user: SessionUser, + req: HttpServletRequest + ): Either[GatewayError, GatewayScope] = { + val ip = Option(req.getRemoteAddr).getOrElse("unknown") + val userKey = s"user:${user.getUid}" + val ipKey = s"ip:$ip" + if (!ctx.perUserLimiter.tryAcquire(userKey)) { + logger.warn(s"observability rate limit hit for user ${user.getUid} (per-user bucket)") + return Left(GatewayError.RateLimited) + } + if (!ctx.perIpLimiter.tryAcquire(ipKey)) { + logger.warn(s"observability rate limit hit for ip $ip (per-ip bucket)") + return Left(GatewayError.RateLimited) + } + val scope = ctx.scopeResolver.resolve(user) + logger.debug( + s"observability preflight ok for user ${user.getUid} from $ip — " + + s"scope: ${scope.allowedWorkflowIds.size} workflow(s), ${scope.allowedProjectIds.size} project(s)" + ) + Right(scope) + } +} + +@Path("/observability/health") +@Produces(Array(MediaType.APPLICATION_JSON)) +class ObservabilityHealthResource(ctx: GatewayContext) extends LazyLogging { + + @GET + def health(@Auth user: SessionUser): Response = { + // Light-touch reachability — used by the dashboard to render + // "Disabled" / "Unreachable" panels. No backend query; just a + // HEAD-style ping that surfaces typed-status only. + val checks = Map( + "logs" -> reachable(ctx.logsClient), + "metrics" -> reachable(ctx.metricsClient), + "traces" -> reachable(ctx.tracesClient), + "profiles" -> reachable(ctx.profilesClient) + ) + val unreachable = checks.collect { case (signal, false) => signal }.toSeq.sorted + if (unreachable.nonEmpty) + logger.warn(s"observability health check: unreachable backend(s): ${unreachable.mkString(", ")}") + else + logger.debug(s"observability health check: all backends reachable") + Respond.json(Map("status" -> "ok", "checks" -> checks)) + } + + private def reachable(client: BackendClient): Boolean = { + client + .get( + "/", + GatewayScope(userId = 0L, allowedWorkflowIds = Set.empty, allowedProjectIds = Set.empty), + "health" + ) + .isRight + } +} diff --git a/amber/src/main/scala/org/apache/texera/web/observability/gateway/RateLimiter.scala b/amber/src/main/scala/org/apache/texera/web/observability/gateway/RateLimiter.scala new file mode 100644 index 00000000000..7bbe9b2185b --- /dev/null +++ b/amber/src/main/scala/org/apache/texera/web/observability/gateway/RateLimiter.scala @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.web.observability.gateway + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicReference + +/** + * Simple token-bucket rate limiter, keyed by an arbitrary string + * (typically userId or remote IP). + * + * Trade-offs vs. a fancier approach: + * - In-memory only. A single instance per JVM. A rolling-restart + * deploy would clear the buckets — acceptable for the + * observability gateway's traffic shape (low QPS, low + * consequences for a single missed limit at restart). + * - No background thread. Refill happens lazily on the next + * [[tryAcquire]] call against the same key. Simpler reasoning, + * no scheduler involved. + * - Keys never expire from the map. The map is bounded informally + * by the number of distinct (user, IP) tuples a deploy sees; + * a future PR can add a periodic clean-up if cardinality grows. + */ +class RateLimiter(capacity: Long, refillPerSecond: Double) { + + require(capacity > 0, "capacity must be positive") + require(refillPerSecond > 0, "refillPerSecond must be positive") + + private val buckets: ConcurrentHashMap[String, AtomicReference[Bucket]] = + new ConcurrentHashMap[String, AtomicReference[Bucket]]() + + /** Try to consume one token. Returns true on success, false on + * rate-limit. Time-aware: the bucket refills based on wall clock + * elapsed since the last call. */ + def tryAcquire(key: String, nowMillis: Long = System.currentTimeMillis()): Boolean = { + val ref = buckets.computeIfAbsent( + key, + _ => new AtomicReference[Bucket](Bucket(capacity.toDouble, nowMillis)) + ) + // CAS loop: take the current bucket, refill against now, + // attempt to debit one token, swap back. + var done = false + var allowed = false + while (!done) { + val cur = ref.get() + val elapsedSec = (nowMillis - cur.lastRefillMs).toDouble / 1000.0 + val refilled = math.min(capacity.toDouble, cur.tokens + elapsedSec * refillPerSecond) + if (refilled >= 1.0) { + val next = Bucket(refilled - 1.0, nowMillis) + if (ref.compareAndSet(cur, next)) { + allowed = true + done = true + } + } else { + val next = Bucket(refilled, nowMillis) + if (ref.compareAndSet(cur, next)) { + allowed = false + done = true + } + } + } + allowed + } + + /** Test-only: drop all in-memory state. */ + private[gateway] def resetForTest(): Unit = buckets.clear() + + private case class Bucket(tokens: Double, lastRefillMs: Long) +} + +object RateLimiter { + /** Default per-user limit per the PR plan: 20 req/s with a 20- + * token burst capacity. */ + def defaultPerUser(): RateLimiter = new RateLimiter(capacity = 20, refillPerSecond = 20.0) + + /** Default per-IP limit: looser per-user limit, tighter per-IP + * to defend against an attacker burning through a fleet of + * accounts. */ + def defaultPerIp(): RateLimiter = new RateLimiter(capacity = 100, refillPerSecond = 50.0) +} diff --git a/amber/src/main/scala/org/apache/texera/web/observability/gateway/ScopeResolver.scala b/amber/src/main/scala/org/apache/texera/web/observability/gateway/ScopeResolver.scala new file mode 100644 index 00000000000..254dd5999cb --- /dev/null +++ b/amber/src/main/scala/org/apache/texera/web/observability/gateway/ScopeResolver.scala @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.web.observability.gateway + +import com.typesafe.scalalogging.LazyLogging +import org.apache.texera.auth.SessionUser +import org.apache.texera.dao.SqlServer +import org.apache.texera.dao.jooq.generated.Tables.{ + PROJECT_USER_ACCESS, + WORKFLOW_OF_USER, + WORKFLOW_USER_ACCESS +} + +import scala.jdk.CollectionConverters._ + +/** + * Resolves a [[GatewayScope]] for a session user. + * + * The single security invariant: there is no public path through + * which a caller can widen the scope. The resolver reads + * authoritative state (jOOQ tables) and returns a closed set of + * allowed workflow / project ids — every backend builder consumes + * this set, and any request that names a workflow id outside it is + * rejected at the resource layer. + * + * Provided in two flavours: + * - [[ScopeResolver.Jooq]] — the production path, queries the + * existing access-control tables. + * - [[ScopeResolver.Stub]] — used by tests so they don't need to + * stand up a real database. + */ +trait ScopeResolver { + def resolve(user: SessionUser): GatewayScope + + /** Membership check: does the caller actually have access to the + * named workflow id? Returns true if no workflowId was supplied + * (defaults to caller's full scope) or if the id is in the + * resolved allow-set. + * + * Implementation note: Jackson Scala module deserializes a JSON + * number that fits in 32 bits as java.lang.Integer regardless of + * the declared `Option[Long]` type (type parameters are erased on + * the JVM). A typed `id: Long` closure then triggers a runtime + * Integer→Long unbox via BoxesRunTime.unboxToLong, which throws a + * ClassCastException. We normalise to a primitive long up front + * via `Number.longValue()` so the contains-check is type-safe. + */ + def assertWorkflowAllowed(scope: GatewayScope, workflowId: Option[Long]): Boolean = + workflowId match { + case None => true + case Some(id) => + // The `id` here may actually be a java.lang.Integer at runtime; + // route through Number.longValue() so the unbox cannot fail. + val asLong = id.asInstanceOf[Any] match { + case n: java.lang.Number => n.longValue() + case other => other.toString.toLong + } + scope.allowedWorkflowIds.contains(asLong) + } +} + +object ScopeResolver { + + /** Production implementation backed by jOOQ. Queries + * WORKFLOW_OF_USER (owned workflows) ∪ WORKFLOW_USER_ACCESS + * (shared workflows) for the user, and PROJECT_USER_ACCESS for + * the set of projects the user can see. */ + class Jooq extends ScopeResolver with LazyLogging { + override def resolve(user: SessionUser): GatewayScope = { + val ctx = SqlServer.getInstance().createDSLContext() + val uid = user.getUid + + // Owned + shared workflow ids. A jOOQ UNION call would be + // ideal but the generated DSL is fussier with type + // inference; two queries collected into a Set keeps this + // straightforward and equally safe. + val ownedWids: Set[Long] = ctx + .selectFrom(WORKFLOW_OF_USER) + .where(WORKFLOW_OF_USER.UID.eq(uid)) + .fetch() + .asScala + .map(_.getWid.longValue()) + .toSet + + val sharedWids: Set[Long] = ctx + .selectFrom(WORKFLOW_USER_ACCESS) + .where(WORKFLOW_USER_ACCESS.UID.eq(uid)) + .fetch() + .asScala + .map(_.getWid.longValue()) + .toSet + + val allowedProjects: Set[Long] = ctx + .selectFrom(PROJECT_USER_ACCESS) + .where(PROJECT_USER_ACCESS.UID.eq(uid)) + .fetch() + .asScala + .map(_.getPid.longValue()) + .toSet + + logger.debug( + s"resolved observability scope for user $uid: ${ownedWids.size} owned + " + + s"${sharedWids.size} shared workflow(s), ${allowedProjects.size} project(s)" + ) + GatewayScope( + userId = uid.longValue(), + allowedWorkflowIds = ownedWids ++ sharedWids, + allowedProjectIds = allowedProjects + ) + } + } + + /** Test double. Constructed with a static scope; ignores the + * caller's SessionUser. */ + class Stub(scope: GatewayScope) extends ScopeResolver { + override def resolve(user: SessionUser): GatewayScope = scope + } +} diff --git a/amber/src/main/scala/org/apache/texera/web/observability/gateway/builders.scala b/amber/src/main/scala/org/apache/texera/web/observability/gateway/builders.scala new file mode 100644 index 00000000000..ea2cd08d4c1 --- /dev/null +++ b/amber/src/main/scala/org/apache/texera/web/observability/gateway/builders.scala @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.web.observability.gateway + +import org.apache.texera.web.observability.gateway.dtos._ + +/** + * Typed query builders. Each takes a validated DTO + the caller's + * resolved scope and returns a backend-specific query string + the + * query parameters that should accompany it. + * + * Security invariant: no field of the input DTO is concatenated into + * the output query without first passing through a typed accessor. + * Even free-text fields are emitted only as escaped *values* in the + * query DSL — never as DSL syntax. + * + * Each builder is pure (no side effects, no I/O). Exhaustive + * injection tests live in BuildersSpec. + */ + +/** Tenancy / scope envelope. Computed by [[ObservabilityScope]]; + * every builder consumes it so the caller cannot widen scope. */ +case class GatewayScope( + userId: Long, + allowedWorkflowIds: Set[Long], + allowedProjectIds: Set[Long] +) { + + /** Allowed list joined as the typed parameter to a backend query. + * Empty allowed-set yields "0" (a workflow id that cannot exist), + * which produces a zero-result query without breaking syntax. */ + def workflowIdsCsv: String = { + if (allowedWorkflowIds.isEmpty) "0" + else allowedWorkflowIds.toSeq.sorted.mkString(",") + } + + /** Allowed list joined as a regex-alternation body (no anchors, no + * parens). For use inside a LogsQL stream filter as + * ``field=~"^()$"``. Empty allow-set yields "0" — a numeric + * literal that matches nothing real and keeps regex syntax valid. */ + def workflowIdsRegexAlt: String = { + if (allowedWorkflowIds.isEmpty) "0" + else allowedWorkflowIds.toSeq.sorted.mkString("|") + } +} diff --git a/amber/src/main/scala/org/apache/texera/web/observability/gateway/dtos.scala b/amber/src/main/scala/org/apache/texera/web/observability/gateway/dtos.scala new file mode 100644 index 00000000000..733834daedd --- /dev/null +++ b/amber/src/main/scala/org/apache/texera/web/observability/gateway/dtos.scala @@ -0,0 +1,375 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.web.observability.gateway + +import java.time.{Duration, Instant} + +/** + * Strongly-typed request / response DTOs for the gateway. + * + * Every field is either a typed primitive (Long, Instant, enum) or + * a length-/range-validated wrapper. There is no public field that + * accepts an arbitrary string and lets it through to a backend + * query language verbatim — the typed builders in [[builders]] + * receive only validated values from these DTOs. + * + * Time-window caps differ per signal, per the PR plan: + * logs ≤ 7 days + * metrics ≤ 90 days + * traces ≤ 24 hours + * profiles ≤ 7 days + * + * Page size is server-clamped at [[MaxPageSize]] for every signal. + */ +object dtos { + + val MaxPageSize: Int = 1000 + val MaxFreeTextLen: Int = 256 + val MaxResponseBytes: Long = 10L * 1024L * 1024L // 10 MiB hard cap + + /** Log severity, exposed to the UI as a closed enum. The wire + * value is the standard OTel severity-text string. */ + sealed abstract class LogLevel(val name: String) + object LogLevel { + case object TRACE extends LogLevel("TRACE") + case object DEBUG extends LogLevel("DEBUG") + case object INFO extends LogLevel("INFO") + case object WARN extends LogLevel("WARN") + case object ERROR extends LogLevel("ERROR") + + val all: Seq[LogLevel] = Seq(TRACE, DEBUG, INFO, WARN, ERROR) + def parse(raw: String): Option[LogLevel] = + Option(raw).flatMap(s => all.find(_.name.equalsIgnoreCase(s.trim))) + } + + /** Signal kind, used by metric/audit code to discriminate. */ + sealed trait Signal + object Signal { + case object Logs extends Signal + case object Metrics extends Signal + case object Traces extends Signal + case object Profiles extends Signal + } + + /** Maximum time window per signal, in seconds. */ + def maxWindowSeconds(signal: Signal): Long = signal match { + case Signal.Logs => 7L * 24L * 3600L + case Signal.Metrics => 90L * 24L * 3600L + case Signal.Traces => 24L * 3600L + case Signal.Profiles => 7L * 24L * 3600L + } + + // ---- typed unions for validator results ------------------------------- + + /** Result of validating an inbound request. Either a clean + * typed value, or a GatewayError with a stable code + message + * shape suitable for serializing to JSON. */ + sealed trait ValidationResult[+T] + case class Valid[T](value: T) extends ValidationResult[T] + case class Invalid(error: GatewayError) extends ValidationResult[Nothing] + + /** Stable error shape returned to the UI. ``code`` is a short + * machine-readable token, ``message`` is human-readable and + * redaction-safe (we never echo raw user input here). */ + case class GatewayError(code: String, message: String, status: Int) + + object GatewayError { + val BadTimeWindow: (Long => GatewayError) = (maxSec: Long) => + GatewayError("bad_time_window", s"time window must be > 0 and <= ${maxSec}s", 400) + val BadPageSize: GatewayError = + GatewayError("bad_page_size", s"pageSize must be in [1, ${MaxPageSize}]", 400) + val BadLevel: GatewayError = + GatewayError("bad_level", "level must be one of TRACE/DEBUG/INFO/WARN/ERROR", 400) + val FreeTextTooLong: GatewayError = + GatewayError("free_text_too_long", s"query must be <= ${MaxFreeTextLen} chars", 400) + val Forbidden: GatewayError = + GatewayError("forbidden", "no access to that scope", 403) + val RateLimited: GatewayError = + GatewayError("rate_limited", "too many requests", 429) + val BackendUnreachable: (String => GatewayError) = (signal: String) => + GatewayError("backend_unreachable", s"$signal backend is unreachable", 503) + val ResponseTooLarge: GatewayError = + GatewayError("response_too_large", s"response exceeded ${MaxResponseBytes} bytes", 502) + // Surfaced when a backend returns a non-2xx status. We deliberately do + // NOT include the backend's body in the message — VictoriaLogs/Jaeger + // echo the rejected query back, which would leak whatever filters the + // caller supplied. Operators see the body in the audit log. + val BackendError: ((String, Int) => GatewayError) = (signal: String, status: Int) => + GatewayError("backend_error", s"$signal backend returned HTTP $status", 502) + val BadSort: GatewayError = + GatewayError("bad_sort", "sort must be one of newest/oldest/severity/service", 400) + val BadCursor: GatewayError = + GatewayError("bad_cursor", "pageCursor must be a non-negative integer", 400) + } + + // ---- shared bits ------------------------------------------------------ + + /** Validated time window. Both ends are Instants, range bounded + * per signal, ``to > from`` strictly. */ + case class TimeWindow(from: Instant, to: Instant) + + object TimeWindow { + def validate( + signal: Signal, + fromMs: Long, + toMs: Long + ): ValidationResult[TimeWindow] = { + val from = Instant.ofEpochMilli(fromMs) + val to = Instant.ofEpochMilli(toMs) + val seconds = Duration.between(from, to).toSeconds + if (seconds <= 0 || seconds > maxWindowSeconds(signal)) + Invalid(GatewayError.BadTimeWindow(maxWindowSeconds(signal))) + else Valid(TimeWindow(from, to)) + } + } + + /** Free text used as a *value* (never as syntax) in a backend + * query. Length-capped and CRLF-stripped before reaching a + * builder. ``None`` for absent input. */ + case class FreeText(value: String) + + object FreeText { + def validate(raw: Option[String]): ValidationResult[Option[FreeText]] = { + raw match { + case None => Valid(None) + case Some(s) => + if (s.length > MaxFreeTextLen) Invalid(GatewayError.FreeTextTooLong) + else { + val stripped = s.filter(c => c >= 0x20 && c != 0x7F) + if (stripped.isEmpty) Valid(None) else Valid(Some(FreeText(stripped))) + } + } + } + } + + /** Validated page size in [1, MaxPageSize]. */ + case class PageSize(value: Int) + + object PageSize { + def validate(raw: Int): ValidationResult[PageSize] = + if (raw < 1 || raw > MaxPageSize) Invalid(GatewayError.BadPageSize) + else Valid(PageSize(raw)) + } + + // ---- per-signal request DTOs (validated) ----------------------------- + + /** Inbound logs search request before validation. Strings/longs + * only — never reaches a query builder unvalidated. */ + case class RawLogsSearchRequest( + workflowId: Option[Long], + executionId: Option[Long], + computingUnitId: Option[Long], + userId: Option[Long], + services: Option[Seq[String]], + level: Option[String], + query: Option[String], + sort: Option[String], + fromMs: Long, + toMs: Long, + pageSize: Int, + pageCursor: Option[String] + ) + + /** Validated and ready to hand to LogsQLBuilder. */ + case class ValidatedLogsRequest( + workflowId: Option[Long], + executionId: Option[Long], + computingUnitId: Option[Long], + userId: Option[Long], + services: Seq[ServiceName], + level: Option[LogLevel], + query: Option[FreeText], + sort: LogSort, + window: TimeWindow, + pageSize: PageSize, + // Page offset (records skipped). The wire shape stays as String + // (it's an opaque cursor on the UI) but we parse it as Long + // here so the builder gets a typed value. + offset: Long, + pageCursor: Option[String] + ) + + /** Closed enum of sort orders. Backed by a LogsQL `| sort by (...)` + * clause; the LogsQL fragment is in [[LogsQLBuilder]] so this DTO + * stays storage-agnostic. */ + sealed abstract class LogSort(val name: String) + object LogSort { + case object NewestFirst extends LogSort("newest") + case object OldestFirst extends LogSort("oldest") + case object SeverityHigh extends LogSort("severity") + case object ServiceAsc extends LogSort("service") + val all: Seq[LogSort] = Seq(NewestFirst, OldestFirst, SeverityHigh, ServiceAsc) + val Default: LogSort = NewestFirst + def parse(raw: String): Option[LogSort] = + Option(raw).flatMap(s => all.find(_.name.equalsIgnoreCase(s.trim))) + } + + /** Validated service name. Texera service names are emitted by the + * OTel resource attribute `service.name` — we know they match the + * pattern `texera-?[a-z0-9-]+` because the JVM bootstrap controls + * them. We enforce that pattern here so a forged value cannot + * inject LogsQL syntax via the service filter. */ + case class ServiceName(value: String) + + object ServiceName { + // Conservative: lowercase letters, digits, dash. Length-capped at + // 64 to keep stream labels bounded. + private val ServicePattern = "^[a-z0-9]([a-z0-9-]{0,62}[a-z0-9])?$".r.pattern + + def parse(raw: String): Option[ServiceName] = + Option(raw) + .map(_.trim.toLowerCase) + .filter(s => s.nonEmpty && s.length <= 64 && ServicePattern.matcher(s).matches()) + .map(ServiceName.apply) + + def validateMany(raws: Option[Seq[String]]): ValidationResult[Seq[ServiceName]] = { + raws match { + case None => Valid(Seq.empty) + case Some(list) => + val parsed = list.iterator.flatMap(s => parse(s).iterator).toSeq.distinct.take(32) + // We accept the parsed subset silently; a caller supplying + // a malformed service name simply gets fewer filters, not + // a 400. The UI's multi-select cannot produce such values + // because its options come from /logs/sources. + Valid(parsed) + } + } + } + + case class LogEntryResponse( + timestampMs: Long, + level: String, + body: String, + traceId: Option[String], + spanId: Option[String], + attributes: Map[String, String] + ) + + case class LogsSearchResponse( + entries: Seq[LogEntryResponse], + total: Long, + nextCursor: Option[String] + ) + + /** Distinct filter values currently present in the logs store. + * Powers the UI's autofill dropdowns for service / workflow id / + * CU id / user id. Service names are returned as raw strings + * (the UI doesn't need typed parsing — it just renders the chip + * and passes the value back). */ + case class LogSourcesResponse( + services: Seq[String], + workflowIds: Seq[Long], + computingUnitIds: Seq[Long], + userIds: Seq[Long] + ) + + // ---- metrics --------------------------------------------------------- + + /** Named server-side metric query. We do not let the client send + * raw MetricsQL — they pick from a fixed enum. */ + sealed abstract class NamedMetric(val name: String) + object NamedMetric { + case object RunsPerDay extends NamedMetric("runsPerDay") + case object TotalRuns extends NamedMetric("totalRuns") + case object ActiveWorkflows extends NamedMetric("activeWorkflows") + case object SuccessRate extends NamedMetric("successRate") + case object FailureRate extends NamedMetric("failureRate") + case object AvgDuration extends NamedMetric("avgDuration") + case object P50Duration extends NamedMetric("p50Duration") + case object P95Duration extends NamedMetric("p95Duration") + case object P99Duration extends NamedMetric("p99Duration") + + val all: Seq[NamedMetric] = + Seq(RunsPerDay, TotalRuns, ActiveWorkflows, SuccessRate, FailureRate, + AvgDuration, P50Duration, P95Duration, P99Duration) + def parse(raw: String): Option[NamedMetric] = + Option(raw).flatMap(s => all.find(_.name == s)) + } + + case class RawMetricsQueryRequest( + name: String, + fromMs: Long, + toMs: Long, + stepSec: Option[Int] + ) + + case class ValidatedMetricsRequest( + metric: NamedMetric, + window: TimeWindow, + stepSec: Int + ) + + case class MetricPoint(timestampMs: Long, value: Double) + + case class MetricsQueryResponse( + metric: String, + points: Seq[MetricPoint] + ) + + // ---- traces ---------------------------------------------------------- + + /** Inbound trace lookup. ``traceId`` must match the regex + * ``^[0-9a-f]{32}$`` (same as W3C trace-id). */ + case class RawTracesGetRequest(traceId: String) + + case class ValidatedTracesGetRequest(traceId: String) + + object ValidatedTracesGetRequest { + private val TraceIdPattern = "^[0-9a-f]{32}$".r.pattern + def validate(raw: RawTracesGetRequest): ValidationResult[ValidatedTracesGetRequest] = { + if (raw.traceId != null && TraceIdPattern.matcher(raw.traceId).matches()) + Valid(ValidatedTracesGetRequest(raw.traceId)) + else + Invalid(GatewayError("bad_trace_id", "traceId must be 32 lowercase hex chars", 400)) + } + } + + case class TraceSpanResponse( + spanId: String, + parentSpanId: Option[String], + name: String, + startMs: Long, + endMs: Long, + attributes: Map[String, String] + ) + + case class TracesGetResponse(traceId: String, spans: Seq[TraceSpanResponse]) + + // ---- profiles -------------------------------------------------------- + + case class RawProfilesQueryRequest( + workflowId: Option[Long], + executionId: Option[Long], + fromMs: Long, + toMs: Long + ) + + case class ValidatedProfilesRequest( + workflowId: Option[Long], + executionId: Option[Long], + window: TimeWindow + ) + + /** Profiles are returned as a tree of frames. We render the tree + * in the UI; the gateway is only responsible for shape + size. */ + case class FlameFrame(name: String, value: Long, children: Seq[FlameFrame]) + + case class ProfilesQueryResponse(root: Option[FlameFrame], totalSamples: Long) +} diff --git a/amber/src/test/scala/org/apache/texera/web/observability/RequestContextMdcFilterSpec.scala b/amber/src/test/scala/org/apache/texera/web/observability/RequestContextMdcFilterSpec.scala new file mode 100644 index 00000000000..fadb8ac948f --- /dev/null +++ b/amber/src/test/scala/org/apache/texera/web/observability/RequestContextMdcFilterSpec.scala @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.web.observability + +import org.apache.texera.observability.LogSanitizer +import org.scalatest.BeforeAndAfterEach +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.slf4j.MDC + +import javax.servlet.{FilterChain, ServletRequest, ServletResponse} +import javax.servlet.http.HttpServletRequest + +class RequestContextMdcFilterSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { + + override def beforeEach(): Unit = MDC.clear() + override def afterEach(): Unit = MDC.clear() + + // Minimal HttpServletRequest stub — only the methods the filter + // actually reads. Hand-rolled instead of pulling Mockito to keep + // the dependency surface narrow. + private class StubReq( + path: String, + headers: Map[String, String] = Map.empty, + params: Map[String, String] = Map.empty + ) extends HttpServletRequestStub { + override def getRequestURI: String = path + override def getHeader(name: String): String = headers.getOrElse(name, null) + override def getParameter(name: String): String = params.getOrElse(name, null) + } + + /** Captures the MDC state from inside the chain so we can assert on + * the per-request context, not the post-finally cleared state. */ + private class CapturingChain extends FilterChain { + var captured: Map[String, String] = Map.empty + override def doFilter(request: ServletRequest, response: ServletResponse): Unit = { + val ctx = MDC.getCopyOfContextMap + captured = if (ctx == null) Map.empty else { + val sb = scala.collection.mutable.Map.empty[String, String] + ctx.forEach((k, v) => sb.put(k, v)) + sb.toMap + } + } + } + + private val filter = new RequestContextMdcFilter() + + "RequestContextMdcFilter" should "extract workflow id from a /workflow/ URL" in { + val chain = new CapturingChain + filter.doFilter(new StubReq("/api/workflow/441/runs"), null, chain) + chain.captured("texera.workflow.id") shouldBe "441" + } + + it should "extract execution id from a /execution/ URL" in { + val chain = new CapturingChain + filter.doFilter(new StubReq("/api/execution/77/status"), null, chain) + chain.captured("texera.execution.id") shouldBe "77" + } + + it should "extract computing-unit id from a /computing-unit/ URL" in { + val chain = new CapturingChain + filter.doFilter(new StubReq("/api/computing-unit/8/health"), null, chain) + chain.captured("texera.computing_unit.id") shouldBe "8" + } + + it should "pick up multiple ids in a single URL" in { + val chain = new CapturingChain + filter.doFilter(new StubReq("/api/workflow/441/execution/77/computing-unit/8"), null, chain) + chain.captured should contain allOf ( + "texera.workflow.id" -> "441", + "texera.execution.id" -> "77", + "texera.computing_unit.id" -> "8" + ) + } + + it should "prefer explicit headers over URL inference" in { + val chain = new CapturingChain + filter.doFilter( + new StubReq( + "/api/workflow/999", + headers = Map("X-Texera-Workflow-Id" -> "123") + ), + null, chain + ) + // Header wins. + chain.captured("texera.workflow.id") shouldBe "123" + } + + it should "reject non-digit headers (log-forging guard)" in { + val chain = new CapturingChain + filter.doFilter( + new StubReq("/", headers = Map("X-Texera-Workflow-Id" -> "evil\r\nINJECT")), + null, chain + ) + chain.captured.get("texera.workflow.id") shouldBe None + } + + it should "reject impossibly-long ids" in { + val chain = new CapturingChain + val twentyDigits = "1" * 20 + filter.doFilter( + new StubReq("/", headers = Map("X-Texera-Computing-Unit-Id" -> twentyDigits)), + null, chain + ) + chain.captured.get("texera.computing_unit.id") shouldBe None + } + + it should "clear MDC entries after the request finishes (no thread leakage)" in { + val chain = new CapturingChain + filter.doFilter(new StubReq("/api/workflow/441"), null, chain) + // Inside the chain it was set: + chain.captured("texera.workflow.id") shouldBe "441" + // After the filter completes, the MDC is back to clean. + MDC.get("texera.workflow.id") shouldBe null + } + + it should "clear MDC entries even when the chain throws" in { + val throwingChain = new FilterChain { + override def doFilter(request: ServletRequest, response: ServletResponse): Unit = + throw new RuntimeException("downstream failure") + } + a[RuntimeException] should be thrownBy { + filter.doFilter(new StubReq("/api/workflow/441"), null, throwingChain) + } + MDC.get("texera.workflow.id") shouldBe null + } + + it should "not push anything to MDC for URLs that don't match any pattern" in { + val chain = new CapturingChain + filter.doFilter(new StubReq("/api/healthcheck"), null, chain) + chain.captured shouldBe empty + } + + // ---- query-string extraction (covers the WS upgrade URL) --------- + + it should "extract ids from query parameters (cuid / wid / eid)" in { + val chain = new CapturingChain + filter.doFilter( + new StubReq( + "/wsapi/workflow-websocket", + params = Map("cuid" -> "8", "wid" -> "441", "eid" -> "9001") + ), + null, chain + ) + chain.captured should contain allOf ( + "texera.computing_unit.id" -> "8", + "texera.workflow.id" -> "441", + "texera.execution.id" -> "9001" + ) + } + + it should "reject non-numeric query-param values (log-forging guard)" in { + val chain = new CapturingChain + filter.doFilter( + new StubReq("/x", params = Map("cuid" -> "abc")), + null, chain + ) + chain.captured.get("texera.computing_unit.id") shouldBe None + } + + it should "prefer URL-path match over query-param when both are present" in { + val chain = new CapturingChain + filter.doFilter( + new StubReq("/api/workflow/441", params = Map("wid" -> "999")), + null, chain + ) + // Path is checked first; param check skips when MDC already set. + chain.captured("texera.workflow.id") shouldBe "441" + } + + it should "stay aligned with LogSanitizer's MDC allowlist (otherwise the OTel appender drops them)" in { + // Every key this filter may push must be allowlisted in + // LogSanitizer.AllowedMdcKeys or the appender silently strips it + // and the dashboard sees nothing. + val filterKeys = Set("texera.workflow.id", "texera.execution.id", "texera.computing_unit.id") + filterKeys.subsetOf(LogSanitizer.AllowedMdcKeys) shouldBe true + } +} + +/** Local trait that gives the unimplemented HttpServletRequest a + * no-op default for every other method — keeps test classes small + * without pulling in a mocking library. */ +private abstract class HttpServletRequestStub extends HttpServletRequest { + // Unused methods throw — surfaces an unexpected dependency before it + // silently returns null and produces a flaky test. + private def stub(): Nothing = throw new UnsupportedOperationException( + "method not stubbed; add to StubReq if a test requires it") + override def getAuthType: String = stub() + override def getCookies: Array[javax.servlet.http.Cookie] = stub() + override def getDateHeader(name: String): Long = stub() + override def getHeaders(name: String): java.util.Enumeration[String] = stub() + override def getHeaderNames: java.util.Enumeration[String] = stub() + override def getIntHeader(name: String): Int = stub() + override def getMethod: String = "GET" + override def getPathInfo: String = null + override def getPathTranslated: String = null + override def getContextPath: String = "" + override def getQueryString: String = null + override def getRemoteUser: String = null + override def isUserInRole(role: String): Boolean = false + override def getUserPrincipal: java.security.Principal = null + override def getRequestedSessionId: String = null + override def getServletPath: String = "" + override def getSession(create: Boolean): javax.servlet.http.HttpSession = null + override def getSession: javax.servlet.http.HttpSession = null + override def changeSessionId(): String = stub() + override def isRequestedSessionIdValid: Boolean = false + override def isRequestedSessionIdFromCookie: Boolean = false + override def isRequestedSessionIdFromURL: Boolean = false + override def isRequestedSessionIdFromUrl: Boolean = false + override def authenticate(response: javax.servlet.http.HttpServletResponse): Boolean = stub() + override def login(username: String, password: String): Unit = stub() + override def logout(): Unit = stub() + override def getParts: java.util.Collection[javax.servlet.http.Part] = stub() + override def getPart(name: String): javax.servlet.http.Part = stub() + override def upgrade[T <: javax.servlet.http.HttpUpgradeHandler](handlerClass: Class[T]): T = stub() + override def getRequestURL: StringBuffer = new StringBuffer(getRequestURI) + override def getAttribute(name: String): AnyRef = null + override def getAttributeNames: java.util.Enumeration[String] = stub() + override def getCharacterEncoding: String = null + override def setCharacterEncoding(env: String): Unit = () + override def getContentLength: Int = -1 + override def getContentLengthLong: Long = -1L + override def getContentType: String = null + override def getInputStream: javax.servlet.ServletInputStream = stub() + override def getParameter(name: String): String = null + override def getParameterNames: java.util.Enumeration[String] = stub() + override def getParameterValues(name: String): Array[String] = null + override def getParameterMap: java.util.Map[String, Array[String]] = stub() + override def getProtocol: String = "HTTP/1.1" + override def getScheme: String = "http" + override def getServerName: String = "localhost" + override def getServerPort: Int = 8080 + override def getReader: java.io.BufferedReader = stub() + override def getRemoteAddr: String = "127.0.0.1" + override def getRemoteHost: String = "localhost" + override def setAttribute(name: String, o: Any): Unit = () + override def removeAttribute(name: String): Unit = () + override def getLocale: java.util.Locale = java.util.Locale.US + override def getLocales: java.util.Enumeration[java.util.Locale] = stub() + override def isSecure: Boolean = false + override def getRequestDispatcher(path: String): javax.servlet.RequestDispatcher = stub() + override def getRealPath(path: String): String = path + override def getRemotePort: Int = 0 + override def getLocalName: String = "localhost" + override def getLocalAddr: String = "127.0.0.1" + override def getLocalPort: Int = 8080 + override def getServletContext: javax.servlet.ServletContext = null + override def startAsync(): javax.servlet.AsyncContext = stub() + override def startAsync(req: ServletRequest, resp: ServletResponse): javax.servlet.AsyncContext = stub() + override def isAsyncStarted: Boolean = false + override def isAsyncSupported: Boolean = false + override def getAsyncContext: javax.servlet.AsyncContext = stub() + override def getDispatcherType: javax.servlet.DispatcherType = javax.servlet.DispatcherType.REQUEST +} diff --git a/amber/src/test/scala/org/apache/texera/web/observability/UserContextMdcFilterSpec.scala b/amber/src/test/scala/org/apache/texera/web/observability/UserContextMdcFilterSpec.scala new file mode 100644 index 00000000000..4a998d3a944 --- /dev/null +++ b/amber/src/test/scala/org/apache/texera/web/observability/UserContextMdcFilterSpec.scala @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.web.observability + +import org.apache.texera.auth.SessionUser +import org.apache.texera.dao.jooq.generated.tables.pojos.User +import org.apache.texera.observability.LogSanitizer +import org.scalatest.BeforeAndAfterEach +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.slf4j.MDC + +import java.security.Principal +import javax.ws.rs.core.SecurityContext + +class UserContextMdcFilterSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { + + override def beforeEach(): Unit = MDC.clear() + override def afterEach(): Unit = MDC.clear() + + private def userPrincipal(uid: Int): SessionUser = { + val u = new User() + u.setUid(uid) + u.setName(s"user-$uid") + u.setEmail(s"user-$uid@example.com") + new SessionUser(u) + } + + private def secCtxWith(principal: Principal): SecurityContext = new SecurityContext { + override def getUserPrincipal: Principal = principal + override def isUserInRole(role: String): Boolean = false + override def isSecure: Boolean = false + override def getAuthenticationScheme: String = "STUB" + } + + /** Minimal ContainerRequestContext that only implements the + * surface the filter actually touches. Avoids pulling Mockito for + * what is effectively a one-method stub. */ + private class StubRequest(secCtx: SecurityContext) + extends StubRequestContextBase { + override def getSecurityContext: SecurityContext = secCtx + } + + // ---- request-filter behaviour -------------------------------------- + + "UserContextMdcFilter" should "push texera.user.id into MDC when SecurityContext has a SessionUser" in { + val filter = new UserContextMdcFilter() + filter.filter(new StubRequest(secCtxWith(userPrincipal(42)))) + MDC.get("texera.user.id") shouldBe "42" + } + + it should "do nothing when the request is anonymous" in { + val filter = new UserContextMdcFilter() + filter.filter(new StubRequest(secCtxWith(null))) + MDC.get("texera.user.id") shouldBe null + } + + it should "do nothing when SecurityContext itself is null" in { + val filter = new UserContextMdcFilter() + filter.filter(new StubRequest(null)) + MDC.get("texera.user.id") shouldBe null + } + + it should "ignore principals that aren't SessionUser" in { + val filter = new UserContextMdcFilter() + val other = new Principal { override def getName: String = "robot" } + filter.filter(new StubRequest(secCtxWith(other))) + MDC.get("texera.user.id") shouldBe null + } + + // ---- response-filter clears MDC ------------------------------------ + + it should "clear texera.user.id after the response filter runs" in { + val filter = new UserContextMdcFilter() + filter.filter(new StubRequest(secCtxWith(userPrincipal(7)))) + MDC.get("texera.user.id") shouldBe "7" + filter.filter(new StubRequest(secCtxWith(userPrincipal(7))), null) + MDC.get("texera.user.id") shouldBe null + } + + // ---- contract with the OTel appender allowlist -------------------- + + it should "use an MDC key that the OTel log appender will actually forward" in { + // If the key here isn't in LogSanitizer.AllowedMdcKeys, the OTel + // bridge silently drops it and the dashboard never sees the user + // id on emitted records. + LogSanitizer.AllowedMdcKeys should contain(UserContextMdcFilter.UserIdKey) + } +} + +/** Base stub that throws on every ContainerRequestContext method; + * subclasses override only what they need. Keeps the spec narrow on + * the actual filter dependencies (just `getSecurityContext`). */ +private abstract class StubRequestContextBase extends javax.ws.rs.container.ContainerRequestContext { + private def stub(): Nothing = + throw new UnsupportedOperationException("not stubbed; override if a test needs it") + override def getProperty(name: String): AnyRef = null + override def getPropertyNames: java.util.Collection[String] = stub() + override def setProperty(name: String, `object`: Any): Unit = () + override def removeProperty(name: String): Unit = () + override def getUriInfo: javax.ws.rs.core.UriInfo = stub() + override def setRequestUri(requestUri: java.net.URI): Unit = stub() + override def setRequestUri(baseUri: java.net.URI, requestUri: java.net.URI): Unit = stub() + override def getRequest: javax.ws.rs.core.Request = stub() + override def getMethod: String = "GET" + override def setMethod(method: String): Unit = stub() + override def getHeaders: javax.ws.rs.core.MultivaluedMap[String, String] = stub() + override def getHeaderString(name: String): String = null + override def getDate: java.util.Date = null + override def getLanguage: java.util.Locale = null + override def getLength: Int = -1 + override def getMediaType: javax.ws.rs.core.MediaType = null + override def getAcceptableMediaTypes: java.util.List[javax.ws.rs.core.MediaType] = stub() + override def getAcceptableLanguages: java.util.List[java.util.Locale] = stub() + override def getCookies: java.util.Map[String, javax.ws.rs.core.Cookie] = stub() + override def hasEntity: Boolean = false + override def getEntityStream: java.io.InputStream = null + override def setEntityStream(input: java.io.InputStream): Unit = () + override def getSecurityContext: javax.ws.rs.core.SecurityContext = null + override def setSecurityContext(context: javax.ws.rs.core.SecurityContext): Unit = stub() + override def abortWith(response: javax.ws.rs.core.Response): Unit = stub() +} diff --git a/amber/src/test/scala/org/apache/texera/web/observability/gateway/BackendClientSpec.scala b/amber/src/test/scala/org/apache/texera/web/observability/gateway/BackendClientSpec.scala new file mode 100644 index 00000000000..55d3b1aba29 --- /dev/null +++ b/amber/src/test/scala/org/apache/texera/web/observability/gateway/BackendClientSpec.scala @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.web.observability.gateway + +import com.sun.net.httpserver.{HttpExchange, HttpHandler, HttpServer} +import org.apache.texera.web.observability.gateway.dtos._ +import org.scalatest.{BeforeAndAfterAll, OptionValues} +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.net.InetSocketAddress +import java.nio.charset.StandardCharsets + +/** + * Tests for [[BackendClient]] using an in-process JDK HttpServer. + * + * We avoid mocking the HttpClient — that would only exercise our + * adapter glue. A real socket on the loopback exercises: + * - URL composition (path concatenation with the base URL) + * - AccountID / ProjectID headers (multi-tenancy guards) + * - Status code propagation into [[BackendResponse.isOk]] + * - Body decoding under UTF-8 + * - The 10 MiB response cap that protects against runaway backends + */ +class BackendClientSpec extends AnyFlatSpec with Matchers with BeforeAndAfterAll with OptionValues { + + private var server: HttpServer = _ + private var baseUrl: String = _ + + // Routing state captured by handlers so tests can assert on what + // arrived at the backend (headers, path, body). + private val lastHeaders = scala.collection.mutable.Map.empty[String, String] + private var lastPath: String = _ + private var lastMethod: String = _ + private var lastBody: Array[Byte] = Array.emptyByteArray + @volatile private var responseStatus: Int = 200 + @volatile private var responseBody: Array[Byte] = Array.emptyByteArray + @volatile private var responseContentType: String = "application/json" + + override def beforeAll(): Unit = { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0) + server.createContext("/", new HttpHandler { + override def handle(ex: HttpExchange): Unit = { + lastHeaders.clear() + ex.getRequestHeaders.keySet().forEach { k => + lastHeaders.put(k, ex.getRequestHeaders.getFirst(k)) + } + lastPath = ex.getRequestURI.toString + lastMethod = ex.getRequestMethod + lastBody = ex.getRequestBody.readAllBytes() + ex.getResponseHeaders.set("Content-Type", responseContentType) + ex.sendResponseHeaders(responseStatus, responseBody.length) + ex.getResponseBody.write(responseBody) + ex.getResponseBody.close() + } + }) + server.setExecutor(null) + server.start() + baseUrl = s"http://127.0.0.1:${server.getAddress.getPort}" + } + + override def afterAll(): Unit = { + if (server != null) server.stop(0) + } + + private def scope = + GatewayScope(userId = 42L, allowedWorkflowIds = Set(7L), allowedProjectIds = Set(9L)) + + // ----- happy-path GET --------------------------------------------- + + "BackendClient.get" should "send GET, propagate path, status, body" in { + responseStatus = 200 + responseBody = """{"ok":true}""".getBytes(StandardCharsets.UTF_8) + val client = new BackendClient(baseUrl) + val r = client.get("/select/foo?bar=1", scope, "logs") + val resp = r.toOption.value + resp.status shouldBe 200 + resp.body shouldBe """{"ok":true}""" + resp.isOk shouldBe true + lastPath shouldBe "/select/foo?bar=1" + lastMethod shouldBe "GET" + } + + it should "send the ProjectID header but NOT AccountID (per single-tenant posture)" in { + responseStatus = 200 + responseBody = "{}".getBytes(StandardCharsets.UTF_8) + val client = new BackendClient(baseUrl) + client.get("/x", scope, "logs") + // JDK HttpServer canonicalises header keys to Pascal-case (case-insensitive). + lastHeaders.find { case (k, _) => k.equalsIgnoreCase("ProjectID") }.value._2 shouldBe "9" + // AccountID is deliberately NOT sent — see BackendClient class comment. + // VictoriaLogs multi-tenancy would otherwise filter every query to an + // empty tenant because OTel ingest does not set AccountID at write time. + lastHeaders.keys.exists(_.equalsIgnoreCase("AccountID")) shouldBe false + } + + it should "default ProjectID to '0' when the allow-set is empty" in { + val emptyProject = GatewayScope(userId = 1L, allowedWorkflowIds = Set.empty, allowedProjectIds = Set.empty) + responseStatus = 200; responseBody = "{}".getBytes + val client = new BackendClient(baseUrl) + client.get("/x", emptyProject, "logs") + lastHeaders.find { case (k, _) => k.equalsIgnoreCase("ProjectID") }.value._2 shouldBe "0" + } + + // ----- non-2xx ---------------------------------------------------- + + it should "surface non-2xx in BackendResponse.status, with isOk=false" in { + responseStatus = 422 + responseBody = "too many points".getBytes(StandardCharsets.UTF_8) + val client = new BackendClient(baseUrl) + val resp = client.get("/api/v1/query_range?query=x", scope, "metrics").toOption.value + resp.status shouldBe 422 + resp.body shouldBe "too many points" + resp.isOk shouldBe false + } + + it should "surface 503/Connection-refused as BackendUnreachable" in { + // Point at a port we know is closed. Pick a high ephemeral port + // unlikely to be bound; localhost-only so no external impact. + val client = new BackendClient("http://127.0.0.1:1") + val r = client.get("/", scope, "logs") + r.isLeft shouldBe true + r.swap.toOption.value.code shouldBe "backend_unreachable" + } + + // ----- body cap --------------------------------------------------- + + it should "surface ResponseTooLarge when the body exceeds MaxResponseBytes" in { + // Build a body just over the cap. Reuse a small buffer to keep + // the test's memory footprint tame. + val len = (MaxResponseBytes + 1L).toInt + val chunk = Array.fill[Byte](1024)('a'.toByte) + val buf = new java.io.ByteArrayOutputStream(len) + while (buf.size() < len) buf.write(chunk, 0, math.min(chunk.length, len - buf.size())) + responseStatus = 200; responseBody = buf.toByteArray + val client = new BackendClient(baseUrl) + val r = client.get("/x", scope, "logs") + r.isLeft shouldBe true + r.swap.toOption.value.code shouldBe "response_too_large" + } + + // ----- POST ------------------------------------------------------- + + it should "POST a byte body with the supplied content-type" in { + responseStatus = 200 + responseBody = "{\"echoed\":true}".getBytes(StandardCharsets.UTF_8) + val client = new BackendClient(baseUrl) + val sent = "logs-payload".getBytes(StandardCharsets.UTF_8) + client.post("/insert", sent, "application/x-ndjson", scope, "logs") + lastMethod shouldBe "POST" + new String(lastBody, StandardCharsets.UTF_8) shouldBe "logs-payload" + lastHeaders.find { case (k, _) => k.equalsIgnoreCase("Content-Type") }.value._2 shouldBe "application/x-ndjson" + } +} diff --git a/amber/src/test/scala/org/apache/texera/web/observability/gateway/DtoValidationSpec.scala b/amber/src/test/scala/org/apache/texera/web/observability/gateway/DtoValidationSpec.scala new file mode 100644 index 00000000000..15f3c48167f --- /dev/null +++ b/amber/src/test/scala/org/apache/texera/web/observability/gateway/DtoValidationSpec.scala @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.web.observability.gateway + +import org.apache.texera.web.observability.gateway.dtos._ +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +class DtoValidationSpec extends AnyFlatSpec with Matchers { + + // ----- TimeWindow ---------------------------------------------------- + + "TimeWindow.validate" should "accept a 1-hour window for logs" in { + val hourMs = 3_600_000L + TimeWindow.validate(Signal.Logs, 0L, hourMs) shouldBe a[Valid[_]] + } + + it should "reject a window > 7 days for logs" in { + val eightDaysMs = 8L * 24L * 3_600_000L + val result = TimeWindow.validate(Signal.Logs, 0L, eightDaysMs) + result shouldBe an[Invalid] + } + + it should "accept a 30-day window for metrics" in { + val thirtyDaysMs = 30L * 24L * 3_600_000L + TimeWindow.validate(Signal.Metrics, 0L, thirtyDaysMs) shouldBe a[Valid[_]] + } + + it should "reject a 25-hour window for traces" in { + val twentyFiveHrMs = 25L * 3_600_000L + TimeWindow.validate(Signal.Traces, 0L, twentyFiveHrMs) shouldBe an[Invalid] + } + + it should "reject zero and negative windows" in { + TimeWindow.validate(Signal.Logs, 0L, 0L) shouldBe an[Invalid] + TimeWindow.validate(Signal.Logs, 100L, 50L) shouldBe an[Invalid] + } + + // ----- PageSize ------------------------------------------------------ + + "PageSize.validate" should "clamp into [1, 1000]" in { + PageSize.validate(1) shouldBe a[Valid[_]] + PageSize.validate(1000) shouldBe a[Valid[_]] + PageSize.validate(0) shouldBe an[Invalid] + PageSize.validate(1001) shouldBe an[Invalid] + PageSize.validate(-5) shouldBe an[Invalid] + } + + // ----- FreeText ------------------------------------------------------ + + "FreeText.validate" should "accept a normal-length query" in { + val r = FreeText.validate(Some("hello world")) + r shouldBe Valid(Some(FreeText("hello world"))) + } + + it should "reject text longer than MaxFreeTextLen" in { + val tooLong = "x" * (MaxFreeTextLen + 1) + FreeText.validate(Some(tooLong)) shouldBe an[Invalid] + } + + it should "strip control characters before reaching a builder" in { + val r = FreeText.validate(Some("hello\r\nworld")) + r match { + case Valid(Some(ft)) => ft.value shouldBe "helloworld" + case other => fail(s"unexpected: $other") + } + } + + it should "return Valid(None) when control-stripping reduces input to empty" in { + FreeText.validate(Some("\r\n\t")) match { + case Valid(None) => succeed + case other => fail(s"unexpected: $other") + } + } + + it should "treat absent input as Valid(None)" in { + FreeText.validate(None) shouldBe Valid(None) + } + + // ----- ValidatedTracesGetRequest ------------------------------------ + + "ValidatedTracesGetRequest.validate" should "accept a 32-hex-char id" in { + val r = ValidatedTracesGetRequest.validate( + RawTracesGetRequest("0af7651916cd43dd8448eb211c80319c") + ) + r shouldBe Valid(ValidatedTracesGetRequest("0af7651916cd43dd8448eb211c80319c")) + } + + it should "reject UPPERCASE hex" in { + val r = ValidatedTracesGetRequest.validate( + RawTracesGetRequest("0AF7651916CD43DD8448EB211C80319C") + ) + r shouldBe an[Invalid] + } + + it should "reject path-traversal-style trace ids" in { + val r = ValidatedTracesGetRequest.validate( + RawTracesGetRequest("../../etc/passwd") + ) + r shouldBe an[Invalid] + } + + it should "reject wrong-length ids" in { + ValidatedTracesGetRequest.validate(RawTracesGetRequest("0af7")) shouldBe an[Invalid] + ValidatedTracesGetRequest.validate( + RawTracesGetRequest("0af7651916cd43dd8448eb211c80319c0") + ) shouldBe an[Invalid] + } + + // ----- LogLevel parsing -------------------------------------------- + + "LogLevel.parse" should "be case-insensitive and reject unknowns" in { + LogLevel.parse("info") shouldBe Some(LogLevel.INFO) + LogLevel.parse("ERROR") shouldBe Some(LogLevel.ERROR) + LogLevel.parse("WaRn") shouldBe Some(LogLevel.WARN) + LogLevel.parse("DELETE") shouldBe None + LogLevel.parse(null) shouldBe None + } + + // ----- NamedMetric parsing ------------------------------------------ + + "NamedMetric.parse" should "accept only the allowlist of names" in { + NamedMetric.parse("runsPerDay") shouldBe Some(NamedMetric.RunsPerDay) + NamedMetric.parse("failureRate") shouldBe Some(NamedMetric.FailureRate) + // Case-sensitive on purpose — UI passes the canonical form. + NamedMetric.parse("runsperday") shouldBe None + NamedMetric.parse("evilQuery") shouldBe None + } +} diff --git a/amber/src/test/scala/org/apache/texera/web/observability/gateway/RateLimiterSpec.scala b/amber/src/test/scala/org/apache/texera/web/observability/gateway/RateLimiterSpec.scala new file mode 100644 index 00000000000..6297b7ae821 --- /dev/null +++ b/amber/src/test/scala/org/apache/texera/web/observability/gateway/RateLimiterSpec.scala @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.web.observability.gateway + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +class RateLimiterSpec extends AnyFlatSpec with Matchers { + + "RateLimiter" should "permit up to `capacity` consecutive calls when no time elapses" in { + val rl = new RateLimiter(capacity = 5, refillPerSecond = 1.0) + (1 to 5).foreach { _ => + rl.tryAcquire("u1", nowMillis = 0L) shouldBe true + } + rl.tryAcquire("u1", nowMillis = 0L) shouldBe false + } + + it should "refill at the specified rate per second" in { + val rl = new RateLimiter(capacity = 2, refillPerSecond = 1.0) + rl.tryAcquire("u1", nowMillis = 0L) shouldBe true + rl.tryAcquire("u1", nowMillis = 0L) shouldBe true + rl.tryAcquire("u1", nowMillis = 0L) shouldBe false + // After 1 second, one token should be back. + rl.tryAcquire("u1", nowMillis = 1000L) shouldBe true + rl.tryAcquire("u1", nowMillis = 1000L) shouldBe false + } + + it should "track buckets per-key independently" in { + val rl = new RateLimiter(capacity = 1, refillPerSecond = 0.1) + rl.tryAcquire("u1", nowMillis = 0L) shouldBe true + rl.tryAcquire("u1", nowMillis = 0L) shouldBe false + // Different key — fresh bucket. + rl.tryAcquire("u2", nowMillis = 0L) shouldBe true + } + + it should "cap refilled tokens at capacity (no overflow)" in { + val rl = new RateLimiter(capacity = 3, refillPerSecond = 100.0) + // After 10 seconds of inactivity, 1000 tokens "would" refill — + // but capacity should clamp to 3. + rl.tryAcquire("u1", nowMillis = 10_000L) shouldBe true + rl.tryAcquire("u1", nowMillis = 10_000L) shouldBe true + rl.tryAcquire("u1", nowMillis = 10_000L) shouldBe true + rl.tryAcquire("u1", nowMillis = 10_000L) shouldBe false + } + + it should "reject zero/negative capacity at construction time" in { + an[IllegalArgumentException] should be thrownBy new RateLimiter(0, 1.0) + an[IllegalArgumentException] should be thrownBy new RateLimiter(-1, 1.0) + an[IllegalArgumentException] should be thrownBy new RateLimiter(10, 0.0) + } + + it should "use the default per-user limit of 20 req/s with capacity 20" in { + val rl = RateLimiter.defaultPerUser() + (1 to 20).foreach { _ => + rl.tryAcquire("u1", nowMillis = 0L) shouldBe true + } + rl.tryAcquire("u1", nowMillis = 0L) shouldBe false + } +} diff --git a/amber/src/test/scala/org/apache/texera/web/observability/gateway/ScopeResolverSpec.scala b/amber/src/test/scala/org/apache/texera/web/observability/gateway/ScopeResolverSpec.scala new file mode 100644 index 00000000000..2cc44c0c462 --- /dev/null +++ b/amber/src/test/scala/org/apache/texera/web/observability/gateway/ScopeResolverSpec.scala @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.web.observability.gateway + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +class ScopeResolverSpec extends AnyFlatSpec with Matchers { + + private val scope = + GatewayScope(userId = 42L, allowedWorkflowIds = Set(7L, 8L), allowedProjectIds = Set(1L)) + private val resolver = new ScopeResolver.Stub(scope) + + // ----- assertWorkflowAllowed ---------------------------------------- + + "assertWorkflowAllowed" should "permit a workflow id inside the allow-set" in { + resolver.assertWorkflowAllowed(scope, Some(7L)) shouldBe true + resolver.assertWorkflowAllowed(scope, Some(8L)) shouldBe true + } + + it should "reject a workflow id outside the allow-set (no widening possible)" in { + resolver.assertWorkflowAllowed(scope, Some(999L)) shouldBe false + resolver.assertWorkflowAllowed(scope, Some(-1L)) shouldBe false + } + + it should "default to permitted when no workflow id is supplied" in { + // Caller's full scope applies — None is the "default to my allowed + // set" path used by LogsQLBuilder via scope.workflowIdsCsv. + resolver.assertWorkflowAllowed(scope, None) shouldBe true + } + + // ----- GatewayScope.workflowIdsCsv ---------------------------------- + + "GatewayScope.workflowIdsCsv" should "render the allowed set as a sorted CSV" in { + scope.workflowIdsCsv shouldBe "7,8" + } + + it should "emit '0' for an empty allow-set so backend queries syntax-check" in { + val empty = GatewayScope(0L, Set.empty, Set.empty) + empty.workflowIdsCsv shouldBe "0" + } + + // ----- GatewayScope.workflowIdsRegexAlt ----------------------------- + + "GatewayScope.workflowIdsRegexAlt" should "render the allowed set as sorted '|'-alternation" in { + scope.workflowIdsRegexAlt shouldBe "7|8" + } + + it should "emit '0' for an empty allow-set" in { + val empty = GatewayScope(0L, Set.empty, Set.empty) + empty.workflowIdsRegexAlt shouldBe "0" + } +} diff --git a/common/config/src/main/resources/observability-gateway.conf b/common/config/src/main/resources/observability-gateway.conf new file mode 100644 index 00000000000..9b6c486687e --- /dev/null +++ b/common/config/src/main/resources/observability-gateway.conf @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Observability query-gateway configuration. +# +# These are the *query* endpoints the dashboard's backend gateway calls to +# read logs, metrics, traces, and profiles. They are distinct from the OTLP +# *export* endpoint (OTEL_EXPORTER_OTLP_ENDPOINT) that services push to. +# +# Defaults target a host-local stack: the single-node docker-compose publishes +# every backend to 127.0.0.1, so a natively-run backend (sbt / nvim) reaches +# them with zero extra configuration — enable the backends and it just works. +# We use the literal 127.0.0.1 rather than "localhost" on purpose: on a +# dual-stack host "localhost" resolves to ::1 (IPv6) first, but docker publishes +# these ports on 127.0.0.1 (IPv4) only, and the gateway's Java HttpClient does +# not fall back ::1 -> 127.0.0.1 — so "localhost" would report every backend +# unreachable. Inside docker-compose the web service overrides each URL through +# the TEXERA_OBS_*_URL entries in bin/single-node/.env so the gateway resolves +# the bridge-network service names instead. +observability-gateway { + # VictoriaLogs LogsQL query API. + logs-url = "http://127.0.0.1:9428" + logs-url = ${?TEXERA_OBS_LOGS_URL} + + # VictoriaMetrics MetricsQL query API. + metrics-url = "http://127.0.0.1:8428" + metrics-url = ${?TEXERA_OBS_METRICS_URL} + + # Jaeger query API. + traces-url = "http://127.0.0.1:16686" + traces-url = ${?TEXERA_OBS_TRACES_URL} + + # Parca pprof query API (gRPC-Web). + profiles-url = "http://127.0.0.1:7070" + profiles-url = ${?TEXERA_OBS_PROFILES_URL} +} diff --git a/common/config/src/main/scala/org/apache/texera/config/ObservabilityGatewayConfig.scala b/common/config/src/main/scala/org/apache/texera/config/ObservabilityGatewayConfig.scala new file mode 100644 index 00000000000..cf5976cd552 --- /dev/null +++ b/common/config/src/main/scala/org/apache/texera/config/ObservabilityGatewayConfig.scala @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.texera.config + +import com.typesafe.config.{Config, ConfigFactory} + +/** + * Query endpoints for the observability dashboard's backend gateway. + * + * Source of truth is `observability-gateway.conf`, which defaults to the + * host-local stack and lets each URL be overridden per deployment via the + * `TEXERA_OBS_*_URL` env vars (docker-compose sets these to the bridge-network + * service names). This is the query side; the OTLP export endpoint that + * services push to is configured separately in [[org.apache.texera.observability.OtelInit]]. + */ +object ObservabilityGatewayConfig { + private val conf: Config = + ConfigFactory.parseResources("observability-gateway.conf").resolve() + + // VictoriaLogs LogsQL query API. + val logsUrl: String = conf.getString("observability-gateway.logs-url") + + // VictoriaMetrics MetricsQL query API. + val metricsUrl: String = conf.getString("observability-gateway.metrics-url") + + // Jaeger query API. + val tracesUrl: String = conf.getString("observability-gateway.traces-url") + + // Parca pprof query API (gRPC-Web). + val profilesUrl: String = conf.getString("observability-gateway.profiles-url") +} diff --git a/frontend/src/app/app-routing.constant.ts b/frontend/src/app/app-routing.constant.ts index 6e06f725201..a8fa9e3f57e 100644 --- a/frontend/src/app/app-routing.constant.ts +++ b/frontend/src/app/app-routing.constant.ts @@ -37,6 +37,7 @@ export const USER_DATASET_CREATE = `${USER_DATASET}/create`; export const USER_COMPUTING_UNIT = `${USER}/compute`; export const USER_QUOTA = `${USER}/quota`; export const USER_DISCUSSION = `${USER}/discussion`; +export const USER_OBSERVABILITY = `${USER}/observability`; export const ADMIN = "/admin"; export const ADMIN_USER = `${ADMIN}/user`; diff --git a/frontend/src/app/app-routing.module.ts b/frontend/src/app/app-routing.module.ts index 8e5a44903e3..f1a570a6e47 100644 --- a/frontend/src/app/app-routing.module.ts +++ b/frontend/src/app/app-routing.module.ts @@ -36,6 +36,7 @@ import { FlarumComponent } from "./dashboard/component/user/flarum/flarum.compon import { AdminGmailComponent } from "./dashboard/component/admin/gmail/admin-gmail.component"; import { DatasetDetailComponent } from "./dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component"; import { UserDatasetComponent } from "./dashboard/component/user/user-dataset/user-dataset.component"; +import { ObservabilityComponent } from "./dashboard/component/user/observability/observability.component"; import { HubWorkflowDetailComponent } from "./hub/component/workflow/detail/hub-workflow-detail.component"; import { LandingPageComponent } from "./hub/component/landing-page/landing-page.component"; import { USER_WORKFLOW } from "./app-routing.constant"; @@ -136,6 +137,10 @@ routes.push({ path: "discussion", component: FlarumComponent, }, + { + path: "observability", + component: ObservabilityComponent, + }, ], }, { diff --git a/frontend/src/app/common/service/blob-error-http-interceptor.service.ts b/frontend/src/app/common/service/blob-error-http-interceptor.service.ts index 9167b6c8df5..392876c13a0 100644 --- a/frontend/src/app/common/service/blob-error-http-interceptor.service.ts +++ b/frontend/src/app/common/service/blob-error-http-interceptor.service.ts @@ -44,11 +44,15 @@ export class BlobErrorHttpInterceptor implements HttpInterceptor { url: err.url !== null ? err.url : undefined, }) ); - } catch (_) { + } catch (parseErr) { + // eslint-disable-next-line no-console + console.error("[http] failed to parse JSON error body delivered as a Blob", parseErr); reject(err); } }; - reader.onerror = _ => { + reader.onerror = readerErr => { + // eslint-disable-next-line no-console + console.error("[http] FileReader could not read the Blob error body", readerErr); reject(err); }; reader.readAsText(err.error); diff --git a/frontend/src/app/dashboard/component/dashboard.component.html b/frontend/src/app/dashboard/component/dashboard.component.html index ba3f74fa3a1..4946fc5e3b1 100644 --- a/frontend/src/app/dashboard/component/dashboard.component.html +++ b/frontend/src/app/dashboard/component/dashboard.component.html @@ -121,6 +121,17 @@ nzType="dashboard"> Quota +
  • + + Observability +
  • +
    +
    +

    Observability

    + +
    + + + + + + + + + + + + + + + + + + + +
    diff --git a/frontend/src/app/dashboard/component/user/observability/observability.component.scss b/frontend/src/app/dashboard/component/user/observability/observability.component.scss new file mode 100644 index 00000000000..9b8864c5d7c --- /dev/null +++ b/frontend/src/app/dashboard/component/user/observability/observability.component.scss @@ -0,0 +1,61 @@ +/* Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +.texera-observability { + padding: 16px; + height: 100%; + display: flex; + flex-direction: column; + overflow: hidden; + + .header { + h2 { + margin: 0 0 8px; + } + + nz-alert { + margin-bottom: 12px; + } + } + + nz-tabs { + flex: 1 1 auto; + min-height: 0; + // Stretch ONLY the active tab pane to fill height so its child + // panel can `height: 100%` and scroll its own contents. Inactive + // panes keep Ant Design's default `display: none` — overriding + // that with a blanket `.ant-tabs-tabpane { display: flex }` rule + // was making every panel render simultaneously and stack + // visually on top of the active one. + ::ng-deep .ant-tabs-content { + height: 100%; + min-height: 0; + } + ::ng-deep .ant-tabs-tabpane-active { + height: 100%; + display: flex; + flex-direction: column; + min-height: 0; + + > * { + flex: 1 1 auto; + min-height: 0; + } + } + } +} diff --git a/frontend/src/app/dashboard/component/user/observability/observability.component.spec.ts b/frontend/src/app/dashboard/component/user/observability/observability.component.spec.ts new file mode 100644 index 00000000000..3cd65cc9418 --- /dev/null +++ b/frontend/src/app/dashboard/component/user/observability/observability.component.spec.ts @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ComponentFixture, TestBed } from "@angular/core/testing"; +import { HttpClientTestingModule } from "@angular/common/http/testing"; +import { NoopAnimationsModule } from "@angular/platform-browser/animations"; +import { of, throwError } from "rxjs"; +import { ObservabilityComponent } from "./observability.component"; +import { ObservabilityService } from "../../../service/user/observability/observability.service"; +import { ObservabilityHealth } from "../../../service/user/observability/observability.types"; + +describe("ObservabilityComponent", () => { + let component: ObservabilityComponent; + let fixture: ComponentFixture; + let mockService: { + health: ReturnType; + logSources: ReturnType; + searchLogs: ReturnType; + queryMetrics: ReturnType; + queryProfiles: ReturnType; + }; + + beforeEach(async () => { + // The shell component constructs child panels (Logs, Metrics, etc.) + // when /health reports a tab as reachable. Each child panel uses + // ObservabilityService methods — stub the minimum surface so the + // children mount without throwing during tab rendering. + mockService = { + health: vi.fn(), + logSources: vi.fn().mockReturnValue(of({ services: [], workflowIds: [], computingUnitIds: [], userIds: [] })), + searchLogs: vi.fn().mockReturnValue(of({ entries: [], total: 0 })), + queryMetrics: vi.fn().mockReturnValue(of({ metric: "stub", points: [] })), + queryProfiles: vi.fn().mockReturnValue(of({ root: null, totalSamples: 0 })), + }; + await TestBed.configureTestingModule({ + imports: [ObservabilityComponent, HttpClientTestingModule, NoopAnimationsModule], + providers: [{ provide: ObservabilityService, useValue: mockService }], + }).compileComponents(); + + fixture = TestBed.createComponent(ObservabilityComponent); + component = fixture.componentInstance; + }); + + it("renders all four tabs as reachable when /health says so", () => { + const ok: ObservabilityHealth = { + status: "ok", + checks: { logs: true, metrics: true, traces: true, profiles: true }, + }; + mockService.health.mockReturnValue(of(ok)); + fixture.detectChanges(); + expect(component.isReachable("logs")).toBe(true); + expect(component.isReachable("metrics")).toBe(true); + expect(component.isReachable("traces")).toBe(true); + expect(component.isReachable("profiles")).toBe(true); + }); + + it("marks individual tabs unreachable per the health response", () => { + const partial: ObservabilityHealth = { + status: "degraded", + checks: { logs: true, metrics: false, traces: true, profiles: false }, + }; + mockService.health.mockReturnValue(of(partial)); + fixture.detectChanges(); + expect(component.isReachable("logs")).toBe(true); + expect(component.isReachable("metrics")).toBe(false); + expect(component.isReachable("profiles")).toBe(false); + }); + + it("falls back to 'all unreachable' + healthError flag when /health fails", () => { + mockService.health.mockReturnValue(throwError(() => new Error("gateway down"))); + fixture.detectChanges(); + expect(component.healthError).toBe(true); + expect(component.isReachable("logs")).toBe(false); + expect(component.isReachable("metrics")).toBe(false); + expect(component.isReachable("traces")).toBe(false); + expect(component.isReachable("profiles")).toBe(false); + }); + + it("starts on the Logs tab (index 0)", () => { + mockService.health.mockReturnValue(of({ + status: "ok", + checks: { logs: true, metrics: true, traces: true, profiles: true }, + } as ObservabilityHealth)); + fixture.detectChanges(); + expect(component.activeTab).toBe(0); + }); +}); diff --git a/frontend/src/app/dashboard/component/user/observability/observability.component.ts b/frontend/src/app/dashboard/component/user/observability/observability.component.ts new file mode 100644 index 00000000000..9c5b56fd586 --- /dev/null +++ b/frontend/src/app/dashboard/component/user/observability/observability.component.ts @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Component, OnInit } from "@angular/core"; +import { NgIf } from "@angular/common"; +import { NzTabsComponent, NzTabComponent } from "ng-zorro-antd/tabs"; +import { NzAlertComponent } from "ng-zorro-antd/alert"; +import { NzEmptyComponent } from "ng-zorro-antd/empty"; +import { FormsModule } from "@angular/forms"; +import { ObservabilityService } from "../../../service/user/observability/observability.service"; +import { ObservabilityHealth } from "../../../service/user/observability/observability.types"; +import { TracesPivotService } from "../../../service/user/observability/traces-pivot.service"; +import { OnDestroy } from "@angular/core"; +import { Subject, takeUntil } from "rxjs"; + +/** + * Shell page for the observability dashboard. Four tabs — Logs, + * Metrics, Traces, Profiles — each guarded by the per-signal + * reachability check from /api/observability/health. Tabs whose + * backend reports unreachable render an explicit "Unreachable" + * card rather than a broken chart. + * + * Logs is the only tab with a populated panel in PR 8. Metrics, + * Traces, Profiles land in PRs 9–11 — each tab body shows a small + * "coming soon in PR X" message so the shell ships without dead UI. + */ +@Component({ + selector: "texera-observability", + templateUrl: "./observability.component.html", + styleUrls: ["./observability.component.scss"], + imports: [ + NgIf, + FormsModule, + NzTabsComponent, + NzTabComponent, + NzAlertComponent, + NzEmptyComponent, + ], +}) +export class ObservabilityComponent implements OnInit, OnDestroy { + /** Reachability state. ``null`` means "still loading". A failed + * /health call sets every check to false so the UI surfaces the + * gateway-down case explicitly. */ + health: ObservabilityHealth | null = null; + healthError = false; + + /** Active tab index — controls which panel is mounted. */ + activeTab = 0; + + /** Trace id forwarded to the traces panel when the user pivots + * from a log row. Null until the first pivot. */ + pivotedTraceId: string | null = null; + + private readonly destroy$ = new Subject(); + + /** Index of the Traces tab — kept as a constant so the + * pivot handler isn't coupled to the ordering by magic number. */ + private static readonly TRACES_TAB_INDEX = 2; + + constructor( + private observabilityService: ObservabilityService, + private tracesPivot: TracesPivotService + ) {} + + ngOnInit(): void { + this.observabilityService.health().subscribe({ + next: h => { + this.health = h; + this.healthError = false; + }, + error: err => { + // eslint-disable-next-line no-console + console.error( + "[observability] health check failed — gateway unreachable; rendering all signals as degraded", + err + ); + this.health = { + status: "degraded", + checks: { logs: false, metrics: false, traces: false, profiles: false }, + }; + this.healthError = true; + }, + }); + + this.tracesPivot.onPivot.pipe(takeUntil(this.destroy$)).subscribe(traceId => { + this.pivotedTraceId = traceId; + this.activeTab = ObservabilityComponent.TRACES_TAB_INDEX; + }); + } + + ngOnDestroy(): void { + this.destroy$.next(); + this.destroy$.complete(); + } + + /** Convenience accessors so the template stays declarative. */ + isReachable(signal: "logs" | "metrics" | "traces" | "profiles"): boolean { + return this.health?.checks[signal] === true; + } +} diff --git a/frontend/src/app/dashboard/service/user/observability/observability.service.spec.ts b/frontend/src/app/dashboard/service/user/observability/observability.service.spec.ts new file mode 100644 index 00000000000..19ccd5f2b9f --- /dev/null +++ b/frontend/src/app/dashboard/service/user/observability/observability.service.spec.ts @@ -0,0 +1,228 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { TestBed } from "@angular/core/testing"; +import { HttpClientTestingModule, HttpTestingController } from "@angular/common/http/testing"; +import { ObservabilityService, ValidationError } from "./observability.service"; +import { + LogsSearchRequest, + MAX_FREE_TEXT_LEN, + MAX_PAGE_SIZE, + MetricsQueryRequest, +} from "./observability.types"; + +describe("ObservabilityService", () => { + let service: ObservabilityService; + let httpMock: HttpTestingController; + + beforeEach(() => { + TestBed.configureTestingModule({ + imports: [HttpClientTestingModule], + providers: [ObservabilityService], + }); + service = TestBed.inject(ObservabilityService); + httpMock = TestBed.inject(HttpTestingController); + }); + + afterEach(() => httpMock.verify()); + + // ----- health ------------------------------------------------------- + + it("health() POSTs to /api/observability/health and returns the checks", () => { + service.health().subscribe(h => { + expect(h.status).toBe("ok"); + expect(h.checks.logs).toBe(true); + }); + const req = httpMock.expectOne(r => r.url.endsWith("/observability/health")); + expect(req.request.method).toBe("GET"); + req.flush({ + status: "ok", + checks: { logs: true, metrics: true, traces: true, profiles: true }, + }); + }); + + // ----- searchLogs validation guards (before HTTP dispatch) ---------- + + it("searchLogs throws ValidationError if toMs <= fromMs (no HTTP dispatch)", () => { + const req: LogsSearchRequest = { + fromMs: 1_000_000, + toMs: 500_000, // before fromMs + pageSize: 50, + }; + expect(() => service.searchLogs(req)).toThrow(); + httpMock.expectNone(r => r.url.endsWith("/observability/logs/search")); + }); + + it("searchLogs throws ValidationError if pageSize is over MAX_PAGE_SIZE", () => { + const req: LogsSearchRequest = { + fromMs: 0, + toMs: 60_000, + pageSize: MAX_PAGE_SIZE + 1, + }; + expect(() => service.searchLogs(req)).toThrow(ValidationError); + httpMock.expectNone(r => r.url.endsWith("/observability/logs/search")); + }); + + it("searchLogs throws ValidationError if query is over MAX_FREE_TEXT_LEN", () => { + const tooLong = "x".repeat(MAX_FREE_TEXT_LEN + 1); + const req: LogsSearchRequest = { + fromMs: 0, + toMs: 60_000, + pageSize: 50, + query: tooLong, + }; + expect(() => service.searchLogs(req)).toThrow(ValidationError); + httpMock.expectNone(r => r.url.endsWith("/observability/logs/search")); + }); + + // ----- searchLogs happy path ---------------------------------------- + + it("searchLogs POSTs the validated body to the gateway", () => { + const req: LogsSearchRequest = { + fromMs: 0, + toMs: 60_000, + pageSize: 50, + level: "ERROR", + workflowId: 42, + }; + service.searchLogs(req).subscribe(resp => { + expect(resp.entries.length).toBe(0); + }); + const http = httpMock.expectOne(r => r.url.endsWith("/observability/logs/search")); + expect(http.request.method).toBe("POST"); + expect(http.request.body.workflowId).toBe(42); + expect(http.request.body.level).toBe("ERROR"); + http.flush({ entries: [], total: 0 }); + }); + + it("searchLogs surfaces server-side 403 unchanged for the caller to render", () => { + const req: LogsSearchRequest = { + fromMs: 0, + toMs: 60_000, + pageSize: 50, + workflowId: 999, + }; + let observedStatus = 0; + service.searchLogs(req).subscribe({ + next: () => {}, + error: err => { + observedStatus = err.status; + }, + }); + const http = httpMock.expectOne(r => r.url.endsWith("/observability/logs/search")); + http.flush({ code: "forbidden", message: "no access to that scope" }, { + status: 403, + statusText: "Forbidden", + }); + expect(observedStatus).toBe(403); + }); + + // ----- queryMetrics (PR 9) ------------------------------------------ + + it("queryMetrics rejects unknown metric names client-side", () => { + const req = { + // Cast through unknown to make the test exercise the runtime + // guard — the type system would otherwise refuse this string. + name: "evilQuery" as unknown as MetricsQueryRequest["name"], + fromMs: 0, + toMs: 60_000, + }; + expect(() => service.queryMetrics(req)).toThrow(ValidationError); + httpMock.expectNone(r => r.url.endsWith("/observability/metrics/query")); + }); + + it("queryMetrics rejects step outside [1, 3600]", () => { + const tooBig: MetricsQueryRequest = { + name: "runsPerDay", + fromMs: 0, + toMs: 60_000, + stepSec: 99999, + }; + expect(() => service.queryMetrics(tooBig)).toThrow(ValidationError); + const tooSmall: MetricsQueryRequest = { + name: "runsPerDay", + fromMs: 0, + toMs: 60_000, + stepSec: 0, + }; + expect(() => service.queryMetrics(tooSmall)).toThrow(ValidationError); + }); + + it("queryMetrics dispatches a valid request to /metrics/query", () => { + const req: MetricsQueryRequest = { + name: "p95Duration", + fromMs: 0, + toMs: 60_000, + stepSec: 60, + }; + service.queryMetrics(req).subscribe(r => { + expect(r.metric).toBe("p95Duration"); + }); + const http = httpMock.expectOne(r => r.url.endsWith("/observability/metrics/query")); + expect(http.request.method).toBe("POST"); + expect(http.request.body.name).toBe("p95Duration"); + http.flush({ metric: "p95Duration", points: [] }); + }); + + // ----- getTrace (PR 10) --------------------------------------------- + + it("getTrace rejects malformed trace ids client-side", () => { + expect(() => service.getTrace("not-a-trace-id")).toThrow(ValidationError); + expect(() => service.getTrace("../../etc/passwd")).toThrow(ValidationError); + expect(() => service.getTrace("0AF7651916CD43DD8448EB211C80319C")).toThrow(); // uppercase + expect(() => service.getTrace("0af7651916cd43dd8448eb211c80319")).toThrow(); // too short + httpMock.expectNone(r => r.url.includes("/observability/traces/")); + }); + + it("getTrace dispatches GET /observability/traces/{id} for a valid id", () => { + const id = "0af7651916cd43dd8448eb211c80319c"; + service.getTrace(id).subscribe(resp => { + expect(resp.traceId).toBe(id); + }); + const http = httpMock.expectOne(r => r.url.endsWith(`/observability/traces/${id}`)); + expect(http.request.method).toBe("GET"); + http.flush({ traceId: id, spans: [] }); + }); + + // ----- queryProfiles (PR 11) ---------------------------------------- + + it("queryProfiles rejects an inverted time window client-side", () => { + expect(() => + service.queryProfiles({ fromMs: 100, toMs: 50 }) + ).toThrow(ValidationError); + httpMock.expectNone(r => r.url.endsWith("/observability/profiles/query")); + }); + + it("queryProfiles dispatches a valid request", () => { + service + .queryProfiles({ + workflowId: 42, + executionId: 7, + fromMs: 0, + toMs: 60_000, + }) + .subscribe(r => { + expect(r.totalSamples).toBe(0); + }); + const http = httpMock.expectOne(r => r.url.endsWith("/observability/profiles/query")); + expect(http.request.method).toBe("POST"); + expect(http.request.body.workflowId).toBe(42); + http.flush({ root: null, totalSamples: 0 }); + }); +}); diff --git a/frontend/src/app/dashboard/service/user/observability/observability.service.ts b/frontend/src/app/dashboard/service/user/observability/observability.service.ts new file mode 100644 index 00000000000..dd47bd527e6 --- /dev/null +++ b/frontend/src/app/dashboard/service/user/observability/observability.service.ts @@ -0,0 +1,174 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { HttpClient } from "@angular/common/http"; +import { Injectable } from "@angular/core"; +import { Observable } from "rxjs"; +import { AppSettings } from "../../../../common/app-setting"; +import { + LogSourcesResponse, + LogsSearchRequest, + LogsSearchResponse, + MAX_FREE_TEXT_LEN, + MAX_PAGE_SIZE, + MetricsQueryRequest, + MetricsQueryResponse, + NAMED_METRICS, + ObservabilityHealth, + TRACE_ID_RE, + TracesGetResponse, + ProfilesQueryRequest, + ProfilesQueryResponse, +} from "./observability.types"; + +const BASE_URL = `${AppSettings.getApiEndpoint()}/observability`; + +/** + * Client for the Texera observability gateway. + * + * JWT is added by Angular's existing HttpClient interceptor — no + * explicit auth handling needed here. + * + * Light client-side validation mirrors the server-side caps so we + * surface obvious errors as form errors rather than as HTTP 400s. + * The server is still the source of truth — these checks are + * usability, not security. + */ +@Injectable({ + providedIn: "root", +}) +export class ObservabilityService { + constructor(private http: HttpClient) {} + + /** + * Light reachability check used by the shell to render + * "Disabled" / "Unreachable" panels. + */ + health(): Observable { + return this.http.get(`${BASE_URL}/health`); + } + + /** + * Search application logs. Throws synchronously (before HTTP + * dispatch) if the request would fail the gateway's own + * validators — keeps the form UX snappy. + */ + searchLogs(req: LogsSearchRequest): Observable { + assertValid(req); + return this.http.post(`${BASE_URL}/logs/search`, req); + } + + /** + * Distinct filter values currently in the logs store. Used by the + * logs panel to populate the service / workflow / CU dropdowns + * with values that actually have data — saves the admin from + * guessing IDs. + */ + logSources(): Observable { + return this.http.get(`${BASE_URL}/logs/sources`); + } + + /** + * Query a named server-side metric. The set of allowed names is + * the same NAMED_METRICS enum the server enforces; we re-check it + * here so a typo in a callsite fails fast rather than after a + * 400 round-trip. + */ + queryMetrics(req: MetricsQueryRequest): Observable { + assertValidMetrics(req); + return this.http.post(`${BASE_URL}/metrics/query`, req); + } + + /** + * Fetch a trace by id. The id is regex-validated against + * TRACE_ID_RE (^[0-9a-f]{32}$) before reaching the network so a + * malformed value never lands in the URL path. The gateway + * applies the same regex server-side. + */ + getTrace(traceId: string): Observable { + assertValidTraceId(traceId); + return this.http.get( + `${BASE_URL}/traces/${encodeURIComponent(traceId)}` + ); + } + + /** + * Query CPU/alloc profiles from Parca. The gateway enforces the + * time-window cap (7d for profiles); we mirror the start { + assertValidProfiles(req); + return this.http.post(`${BASE_URL}/profiles/query`, req); + } +} + +/** Light client-side validation. Throws Error with a stable + * ``code`` field that the UI can branch on. */ +function assertValid(req: LogsSearchRequest): void { + if (req.toMs <= req.fromMs) { + throw new ValidationError("bad_time_window", "End time must be after start time."); + } + if (req.pageSize < 1 || req.pageSize > MAX_PAGE_SIZE) { + throw new ValidationError("bad_page_size", `pageSize must be between 1 and ${MAX_PAGE_SIZE}.`); + } + if (req.query !== undefined && req.query.length > MAX_FREE_TEXT_LEN) { + throw new ValidationError( + "free_text_too_long", + `Query text must be ${MAX_FREE_TEXT_LEN} characters or fewer.` + ); + } +} + +function assertValidMetrics(req: MetricsQueryRequest): void { + if (!NAMED_METRICS.includes(req.name)) { + throw new ValidationError("bad_metric_name", `unknown metric '${req.name}'.`); + } + if (req.toMs <= req.fromMs) { + throw new ValidationError("bad_time_window", "End time must be after start time."); + } + if (req.stepSec !== undefined && (req.stepSec < 1 || req.stepSec > 3600)) { + throw new ValidationError("bad_step", "Step must be between 1 and 3600 seconds."); + } +} + +function assertValidProfiles(req: ProfilesQueryRequest): void { + if (req.toMs <= req.fromMs) { + throw new ValidationError("bad_time_window", "End time must be after start time."); + } +} + +function assertValidTraceId(traceId: string): void { + if (typeof traceId !== "string" || !TRACE_ID_RE.test(traceId)) { + throw new ValidationError( + "bad_trace_id", + "Trace id must be 32 lowercase hex characters." + ); + } +} + +export class ValidationError extends Error { + constructor( + public readonly code: string, + message: string + ) { + super(message); + this.name = "ValidationError"; + } +} diff --git a/frontend/src/app/dashboard/service/user/observability/observability.types.ts b/frontend/src/app/dashboard/service/user/observability/observability.types.ts new file mode 100644 index 00000000000..c0dc8849e77 --- /dev/null +++ b/frontend/src/app/dashboard/service/user/observability/observability.types.ts @@ -0,0 +1,197 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Mirrors the Scala gateway DTOs in + * amber/.../web/observability/gateway/dtos.scala. + * + * Kept as discriminated unions + readonly types so the compiler + * catches drift between the wire format and the UI's assumptions. + */ + +export type LogLevel = "TRACE" | "DEBUG" | "INFO" | "WARN" | "ERROR"; + +/** Closed set — must match the LogLevel enum in dtos.scala. */ +export const LOG_LEVELS: readonly LogLevel[] = ["TRACE", "DEBUG", "INFO", "WARN", "ERROR"] as const; + +/** Server-enforced per-page maximum from the gateway. */ +export const MAX_PAGE_SIZE = 1000; + +/** Server-enforced max free-text query length. */ +export const MAX_FREE_TEXT_LEN = 256; + +/** Inbound request shape — matches RawLogsSearchRequest. */ +export interface LogsSearchRequest { + readonly workflowId?: number; + readonly executionId?: number; + readonly computingUnitId?: number; + readonly userId?: number; + /** Optional whitelist of service names — overrides the default + * texera-* stream prefix when present. Use the values returned + * by GET /observability/logs/sources. */ + readonly services?: ReadonlyArray; + readonly level?: LogLevel; + readonly query?: string; + readonly sort?: LogSort; + readonly fromMs: number; + readonly toMs: number; + readonly pageSize: number; + /** Opaque cursor returned in the previous page's response. */ + readonly pageCursor?: string; +} + +/** Closed enum of sort orders — must match LogSort in dtos.scala. */ +export type LogSort = "newest" | "oldest" | "severity" | "service"; + +export const LOG_SORTS: ReadonlyArray<{ value: LogSort; label: string }> = [ + { value: "newest", label: "Newest first" }, + { value: "oldest", label: "Oldest first" }, + { value: "severity", label: "Severity (high → low)" }, + { value: "service", label: "Service (A → Z)" }, +] as const; + +/** Distinct filter values currently in the logs store — backing for + * the panel's autofill dropdowns. */ +export interface LogSourcesResponse { + readonly services: ReadonlyArray; + readonly workflowIds: ReadonlyArray; + readonly computingUnitIds: ReadonlyArray; + readonly userIds: ReadonlyArray; +} + +export interface LogEntry { + readonly timestampMs: number; + readonly level: string; + readonly body: string; + readonly traceId?: string; + readonly spanId?: string; + readonly attributes: Record; +} + +export interface LogsSearchResponse { + readonly entries: ReadonlyArray; + readonly total: number; + readonly nextCursor?: string; +} + +export interface ObservabilityHealth { + readonly status: "ok" | "degraded"; + readonly checks: { + readonly logs: boolean; + readonly metrics: boolean; + readonly traces: boolean; + readonly profiles: boolean; + }; +} + +/** Stable error shape returned by the gateway. */ +export interface GatewayErrorBody { + readonly code: string; + readonly message: string; +} + +// ---- Metrics (PR 9) --------------------------------------------------- + +/** Named server-side queries. Mirrors NamedMetric in dtos.scala — + * any string the UI sends that isn't in this enum is rejected + * client-side before HTTP dispatch (the gateway rejects it again + * on the server). */ +export type NamedMetric = + | "runsPerDay" + | "totalRuns" + | "activeWorkflows" + | "successRate" + | "failureRate" + | "avgDuration" + | "p50Duration" + | "p95Duration" + | "p99Duration"; + +export const NAMED_METRICS: readonly NamedMetric[] = [ + "runsPerDay", + "totalRuns", + "activeWorkflows", + "successRate", + "failureRate", + "avgDuration", + "p50Duration", + "p95Duration", + "p99Duration", +] as const; + +export interface MetricsQueryRequest { + readonly name: NamedMetric; + readonly fromMs: number; + readonly toMs: number; + readonly stepSec?: number; +} + +export interface MetricPoint { + readonly timestampMs: number; + readonly value: number; +} + +export interface MetricsQueryResponse { + readonly metric: string; + readonly points: ReadonlyArray; +} + +// ---- Traces (PR 10) --------------------------------------------------- + +/** W3C trace-id format. Lowercase hex, exactly 32 chars. The + * service rejects anything else before HTTP dispatch and the + * gateway rejects it again on the server. */ +export const TRACE_ID_RE = /^[0-9a-f]{32}$/; + +export interface TraceSpan { + readonly spanId: string; + readonly parentSpanId?: string; + readonly name: string; + readonly startMs: number; + readonly endMs: number; + readonly attributes: Record; +} + +export interface TracesGetResponse { + readonly traceId: string; + readonly spans: ReadonlyArray; +} + +// ---- Profiles (PR 11) ------------------------------------------------- + +export interface ProfilesQueryRequest { + readonly workflowId?: number; + readonly executionId?: number; + readonly fromMs: number; + readonly toMs: number; +} + +/** Recursive flame-graph frame. value is the sample count (or any + * positive measure) at this node; children's values sum to <= value + * per pprof convention. */ +export interface FlameFrame { + readonly name: string; + readonly value: number; + readonly children: ReadonlyArray; +} + +export interface ProfilesQueryResponse { + readonly root: FlameFrame | null; + readonly totalSamples: number; +} diff --git a/frontend/src/app/dashboard/service/user/observability/traces-pivot.service.spec.ts b/frontend/src/app/dashboard/service/user/observability/traces-pivot.service.spec.ts new file mode 100644 index 00000000000..68b531c21f7 --- /dev/null +++ b/frontend/src/app/dashboard/service/user/observability/traces-pivot.service.spec.ts @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { TestBed } from "@angular/core/testing"; +import { TracesPivotService } from "./traces-pivot.service"; + +describe("TracesPivotService", () => { + let service: TracesPivotService; + + beforeEach(() => { + TestBed.configureTestingModule({ providers: [TracesPivotService] }); + service = TestBed.inject(TracesPivotService); + }); + + it("publishes a valid trace id to subscribers", () => { + const valid = "0af7651916cd43dd8448eb211c80319c"; + const observed: string[] = []; + service.onPivot.subscribe(id => observed.push(id)); + service.pivot(valid); + expect(observed).toEqual([valid]); + }); + + it("silently drops invalid trace ids (no subscriber notification)", () => { + const observed: string[] = []; + service.onPivot.subscribe(id => observed.push(id)); + // Each of these should be a no-op. + service.pivot("not-a-trace-id"); + service.pivot("../../etc/passwd"); + service.pivot("0AF7651916CD43DD8448EB211C80319C"); // uppercase + service.pivot("0af7651916cd43dd8448eb211c8031"); // too short + service.pivot(""); + service.pivot(undefined as unknown as string); + expect(observed).toEqual([]); + }); +}); diff --git a/frontend/src/app/dashboard/service/user/observability/traces-pivot.service.ts b/frontend/src/app/dashboard/service/user/observability/traces-pivot.service.ts new file mode 100644 index 00000000000..63b669ac5ef --- /dev/null +++ b/frontend/src/app/dashboard/service/user/observability/traces-pivot.service.ts @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Injectable } from "@angular/core"; +import { Observable, Subject } from "rxjs"; +import { TRACE_ID_RE } from "./observability.types"; + +/** + * Small coordinator that lets the Logs panel ask the shell to switch + * to the Traces tab and pre-fill a trace id. Singleton (providedIn: + * "root"), Subject-based — no router state, no global mutation. + * + * The trace id is regex-validated at the boundary so a corrupted log + * entry can never push a malformed value at downstream subscribers. + */ +@Injectable({ providedIn: "root" }) +export class TracesPivotService { + private readonly pivot$ = new Subject(); + + /** Stream of trace ids the user has clicked-to-open from another + * panel. Subscribers (the shell, the traces panel) receive only + * values that pass the W3C trace-id regex. */ + readonly onPivot: Observable = this.pivot$.asObservable(); + + /** Request a pivot. Silently no-op for invalid input so a caller + * cannot wedge the UI by emitting nonsense — same posture as the + * service's HTTP guards. */ + pivot(traceId: string): void { + if (typeof traceId === "string" && TRACE_ID_RE.test(traceId)) { + this.pivot$.next(traceId); + } + } +} From df10b8717e7f57087dd03a66962121e448c15632 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 5 Jun 2026 04:49:02 -0700 Subject: [PATCH 03/26] feat(observability): deployment - docker-compose OTel collector + Parca/eBPF profiling Co-Authored-By: Claude Opus 4.8 (1M context) --- bin/observability/otel-collector/config.yaml | 132 +++++++++++ bin/observability/parca/README.md | 98 ++++++++ bin/observability/parca/parca-agent.env | 62 +++++ bin/observability/parca/parca.yaml | 42 ++++ bin/single-node/.env | 34 +++ bin/single-node/docker-compose.yml | 191 +++++++++++++++- bin/single-node/up.sh | 65 ++++++ .../ObservabilityComposeSpec.scala | 216 ++++++++++++++++++ .../observability/ParcaConfigSpec.scala | 129 +++++++++++ 9 files changed, 968 insertions(+), 1 deletion(-) create mode 100644 bin/observability/otel-collector/config.yaml create mode 100644 bin/observability/parca/README.md create mode 100644 bin/observability/parca/parca-agent.env create mode 100644 bin/observability/parca/parca.yaml create mode 100755 bin/single-node/up.sh create mode 100644 common/config/src/test/scala/org/apache/texera/observability/ObservabilityComposeSpec.scala create mode 100644 common/config/src/test/scala/org/apache/texera/observability/ParcaConfigSpec.scala diff --git a/bin/observability/otel-collector/config.yaml b/bin/observability/otel-collector/config.yaml new file mode 100644 index 00000000000..7841e37f2f7 --- /dev/null +++ b/bin/observability/otel-collector/config.yaml @@ -0,0 +1,132 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# OpenTelemetry Collector configuration for the Texera bundled +# observability stack. +# +# Bundled image: otel/opentelemetry-collector-contrib:0.153.0 +# (Apache-2.0). The contrib distribution is needed for the +# prometheusremotewrite exporter that fans metrics to VictoriaMetrics. +# +# Pipeline topology: +# +# Texera services +# │ OTLP/gRPC (4317) or OTLP/HTTP (4318) +# ▼ +# otel-collector +# ├── logs ── OTLP ──────────────────► victorialogs:9428 +# ├── metrics ── prometheusremotewrite ─► victoriametrics:8428 +# └── traces ── OTLP ──────────────────► jaeger:4317 +# +# Network: every receiver and exporter is reachable only on the +# compose bridge network. No host-port mapping for the receivers in +# docker-compose.yml — that keeps OTLP ingest cluster-local, which +# is the security posture the PR plan calls for. +# +# Hostnames below are docker-compose service names — they resolve via +# docker's embedded DNS inside the network. + +receivers: + otlp: + protocols: + grpc: + # 0.0.0.0 binds inside the container only; the compose service + # does NOT publish this port to the host. Reachable only via + # the texera-single-node bridge network. + endpoint: 0.0.0.0:4317 + # 4 MiB cap on a single OTLP message — anyone trying to spam + # the collector with a huge payload gets a clean reject. + max_recv_msg_size_mib: 4 + http: + endpoint: 0.0.0.0:4318 + +processors: + # batch is the upstream-recommended first processor. Caps memory + # per export by batching by either size or timeout, whichever hits + # first. Defaults are sensible for a single-node deployment. + batch: + send_batch_size: 1024 + timeout: 5s + + # memory_limiter prevents the collector OOMing under a sudden burst + # of telemetry. We set a soft limit, not a hard percentage, so the + # numbers are reviewable. + memory_limiter: + check_interval: 1s + limit_mib: 512 + spike_limit_mib: 128 + +exporters: + # VictoriaLogs accepts OTLP for logs natively (verify on every + # version bump). The endpoint path includes /opentelemetry/v1/logs + # per VictoriaLogs's OTLP ingest convention. + otlphttp/victorialogs: + endpoint: http://victorialogs:9428/insert/opentelemetry + compression: gzip + timeout: 10s + # tls is disabled for the compose bridge network. Any deploy + # that opens these ports beyond the bridge must enable TLS here. + tls: + insecure: true + + # VictoriaMetrics consumes Prometheus remote-write natively. The + # prometheusremotewrite exporter is the upstream-recommended way + # to bridge OTel metrics into Prom-ecosystem stores. + prometheusremotewrite: + endpoint: http://victoriametrics:8428/api/v1/write + tls: + insecure: true + # Keep the remote-write batch bounded so a metric burst can't + # become an oversized HTTP request. + remote_write_queue: + queue_size: 1000 + num_consumers: 4 + + # Jaeger v2 accepts OTLP directly — no jaeger exporter type needed. + otlp/jaeger: + endpoint: jaeger:4317 + tls: + insecure: true + +service: + pipelines: + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/victorialogs] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheusremotewrite] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp/jaeger] + + # Collector telemetry binds to loopback inside the container only + # so we can scrape /metrics from a sidecar if needed without + # exposing the collector's own metrics to the host. + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 127.0.0.1 + port: 8888 + logs: + level: info diff --git a/bin/observability/parca/README.md b/bin/observability/parca/README.md new file mode 100644 index 00000000000..27ef602dfb7 --- /dev/null +++ b/bin/observability/parca/README.md @@ -0,0 +1,98 @@ + + +# Parca profiles (PR 5) + +This directory holds configuration for the **profiles** signal in the +Texera observability stack. The compose service definitions that +consume these files land in PR 6; PR 5 ships the configuration only, +so the agent's deploy posture can be reviewed in isolation from the +broader compose changes. + +Components — both Apache-2.0 (see +[`docs/observability/LICENSING.md`](../../../docs/observability/LICENSING.md)): + +| File | Component | Image | +|---|---|---| +| `parca.yaml` | Parca server v0.28.0 | `ghcr.io/parca-dev/parca:v0.28.0` | +| `parca-agent.env` | Parca eBPF agent v0.47.1 | `ghcr.io/parca-dev/parca-agent:v0.47.1` | + +## Deploy posture + +The Parca agent uses eBPF to sample stack traces from running +processes. That puts a few non-negotiable requirements on the host: + +- **Linux only.** eBPF is a Linux kernel feature. macOS and Windows + developers cannot run the agent; the rest of the observability + stack (logs, metrics, traces) works on all platforms. +- **Privileged container.** The agent needs `CAP_SYS_ADMIN`-class + permissions to load eBPF programs and mount the perf-event + facility. The PR 6 compose service will set `privileged: true` + and bind-mount `/sys/kernel/debug`, `/proc`, and `/sys` read-only + into the container. +- **Read-only on host filesystems.** The bind-mounts above are + `ro` — the agent reads kernel state but cannot write to it. No + network exposure outside the cluster: the agent only opens an + outbound connection to the bundled Parca server on + `parca:7070`. + +## Opt-out + +For developers on non-Linux dev machines, or for any deploy that +chooses not to run profiles, set this in the host environment before +`docker compose up`: + +``` +TEXERA_OBSERVABILITY_PROFILES=disabled +``` + +PR 6's compose file gates the `parca-agent` (and optionally the +`parca` server too) on this flag — the rest of the stack continues +to run with `disabled` panels in the UI. + +## What gets profiled + +The agent's default behaviour is to discover and profile every +process on the host. We attach two static labels via +`parca-agent.env`: + +- `deployment=texera` +- `cluster=local` (override per env) + +When the PR 7 Texera query gateway runs Parca queries, it filters on +`deployment=texera` so the dashboard only ever shows Texera-process +profiles, never the operator's other workloads. + +We do **not** label profiles with `workflow.id` / `execution.id`. As +with metrics, those are unbounded identifiers and would blow up +Parca's storage cardinality. Per-execution profile views are reached +by joining on `trace_id` at query time (the Parca query API supports +this). + +## What is not in PR 5 + +- The docker-compose service definitions (PR 6). +- The Angular flame-graph panel that renders pprof data (PR 11 in + the [PR plan](../../../docs/observability/PR-PLAN.md)). +- Kubernetes Helm templates for the agent DaemonSet (deferred to a + later series — single-node compose first). +- TLS between agent and server. The agent dials Parca over plain + gRPC because the bundled deployment binds both to the docker + bridge network. Any deploy that opens those ports to a wider + network must enable TLS at the compose / k8s layer. diff --git a/bin/observability/parca/parca-agent.env b/bin/observability/parca/parca-agent.env new file mode 100644 index 00000000000..5ff164a20da --- /dev/null +++ b/bin/observability/parca/parca-agent.env @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Parca eBPF agent environment. +# +# Bundled image: ghcr.io/parca-dev/parca-agent:v0.47.1 (Apache-2.0). +# +# Consumed by the PR 6 docker-compose service via env_file:. The agent +# itself reads its config as CLI args; we stage the args here so they +# are version-controlled, reviewed, and reused across compose + +# (future) Helm. +# +# Deploy requirements (Linux-only, privileged): +# * The agent uses eBPF and must run on a Linux host kernel. +# * Requires CAP_SYS_ADMIN-class permissions to load eBPF programs. +# The compose service runs the agent container with +# `privileged: true` and mounts /sys/kernel/debug, /proc, /sys +# read-only. macOS / Windows developers must run with +# TEXERA_OBSERVABILITY_PROFILES=disabled (see README.md). +# +# Labels: +# * node: host identifier — set by the compose layer to the docker +# host's hostname. Overridable. +# * metadata-external-labels: static labels attached to every +# profile. We attach service.name so the Parca query layer can +# group profiles by Texera service the same way logs/traces are +# grouped (matches the OTel resource attr). + +PARCA_AGENT_REMOTE_STORE_ADDRESS=parca:7070 +PARCA_AGENT_REMOTE_STORE_INSECURE=true + +# Per-host identification. Compose can override at deploy time: +# `--node=${HOSTNAME:-texera-dev}` +PARCA_AGENT_NODE=texera-dev + +# Static labels attached to every profile this agent emits. +# Must NOT include workflow.id / execution.id — cardinality DoS in +# Parca's storage. Keep to coarse fields only. +PARCA_AGENT_METADATA_EXTERNAL_LABELS=deployment=texera;cluster=local + +# CPU sample rate. 19 Hz is upstream default and a sensible balance +# between overhead and resolution; we keep it explicit so an +# accidental upstream change doesn't silently shift it. +PARCA_AGENT_PROFILING_CPU_SAMPLING_FREQUENCY=19 + +# Log level — info by default; switch to warn in production to keep +# the agent quiet. +PARCA_AGENT_LOG_LEVEL=info diff --git a/bin/observability/parca/parca.yaml b/bin/observability/parca/parca.yaml new file mode 100644 index 00000000000..0af9e88a393 --- /dev/null +++ b/bin/observability/parca/parca.yaml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Parca server configuration. +# +# This is the minimal config the Texera bundled observability stack +# uses. We deliberately keep scrape_configs empty — profiles arrive +# from the Parca eBPF agent (push model) rather than pull-mode scraping. +# Bundled image: ghcr.io/parca-dev/parca:v0.28.0 (Apache-2.0). +# +# Storage is intentionally filesystem-backed for the single-node +# docker-compose deployment. Replace with an object-storage backend +# (S3/GCS/Azure) before any multi-node or production use. + +object_storage: + bucket: + # FILESYSTEM is the simplest backend for single-node development. + # Data lives inside the container at this path; mount a volume + # in docker-compose to persist across container restarts. + type: "FILESYSTEM" + config: + directory: "/var/lib/parca" + +# scrape_configs is intentionally empty: the Texera deployment pushes +# profiles from the parca-agent via OTLP/gRPC. Adding scrape targets +# here would generate significantly more data than agent-based +# profiling per the upstream recommendation. +scrape_configs: [] diff --git a/bin/single-node/.env b/bin/single-node/.env index 54aa2f5b322..4721a1a190f 100644 --- a/bin/single-node/.env +++ b/bin/single-node/.env @@ -97,3 +97,37 @@ LLM_API_KEY=dummy TEXERA_DASHBOARD_SERVICE_ENDPOINT=http://dashboard-service:8080 WORKFLOW_COMPILING_SERVICE_ENDPOINT=http://workflow-compiling-service:9090 WORKFLOW_EXECUTION_SERVICE_ENDPOINT=http://workflow-runtime-coordinator-service:8085 + +# ============================================================================ +# Observability (PR 6) +# ============================================================================ +# Each backend lives behind its own docker-compose profile. By default +# COMPOSE_PROFILES enables every observability profile so `docker compose up` +# brings the whole stack online — there is no separate observability compose +# file or monolithic observability profile. +# +# To disable a signal: +# * Prefer using bin/single-node/up.sh, which translates the +# TEXERA_OBSERVABILITY_* env vars below into the right COMPOSE_PROFILES. +# * Or edit COMPOSE_PROFILES directly here. +# +# Disable env-var conventions (consumed by up.sh): +# TEXERA_OBSERVABILITY_LOGS=disabled drops victorialogs +# TEXERA_OBSERVABILITY_METRICS=disabled drops victoriametrics +# TEXERA_OBSERVABILITY_TRACES=disabled drops jaeger +# TEXERA_OBSERVABILITY_PROFILES=disabled drops parca + parca-agent +# TEXERA_OBSERVABILITY_COLLECTOR=disabled drops the otel-collector (rare) +# +# Parca eBPF agent needs Linux + privileged container; macOS/Windows +# developers must set TEXERA_OBSERVABILITY_PROFILES=disabled. +COMPOSE_PROFILES=observability-collector,observability-logs,observability-metrics,observability-traces,observability-profiles + +# Query endpoints the dashboard gateway calls to read each signal. The +# defaults in observability-gateway.conf are host-local (127.0.0.1), which +# is what a natively-run backend (sbt / IntelliJ) needs. Inside this compose +# the gateway runs in a container, so it must reach the backends by their +# bridge-network service names — these overrides do that. +TEXERA_OBS_LOGS_URL=http://victorialogs:9428 +TEXERA_OBS_METRICS_URL=http://victoriametrics:8428 +TEXERA_OBS_TRACES_URL=http://jaeger:16686 +TEXERA_OBS_PROFILES_URL=http://parca:7070 diff --git a/bin/single-node/docker-compose.yml b/bin/single-node/docker-compose.yml index e26fe8aa957..7930516746e 100644 --- a/bin/single-node/docker-compose.yml +++ b/bin/single-node/docker-compose.yml @@ -487,6 +487,189 @@ services: command: > sh -c 'apk add --no-cache curl jq bash > /dev/null 2>&1 && bash /examples/load-examples.sh' + # ======================================================================== + # Part 5: Observability stack (PR 6). + # + # All six services live in this single compose file (no separate + # observability compose file, no monolithic observability profile). + # Per-signal profiles let an operator drop one without disturbing + # the others — see bin/single-node/up.sh and bin/single-node/.env + # (COMPOSE_PROFILES default includes every observability profile so + # `docker compose up` runs the whole stack). + # + # Network posture: every receiver/HTTP port binds to loopback + # (127.0.0.1) on the host or stays inside the texera-single-node + # bridge network entirely. No 0.0.0.0 host bindings, no Ingress. + # ======================================================================== + + # OpenTelemetry Collector — the single OTLP ingress for all three + # signals. Reads bin/observability/otel-collector/config.yaml. + # Apache-2.0; pinned to the contrib distribution. + otel-collector: + image: otel/opentelemetry-collector-contrib:0.153.0 + container_name: texera-otel-collector + profiles: [observability-collector] + restart: always + user: "10001:10001" + read_only: true + security_opt: + - no-new-privileges:true + volumes: + - ../observability/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml:ro + command: ["--config=/etc/otelcol-contrib/config.yaml"] + # Local dev override: publish OTLP receiver ports on the host + # loopback so a Scala backend running outside docker (sbt / + # IntelliJ) can emit telemetry. In compose-only deploys, services + # talk to otel-collector:4317/:4318 via the bridge network. + ports: + - "127.0.0.1:4317:4317" + - "127.0.0.1:4318:4318" + deploy: + resources: + limits: + memory: 768M + cpus: "1.0" + + # VictoriaLogs — Apache-2.0 log store. LogsQL query API on 9428. + victorialogs: + image: victoriametrics/victoria-logs:v1.50.0 + container_name: texera-victorialogs + profiles: [observability-logs] + restart: always + user: "10002:10002" + read_only: true + security_opt: + - no-new-privileges:true + command: + - "-storageDataPath=/data" + - "-retentionPeriod=30d" + - "-httpListenAddr=:9428" + volumes: + - victorialogs_data:/data + # Loopback-only host binding so an operator can curl the query + # API from the host for ad-hoc debugging without exposing it to + # the network. + ports: + - "127.0.0.1:9428:9428" + deploy: + resources: + limits: + memory: 1G + cpus: "1.0" + + # VictoriaMetrics — Apache-2.0 metrics store. Accepts Prometheus + # remote-write from the collector at /api/v1/write, MetricsQL on + # /api/v1/query. + victoriametrics: + image: victoriametrics/victoria-metrics:v1.144.0 + container_name: texera-victoriametrics + profiles: [observability-metrics] + restart: always + user: "10003:10003" + read_only: true + security_opt: + - no-new-privileges:true + command: + - "-storageDataPath=/data" + - "-retentionPeriod=90d" + - "-httpListenAddr=:8428" + volumes: + - victoriametrics_data:/data + ports: + - "127.0.0.1:8428:8428" + deploy: + resources: + limits: + memory: 1G + cpus: "1.0" + + # Jaeger v2 — Apache-2.0 traces backend + UI. v2 accepts OTLP + # natively. The query API lives at :16686, OTLP ingest at :4317 + # (kept inside the bridge network only — only the collector talks + # to it). + jaeger: + image: jaegertracing/jaeger:2.18.0 + container_name: texera-jaeger + profiles: [observability-traces] + restart: always + security_opt: + - no-new-privileges:true + # In-memory storage for the single-node deployment — restarts + # wipe traces. Any deploy needing trace persistence must swap + # in Cassandra or OpenSearch-backed storage here. + ports: + - "127.0.0.1:16686:16686" + deploy: + resources: + limits: + memory: 1G + cpus: "1.0" + + # Parca server — Apache-2.0 profiles backend. Reads + # bin/observability/parca/parca.yaml (PR 5). FILESYSTEM storage. + parca: + image: ghcr.io/parca-dev/parca:v0.28.0 + container_name: texera-parca + profiles: [observability-profiles] + restart: always + security_opt: + - no-new-privileges:true + volumes: + - ../observability/parca/parca.yaml:/parca.yaml:ro + - parca_data:/var/lib/parca + command: + - "/parca" + - "--config-path=/parca.yaml" + ports: + - "127.0.0.1:7070:7070" + deploy: + resources: + limits: + memory: 1G + cpus: "1.0" + + # Parca eBPF agent — Apache-2.0. Linux-only, privileged. + # See bin/observability/parca/README.md for the full deploy + # posture. The privileged + bind-mount block here is the ONLY + # observability service that requires elevated permissions; the + # surface is documented and reviewed. + parca-agent: + image: ghcr.io/parca-dev/parca-agent:v0.47.1 + container_name: texera-parca-agent + profiles: [observability-profiles] + restart: always + depends_on: + parca: + condition: service_started + env_file: + - ../observability/parca/parca-agent.env + # eBPF requires CAP_SYS_ADMIN-class privileges. macOS / Windows + # developers cannot run this — they should drop the + # observability-profiles profile from COMPOSE_PROFILES. + privileged: true + pid: "host" + # Bind-mount kernel state read-only — the agent reads /proc and + # /sys for stack-trace symbolization but cannot write to either. + volumes: + - /sys/kernel/debug:/sys/kernel/debug:ro + - /proc:/host/proc:ro + - /sys:/host/sys:ro + # parca-agent ships as a distroless image (no /bin/sh), so flags + # are passed directly. env_file values above are kept as the + # source of truth — edit both if you change a value. + command: + - "--remote-store-address=parca:7070" + - "--remote-store-insecure=true" + - "--node=texera-dev" + - "--metadata-external-labels=deployment=texera;cluster=local" + - "--profiling-cpu-sampling-frequency=19" + - "--log-level=info" + deploy: + resources: + limits: + memory: 512M + cpus: "0.5" + networks: default: name: texera-single-node @@ -495,4 +678,10 @@ networks: volumes: minio_data: postgres_data: - workflow_result_data: \ No newline at end of file + workflow_result_data: + # Observability storage. Each backend gets its own named volume so + # an operator can drop one signal's history (e.g. logs) without + # touching the others. + victorialogs_data: + victoriametrics_data: + parca_data: \ No newline at end of file diff --git a/bin/single-node/up.sh b/bin/single-node/up.sh new file mode 100755 index 00000000000..2af7abaf191 --- /dev/null +++ b/bin/single-node/up.sh @@ -0,0 +1,65 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Wrapper around `docker compose up` that honors the per-backend +# observability disable env vars described in .env. Run from +# bin/single-node/. +# +# Usage: +# ./up.sh # bring up everything +# TEXERA_OBSERVABILITY_PROFILES=disabled ./up.sh # no Parca / agent +# ./up.sh -d --remove-orphans # extra args forwarded to compose +# +# Why this exists: docker-compose has no first-class way to say +# "default-on, disable per env var". We translate the TEXERA_OBSERVABILITY_* +# envs into the COMPOSE_PROFILES list, then exec `docker compose`. + +set -eu + +cd "$(dirname "$0")" + +# Start from the full set; drop entries as disable envs are set. +PROFILES="observability-collector observability-logs observability-metrics observability-traces observability-profiles" + +drop_profile() { + # $1 = profile name to remove from PROFILES + PROFILES=$(printf '%s\n' $PROFILES | grep -vx "$1" | tr '\n' ' ') +} + +case "${TEXERA_OBSERVABILITY_LOGS:-enabled}" in + disabled|off|false|0) drop_profile observability-logs ;; +esac +case "${TEXERA_OBSERVABILITY_METRICS:-enabled}" in + disabled|off|false|0) drop_profile observability-metrics ;; +esac +case "${TEXERA_OBSERVABILITY_TRACES:-enabled}" in + disabled|off|false|0) drop_profile observability-traces ;; +esac +case "${TEXERA_OBSERVABILITY_PROFILES:-enabled}" in + disabled|off|false|0) drop_profile observability-profiles ;; +esac +case "${TEXERA_OBSERVABILITY_COLLECTOR:-enabled}" in + disabled|off|false|0) drop_profile observability-collector ;; +esac + +# Comma-separated for COMPOSE_PROFILES. +COMPOSE_PROFILES=$(printf '%s\n' $PROFILES | paste -sd, -) +export COMPOSE_PROFILES + +echo "Bringing up Texera with COMPOSE_PROFILES=${COMPOSE_PROFILES:-}" +exec docker compose up "$@" diff --git a/common/config/src/test/scala/org/apache/texera/observability/ObservabilityComposeSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/ObservabilityComposeSpec.scala new file mode 100644 index 00000000000..5f21df9b407 --- /dev/null +++ b/common/config/src/test/scala/org/apache/texera/observability/ObservabilityComposeSpec.scala @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path, Paths} + +/** + * Smoke tests for the PR 6 docker-compose + collector config. Same + * design as [[ParcaConfigSpec]] — string-level assertions, no YAML + * parser, because the goal is to catch typos and licence-pin drift, + * not to validate the upstream schemas. + */ +class ObservabilityComposeSpec extends AnyFlatSpec with Matchers { + + private def resolveBundled(relative: String): Path = { + var dir = Paths.get("").toAbsolutePath + var hops = 0 + while (hops < 10) { + val candidate = dir.resolve(relative) + if (Files.exists(candidate)) return candidate + val parent = dir.getParent + if (parent == null) return candidate + dir = parent + hops += 1 + } + Paths.get(relative) + } + + private def read(p: Path): String = + new String(Files.readAllBytes(p), StandardCharsets.UTF_8) + + // ----- bundled paths -------------------------------------------------- + + private val compose = resolveBundled("bin/single-node/docker-compose.yml") + private val envFile = resolveBundled("bin/single-node/.env") + private val upScript = resolveBundled("bin/single-node/up.sh") + private val collector = resolveBundled("bin/observability/otel-collector/config.yaml") + + // ----- docker-compose: pinned images --------------------------------- + + "docker-compose.yml" should "exist" in { + Files.exists(compose) shouldBe true + } + + it should "pin the OSS observability image tags from LICENSING.md" in { + val text = read(compose) + text should include("otel/opentelemetry-collector-contrib:0.153.0") + text should include("victoriametrics/victoria-logs:v1.50.0") + text should include("victoriametrics/victoria-metrics:v1.144.0") + text should include("jaegertracing/jaeger:2.18.0") + text should include("ghcr.io/parca-dev/parca:v0.28.0") + text should include("ghcr.io/parca-dev/parca-agent:v0.47.1") + } + + it should "NOT reference the VictoriaMetrics enterprise images" in { + // Tripwire: '-enterprise' images are not Apache-2.0. Texera ships + // OSS only — anyone editing the tags must be reminded of that. + val text = read(compose) + text should not include "-enterprise" + } + + it should "bind every observability host port to loopback (127.0.0.1)" in { + val text = read(compose) + // The four query/UI endpoints we expose to the host MUST use + // the 127.0.0.1: prefix. Any "0.0.0.0:9428" or naked "9428:9428" + // would publish to all interfaces — a misconfig that would + // expose the data store to the network. + Seq("9428", "8428", "16686", "7070").foreach { port => + val pattern = s""""127.0.0.1:$port:$port"""" + withClue(s"port $port should be loopback-only: ") { + text should include(pattern) + } + } + } + + it should "only ever bind the OTel collector OTLP receivers to loopback" in { + // Dev mode publishes 127.0.0.1:4317-4318 so a Scala backend running + // outside docker (sbt / IntelliJ) can emit telemetry. The hard + // requirement is that the receiver is NEVER bound on a non-loopback + // address — otherwise OTLP would be reachable from the LAN. + val text = read(compose) + text should not include "0.0.0.0:4317" + text should not include "0.0.0.0:4318" + // Also reject the bare `"4317:4317"` form which docker treats as + // "bind on all interfaces". Only the explicit loopback form is OK. + text should not include "\"4317:4317\"" + text should not include "\"4318:4318\"" + } + + it should "only mark the parca-agent privileged (eBPF needs CAP_SYS_ADMIN)" in { + // privileged: true is dangerous; we want it on exactly one + // service. If a future contributor copies the agent block as a + // template for another service, this test trips. + val text = read(compose) + val priv = "privileged: true".r.findAllIn(text).length + priv shouldBe 1 + } + + it should "give every observability backend a profile so it can be disabled" in { + val text = read(compose) + Seq( + "observability-collector", + "observability-logs", + "observability-metrics", + "observability-traces", + "observability-profiles" + ).foreach { profile => + text should include(profile) + } + } + + it should "set a memory limit on every observability service" in { + // The new section lives below "Part 5: Observability stack". + val text = read(compose).split("Part 5: Observability stack").last + // Six services, each gets a `deploy.resources.limits.memory:` line. + val memoryLimits = "memory:".r.findAllIn(text).length + memoryLimits should be >= 6 + } + + // ----- .env defaults ------------------------------------------------- + + ".env" should "default COMPOSE_PROFILES to include every observability profile" in { + val text = read(envFile) + text should include("COMPOSE_PROFILES=") + Seq( + "observability-collector", + "observability-logs", + "observability-metrics", + "observability-traces", + "observability-profiles" + ).foreach { profile => + val grepCount = text.split('\n').count(line => + !line.trim.startsWith("#") && line.contains(profile) + ) + withClue(s"$profile should be in default COMPOSE_PROFILES (non-comment): ")( + grepCount should be >= 1 + ) + } + } + + // ----- up.sh ---------------------------------------------------------- + + "up.sh" should "honor each per-signal disable env var" in { + val text = read(upScript) + Seq( + "TEXERA_OBSERVABILITY_LOGS", + "TEXERA_OBSERVABILITY_METRICS", + "TEXERA_OBSERVABILITY_TRACES", + "TEXERA_OBSERVABILITY_PROFILES", + "TEXERA_OBSERVABILITY_COLLECTOR" + ).foreach { v => + text should include(v) + } + } + + // ----- otel-collector config ----------------------------------------- + + "otel-collector/config.yaml" should "exist and declare all three signal pipelines" in { + Files.exists(collector) shouldBe true + val text = read(collector) + // The Service block defines exactly three pipelines. + text should include("logs:") + text should include("metrics:") + text should include("traces:") + } + + it should "route metrics via prometheusremotewrite to VictoriaMetrics" in { + val text = read(collector) + text should include("prometheusremotewrite") + text should include("victoriametrics:8428") + } + + it should "route logs to VictoriaLogs over OTLP HTTP" in { + val text = read(collector) + text should include("otlphttp/victorialogs") + text should include("victorialogs:9428") + } + + it should "route traces to Jaeger over OTLP gRPC" in { + val text = read(collector) + text should include("otlp/jaeger") + text should include("jaeger:4317") + } + + it should "cap incoming OTLP message size (DoS guard)" in { + val text = read(collector) + text should include("max_recv_msg_size_mib") + } + + it should "configure memory_limiter to bound collector memory" in { + val text = read(collector) + text should include("memory_limiter") + text should include("limit_mib") + } +} diff --git a/common/config/src/test/scala/org/apache/texera/observability/ParcaConfigSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/ParcaConfigSpec.scala new file mode 100644 index 00000000000..8e0d162b23f --- /dev/null +++ b/common/config/src/test/scala/org/apache/texera/observability/ParcaConfigSpec.scala @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path, Paths} + +/** + * Smoke test for the bundled Parca configuration files. + * + * Intentionally lightweight: we are guarding against accidental + * deletion / emptying / tag drift, not validating Parca's schema. + * The real config validation happens when the agent starts up + * inside its container — but a unit-level smoke test catches typos + * before a developer pushes them. + */ +class ParcaConfigSpec extends AnyFlatSpec with Matchers { + + // sbt runs tests with the module dir as CWD, but a developer who + // runs `sbt test` from the project root has a different CWD. Walk + // upwards until we find the file, bounded so a missing file fails + // loudly rather than infinite-looping. + private def resolveBundled(relative: String): Path = { + var dir = Paths.get("").toAbsolutePath + var hops = 0 + while (hops < 10) { + val candidate = dir.resolve(relative) + if (Files.exists(candidate)) return candidate + val parent = dir.getParent + if (parent == null) return candidate + dir = parent + hops += 1 + } + Paths.get(relative) // will fail the existence assert below + } + + private val parcaYaml = resolveBundled("bin/observability/parca/parca.yaml") + private val agentEnv = resolveBundled("bin/observability/parca/parca-agent.env") + private val readme = resolveBundled("bin/observability/parca/README.md") + + // ----- parca.yaml ----------------------------------------------------- + + "parca.yaml" should "exist in bin/observability/parca/" in { + Files.exists(parcaYaml) shouldBe true + } + + it should "declare object_storage with the FILESYSTEM bucket type" in { + val text = new String(Files.readAllBytes(parcaYaml), StandardCharsets.UTF_8) + text should include("object_storage") + text should include("FILESYSTEM") + text should include("/var/lib/parca") + } + + it should "include the ASF license header" in { + val text = new String(Files.readAllBytes(parcaYaml), StandardCharsets.UTF_8) + text should include("Apache License, Version 2.0") + } + + // ----- parca-agent.env ------------------------------------------------ + + "parca-agent.env" should "exist and pin the bundled image version reference" in { + Files.exists(agentEnv) shouldBe true + val text = new String(Files.readAllBytes(agentEnv), StandardCharsets.UTF_8) + text should include("v0.47.1") + } + + it should "point the agent at the bundled Parca server hostname" in { + val text = new String(Files.readAllBytes(agentEnv), StandardCharsets.UTF_8) + text should include("PARCA_AGENT_REMOTE_STORE_ADDRESS=parca:7070") + } + + it should "carry the deployment label so the gateway can filter Texera processes" in { + val text = new String(Files.readAllBytes(agentEnv), StandardCharsets.UTF_8) + text should include("deployment=texera") + } + + it should "NOT include high-cardinality labels (workflow.id / execution.id)" in { + // Tripwire: a future contributor might be tempted to add + // workflow.id as a static label. That blows up Parca storage. + // This assertion makes the design intent enforceable. Comments + // are skipped — they're allowed (and required) to explain the + // rule. + val configLines = new String(Files.readAllBytes(agentEnv), StandardCharsets.UTF_8) + .linesIterator + .map(_.trim) + .filter(line => line.nonEmpty && !line.startsWith("#")) + .toSeq + configLines.foreach { line => + line should not include "workflow.id" + line should not include "workflow_id" + line should not include "execution.id" + line should not include "execution_id" + } + } + + // ----- README --------------------------------------------------------- + + "README.md" should "exist and document the opt-out env var" in { + Files.exists(readme) shouldBe true + val text = new String(Files.readAllBytes(readme), StandardCharsets.UTF_8) + text should include("TEXERA_OBSERVABILITY_PROFILES=disabled") + } + + it should "document the Linux-only / privileged-container requirement" in { + val text = new String(Files.readAllBytes(readme), StandardCharsets.UTF_8) + text.toLowerCase should include("linux") + text.toLowerCase should include("privileged") + } +} From 71cfbf64a0133c35da213e43abd7d5efb1472fae Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 5 Jun 2026 04:49:02 -0700 Subject: [PATCH 04/26] feat(observability): backend signal emission - metrics + distributed tracing primitives Co-Authored-By: Claude Opus 4.8 (1M context) --- .../texera/observability/SpanAttrs.scala | 116 ++++++++++ .../texera/observability/TexeraMetrics.scala | 216 ++++++++++++++++++ .../texera/observability/TexeraTracer.scala | 84 +++++++ .../observability/TraceparentValidator.scala | 94 ++++++++ .../texera/observability/SpanAttrsSpec.scala | Bin 0 -> 5448 bytes .../observability/TexeraMetricsSpec.scala | 189 +++++++++++++++ .../TraceparentValidatorSpec.scala | Bin 0 -> 5085 bytes 7 files changed, 699 insertions(+) create mode 100644 common/config/src/main/scala/org/apache/texera/observability/SpanAttrs.scala create mode 100644 common/config/src/main/scala/org/apache/texera/observability/TexeraMetrics.scala create mode 100644 common/config/src/main/scala/org/apache/texera/observability/TexeraTracer.scala create mode 100644 common/config/src/main/scala/org/apache/texera/observability/TraceparentValidator.scala create mode 100644 common/config/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala create mode 100644 common/config/src/test/scala/org/apache/texera/observability/TexeraMetricsSpec.scala create mode 100644 common/config/src/test/scala/org/apache/texera/observability/TraceparentValidatorSpec.scala diff --git a/common/config/src/main/scala/org/apache/texera/observability/SpanAttrs.scala b/common/config/src/main/scala/org/apache/texera/observability/SpanAttrs.scala new file mode 100644 index 00000000000..c66db23f652 --- /dev/null +++ b/common/config/src/main/scala/org/apache/texera/observability/SpanAttrs.scala @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import io.opentelemetry.api.common.AttributeKey +import io.opentelemetry.api.trace.{Span, SpanBuilder} + +/** + * Thin helper for setting span attributes safely. + * + * Three rules: + * 1. Typed setters only — no public escape hatch for arbitrary + * untyped strings to land on a span as untrusted free text. + * 2. Free-text values are CRLF-stripped + capped at + * [[FreeTextMaxLen]] to prevent log/span forging via embedded + * newlines. + * 3. Operator IDs and workflow/execution IDs must match a strict + * character set — otherwise dropped silently (the operator + * identifier should be a stable internal value, not user free + * text). + */ +object SpanAttrs { + + /** Maximum length for free-text span attribute values. */ + val FreeTextMaxLen: Int = 256 + + /** Validates the shape we accept for operator IDs: alnum + `_.-`, + * 1–64 chars. Anything else is dropped (not coerced — we'd rather + * miss a label than leak an unbounded string into a span). */ + private val OperatorIdPattern = "^[A-Za-z0-9_.\\-]{1,64}$".r.pattern + + // ---- Standard Texera correlation labels ------------------------------ + + val WorkflowId: AttributeKey[java.lang.Long] = AttributeKey.longKey("texera.workflow.id") + val ExecutionId: AttributeKey[java.lang.Long] = AttributeKey.longKey("texera.execution.id") + val ProjectId: AttributeKey[java.lang.Long] = AttributeKey.longKey("texera.project.id") + val UserId: AttributeKey[java.lang.Long] = AttributeKey.longKey("texera.user.id") + val OperatorId: AttributeKey[String] = AttributeKey.stringKey("texera.operator.id") + val OperatorName: AttributeKey[String] = AttributeKey.stringKey("texera.operator.name") + val Outcome: AttributeKey[String] = AttributeKey.stringKey("texera.outcome") + + // ---- Typed setters for SpanBuilder (used at span-start time) --------- + + def withWorkflowId(b: SpanBuilder, id: Long): SpanBuilder = + b.setAttribute(WorkflowId, java.lang.Long.valueOf(id)) + + def withExecutionId(b: SpanBuilder, id: Long): SpanBuilder = + b.setAttribute(ExecutionId, java.lang.Long.valueOf(id)) + + def withProjectId(b: SpanBuilder, id: Long): SpanBuilder = + b.setAttribute(ProjectId, java.lang.Long.valueOf(id)) + + def withUserId(b: SpanBuilder, id: Long): SpanBuilder = + b.setAttribute(UserId, java.lang.Long.valueOf(id)) + + /** Sets the operator id only if it passes the strict character + * check; otherwise the attribute is omitted. Returns the same + * builder either way for fluent chaining. */ + def withOperatorId(b: SpanBuilder, id: String): SpanBuilder = { + if (id != null && OperatorIdPattern.matcher(id).matches()) { + b.setAttribute(OperatorId, id) + } + b + } + + /** Sets a free-text label after stripping CRLF and capping length. */ + def withOperatorName(b: SpanBuilder, name: String): SpanBuilder = { + val safe = sanitizeFreeText(name) + if (safe != null) b.setAttribute(OperatorName, safe) else b + } + + // ---- Typed setters for Span (used after a span is active) ------------ + + def setWorkflowId(s: Span, id: Long): Span = s.setAttribute(WorkflowId, java.lang.Long.valueOf(id)) + def setExecutionId(s: Span, id: Long): Span = s.setAttribute(ExecutionId, java.lang.Long.valueOf(id)) + def setOperatorId(s: Span, id: String): Span = { + if (id != null && OperatorIdPattern.matcher(id).matches()) s.setAttribute(OperatorId, id) + else s + } + def setOutcome(s: Span, outcome: String): Span = { + val safe = sanitizeFreeText(outcome) + if (safe != null) s.setAttribute(Outcome, safe) else s + } + + // ---- Pure helpers (exposed for testing) ------------------------------ + + /** + * Strip CR/LF and other C0 control characters from a free-text + * value, then cap at [[FreeTextMaxLen]]. Returns null for + * null/empty input (caller skips the setAttribute call). + */ + def sanitizeFreeText(value: String): String = { + if (value == null || value.isEmpty) return null + val stripped = value.filter(c => c >= 0x20 && c != 0x7F) + if (stripped.isEmpty) null + else if (stripped.length <= FreeTextMaxLen) stripped + else stripped.substring(0, FreeTextMaxLen) + } +} diff --git a/common/config/src/main/scala/org/apache/texera/observability/TexeraMetrics.scala b/common/config/src/main/scala/org/apache/texera/observability/TexeraMetrics.scala new file mode 100644 index 00000000000..c850c055392 --- /dev/null +++ b/common/config/src/main/scala/org/apache/texera/observability/TexeraMetrics.scala @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import com.typesafe.scalalogging.LazyLogging +import io.opentelemetry.api.GlobalOpenTelemetry +import io.opentelemetry.api.common.{AttributeKey, Attributes} +import io.opentelemetry.api.metrics.Meter + +/** + * Strongly-typed façade for Texera-emitted metrics. + * + * Cardinality safety is enforced by the API surface, not by + * documentation: there is no public method that accepts an arbitrary + * string as a label key or value. The only labels that ever land on + * an instrument are the two enums [[Outcome]] and [[WorkflowKind]], + * each restricted to a fixed set. ``workflow.id`` / ``execution.id`` + * are deliberately NOT metric labels — per-execution detail belongs + * in traces and logs, joined on ``trace_id`` at query time. + * + * Histogram bucket bounds are hard-coded constants so they can't be + * coerced by request input. The OTel SDK applies its own default + * attribute-value-length cap to anything that does slip through. + */ +object TexeraMetrics extends LazyLogging { + + /** Outcome enum, the only mutable label on lifecycle counters. */ + sealed abstract class Outcome(val name: String) + object Outcome { + case object Success extends Outcome("success") + case object Failure extends Outcome("failure") + case object Cancelled extends Outcome("cancelled") + } + + /** Workflow kind enum. Distinguishes interactive vs. scheduled + * workflows for the dashboard's basic split — extend deliberately. */ + sealed abstract class WorkflowKind(val name: String) + object WorkflowKind { + case object Interactive extends WorkflowKind("interactive") + case object Scheduled extends WorkflowKind("scheduled") + case object Unknown extends WorkflowKind("unknown") + } + + private val OutcomeKey: AttributeKey[String] = AttributeKey.stringKey("texera.outcome") + private val WorkflowKindKey: AttributeKey[String] = AttributeKey.stringKey("texera.workflow.kind") + + /** Histogram bucket bounds in seconds. Hard-coded — constants so + * request input can't reshape the histogram. Range covers + * fast (<1s) to long (>1h) workflows. */ + private val DurationBuckets: java.util.List[java.lang.Double] = { + val builder = new java.util.ArrayList[java.lang.Double]() + Seq(0.1, 0.5, 1.0, 5.0, 10.0, 30.0, 60.0, 300.0, 600.0, 1800.0, 3600.0) + .foreach(b => builder.add(java.lang.Double.valueOf(b))) + builder + } + + private val InstrumentationScope = "org.apache.texera" + + // Instruments are lazy + memoised. The first call after SDK init + // builds them against the active GlobalOpenTelemetry meter; once + // built they hold the meter instance, so a later GlobalOpenTelemetry + // reset (in tests) wouldn't be visible here — see [[resetForTest]]. + @volatile private var _starts: io.opentelemetry.api.metrics.LongCounter = _ + @volatile private var _completions: io.opentelemetry.api.metrics.LongCounter = _ + @volatile private var _cancellations: io.opentelemetry.api.metrics.LongCounter = _ + // `texera.workflow.active` is an OBSERVABLE gauge, not a manual up/down + // counter. A manual +1/-1 counter leaks whenever a run starts but its + // terminal event never fires (the controller is killed, the process + // restarts mid-run, or the +1 and -1 are split across the engine and web + // tiers and one side is missed) — the gauge then drifts upward forever and + // the dashboard shows phantom "active" executions. An observable gauge + // instead reports the TRUE in-progress count from the live execution + // registry on every collection, so it cannot leak. The count is supplied + // by the host process via [[setActiveExecutionsSupplier]]. + @volatile private var _active: io.opentelemetry.api.metrics.ObservableLongGauge = _ + @volatile private var _duration: io.opentelemetry.api.metrics.DoubleHistogram = _ + + // Supplier of the current in-progress execution count. Defaults to 0 until + // the host process registers the real source (so a process that never + // registers reports a flat 0 rather than a wrong number). Read by the gauge + // callback on every metric collection. + @volatile private var activeExecutionsSupplier: () => Long = () => 0L + + /** Register the authoritative source of "currently active executions". + * The supplier is polled on every metric collection, so the gauge always + * reflects ground truth and can never leak. Called once at process + * startup (e.g. by ComputingUnitMaster). */ + def setActiveExecutionsSupplier(supplier: () => Long): Unit = synchronized { + activeExecutionsSupplier = supplier + ensureBound() + } + + /** Bind instruments to the current global meter. Idempotent — the + * first call wins; later calls are no-ops. Tests can call + * [[bindForTest]] with an explicit Meter, then [[resetForTest]] to + * rebind. */ + def ensureBound(): Unit = synchronized { + if (_starts == null) bind(GlobalOpenTelemetry.getMeter(InstrumentationScope)) + } + + private[observability] def bindForTest(meter: Meter): Unit = synchronized { + bind(meter) + } + + private[observability] def resetForTest(): Unit = synchronized { + _starts = null + _completions = null + _cancellations = null + // The observable gauge registered a collection callback — close it so the + // previous test's meter provider stops being polled after it's discarded. + if (_active != null) _active.close() + _active = null + _duration = null + } + + private def bind(meter: Meter): Unit = { + _starts = meter + .counterBuilder("texera.workflow.starts") + .setDescription("Number of workflow executions started.") + .build() + // Completions carry texera.outcome={success|failure}. Both success + // and self-terminating failures land here so the success/failure-rate + // queries (non-success ÷ all completions) have a denominator that + // means "runs that finished on their own". User-initiated kills are + // NOT completions — see _cancellations. + _completions = meter + .counterBuilder("texera.workflow.completions") + .setDescription("Number of workflow executions that ran to completion (success or failure).") + .build() + // Cancellations (user kills) are tracked separately so they decrement + // the active gauge without polluting the success/failure-rate + // denominator. Deliberately label-free apart from workflow.kind. + _cancellations = meter + .counterBuilder("texera.workflow.cancellations") + .setDescription("Number of workflow executions cancelled/killed before finishing.") + .build() + // Observable gauge: the callback runs on each collection and reports the + // live in-progress count, so the value cannot leak. No per-execution + // labels (cardinality-safe); the dashboard queries sum(texera_workflow_active). + _active = meter + .gaugeBuilder("texera.workflow.active") + .ofLongs() + .setDescription("Number of workflow executions currently in progress (observed from the live registry).") + .buildWithCallback(obs => obs.record(activeExecutionsSupplier())) + _duration = meter + .histogramBuilder("texera.workflow.duration") + .setDescription("End-to-end duration of a workflow execution.") + .setUnit("s") + .setExplicitBucketBoundariesAdvice(DurationBuckets) + .build() + logger.info(s"Texera metric instruments bound to meter scope '$InstrumentationScope'") + } + + // ---- Public emitters — typed, no untyped escape hatch -------------- + + def recordStart(kind: WorkflowKind): Unit = { + ensureBound() + logger.debug(s"metric: workflow started (kind=${kind.name}) — starts +1") + _starts.add(1L, Attributes.of(WorkflowKindKey, kind.name)) + // `active` is no longer mutated here — it is an observable gauge sourced + // from the live execution registry (see setActiveExecutionsSupplier). + } + + def recordCompletion(kind: WorkflowKind, durationSec: Double): Unit = { + ensureBound() + logger.debug( + f"metric: workflow completed successfully (kind=${kind.name}, ${durationSec}%.1fs) — completions +1" + ) + val attrs = Attributes.of(OutcomeKey, Outcome.Success.name, WorkflowKindKey, kind.name) + _completions.add(1L, attrs) + _duration.record(durationSec, attrs) + } + + def recordFailure(kind: WorkflowKind, durationSec: Double): Unit = { + ensureBound() + logger.debug( + f"metric: workflow failed (kind=${kind.name}, ${durationSec}%.1fs) — completions +1 (outcome=failure)" + ) + // A failure is a completion with outcome=failure — it shares the + // completions counter (and duration histogram) with successes so the + // failure-rate query has both numerator and denominator. + val attrs = Attributes.of(OutcomeKey, Outcome.Failure.name, WorkflowKindKey, kind.name) + _completions.add(1L, attrs) + _duration.record(durationSec, attrs) + } + + /** A user-initiated kill/cancel. Bumps a dedicated counter. Deliberately + * NOT recorded as a completion: a cancelled run never finished on its own, + * so it must not drag down the success rate. No duration is recorded for + * the same reason — a killed run's wall-clock time is not a real runtime + * and would skew the duration percentiles. (The active gauge is observed + * from the live registry and needs no decrement here.) */ + def recordCancellation(kind: WorkflowKind): Unit = { + ensureBound() + logger.debug(s"metric: workflow cancelled (kind=${kind.name}) — cancellations +1") + val attrs = Attributes.of(WorkflowKindKey, kind.name) + _cancellations.add(1L, attrs) + } +} diff --git a/common/config/src/main/scala/org/apache/texera/observability/TexeraTracer.scala b/common/config/src/main/scala/org/apache/texera/observability/TexeraTracer.scala new file mode 100644 index 00000000000..8941d97aefa --- /dev/null +++ b/common/config/src/main/scala/org/apache/texera/observability/TexeraTracer.scala @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import io.opentelemetry.api.GlobalOpenTelemetry +import io.opentelemetry.api.trace.{Span, SpanBuilder, StatusCode, Tracer} +import io.opentelemetry.context.{Context, Scope} + +/** + * Thin convenience wrapper around the global OTel tracer. + * + * Two reasons to go through this rather than calling + * ``GlobalOpenTelemetry.getTracer`` directly at every callsite: + * + * 1. Single instrumentation scope name (``org.apache.texera``) — so + * every Texera-produced span shows up under one logical scope in + * the backend, separable from anything emitted by transitive + * libraries. + * 2. One ergonomic ``withSpan`` API that handles exception → status, + * scope cleanup, and span end in a single try/finally. Callers + * don't have to remember the ceremony at every site. + * + * When the SDK is disabled, ``GlobalOpenTelemetry.getTracer`` returns + * a no-op tracer, so calling these methods is safe at any time. + */ +object TexeraTracer { + + private val InstrumentationScope = "org.apache.texera" + + def tracer: Tracer = GlobalOpenTelemetry.getTracer(InstrumentationScope) + + def spanBuilder(name: String): SpanBuilder = tracer.spanBuilder(name) + + /** + * Run ``block`` inside a fresh span; record exceptions, propagate + * the right span status, and ensure the span is ended exactly once. + * + * Use this for synchronous critical sections. For async (Future- + * returning) code paths use ``withAsyncSpan`` so the span doesn't + * close before the async work completes. + */ + def withSpan[T](name: String, configure: SpanBuilder => SpanBuilder = identity)( + block: Span => T + ): T = { + val span = configure(spanBuilder(name)).startSpan() + val scope: Scope = span.makeCurrent() + try { + block(span) + } catch { + case t: Throwable => + span.recordException(t) + span.setStatus(StatusCode.ERROR) + throw t + } finally { + scope.close() + span.end() + } + } + + /** + * Snapshot the current OTel ``Context`` so async callbacks can + * re-attach it via ``Context.makeCurrent`` later. Useful at the + * Scala↔Python boundary where the calling thread is not the + * receiving thread. + */ + def currentContext: Context = Context.current() +} diff --git a/common/config/src/main/scala/org/apache/texera/observability/TraceparentValidator.scala b/common/config/src/main/scala/org/apache/texera/observability/TraceparentValidator.scala new file mode 100644 index 00000000000..9577a9f9438 --- /dev/null +++ b/common/config/src/main/scala/org/apache/texera/observability/TraceparentValidator.scala @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +/** + * Pure validators for W3C Trace Context headers crossing the + * Scala↔Python boundary. The single rule: if the inbound bytes do + * not match the strict regex, we discard the value and start a fresh + * trace. We never echo a rejected value back into a span, log, or + * error message. + * + * Spec reference: https://www.w3.org/TR/trace-context/ + * + * The regexes here intentionally do NOT use any context-sensitive + * grouping or backreferences — keeps the validators safe against + * pathological inputs (ReDoS) and trivially fast. + */ +object TraceparentValidator { + + /** W3C traceparent format: `---`. + * We accept only version `00` (the only published version) with the + * canonical 32-hex / 16-hex / 2-hex layout. All hex lowercase per + * spec — uppercase is invalid. */ + private val TraceparentPattern = + "^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$".r.pattern + + /** Bounded tracestate length. Spec recommends ≤512 chars. We are + * stricter to remove a small DoS surface — an attacker can't send + * a 1 MiB tracestate to balloon downstream context allocations. */ + val MaxTracestateLength: Int = 512 + + /** Validate a traceparent header. Returns the input unchanged on + * success, None on any failure (rejected — caller starts a fresh + * trace). Null and empty are silent failures. */ + def validateTraceparent(header: String): Option[String] = { + if (header == null || header.isEmpty) return None + // Trace-id and parent-id must not be all-zero per spec — an + // all-zero ID is a sentinel for "no value" and MUST be rejected. + if (!TraceparentPattern.matcher(header).matches()) return None + val parts = header.split('-') + val traceId = parts(1) + val spanId = parts(2) + if (isAllZero(traceId) || isAllZero(spanId)) return None + Some(header) + } + + /** Validate a tracestate header. Spec: comma-separated list of + * key=value pairs, ASCII-printable only, total length capped. + * Returns the input unchanged on success, None on rejection. */ + def validateTracestate(header: String): Option[String] = { + if (header == null || header.isEmpty) return None + if (header.length > MaxTracestateLength) return None + if (!isAsciiPrintable(header)) return None + Some(header) + } + + private def isAllZero(s: String): Boolean = { + var i = 0 + while (i < s.length) { + if (s.charAt(i) != '0') return false + i += 1 + } + true + } + + private def isAsciiPrintable(s: String): Boolean = { + var i = 0 + while (i < s.length) { + val c = s.charAt(i) + // Allow printable ASCII range (0x20-0x7E). Tab and CR/LF are + // rejected — a tracestate must not span lines. + if (c < 0x20 || c > 0x7E) return false + i += 1 + } + true + } +} diff --git a/common/config/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala new file mode 100644 index 0000000000000000000000000000000000000000..388ee1d2aaa3a7f102742d7ea6bc083c1e6a00e8 GIT binary patch literal 5448 zcmc&&e{b8y8E*g8m)Hwpz^FhvIY!qO=`uiZEKG2sqg9w(DN;is}>v|?#vX>G77HQL*JQF%NlYKeuXu;qm8lGbOlv1%W^g?3(7Sftb-$L*s7(pS+18bU2bi@TzN7Z zt*l$+CG5>Plw{I6<7&mNS;V?ERTD}l>LwWj`VNXXxl*P1Rh}X|y{zO?(bC)~s~IY#vg_P&VI7>z0FHI;1Ed_=J%~-<>8RI3uCG}U zvfU=$t6OEIFbk*bja=kK?zg=j95fcSH^P*_y($2IZ@0o_*WwaS=IxUtyZ+!}uN;>_ zAb2QLR!sG4g^jjNN~zDAhac8pWvT@1%=wG`t2H+{_U$8NeyWZBBofZIVvCSYQ3JCR z=S;X%7LpwmSNP<#i?QBL@IArz*3jLR*uNkb>x5gGs>sCa=JzgjL8%9i3wW^&MBG;x zvs}N?`a%VL9}|h9LxK0~Zti#PG2xbU_GWZ337f!#Z59aJ6qK$c0K>_CoRiT?6@~f1 z{-8nHMb@=nzp5X9)N%RQ;`8)(*+11C4RVoYDuVF+Aea-d^Awdf*FMl0wj|STT2!Ge zIP6;W*dW&1p(*L+DaEoVSF(S{iG!C%{7GdArluB9ujTE9(wz+6po0|#6!e^;+X!3} z?+brd2=Bh=hjg)wsXRpXYY#xeO6%oVDhj2SXreU|ftkJFT_Z;#=%RMBQknuzN~TS; zjtaR9Z7JZFXN3UgPK0|?2R=p&kIz2uCY7<>IOd`{4zbR!J+Onh#@H*sC0_wriVIk#Mx!aPQ_qb)=0r^0n zeUw3~P0gn9zCaH97q99?f`7fJ@&Zg7N2ZMGb^FxZ^)BY+1B%w$A5A&Dc`8n$zHnYz zFsRgVUmz1OwFg5v&0wo_O)Qn4rDVn{m?e2IQ;vUiV7Jdw zgCEwuLr&MJwon}s$t?DZ#Pm`erji`yAdNcxKXWR?2!G0_qZ0jM17iA6W+k_hPe^Sj z(PhcqDh|3H&UJ^NXJLpw;I=xqns{t+=u&rZ`pL=l##}T+B$GeX-3(FPX^C{}@OnD2PHDT?c6Vkl{07hpVntTw4xc3jr=T3fM@sR-ujfvTfgnucz8^oNRdLXI-{{KvGni;f=|6fV$qDrbsq5mcYRC(ySs+? z$o)fWr1ho0lMd=S#&P8W>-R5R_Ex#Hb(HPZNbJjg{k~Dt=|IrNy|EUEBZg5FY`^cl G@BJ4E#T2pt literal 0 HcmV?d00001 diff --git a/common/config/src/test/scala/org/apache/texera/observability/TexeraMetricsSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/TexeraMetricsSpec.scala new file mode 100644 index 00000000000..10a9eb7651a --- /dev/null +++ b/common/config/src/test/scala/org/apache/texera/observability/TexeraMetricsSpec.scala @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import io.opentelemetry.sdk.metrics.SdkMeterProvider +import io.opentelemetry.sdk.metrics.data.MetricData +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader +import org.scalatest.BeforeAndAfterEach +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import scala.jdk.CollectionConverters._ + +class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { + + private var reader: InMemoryMetricReader = _ + private var provider: SdkMeterProvider = _ + + override def beforeEach(): Unit = { + reader = InMemoryMetricReader.create() + provider = SdkMeterProvider.builder().registerMetricReader(reader).build() + TexeraMetrics.resetForTest() + TexeraMetrics.bindForTest(provider.get("org.apache.texera")) + } + + override def afterEach(): Unit = { + TexeraMetrics.resetForTest() + provider.close() + } + + private def collectAll(): Map[String, MetricData] = { + reader.collectAllMetrics().asScala.map(m => m.getName -> m).toMap + } + + // ----- positive: lifecycle emissions ---------------------------------- + + "TexeraMetrics" should "increment workflow.starts on recordStart" in { + TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) + + val metrics = collectAll() + metrics.keySet should contain("texera.workflow.starts") + + val starts = metrics("texera.workflow.starts").getLongSumData.getPoints.asScala.head + starts.getValue shouldBe 1L + // recordStart no longer mutates `active` — that gauge is observed from the + // live registry (see "report the active-execution count…" below). + } + + it should "report the active-execution count from the registered supplier, never a manual counter" in { + // The whole point of the fix: `active` is an observable gauge sourced from + // ground truth, so it cannot leak. Drive it with a stub supplier and + // confirm the gauge reports exactly what the supplier returns — regardless + // of how many starts/completions were recorded. + @volatile var live = 3L + TexeraMetrics.setActiveExecutionsSupplier(() => live) + TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) + TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) + + val first = collectAll()("texera.workflow.active").getLongGaugeData.getPoints.asScala.head + first.getValue shouldBe 3L + + // It tracks the live source on the next collection — a manual +1/-1 + // counter could never drop like this without an explicit decrement. + live = 0L + val second = collectAll()("texera.workflow.active").getLongGaugeData.getPoints.asScala.head + second.getValue shouldBe 0L + } + + it should "record a completion and duration sample on recordCompletion" in { + TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) + TexeraMetrics.recordCompletion(TexeraMetrics.WorkflowKind.Interactive, durationSec = 2.5) + + val metrics = collectAll() + metrics.keySet should contain allOf ( + "texera.workflow.completions", + "texera.workflow.duration" + ) + + metrics("texera.workflow.completions").getLongSumData.getPoints.asScala.head.getValue shouldBe 1L + + val histogram = metrics("texera.workflow.duration").getHistogramData.getPoints.asScala.head + histogram.getCount shouldBe 1L + histogram.getSum shouldBe 2.5 + } + + it should "record a failure as a non-success completion (so failure-rate queries work)" in { + TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Scheduled) + TexeraMetrics.recordFailure(TexeraMetrics.WorkflowKind.Scheduled, durationSec = 12.0) + + val metrics = collectAll() + // A failure shares the completions counter with successes — the + // failure-rate query divides non-success completions by all + // completions, so failures must live here (not in a separate + // series the query never reads). + val completion = metrics("texera.workflow.completions").getLongSumData.getPoints.asScala.head + completion.getValue shouldBe 1L + completion.getAttributes.asMap.asScala.map { case (k, v) => k.getKey -> v.toString } should contain( + "texera.outcome" -> "failure" + ) + metrics("texera.workflow.duration").getHistogramData.getPoints.asScala.head.getSum shouldBe 12.0 + // The orphan counter the old wiring used must be gone. + metrics.keySet should not contain "texera.workflow.failures" + } + + it should "record a cancellation that is not a completion" in { + TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) + TexeraMetrics.recordCancellation(TexeraMetrics.WorkflowKind.Interactive) + + val metrics = collectAll() + metrics("texera.workflow.cancellations").getLongSumData.getPoints.asScala.head.getValue shouldBe 1L + // A kill is not a completion and records no duration: it must not + // drag down the success rate nor skew the duration percentiles. + metrics.keySet should not contain "texera.workflow.completions" + metrics.keySet should not contain "texera.workflow.duration" + } + + // ----- security: cardinality safety ----------------------------------- + + it should "only emit the texera.outcome and texera.workflow.kind labels" in { + TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) + TexeraMetrics.recordCompletion(TexeraMetrics.WorkflowKind.Interactive, durationSec = 1.0) + + val attrKeys = collectAll().values.flatMap { md => + val pointSet = md.getType.name() match { + case "HISTOGRAM" => md.getHistogramData.getPoints.asScala + case "LONG_GAUGE" => md.getLongGaugeData.getPoints.asScala + case _ => md.getLongSumData.getPoints.asScala + } + pointSet.flatMap(_.getAttributes.asMap.keySet.asScala.map(_.getKey)) + }.toSet + + attrKeys.foreach { key => + Set("texera.outcome", "texera.workflow.kind") should contain(key) + } + attrKeys should not contain "texera.workflow.id" + attrKeys should not contain "texera.execution.id" + } + + it should "expose no public API to attach an arbitrary string label" in { + // The class has only typed emitters whose attribute set is + // hard-coded. This test is intentionally a compile-time check + // disguised as a runtime one — if a future contributor adds an + // untyped public method like recordStart(attrs: Attributes), + // this assertion still passes but the design intent is broken. + // Make the intent explicit: + val methodNames = classOf[TexeraMetrics.type].getDeclaredMethods + .map(_.getName) + .toSet + methodNames should contain allOf ("recordStart", "recordCompletion", "recordFailure") + methodNames should not contain "recordWithAttributes" + } + + // ----- histogram buckets are constants -------------------------------- + + it should "use the hard-coded explicit bucket boundaries for duration" in { + TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) + // Hit a few bucket bounds. + Seq(0.05, 0.6, 7.0, 65.0, 400.0).foreach { d => + TexeraMetrics.recordCompletion(TexeraMetrics.WorkflowKind.Interactive, durationSec = d) + } + + val histogram = collectAll()("texera.workflow.duration").getHistogramData + val point = histogram.getPoints.asScala.head + point.getCount shouldBe 5L + // The point exposes the SDK-configured boundaries; we don't + // assert exact values here (would couple the test to the impl) + // but we do assert there ARE explicit boundaries — anything + // empty would mean the .setExplicitBucketBoundariesAdvice call + // was lost during a refactor. + point.getBoundaries.size should be > 0 + } +} diff --git a/common/config/src/test/scala/org/apache/texera/observability/TraceparentValidatorSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/TraceparentValidatorSpec.scala new file mode 100644 index 0000000000000000000000000000000000000000..d95b61239ba4dfcf2dc87ff4fbbbc19b3d022ab5 GIT binary patch literal 5085 zcmcIo+j84B5bd+RVuw$1V^WgkI+uc|;OxpndalAweJ0$2dDtYq?Sedv#M z7bG3pk`l>vJs#VnfZcO;&n^~j{oH_`;X))_S)PK^;AR|pIZN=@SZ8j@42OYUsFXRO z6*PO}!2#eSH=q@#-P*uh?fiW>Iy(f8azLT+o7@lyL@FqrJgupz!a!Onvt)Cg zh)g6%PfZtW%3-RnxltsQoSV6@lvs-(QxwNsI4>zne%H7~__oz(V5~opA&hnvc*k$J zVUdn4H`go{Qn*#4fdI8Z9pXBhFv%RZF3RvXfr@%+HNd~e`1j_OsBSDhVP^DCZolp~pbGhdh5*nK=18Y- z3PHOawpsS{R?4Ha^h?oK5m}P=b9)y>8BncWCA-u@mk-70Xb3JUwj#=5zX}=-ZEtesi7`cFg_r zT#JO^fAYkET1e@uhtG4j!ZhX!SBuBaNgzA5B#iutf^yi|KoG2Nzn{GlGf6vihG8SH zOorC2u>aXy>x|k0bB?&!hp-5usDvNwjM0x2bD!}3{eLe;Z(hGX zzv}nK=P=_pm>@cE%ZbW@)Jtv>X4g`*$Z^=HPPE>jJkk1R$KA6tI?>Kw;)&MxCtCjz za^&6ZoFqJxLn4#R7z)o}BcRi`65&0up`=S5??3$Uf1k|gY-bj}jfIk{Dc&0_pZcpt z4Lo+TiV|H{JX<=}4NlZ$r}9rFyC2DJ>)3&QDzV?Xk=S8xz0d=+}nvd#zIKWAp zzkn6DdyY_qVGF;z!;{790pDdNUK^Izhh%T-yEC$vYq{*sc)WM z(ajk4HXM5rme;V;IZ78pD?W1yE;|k22{eN^z?)!mBePfR#=rNVE{eYM1Ud&4x?PH^ zAX7Q5rJ$UBw@{*P-5d9ZLueKMXVJ3Oxoea*f<~TnQ!MUjWjG7z6gv9n??OQ9aCWCu zDzh+1rTNGvf7)j6e#FXqXH5jm;q4gDIGkuVYk;{v%SiyAN&#H&Cx8%+N&$FFQFR{q dD?DB Date: Fri, 5 Jun 2026 07:04:36 -0700 Subject: [PATCH 05/26] apply scalafmt to logging foundations (pr1) --- .../texera/observability/LogSanitizer.scala | 60 ++-- .../texera/observability/OtelInit.scala | 260 ++++++++++-------- .../observability/TexeraOtelLogAppender.scala | 35 +-- .../texera/observability/OtelInitSpec.scala | 29 +- .../TexeraOtelLogAppenderSpec.scala | 13 +- 5 files changed, 221 insertions(+), 176 deletions(-) diff --git a/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala b/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala index 781d8bd776d..aa92c558c27 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala @@ -22,39 +22,43 @@ package org.apache.texera.observability import scala.jdk.CollectionConverters._ /** - * Pure functions that sanitize log records before they leave the - * process via the OTel logs bridge. Lives in its own object so the - * security-critical behaviour can be unit-tested without a Logback - * fixture. - * - * Three invariants: - * 1. No control characters in the body (prevents log forging via - * CR/LF injection in user-supplied strings). - * 2. No oversized bodies (a 1 GiB log line must never reach the - * exporter). - * 3. No secrets in plain text (Bearer tokens, password=, AWS keys). - * - * Plus an MDC allowlist so accidental MDC pollution from a downstream - * library cannot leak unintended fields into the exporter. - */ + * Pure functions that sanitize log records before they leave the + * process via the OTel logs bridge. Lives in its own object so the + * security-critical behaviour can be unit-tested without a Logback + * fixture. + * + * Three invariants: + * 1. No control characters in the body (prevents log forging via + * CR/LF injection in user-supplied strings). + * 2. No oversized bodies (a 1 GiB log line must never reach the + * exporter). + * 3. No secrets in plain text (Bearer tokens, password=, AWS keys). + * + * Plus an MDC allowlist so accidental MDC pollution from a downstream + * library cannot leak unintended fields into the exporter. + */ object LogSanitizer { /** Per-record body cap. The OTel SDK and OTLP have higher limits, - * but 16 KiB is plenty for a useful log line and protects the - * collector from a runaway log. */ + * but 16 KiB is plenty for a useful log line and protects the + * collector from a runaway log. + */ val MaxBodyBytes: Int = 16 * 1024 /** Suffix appended to truncated bodies. Chosen to be visually - * obvious in a UI but short enough not to dominate the cap. */ + * obvious in a UI but short enough not to dominate the cap. + */ val TruncatedMarker: String = "...[truncated]" /** C0 control characters except TAB (0x09). Stripping CR/LF here - * prevents log forging via newline injection in user-supplied - * message bodies. DEL (0x7F) included for the same reason. */ + * prevents log forging via newline injection in user-supplied + * message bodies. DEL (0x7F) included for the same reason. + */ private val C0ControlRegex = "[\\x00-\\x08\\x0A-\\x1F\\x7F]".r /** Secret patterns. Order is significant: most specific first so a - * partial match doesn't shadow a tighter pattern. */ + * partial match doesn't shadow a tighter pattern. + */ private val SecretPatterns: Seq[scala.util.matching.Regex] = Seq( // Authorization: Bearer — bearer token in header form. """(?i)Bearer\s+[A-Za-z0-9._\-/+=]{8,}""".r, @@ -67,8 +71,9 @@ object LogSanitizer { ) /** MDC keys we will forward to OTel log attributes. Anything else - * is dropped — additions require a code change + reviewer - * acknowledgement of the privacy implications. */ + * is dropped — additions require a code change + reviewer + * acknowledgement of the privacy implications. + */ val AllowedMdcKeys: Set[String] = Set( "trace_id", "span_id", @@ -85,7 +90,8 @@ object LogSanitizer { ) /** Apply all three body-side transformations. Idempotent — running - * sanitize on already-sanitized output is a no-op. */ + * sanitize on already-sanitized output is a no-op. + */ def sanitize(body: String): String = { if (body == null || body.isEmpty) return "" val stripped = C0ControlRegex.replaceAllIn(body, "") @@ -104,8 +110,8 @@ object LogSanitizer { /** Filter an MDC map to the allowlist. Null-safe. */ def filterMdc(mdc: java.util.Map[String, String]): Map[String, String] = { if (mdc == null) return Map.empty - mdc.asScala.iterator - .collect { case (k, v) if k != null && AllowedMdcKeys.contains(k) && v != null => k -> v } - .toMap + mdc.asScala.iterator.collect { + case (k, v) if k != null && AllowedMdcKeys.contains(k) && v != null => k -> v + }.toMap } } diff --git a/common/config/src/main/scala/org/apache/texera/observability/OtelInit.scala b/common/config/src/main/scala/org/apache/texera/observability/OtelInit.scala index 35e328c49b0..d7c803fc79e 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/OtelInit.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/OtelInit.scala @@ -39,34 +39,35 @@ import java.time.Duration import scala.util.{Failure, Success, Try} /** - * Bootstraps the OpenTelemetry SDK for a Texera service. - * - * Design notes: - * - Default-disabled. Sets up nothing unless `OTEL_SDK_DISABLED=false`. - * - We deliberately do not use the autoconfigure SPI: the security model - * requires endpoint + resource-attribute filtering to happen BEFORE - * any exporter is constructed. Autoconfigure would parse env vars - * behind our back. - * - Validation is a single pure function so it can be unit-tested - * without spinning the SDK. - * - On any validation failure we log one WARN and return None. We - * do NOT throw — observability is opt-in plumbing; misconfiguration - * must never crash the service. - * - This is the only place in Texera that reads `OTEL_*` environment - * variables. Other modules consume the returned `OpenTelemetry` - * instance directly. - */ + * Bootstraps the OpenTelemetry SDK for a Texera service. + * + * Design notes: + * - Default-disabled. Sets up nothing unless `OTEL_SDK_DISABLED=false`. + * - We deliberately do not use the autoconfigure SPI: the security model + * requires endpoint + resource-attribute filtering to happen BEFORE + * any exporter is constructed. Autoconfigure would parse env vars + * behind our back. + * - Validation is a single pure function so it can be unit-tested + * without spinning the SDK. + * - On any validation failure we log one WARN and return None. We + * do NOT throw — observability is opt-in plumbing; misconfiguration + * must never crash the service. + * - This is the only place in Texera that reads `OTEL_*` environment + * variables. Other modules consume the returned `OpenTelemetry` + * instance directly. + */ object OtelInit extends LazyLogging { /** Resource attribute keys we accept from OTEL_RESOURCE_ATTRIBUTES. - * Resource attrs ride on every record this JVM emits (logs, - * metrics, traces) — so for a per-CU JVM (ComputingUnitMaster / - * ComputingUnitWorker) setting `texera.computing_unit.id=N` at - * boot is enough to tag every record without per-request MDC - * plumbing. Workflow/execution ids vary per task and still need - * MDC at the message boundary, but exposing them in the allowlist - * lets test harnesses + future per-task code populate them via - * the same mechanism. */ + * Resource attrs ride on every record this JVM emits (logs, + * metrics, traces) — so for a per-CU JVM (ComputingUnitMaster / + * ComputingUnitWorker) setting `texera.computing_unit.id=N` at + * boot is enough to tag every record without per-request MDC + * plumbing. Workflow/execution ids vary per task and still need + * MDC at the message boundary, but exposing them in the allowlist + * lets test harnesses + future per-task code populate them via + * the same mechanism. + */ private[observability] val AllowedResourceKeys: Set[String] = Set( "service.name", "service.version", @@ -88,16 +89,18 @@ object OtelInit extends LazyLogging { ) /** Default endpoint when SDK is enabled but no endpoint set explicitly. - * Uses 127.0.0.1 (not "localhost") so a natively-run service reaches the - * IPv4-only collector port published by docker-compose — on dual-stack - * hosts "localhost" resolves to ::1 first and the OTLP export silently - * fails. Inside docker the endpoint is overridden to otel-collector:4317. */ + * Uses 127.0.0.1 (not "localhost") so a natively-run service reaches the + * IPv4-only collector port published by docker-compose — on dual-stack + * hosts "localhost" resolves to ::1 first and the OTLP export silently + * fails. Inside docker the endpoint is overridden to otel-collector:4317. + */ private val DefaultEndpoint = "http://127.0.0.1:4317" /** Metric export interval bounds. Values outside this range get - * clamped to the default with a one-shot WARN. The lower bound - * prevents an attacker tipping the exporter into busy-loop mode; - * the upper bound keeps metrics useful for human operators. */ + * clamped to the default with a one-shot WARN. The lower bound + * prevents an attacker tipping the exporter into busy-loop mode; + * the upper bound keeps metrics useful for human operators. + */ private[observability] val MinMetricIntervalMs: Long = 1000L private[observability] val MaxMetricIntervalMs: Long = 10L * 60L * 1000L private[observability] val DefaultMetricIntervalMs: Long = 60L * 1000L @@ -107,78 +110,82 @@ object OtelInit extends LazyLogging { @volatile private var initialized: Option[OpenTelemetry] = None /** - * Initialize the SDK for the given service name. - * Returns Some(sdk) on success, None on disabled / invalid config. - * - * Side effect when enabled: attaches a [[TexeraOtelLogAppender]] to - * the Logback ROOT logger so application logs are mirrored to the - * OTel collector, with the security guards in [[LogSanitizer]] - * applied to every record. - */ - def init(serviceName: String): Option[OpenTelemetry] = synchronized { - if (initialized.isDefined) return initialized - - val env = (key: String) => Option(System.getenv(key)) - val result = initInternal( - serviceName = serviceName, - envProvider = env, - spanExporterFactory = buildOtlpSpanExporter, - logExporterFactory = endpoint => Some(buildOtlpLogExporter(endpoint)), - metricExporterFactory = endpoint => Some(buildOtlpMetricExporter(endpoint)), - logbackAttacher = LogbackBinder.attach - ) - // Register globally so [[TexeraTracer]] and any other OTel-aware - // code can call ``GlobalOpenTelemetry.getTracer(...)`` without - // threading the SDK through every callsite. set() throws on a - // second call within the same JVM — our outer ``initialized`` - // guard makes that unreachable, but wrap defensively. The test - // path deliberately skips this so multiple isolated SDKs can be - // built within one JVM. - result.foreach { sdk => - Try(GlobalOpenTelemetry.set(sdk)).failed.foreach { t => - logger.warn( - s"GlobalOpenTelemetry already set; using the existing instance: ${t.getMessage}" - ) + * Initialize the SDK for the given service name. + * Returns Some(sdk) on success, None on disabled / invalid config. + * + * Side effect when enabled: attaches a [[TexeraOtelLogAppender]] to + * the Logback ROOT logger so application logs are mirrored to the + * OTel collector, with the security guards in [[LogSanitizer]] + * applied to every record. + */ + def init(serviceName: String): Option[OpenTelemetry] = + synchronized { + if (initialized.isDefined) return initialized + + val env = (key: String) => Option(System.getenv(key)) + val result = initInternal( + serviceName = serviceName, + envProvider = env, + spanExporterFactory = buildOtlpSpanExporter, + logExporterFactory = endpoint => Some(buildOtlpLogExporter(endpoint)), + metricExporterFactory = endpoint => Some(buildOtlpMetricExporter(endpoint)), + logbackAttacher = LogbackBinder.attach + ) + // Register globally so [[TexeraTracer]] and any other OTel-aware + // code can call ``GlobalOpenTelemetry.getTracer(...)`` without + // threading the SDK through every callsite. set() throws on a + // second call within the same JVM — our outer ``initialized`` + // guard makes that unreachable, but wrap defensively. The test + // path deliberately skips this so multiple isolated SDKs can be + // built within one JVM. + result.foreach { sdk => + Try(GlobalOpenTelemetry.set(sdk)).failed.foreach { t => + logger.warn( + s"GlobalOpenTelemetry already set; using the existing instance: ${t.getMessage}" + ) + } } + result } - result - } /** - * Test-only entry point. Allows the test to inject an env-var map - * and a span exporter so the SDK does not attempt a real network - * connection. The Logback appender is NOT attached in tests — - * appender tests construct it directly with an in-memory log - * exporter. - */ + * Test-only entry point. Allows the test to inject an env-var map + * and a span exporter so the SDK does not attempt a real network + * connection. The Logback appender is NOT attached in tests — + * appender tests construct it directly with an in-memory log + * exporter. + */ private[observability] def initForTest( serviceName: String, envOverride: Map[String, String], exporter: SpanExporter, metricExporter: Option[MetricExporter] = None - ): Option[OpenTelemetry] = synchronized { - initInternal( - serviceName = serviceName, - envProvider = envOverride.get, - spanExporterFactory = _ => exporter, - logExporterFactory = _ => None, - metricExporterFactory = _ => metricExporter, - logbackAttacher = (_, _) => () // no-op in tests - ) - } + ): Option[OpenTelemetry] = + synchronized { + initInternal( + serviceName = serviceName, + envProvider = envOverride.get, + spanExporterFactory = _ => exporter, + logExporterFactory = _ => None, + metricExporterFactory = _ => metricExporter, + logbackAttacher = (_, _) => () // no-op in tests + ) + } /** Test-only: forget any previously-installed SDK. Does not unregister - * shutdown hooks (the previous SDK is closed instead). */ - private[observability] def resetForTest(): Unit = synchronized { - initialized.foreach { - case sdk: OpenTelemetrySdk => - Try(sdk.getSdkTracerProvider.close()) - Try(sdk.getSdkLoggerProvider.close()) - Try(sdk.getSdkMeterProvider.close()) - case _ => () + * shutdown hooks (the previous SDK is closed instead). + */ + private[observability] def resetForTest(): Unit = + synchronized { + initialized.foreach { + case sdk: OpenTelemetrySdk => + Try(sdk.getSdkTracerProvider.close()) + Try(sdk.getSdkLoggerProvider.close()) + Try(sdk.getSdkMeterProvider.close()) + case _ => () + } + initialized = None } - initialized = None - } private def initInternal( serviceName: String, @@ -200,7 +207,9 @@ object OtelInit extends LazyLogging { // dev time is a quiet no-op rather than a startup failure. val disabled = envProvider("OTEL_SDK_DISABLED").getOrElse("false") if (disabled.equalsIgnoreCase("true")) { - logger.info("OpenTelemetry SDK disabled (OTEL_SDK_DISABLED=true). No telemetry will be emitted.") + logger.info( + "OpenTelemetry SDK disabled (OTEL_SDK_DISABLED=true). No telemetry will be emitted." + ) return None } @@ -284,12 +293,17 @@ object OtelInit extends LazyLogging { // Make sure providers flush on shutdown. We add the hook only after // the SDK has been fully built so a panic during init doesn't leave // a dangling hook pointing at a half-constructed provider. - Runtime.getRuntime.addShutdownHook(new Thread(() => { - Try(tracerProvider.close()) - loggerProviderOpt.foreach(lp => Try(lp.close())) - meterProviderOpt.foreach(mp => Try(mp.close())) - () - }, "otel-shutdown")) + Runtime.getRuntime.addShutdownHook( + new Thread( + () => { + Try(tracerProvider.close()) + loggerProviderOpt.foreach(lp => Try(lp.close())) + meterProviderOpt.foreach(mp => Try(mp.close())) + () + }, + "otel-shutdown" + ) + ) initialized = Some(sdk) logger.info(s"OpenTelemetry SDK initialized for service '$serviceName' (endpoint=$endpoint).") @@ -297,10 +311,10 @@ object OtelInit extends LazyLogging { } /** - * Validate that the endpoint is parseable, uses an allowlisted scheme, - * and resolves to an allowlisted host. Pure function — safe to test - * without standing up the SDK. - */ + * Validate that the endpoint is parseable, uses an allowlisted scheme, + * and resolves to an allowlisted host. Pure function — safe to test + * without standing up the SDK. + */ private[observability] def validateEndpoint( endpoint: String, allowedHosts: Set[String] @@ -313,7 +327,9 @@ object OtelInit extends LazyLogging { if (scheme.isEmpty) { Left("missing scheme") } else if (!AllowedSchemes.contains(scheme)) { - Left(s"scheme '$scheme' not in allowlist ${AllowedSchemes.toSeq.sorted.mkString("{", ",", "}")}") + Left( + s"scheme '$scheme' not in allowlist ${AllowedSchemes.toSeq.sorted.mkString("{", ",", "}")}" + ) } else { val host = Option(uri.getHost).map(_.toLowerCase).getOrElse("") if (host.isEmpty) { @@ -328,10 +344,10 @@ object OtelInit extends LazyLogging { } /** - * Build a Resource from the service name plus the allowlisted subset - * of OTEL_RESOURCE_ATTRIBUTES. Unknown keys are dropped silently; - * service.name from env is ignored in favour of the argument. - */ + * Build a Resource from the service name plus the allowlisted subset + * of OTEL_RESOURCE_ATTRIBUTES. Unknown keys are dropped silently; + * service.name from env is ignored in favour of the argument. + */ private[observability] def buildResource(serviceName: String, rawAttrs: String): Resource = { val builder = Attributes.builder() builder.put(AttributeKey.stringKey("service.name"), serviceName) @@ -371,11 +387,11 @@ object OtelInit extends LazyLogging { OtlpGrpcMetricExporter.builder().setEndpoint(endpoint).build() /** - * Parse and clamp OTEL_METRIC_EXPORT_INTERVAL (milliseconds). - * Out-of-range or unparseable input falls back to the default and - * emits a single WARN. Pure-ish — easy to test without standing up - * the meter SDK. - */ + * Parse and clamp OTEL_METRIC_EXPORT_INTERVAL (milliseconds). + * Out-of-range or unparseable input falls back to the default and + * emits a single WARN. Pure-ish — easy to test without standing up + * the meter SDK. + */ private[observability] def clampIntervalMs(raw: Option[String]): Long = { raw match { case None => DefaultMetricIntervalMs @@ -401,18 +417,18 @@ object OtelInit extends LazyLogging { } /** - * Hides the Logback attach step behind a small object so [[OtelInit]] - * doesn't import Logback types directly (keeps the SDK init testable - * without a Logback dependency on the classpath in test runs that - * inject a mock attacher). - */ + * Hides the Logback attach step behind a small object so [[OtelInit]] + * doesn't import Logback types directly (keeps the SDK init testable + * without a Logback dependency on the classpath in test runs that + * inject a mock attacher). + */ private[observability] object LogbackBinder extends LazyLogging { /** Attempts to find the Logback ROOT logger, attach a fresh - * [[TexeraOtelLogAppender]] bound to `otel`, and start it. If - * Logback is not the active SLF4J binding (or for any other - * classpath issue), emits one WARN and returns — never throws. - */ + * [[TexeraOtelLogAppender]] bound to `otel`, and start it. If + * Logback is not the active SLF4J binding (or for any other + * classpath issue), emits one WARN and returns — never throws. + */ def attach(serviceName: String, otel: OpenTelemetry): Unit = { val factory = org.slf4j.LoggerFactory.getILoggerFactory factory match { diff --git a/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala b/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala index 37d04a303e6..2dbf160bcfb 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala @@ -31,21 +31,21 @@ import io.opentelemetry.context.Context import java.util.concurrent.TimeUnit /** - * Logback appender that forwards every event through [[LogSanitizer]] - * before emitting it as an OTel LogRecord. - * - * Lifecycle: - * - Construct with no args (Logback / programmatic instantiation). - * - Call [[bind]] once with the active [[OpenTelemetry]] instance - * (done by [[OtelInit]] after the SDK is built). Until then, - * [[append]] is a silent no-op — log events keep flowing to - * stdout/file unimpeded. - * - Stopping the appender unbinds; subsequent events drop. - * - * This is intentionally a thin shim. All security-critical logic - * lives in [[LogSanitizer]] so it can be tested without a Logback - * fixture. - */ + * Logback appender that forwards every event through [[LogSanitizer]] + * before emitting it as an OTel LogRecord. + * + * Lifecycle: + * - Construct with no args (Logback / programmatic instantiation). + * - Call [[bind]] once with the active [[OpenTelemetry]] instance + * (done by [[OtelInit]] after the SDK is built). Until then, + * [[append]] is a silent no-op — log events keep flowing to + * stdout/file unimpeded. + * - Stopping the appender unbinds; subsequent events drop. + * + * This is intentionally a thin shim. All security-critical logic + * lives in [[LogSanitizer]] so it can be tested without a Logback + * fixture. + */ class TexeraOtelLogAppender extends UnsynchronizedAppenderBase[ILoggingEvent] { // @volatile so a late [[bind]] is visible to appender threads @@ -121,8 +121,9 @@ class TexeraOtelLogAppender extends UnsynchronizedAppenderBase[ILoggingEvent] { } /** Pretty-print a Logback throwable proxy. Matches what Logback's - * default pattern layout would produce for `%ex` — class name, - * message, full stack frames, then walks the cause chain. */ + * default pattern layout would produce for `%ex` — class name, + * message, full stack frames, then walks the cause chain. + */ private def formatThrowable(proxy: IThrowableProxy): String = ThrowableProxyUtil.asString(proxy) diff --git a/common/config/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala index 5ebd0eb3f4a..45bbac119af 100644 --- a/common/config/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala +++ b/common/config/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala @@ -39,9 +39,16 @@ class OtelInitSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { // ----- validateEndpoint: pure function, exhaustive cases ------------- "validateEndpoint" should "accept a loopback OTLP gRPC URL" in { - OtelInit.validateEndpoint("http://localhost:4317", OtelInit.DefaultAllowedHosts) shouldBe Right(()) - OtelInit.validateEndpoint("grpc://127.0.0.1:4317", OtelInit.DefaultAllowedHosts) shouldBe Right(()) - OtelInit.validateEndpoint("https://localhost:4318", OtelInit.DefaultAllowedHosts) shouldBe Right(()) + OtelInit.validateEndpoint("http://localhost:4317", OtelInit.DefaultAllowedHosts) shouldBe Right( + () + ) + OtelInit.validateEndpoint("grpc://127.0.0.1:4317", OtelInit.DefaultAllowedHosts) shouldBe Right( + () + ) + OtelInit.validateEndpoint( + "https://localhost:4318", + OtelInit.DefaultAllowedHosts + ) shouldBe Right(()) } it should "reject file:// schemes (path traversal style attack)" in { @@ -83,14 +90,18 @@ class OtelInitSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { "buildResource" should "always include the service.name from the argument" in { val r = OtelInit.buildResource("my-service", "") - Option(r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.name"))) shouldBe Some( + Option( + r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.name")) + ) shouldBe Some( "my-service" ) } it should "honor allowlisted keys from OTEL_RESOURCE_ATTRIBUTES" in { val r = OtelInit.buildResource("svc", "service.version=1.2.3,deployment.environment=staging") - Option(r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.version"))) shouldBe Some( + Option( + r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.version")) + ) shouldBe Some( "1.2.3" ) Option( @@ -115,7 +126,9 @@ class OtelInitSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { it should "refuse to let OTEL_RESOURCE_ATTRIBUTES override service.name" in { val r = OtelInit.buildResource("real-svc", "service.name=spoofed") - Option(r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.name"))) shouldBe Some( + Option( + r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.name")) + ) shouldBe Some( "real-svc" ) } @@ -140,7 +153,9 @@ class OtelInitSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { it should "ignore malformed pairs without crashing" in { val r = OtelInit.buildResource("svc", ",,,=,foo,service.version=,=bar,service.version=1.0,") - Option(r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.version"))) shouldBe Some( + Option( + r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.version")) + ) shouldBe Some( "1.0" ) } diff --git a/common/config/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala index 83b9ca9d469..c879d1f9d56 100644 --- a/common/config/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala +++ b/common/config/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala @@ -36,8 +36,9 @@ import scala.jdk.CollectionConverters._ class TexeraOtelLogAppenderSpec extends AnyFlatSpec with Matchers { /** Build an OpenTelemetry SDK whose LoggerProvider drains to the - * given in-memory exporter via the synchronous SimpleLogRecordProcessor, - * so tests don't depend on batch timing. */ + * given in-memory exporter via the synchronous SimpleLogRecordProcessor, + * so tests don't depend on batch timing. + */ private def newFixture(): (OpenTelemetry, InMemoryLogRecordExporter, TexeraOtelLogAppender) = { val exporter = InMemoryLogRecordExporter.create() val lp = SdkLoggerProvider @@ -83,7 +84,13 @@ class TexeraOtelLogAppenderSpec extends AnyFlatSpec with Matchers { appender.doAppend(makeEvent(s"msg-$lvl", lvl)) } val severities = exporter.getFinishedLogRecordItems.asScala.map(_.getSeverity).toSet - severities shouldBe Set(Severity.TRACE, Severity.DEBUG, Severity.INFO, Severity.WARN, Severity.ERROR) + severities shouldBe Set( + Severity.TRACE, + Severity.DEBUG, + Severity.INFO, + Severity.WARN, + Severity.ERROR + ) } // ----- security: sanitisation happens at the boundary ----------------- From 17b1bdc9c1a8f9d711bceaeb476b23d3755f51a4 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 5 Jun 2026 07:16:20 -0700 Subject: [PATCH 06/26] apply scalafmt to metrics and tracing primitives (pr2) --- .../texera/observability/SpanAttrs.scala | 52 ++++---- .../texera/observability/TexeraMetrics.scala | 113 ++++++++++-------- .../texera/observability/TexeraTracer.scala | 56 ++++----- .../observability/TraceparentValidator.scala | 48 ++++---- .../texera/observability/SpanAttrsSpec.scala | Bin 5448 -> 5460 bytes .../observability/TexeraMetricsSpec.scala | 12 +- .../TraceparentValidatorSpec.scala | Bin 5085 -> 5084 bytes 7 files changed, 153 insertions(+), 128 deletions(-) diff --git a/common/config/src/main/scala/org/apache/texera/observability/SpanAttrs.scala b/common/config/src/main/scala/org/apache/texera/observability/SpanAttrs.scala index c66db23f652..a1ac0bbc18b 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/SpanAttrs.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/SpanAttrs.scala @@ -23,27 +23,28 @@ import io.opentelemetry.api.common.AttributeKey import io.opentelemetry.api.trace.{Span, SpanBuilder} /** - * Thin helper for setting span attributes safely. - * - * Three rules: - * 1. Typed setters only — no public escape hatch for arbitrary - * untyped strings to land on a span as untrusted free text. - * 2. Free-text values are CRLF-stripped + capped at - * [[FreeTextMaxLen]] to prevent log/span forging via embedded - * newlines. - * 3. Operator IDs and workflow/execution IDs must match a strict - * character set — otherwise dropped silently (the operator - * identifier should be a stable internal value, not user free - * text). - */ + * Thin helper for setting span attributes safely. + * + * Three rules: + * 1. Typed setters only — no public escape hatch for arbitrary + * untyped strings to land on a span as untrusted free text. + * 2. Free-text values are CRLF-stripped + capped at + * [[FreeTextMaxLen]] to prevent log/span forging via embedded + * newlines. + * 3. Operator IDs and workflow/execution IDs must match a strict + * character set — otherwise dropped silently (the operator + * identifier should be a stable internal value, not user free + * text). + */ object SpanAttrs { /** Maximum length for free-text span attribute values. */ val FreeTextMaxLen: Int = 256 /** Validates the shape we accept for operator IDs: alnum + `_.-`, - * 1–64 chars. Anything else is dropped (not coerced — we'd rather - * miss a label than leak an unbounded string into a span). */ + * 1–64 chars. Anything else is dropped (not coerced — we'd rather + * miss a label than leak an unbounded string into a span). + */ private val OperatorIdPattern = "^[A-Za-z0-9_.\\-]{1,64}$".r.pattern // ---- Standard Texera correlation labels ------------------------------ @@ -71,8 +72,9 @@ object SpanAttrs { b.setAttribute(UserId, java.lang.Long.valueOf(id)) /** Sets the operator id only if it passes the strict character - * check; otherwise the attribute is omitted. Returns the same - * builder either way for fluent chaining. */ + * check; otherwise the attribute is omitted. Returns the same + * builder either way for fluent chaining. + */ def withOperatorId(b: SpanBuilder, id: String): SpanBuilder = { if (id != null && OperatorIdPattern.matcher(id).matches()) { b.setAttribute(OperatorId, id) @@ -88,8 +90,10 @@ object SpanAttrs { // ---- Typed setters for Span (used after a span is active) ------------ - def setWorkflowId(s: Span, id: Long): Span = s.setAttribute(WorkflowId, java.lang.Long.valueOf(id)) - def setExecutionId(s: Span, id: Long): Span = s.setAttribute(ExecutionId, java.lang.Long.valueOf(id)) + def setWorkflowId(s: Span, id: Long): Span = + s.setAttribute(WorkflowId, java.lang.Long.valueOf(id)) + def setExecutionId(s: Span, id: Long): Span = + s.setAttribute(ExecutionId, java.lang.Long.valueOf(id)) def setOperatorId(s: Span, id: String): Span = { if (id != null && OperatorIdPattern.matcher(id).matches()) s.setAttribute(OperatorId, id) else s @@ -102,13 +106,13 @@ object SpanAttrs { // ---- Pure helpers (exposed for testing) ------------------------------ /** - * Strip CR/LF and other C0 control characters from a free-text - * value, then cap at [[FreeTextMaxLen]]. Returns null for - * null/empty input (caller skips the setAttribute call). - */ + * Strip CR/LF and other C0 control characters from a free-text + * value, then cap at [[FreeTextMaxLen]]. Returns null for + * null/empty input (caller skips the setAttribute call). + */ def sanitizeFreeText(value: String): String = { if (value == null || value.isEmpty) return null - val stripped = value.filter(c => c >= 0x20 && c != 0x7F) + val stripped = value.filter(c => c >= 0x20 && c != 0x7f) if (stripped.isEmpty) null else if (stripped.length <= FreeTextMaxLen) stripped else stripped.substring(0, FreeTextMaxLen) diff --git a/common/config/src/main/scala/org/apache/texera/observability/TexeraMetrics.scala b/common/config/src/main/scala/org/apache/texera/observability/TexeraMetrics.scala index c850c055392..ef7ad78aeef 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/TexeraMetrics.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/TexeraMetrics.scala @@ -25,20 +25,20 @@ import io.opentelemetry.api.common.{AttributeKey, Attributes} import io.opentelemetry.api.metrics.Meter /** - * Strongly-typed façade for Texera-emitted metrics. - * - * Cardinality safety is enforced by the API surface, not by - * documentation: there is no public method that accepts an arbitrary - * string as a label key or value. The only labels that ever land on - * an instrument are the two enums [[Outcome]] and [[WorkflowKind]], - * each restricted to a fixed set. ``workflow.id`` / ``execution.id`` - * are deliberately NOT metric labels — per-execution detail belongs - * in traces and logs, joined on ``trace_id`` at query time. - * - * Histogram bucket bounds are hard-coded constants so they can't be - * coerced by request input. The OTel SDK applies its own default - * attribute-value-length cap to anything that does slip through. - */ + * Strongly-typed façade for Texera-emitted metrics. + * + * Cardinality safety is enforced by the API surface, not by + * documentation: there is no public method that accepts an arbitrary + * string as a label key or value. The only labels that ever land on + * an instrument are the two enums [[Outcome]] and [[WorkflowKind]], + * each restricted to a fixed set. ``workflow.id`` / ``execution.id`` + * are deliberately NOT metric labels — per-execution detail belongs + * in traces and logs, joined on ``trace_id`` at query time. + * + * Histogram bucket bounds are hard-coded constants so they can't be + * coerced by request input. The OTel SDK applies its own default + * attribute-value-length cap to anything that does slip through. + */ object TexeraMetrics extends LazyLogging { /** Outcome enum, the only mutable label on lifecycle counters. */ @@ -50,7 +50,8 @@ object TexeraMetrics extends LazyLogging { } /** Workflow kind enum. Distinguishes interactive vs. scheduled - * workflows for the dashboard's basic split — extend deliberately. */ + * workflows for the dashboard's basic split — extend deliberately. + */ sealed abstract class WorkflowKind(val name: String) object WorkflowKind { case object Interactive extends WorkflowKind("interactive") @@ -62,8 +63,9 @@ object TexeraMetrics extends LazyLogging { private val WorkflowKindKey: AttributeKey[String] = AttributeKey.stringKey("texera.workflow.kind") /** Histogram bucket bounds in seconds. Hard-coded — constants so - * request input can't reshape the histogram. Range covers - * fast (<1s) to long (>1h) workflows. */ + * request input can't reshape the histogram. Range covers + * fast (<1s) to long (>1h) workflows. + */ private val DurationBuckets: java.util.List[java.lang.Double] = { val builder = new java.util.ArrayList[java.lang.Double]() Seq(0.1, 0.5, 1.0, 5.0, 10.0, 30.0, 60.0, 300.0, 600.0, 1800.0, 3600.0) @@ -99,36 +101,42 @@ object TexeraMetrics extends LazyLogging { @volatile private var activeExecutionsSupplier: () => Long = () => 0L /** Register the authoritative source of "currently active executions". - * The supplier is polled on every metric collection, so the gauge always - * reflects ground truth and can never leak. Called once at process - * startup (e.g. by ComputingUnitMaster). */ - def setActiveExecutionsSupplier(supplier: () => Long): Unit = synchronized { - activeExecutionsSupplier = supplier - ensureBound() - } + * The supplier is polled on every metric collection, so the gauge always + * reflects ground truth and can never leak. Called once at process + * startup (e.g. by ComputingUnitMaster). + */ + def setActiveExecutionsSupplier(supplier: () => Long): Unit = + synchronized { + activeExecutionsSupplier = supplier + ensureBound() + } /** Bind instruments to the current global meter. Idempotent — the - * first call wins; later calls are no-ops. Tests can call - * [[bindForTest]] with an explicit Meter, then [[resetForTest]] to - * rebind. */ - def ensureBound(): Unit = synchronized { - if (_starts == null) bind(GlobalOpenTelemetry.getMeter(InstrumentationScope)) - } - - private[observability] def bindForTest(meter: Meter): Unit = synchronized { - bind(meter) - } - - private[observability] def resetForTest(): Unit = synchronized { - _starts = null - _completions = null - _cancellations = null - // The observable gauge registered a collection callback — close it so the - // previous test's meter provider stops being polled after it's discarded. - if (_active != null) _active.close() - _active = null - _duration = null - } + * first call wins; later calls are no-ops. Tests can call + * [[bindForTest]] with an explicit Meter, then [[resetForTest]] to + * rebind. + */ + def ensureBound(): Unit = + synchronized { + if (_starts == null) bind(GlobalOpenTelemetry.getMeter(InstrumentationScope)) + } + + private[observability] def bindForTest(meter: Meter): Unit = + synchronized { + bind(meter) + } + + private[observability] def resetForTest(): Unit = + synchronized { + _starts = null + _completions = null + _cancellations = null + // The observable gauge registered a collection callback — close it so the + // previous test's meter provider stops being polled after it's discarded. + if (_active != null) _active.close() + _active = null + _duration = null + } private def bind(meter: Meter): Unit = { _starts = meter @@ -157,7 +165,9 @@ object TexeraMetrics extends LazyLogging { _active = meter .gaugeBuilder("texera.workflow.active") .ofLongs() - .setDescription("Number of workflow executions currently in progress (observed from the live registry).") + .setDescription( + "Number of workflow executions currently in progress (observed from the live registry)." + ) .buildWithCallback(obs => obs.record(activeExecutionsSupplier())) _duration = meter .histogramBuilder("texera.workflow.duration") @@ -202,11 +212,12 @@ object TexeraMetrics extends LazyLogging { } /** A user-initiated kill/cancel. Bumps a dedicated counter. Deliberately - * NOT recorded as a completion: a cancelled run never finished on its own, - * so it must not drag down the success rate. No duration is recorded for - * the same reason — a killed run's wall-clock time is not a real runtime - * and would skew the duration percentiles. (The active gauge is observed - * from the live registry and needs no decrement here.) */ + * NOT recorded as a completion: a cancelled run never finished on its own, + * so it must not drag down the success rate. No duration is recorded for + * the same reason — a killed run's wall-clock time is not a real runtime + * and would skew the duration percentiles. (The active gauge is observed + * from the live registry and needs no decrement here.) + */ def recordCancellation(kind: WorkflowKind): Unit = { ensureBound() logger.debug(s"metric: workflow cancelled (kind=${kind.name}) — cancellations +1") diff --git a/common/config/src/main/scala/org/apache/texera/observability/TexeraTracer.scala b/common/config/src/main/scala/org/apache/texera/observability/TexeraTracer.scala index 8941d97aefa..ae8f71c82e6 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/TexeraTracer.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/TexeraTracer.scala @@ -24,22 +24,22 @@ import io.opentelemetry.api.trace.{Span, SpanBuilder, StatusCode, Tracer} import io.opentelemetry.context.{Context, Scope} /** - * Thin convenience wrapper around the global OTel tracer. - * - * Two reasons to go through this rather than calling - * ``GlobalOpenTelemetry.getTracer`` directly at every callsite: - * - * 1. Single instrumentation scope name (``org.apache.texera``) — so - * every Texera-produced span shows up under one logical scope in - * the backend, separable from anything emitted by transitive - * libraries. - * 2. One ergonomic ``withSpan`` API that handles exception → status, - * scope cleanup, and span end in a single try/finally. Callers - * don't have to remember the ceremony at every site. - * - * When the SDK is disabled, ``GlobalOpenTelemetry.getTracer`` returns - * a no-op tracer, so calling these methods is safe at any time. - */ + * Thin convenience wrapper around the global OTel tracer. + * + * Two reasons to go through this rather than calling + * ``GlobalOpenTelemetry.getTracer`` directly at every callsite: + * + * 1. Single instrumentation scope name (``org.apache.texera``) — so + * every Texera-produced span shows up under one logical scope in + * the backend, separable from anything emitted by transitive + * libraries. + * 2. One ergonomic ``withSpan`` API that handles exception → status, + * scope cleanup, and span end in a single try/finally. Callers + * don't have to remember the ceremony at every site. + * + * When the SDK is disabled, ``GlobalOpenTelemetry.getTracer`` returns + * a no-op tracer, so calling these methods is safe at any time. + */ object TexeraTracer { private val InstrumentationScope = "org.apache.texera" @@ -49,13 +49,13 @@ object TexeraTracer { def spanBuilder(name: String): SpanBuilder = tracer.spanBuilder(name) /** - * Run ``block`` inside a fresh span; record exceptions, propagate - * the right span status, and ensure the span is ended exactly once. - * - * Use this for synchronous critical sections. For async (Future- - * returning) code paths use ``withAsyncSpan`` so the span doesn't - * close before the async work completes. - */ + * Run ``block`` inside a fresh span; record exceptions, propagate + * the right span status, and ensure the span is ended exactly once. + * + * Use this for synchronous critical sections. For async (Future- + * returning) code paths use ``withAsyncSpan`` so the span doesn't + * close before the async work completes. + */ def withSpan[T](name: String, configure: SpanBuilder => SpanBuilder = identity)( block: Span => T ): T = { @@ -75,10 +75,10 @@ object TexeraTracer { } /** - * Snapshot the current OTel ``Context`` so async callbacks can - * re-attach it via ``Context.makeCurrent`` later. Useful at the - * Scala↔Python boundary where the calling thread is not the - * receiving thread. - */ + * Snapshot the current OTel ``Context`` so async callbacks can + * re-attach it via ``Context.makeCurrent`` later. Useful at the + * Scala↔Python boundary where the calling thread is not the + * receiving thread. + */ def currentContext: Context = Context.current() } diff --git a/common/config/src/main/scala/org/apache/texera/observability/TraceparentValidator.scala b/common/config/src/main/scala/org/apache/texera/observability/TraceparentValidator.scala index 9577a9f9438..91868449027 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/TraceparentValidator.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/TraceparentValidator.scala @@ -20,35 +20,38 @@ package org.apache.texera.observability /** - * Pure validators for W3C Trace Context headers crossing the - * Scala↔Python boundary. The single rule: if the inbound bytes do - * not match the strict regex, we discard the value and start a fresh - * trace. We never echo a rejected value back into a span, log, or - * error message. - * - * Spec reference: https://www.w3.org/TR/trace-context/ - * - * The regexes here intentionally do NOT use any context-sensitive - * grouping or backreferences — keeps the validators safe against - * pathological inputs (ReDoS) and trivially fast. - */ + * Pure validators for W3C Trace Context headers crossing the + * Scala↔Python boundary. The single rule: if the inbound bytes do + * not match the strict regex, we discard the value and start a fresh + * trace. We never echo a rejected value back into a span, log, or + * error message. + * + * Spec reference: https://www.w3.org/TR/trace-context/ + * + * The regexes here intentionally do NOT use any context-sensitive + * grouping or backreferences — keeps the validators safe against + * pathological inputs (ReDoS) and trivially fast. + */ object TraceparentValidator { /** W3C traceparent format: `---`. - * We accept only version `00` (the only published version) with the - * canonical 32-hex / 16-hex / 2-hex layout. All hex lowercase per - * spec — uppercase is invalid. */ + * We accept only version `00` (the only published version) with the + * canonical 32-hex / 16-hex / 2-hex layout. All hex lowercase per + * spec — uppercase is invalid. + */ private val TraceparentPattern = "^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$".r.pattern /** Bounded tracestate length. Spec recommends ≤512 chars. We are - * stricter to remove a small DoS surface — an attacker can't send - * a 1 MiB tracestate to balloon downstream context allocations. */ + * stricter to remove a small DoS surface — an attacker can't send + * a 1 MiB tracestate to balloon downstream context allocations. + */ val MaxTracestateLength: Int = 512 /** Validate a traceparent header. Returns the input unchanged on - * success, None on any failure (rejected — caller starts a fresh - * trace). Null and empty are silent failures. */ + * success, None on any failure (rejected — caller starts a fresh + * trace). Null and empty are silent failures. + */ def validateTraceparent(header: String): Option[String] = { if (header == null || header.isEmpty) return None // Trace-id and parent-id must not be all-zero per spec — an @@ -62,8 +65,9 @@ object TraceparentValidator { } /** Validate a tracestate header. Spec: comma-separated list of - * key=value pairs, ASCII-printable only, total length capped. - * Returns the input unchanged on success, None on rejection. */ + * key=value pairs, ASCII-printable only, total length capped. + * Returns the input unchanged on success, None on rejection. + */ def validateTracestate(header: String): Option[String] = { if (header == null || header.isEmpty) return None if (header.length > MaxTracestateLength) return None @@ -86,7 +90,7 @@ object TraceparentValidator { val c = s.charAt(i) // Allow printable ASCII range (0x20-0x7E). Tab and CR/LF are // rejected — a tracestate must not span lines. - if (c < 0x20 || c > 0x7E) return false + if (c < 0x20 || c > 0x7e) return false i += 1 } true diff --git a/common/config/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala index 388ee1d2aaa3a7f102742d7ea6bc083c1e6a00e8..1afca195533073f1d8ff2b0b8c8b4dd524da77e2 100644 GIT binary patch delta 27 gcmX@1bwz7KH4ht?0tigb7q*=Ifm43-S)OVR0DayF-~a#s delta 20 ccmcbjbwX=HHP7Tke*ekE+)|rA@T71609h6XzyJUM diff --git a/common/config/src/test/scala/org/apache/texera/observability/TexeraMetricsSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/TexeraMetricsSpec.scala index 10a9eb7651a..8bed93d6b01 100644 --- a/common/config/src/test/scala/org/apache/texera/observability/TexeraMetricsSpec.scala +++ b/common/config/src/test/scala/org/apache/texera/observability/TexeraMetricsSpec.scala @@ -93,7 +93,9 @@ class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEac "texera.workflow.duration" ) - metrics("texera.workflow.completions").getLongSumData.getPoints.asScala.head.getValue shouldBe 1L + metrics( + "texera.workflow.completions" + ).getLongSumData.getPoints.asScala.head.getValue shouldBe 1L val histogram = metrics("texera.workflow.duration").getHistogramData.getPoints.asScala.head histogram.getCount shouldBe 1L @@ -111,7 +113,9 @@ class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEac // series the query never reads). val completion = metrics("texera.workflow.completions").getLongSumData.getPoints.asScala.head completion.getValue shouldBe 1L - completion.getAttributes.asMap.asScala.map { case (k, v) => k.getKey -> v.toString } should contain( + completion.getAttributes.asMap.asScala.map { + case (k, v) => k.getKey -> v.toString + } should contain( "texera.outcome" -> "failure" ) metrics("texera.workflow.duration").getHistogramData.getPoints.asScala.head.getSum shouldBe 12.0 @@ -124,7 +128,9 @@ class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEac TexeraMetrics.recordCancellation(TexeraMetrics.WorkflowKind.Interactive) val metrics = collectAll() - metrics("texera.workflow.cancellations").getLongSumData.getPoints.asScala.head.getValue shouldBe 1L + metrics( + "texera.workflow.cancellations" + ).getLongSumData.getPoints.asScala.head.getValue shouldBe 1L // A kill is not a completion and records no duration: it must not // drag down the success rate nor skew the duration percentiles. metrics.keySet should not contain "texera.workflow.completions" diff --git a/common/config/src/test/scala/org/apache/texera/observability/TraceparentValidatorSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/TraceparentValidatorSpec.scala index d95b61239ba4dfcf2dc87ff4fbbbc19b3d022ab5..8c4bd670fc1994160c1bb916f1443f08d9d3f20c 100644 GIT binary patch delta 16 Ycmcbsen)*n9`EFdEasb+@tU&%06lXCH2?qr delta 16 Ycmcbkeph`%9`ED{EEb!W@|v>&06l94Gynhq From c89c985c4231eb31e2afdb42995254b5683ea291 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 5 Jun 2026 07:23:58 -0700 Subject: [PATCH 07/26] apply scalafmt to deployment config specs (pr3) --- .../ObservabilityComposeSpec.scala | 15 +++++++-------- .../observability/ParcaConfigSpec.scala | 19 +++++++++---------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/common/config/src/test/scala/org/apache/texera/observability/ObservabilityComposeSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/ObservabilityComposeSpec.scala index 5f21df9b407..af8fe6792bc 100644 --- a/common/config/src/test/scala/org/apache/texera/observability/ObservabilityComposeSpec.scala +++ b/common/config/src/test/scala/org/apache/texera/observability/ObservabilityComposeSpec.scala @@ -26,11 +26,11 @@ import java.nio.charset.StandardCharsets import java.nio.file.{Files, Path, Paths} /** - * Smoke tests for the PR 6 docker-compose + collector config. Same - * design as [[ParcaConfigSpec]] — string-level assertions, no YAML - * parser, because the goal is to catch typos and licence-pin drift, - * not to validate the upstream schemas. - */ + * Smoke tests for the PR 6 docker-compose + collector config. Same + * design as [[ParcaConfigSpec]] — string-level assertions, no YAML + * parser, because the goal is to catch typos and licence-pin drift, + * not to validate the upstream schemas. + */ class ObservabilityComposeSpec extends AnyFlatSpec with Matchers { private def resolveBundled(relative: String): Path = { @@ -150,9 +150,8 @@ class ObservabilityComposeSpec extends AnyFlatSpec with Matchers { "observability-traces", "observability-profiles" ).foreach { profile => - val grepCount = text.split('\n').count(line => - !line.trim.startsWith("#") && line.contains(profile) - ) + val grepCount = + text.split('\n').count(line => !line.trim.startsWith("#") && line.contains(profile)) withClue(s"$profile should be in default COMPOSE_PROFILES (non-comment): ")( grepCount should be >= 1 ) diff --git a/common/config/src/test/scala/org/apache/texera/observability/ParcaConfigSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/ParcaConfigSpec.scala index 8e0d162b23f..1b6a15abcec 100644 --- a/common/config/src/test/scala/org/apache/texera/observability/ParcaConfigSpec.scala +++ b/common/config/src/test/scala/org/apache/texera/observability/ParcaConfigSpec.scala @@ -26,14 +26,14 @@ import java.nio.charset.StandardCharsets import java.nio.file.{Files, Path, Paths} /** - * Smoke test for the bundled Parca configuration files. - * - * Intentionally lightweight: we are guarding against accidental - * deletion / emptying / tag drift, not validating Parca's schema. - * The real config validation happens when the agent starts up - * inside its container — but a unit-level smoke test catches typos - * before a developer pushes them. - */ + * Smoke test for the bundled Parca configuration files. + * + * Intentionally lightweight: we are guarding against accidental + * deletion / emptying / tag drift, not validating Parca's schema. + * The real config validation happens when the agent starts up + * inside its container — but a unit-level smoke test catches typos + * before a developer pushes them. + */ class ParcaConfigSpec extends AnyFlatSpec with Matchers { // sbt runs tests with the module dir as CWD, but a developer who @@ -100,8 +100,7 @@ class ParcaConfigSpec extends AnyFlatSpec with Matchers { // This assertion makes the design intent enforceable. Comments // are skipped — they're allowed (and required) to explain the // rule. - val configLines = new String(Files.readAllBytes(agentEnv), StandardCharsets.UTF_8) - .linesIterator + val configLines = new String(Files.readAllBytes(agentEnv), StandardCharsets.UTF_8).linesIterator .map(_.trim) .filter(line => line.nonEmpty && !line.startsWith("#")) .toSeq From 437848b07b753d783313f597604c86c3b2ab7e20 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 5 Jun 2026 07:35:15 -0700 Subject: [PATCH 08/26] satisfy scalafmt + eslint (takeUntil, unknown errors) for gateway core and shell (pr4) --- .../texera/web/TexeraWebApplication.scala | 5 +- .../RequestContextMdcFilter.scala | 63 +++++----- .../observability/UserContextMdcFilter.scala | 36 +++--- .../observability/gateway/AuditLogger.scala | 32 ++--- .../observability/gateway/BackendClient.scala | 90 +++++++------- .../gateway/GatewayContext.scala | 21 ++-- .../gateway/ObservabilityResources.scala | 37 +++--- .../observability/gateway/RateLimiter.scala | 44 ++++--- .../observability/gateway/ScopeResolver.scala | 66 +++++----- .../web/observability/gateway/builders.scala | 39 +++--- .../web/observability/gateway/dtos.scala | 116 +++++++++++------- .../RequestContextMdcFilterSpec.scala | 53 +++++--- .../UserContextMdcFilterSpec.scala | 29 +++-- .../gateway/BackendClientSpec.scala | 57 +++++---- .../config/ObservabilityGatewayConfig.scala | 16 +-- .../observability.component.spec.ts | 10 +- .../observability/observability.component.ts | 48 ++++---- .../observability.service.spec.ts | 29 ++--- .../observability/observability.service.ts | 14 +-- .../traces-pivot.service.spec.ts | 2 +- 20 files changed, 431 insertions(+), 376 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/web/TexeraWebApplication.scala b/amber/src/main/scala/org/apache/texera/web/TexeraWebApplication.scala index 4a185f76e52..69f3e64bec5 100644 --- a/amber/src/main/scala/org/apache/texera/web/TexeraWebApplication.scala +++ b/amber/src/main/scala/org/apache/texera/web/TexeraWebApplication.scala @@ -33,10 +33,7 @@ import org.apache.texera.auth.SessionUser import org.apache.texera.dao.SqlServer import org.apache.texera.observability.OtelInit import org.apache.texera.web.auth.JwtAuth.setupJwtAuth -import org.apache.texera.web.observability.gateway.{ - GatewayContext, - ObservabilityHealthResource -} +import org.apache.texera.web.observability.gateway.{GatewayContext, ObservabilityHealthResource} import org.apache.texera.web.resource._ import org.apache.texera.web.resource.auth.{AuthResource, GoogleAuthResource} import org.apache.texera.web.resource.dashboard.DashboardResource diff --git a/amber/src/main/scala/org/apache/texera/web/observability/RequestContextMdcFilter.scala b/amber/src/main/scala/org/apache/texera/web/observability/RequestContextMdcFilter.scala index 054360c1c44..4f780feb65a 100644 --- a/amber/src/main/scala/org/apache/texera/web/observability/RequestContextMdcFilter.scala +++ b/amber/src/main/scala/org/apache/texera/web/observability/RequestContextMdcFilter.scala @@ -25,32 +25,32 @@ import javax.servlet._ import javax.servlet.http.HttpServletRequest /** - * Servlet filter that pushes request-scoped IDs into the SLF4J MDC - * so every log record emitted while handling a request carries - * `texera.workflow.id`, `texera.execution.id`, and - * `texera.computing_unit.id` when those are visible in the URL or - * request headers. - * - * Without this, the OTel log appender bridges every Logback event - * with no per-request context, so the dashboard's CU/workflow - * filters match nothing live — the only records carrying those keys - * are seed data pushed via the VictoriaLogs ingest API. - * - * Sources of IDs, in priority order: - * 1. HTTP headers `X-Texera-Workflow-Id`, `X-Texera-Execution-Id`, - * `X-Texera-Computing-Unit-Id` — set explicitly by the - * Angular client when known. - * 2. URL path segments matching `/workflow/`, - * `/execution/`, `/computing-unit/` (also the - * `/{wid}` / `/{cuid}` patterns used by some resources). - * - * MDC is ALWAYS cleared in a `finally` block so a thread reused for - * a different request doesn't leak the previous request's labels. - * - * Keys here must stay in sync with [[LogSanitizer.AllowedMdcKeys]] — - * any new key added here must be allowlisted there or the OTel - * appender will silently drop it. - */ + * Servlet filter that pushes request-scoped IDs into the SLF4J MDC + * so every log record emitted while handling a request carries + * `texera.workflow.id`, `texera.execution.id`, and + * `texera.computing_unit.id` when those are visible in the URL or + * request headers. + * + * Without this, the OTel log appender bridges every Logback event + * with no per-request context, so the dashboard's CU/workflow + * filters match nothing live — the only records carrying those keys + * are seed data pushed via the VictoriaLogs ingest API. + * + * Sources of IDs, in priority order: + * 1. HTTP headers `X-Texera-Workflow-Id`, `X-Texera-Execution-Id`, + * `X-Texera-Computing-Unit-Id` — set explicitly by the + * Angular client when known. + * 2. URL path segments matching `/workflow/`, + * `/execution/`, `/computing-unit/` (also the + * `/{wid}` / `/{cuid}` patterns used by some resources). + * + * MDC is ALWAYS cleared in a `finally` block so a thread reused for + * a different request doesn't leak the previous request's labels. + * + * Keys here must stay in sync with [[LogSanitizer.AllowedMdcKeys]] — + * any new key added here must be allowlisted there or the OTel + * appender will silently drop it. + */ class RequestContextMdcFilter extends Filter { override def init(filterConfig: FilterConfig): Unit = () @@ -87,8 +87,8 @@ class RequestContextMdcFilter extends Filter { applyParam(http, "eid", "texera.execution.id", pushed) applyParam(http, "cuid", "texera.computing_unit.id", pushed) case _ => - // Non-HTTP request (websocket upgrades fall through here on - // some Servlet versions). No MDC context to add. + // Non-HTTP request (websocket upgrades fall through here on + // some Servlet versions). No MDC context to add. } try { chain.doFilter(request, response) @@ -132,9 +132,10 @@ class RequestContextMdcFilter extends Filter { } /** Read a query parameter, validate it's a positive integer, push - * to MDC. Same shape as applyHeader: numeric-only allowlist and a - * length cap so a forged value can't inject newlines into the - * rendered log line. */ + * to MDC. Same shape as applyHeader: numeric-only allowlist and a + * length cap so a forged value can't inject newlines into the + * rendered log line. + */ private def applyParam( req: HttpServletRequest, paramName: String, diff --git a/amber/src/main/scala/org/apache/texera/web/observability/UserContextMdcFilter.scala b/amber/src/main/scala/org/apache/texera/web/observability/UserContextMdcFilter.scala index 4c0dd3df468..80594332108 100644 --- a/amber/src/main/scala/org/apache/texera/web/observability/UserContextMdcFilter.scala +++ b/amber/src/main/scala/org/apache/texera/web/observability/UserContextMdcFilter.scala @@ -33,21 +33,21 @@ import javax.ws.rs.ext.Provider import javax.ws.rs.Priorities /** - * Jersey filter that pushes the authenticated user's id into the - * SLF4J MDC so every log record emitted while handling the request - * carries `texera.user.id`. - * - * Why Jersey and not a Servlet filter: - * - Authentication is wired as a Jersey `ContainerRequestFilter` - * (see [[org.apache.texera.auth.JwtAuthFilter]]) — by the time - * Servlet filters run, `SecurityContext.getUserPrincipal` is - * still null. A Jersey filter ordered after auth picks it up. - * - Priorities.AUTHENTICATION + 100 places us in the - * AUTHORIZATION bucket which is guaranteed to run AFTER auth. - * - * Symmetric request/response interfaces let us pair set + clear - * without leaking MDC across threads in the worker pool. - */ + * Jersey filter that pushes the authenticated user's id into the + * SLF4J MDC so every log record emitted while handling the request + * carries `texera.user.id`. + * + * Why Jersey and not a Servlet filter: + * - Authentication is wired as a Jersey `ContainerRequestFilter` + * (see [[org.apache.texera.auth.JwtAuthFilter]]) — by the time + * Servlet filters run, `SecurityContext.getUserPrincipal` is + * still null. A Jersey filter ordered after auth picks it up. + * - Priorities.AUTHENTICATION + 100 places us in the + * AUTHORIZATION bucket which is guaranteed to run AFTER auth. + * + * Symmetric request/response interfaces let us pair set + clear + * without leaking MDC across threads in the worker pool. + */ @Provider @Priority(Priorities.AUTHORIZATION) class UserContextMdcFilter extends ContainerRequestFilter with ContainerResponseFilter { @@ -75,8 +75,10 @@ class UserContextMdcFilter extends ContainerRequestFilter with ContainerResponse } object UserContextMdcFilter { + /** MDC key — must stay in sync with - * [[org.apache.texera.observability.LogSanitizer.AllowedMdcKeys]] - * or the OTel log appender drops it before emit. */ + * [[org.apache.texera.observability.LogSanitizer.AllowedMdcKeys]] + * or the OTel log appender drops it before emit. + */ val UserIdKey: String = "texera.user.id" } diff --git a/amber/src/main/scala/org/apache/texera/web/observability/gateway/AuditLogger.scala b/amber/src/main/scala/org/apache/texera/web/observability/gateway/AuditLogger.scala index 5a0daa9ee82..43e173fc255 100644 --- a/amber/src/main/scala/org/apache/texera/web/observability/gateway/AuditLogger.scala +++ b/amber/src/main/scala/org/apache/texera/web/observability/gateway/AuditLogger.scala @@ -23,18 +23,18 @@ import org.apache.texera.observability.LogSanitizer import org.slf4j.{Logger, LoggerFactory} /** - * Per-query audit log for the gateway. - * - * Lives on its own SLF4J logger name so operators can route audit - * events to a dedicated appender (file/syslog/SIEM) separate from - * application logs. The logback.xml in each service can selectively - * disable propagation to the root appender if audit events should - * NOT flow into the OTel collector. - * - * The audit line is one JSON-shaped string per query. Free-text - * fields go through [[LogSanitizer.sanitize]] first so secret - * patterns and CRLF injections cannot land in audit storage. - */ + * Per-query audit log for the gateway. + * + * Lives on its own SLF4J logger name so operators can route audit + * events to a dedicated appender (file/syslog/SIEM) separate from + * application logs. The logback.xml in each service can selectively + * disable propagation to the root appender if audit events should + * NOT flow into the OTel collector. + * + * The audit line is one JSON-shaped string per query. Free-text + * fields go through [[LogSanitizer.sanitize]] first so secret + * patterns and CRLF injections cannot land in audit storage. + */ object AuditLogger { private val log: Logger = LoggerFactory.getLogger("texera.audit.observability") @@ -78,10 +78,10 @@ object AuditLogger { while (i < s.length) { val c = s.charAt(i) c match { - case '"' => out.append("\\\"") - case '\\' => out.append("\\\\") - case _ if c < 0x20 || c == 0x7F => // control chars stripped (defense in depth) - case _ => out.append(c) + case '"' => out.append("\\\"") + case '\\' => out.append("\\\\") + case _ if c < 0x20 || c == 0x7f => // control chars stripped (defense in depth) + case _ => out.append(c) } i += 1 } diff --git a/amber/src/main/scala/org/apache/texera/web/observability/gateway/BackendClient.scala b/amber/src/main/scala/org/apache/texera/web/observability/gateway/BackendClient.scala index 3de66f350a4..13785991f8a 100644 --- a/amber/src/main/scala/org/apache/texera/web/observability/gateway/BackendClient.scala +++ b/amber/src/main/scala/org/apache/texera/web/observability/gateway/BackendClient.scala @@ -20,10 +20,7 @@ package org.apache.texera.web.observability.gateway import com.typesafe.scalalogging.LazyLogging -import org.apache.texera.web.observability.gateway.dtos.{ - GatewayError, - MaxResponseBytes -} +import org.apache.texera.web.observability.gateway.dtos.{GatewayError, MaxResponseBytes} import java.net.URI import java.net.http.HttpResponse.BodyHandlers @@ -33,31 +30,31 @@ import java.time.Duration import scala.util.{Failure, Success, Try} /** - * Thin HTTP wrapper used by the per-backend clients. We deliberately - * use the JDK 11 HttpClient (no new dependency) and keep the surface - * minimal: one ``get`` / one ``post`` taking a typed body, with - * VictoriaLogs/Metrics tenancy headers injected at the boundary. - * - * Two security rails enforced here: - * 1. Hard response-byte cap. We read the body into a buffer that - * stops at [[dtos.MaxResponseBytes]] — an attacker / runaway - * backend cannot make us swallow a 1 GiB body. - * 2. Per-request scope: the GatewayScope arrives validated. Tenant - * isolation against VictoriaLogs / VictoriaMetrics is enforced - * by the LogsQL / MetricsQL stream filters that the builders - * derive from `scope.allowedWorkflowIds` — NOT by header - * multi-tenancy. We previously sent an `AccountID: ` - * header for defence-in-depth, but the OTel collector does not - * set AccountID at ingest (every record lands in tenant 0), so - * that header caused every authenticated query to return zero - * results. TODO(observability/multi-tenant): wire per-user - * AccountID through the collector + ingest pipeline, then - * re-introduce the header here. - * - * Secret redaction is intentionally NOT done at this layer: it runs - * per-field inside the parsers (parseLogs / parseTraces), so a single - * oversized value can't truncate and corrupt the whole JSON response. - */ + * Thin HTTP wrapper used by the per-backend clients. We deliberately + * use the JDK 11 HttpClient (no new dependency) and keep the surface + * minimal: one ``get`` / one ``post`` taking a typed body, with + * VictoriaLogs/Metrics tenancy headers injected at the boundary. + * + * Two security rails enforced here: + * 1. Hard response-byte cap. We read the body into a buffer that + * stops at [[dtos.MaxResponseBytes]] — an attacker / runaway + * backend cannot make us swallow a 1 GiB body. + * 2. Per-request scope: the GatewayScope arrives validated. Tenant + * isolation against VictoriaLogs / VictoriaMetrics is enforced + * by the LogsQL / MetricsQL stream filters that the builders + * derive from `scope.allowedWorkflowIds` — NOT by header + * multi-tenancy. We previously sent an `AccountID: ` + * header for defence-in-depth, but the OTel collector does not + * set AccountID at ingest (every record lands in tenant 0), so + * that header caused every authenticated query to return zero + * results. TODO(observability/multi-tenant): wire per-user + * AccountID through the collector + ingest pipeline, then + * re-introduce the header here. + * + * Secret redaction is intentionally NOT done at this layer: it runs + * per-field inside the parsers (parseLogs / parseTraces), so a single + * oversized value can't truncate and corrupt the whole JSON response. + */ class BackendClient( baseUrl: String, timeoutMs: Long = 5000L @@ -66,13 +63,20 @@ class BackendClient( private val http: HttpClient = HttpClient .newBuilder() .connectTimeout(Duration.ofMillis(timeoutMs)) - .followRedirects(HttpClient.Redirect.NEVER) // backends shouldn't redirect; if they do, treat as error + .followRedirects( + HttpClient.Redirect.NEVER + ) // backends shouldn't redirect; if they do, treat as error .build() /** GET that returns a (status, body) tuple or a typed error. The - * body is decoded as UTF-8 and is truncated at MaxResponseBytes - * with a [[GatewayError.ResponseTooLarge]] surfaced. */ - def get(path: String, scope: GatewayScope, signal: String): Either[GatewayError, BackendResponse] = { + * body is decoded as UTF-8 and is truncated at MaxResponseBytes + * with a [[GatewayError.ResponseTooLarge]] surfaced. + */ + def get( + path: String, + scope: GatewayScope, + signal: String + ): Either[GatewayError, BackendResponse] = { val uri = URI.create(baseUrl + path) val req = HttpRequest .newBuilder(uri) @@ -88,8 +92,9 @@ class BackendClient( } /** POST a typed body. ``contentType`` is the only place we accept - * an arbitrary string — but it's a CONST passed by the caller, - * never from request input. */ + * an arbitrary string — but it's a CONST passed by the caller, + * never from request input. + */ def post( path: String, body: Array[Byte], @@ -129,7 +134,9 @@ class BackendClient( val elapsedMs = (System.nanoTime() - startNanos) / 1000000L val raw = resp.body() if (raw == null) { - logger.debug(s"[$signal] ${resp.statusCode()} from $baseUrl in ${elapsedMs}ms (empty body)") + logger.debug( + s"[$signal] ${resp.statusCode()} from $baseUrl in ${elapsedMs}ms (empty body)" + ) Right(BackendResponse(resp.statusCode(), "")) } else if (raw.length.toLong > MaxResponseBytes) { logger.warn( @@ -148,11 +155,12 @@ class BackendClient( } /** Wrapped backend response — status + body. Secret redaction is NOT - * applied here: it happens per-field inside the parsers (parseLogs / - * parseTraces sanitize individual message/attribute values). A - * whole-body LogSanitizer.sanitize pass is unsafe on these JSON - * payloads — its 16 KiB cap truncates large responses mid-value and - * corrupts the JSON. */ + * applied here: it happens per-field inside the parsers (parseLogs / + * parseTraces sanitize individual message/attribute values). A + * whole-body LogSanitizer.sanitize pass is unsafe on these JSON + * payloads — its 16 KiB cap truncates large responses mid-value and + * corrupts the JSON. + */ case class BackendResponse(status: Int, body: String) { def isOk: Boolean = status >= 200 && status < 300 } diff --git a/amber/src/main/scala/org/apache/texera/web/observability/gateway/GatewayContext.scala b/amber/src/main/scala/org/apache/texera/web/observability/gateway/GatewayContext.scala index b38f0ed0476..cb94252c201 100644 --- a/amber/src/main/scala/org/apache/texera/web/observability/gateway/GatewayContext.scala +++ b/amber/src/main/scala/org/apache/texera/web/observability/gateway/GatewayContext.scala @@ -22,11 +22,11 @@ package org.apache.texera.web.observability.gateway import org.apache.texera.config.ObservabilityGatewayConfig /** - * Single bag of collaborators every observability resource needs. - * Centralises wiring so the resource constructors stay short and so - * tests can override one collaborator (e.g. stub the scope resolver) - * without touching the rest. - */ + * Single bag of collaborators every observability resource needs. + * Centralises wiring so the resource constructors stay short and so + * tests can override one collaborator (e.g. stub the scope resolver) + * without touching the rest. + */ case class GatewayContext( scopeResolver: ScopeResolver, perUserLimiter: RateLimiter, @@ -46,11 +46,12 @@ case class GatewayContext( object GatewayContext { /** Build the production context. The backend query URLs come from - * [[ObservabilityGatewayConfig]] (`observability-gateway.conf`), which - * defaults to the host-local stack and is overridden inside docker via - * the TEXERA_OBS_*_URL env vars set in bin/single-node/.env. A natively - * run backend therefore reaches the loopback-published backends with no - * extra configuration. */ + * [[ObservabilityGatewayConfig]] (`observability-gateway.conf`), which + * defaults to the host-local stack and is overridden inside docker via + * the TEXERA_OBS_*_URL env vars set in bin/single-node/.env. A natively + * run backend therefore reaches the loopback-published backends with no + * extra configuration. + */ def default(): GatewayContext = { GatewayContext( scopeResolver = new ScopeResolver.Jooq(), diff --git a/amber/src/main/scala/org/apache/texera/web/observability/gateway/ObservabilityResources.scala b/amber/src/main/scala/org/apache/texera/web/observability/gateway/ObservabilityResources.scala index 12ab7618eec..ddab2445989 100644 --- a/amber/src/main/scala/org/apache/texera/web/observability/gateway/ObservabilityResources.scala +++ b/amber/src/main/scala/org/apache/texera/web/observability/gateway/ObservabilityResources.scala @@ -29,24 +29,25 @@ import org.apache.texera.auth.SessionUser import org.apache.texera.web.observability.gateway.dtos._ /** - * Dropwizard resources for the observability gateway. - * - * Every endpoint runs the same five-step skeleton (see DESIGN.md): - * 1. Auth — handled by @Auth + @RolesAllowed. - * 2. Rate limit — token bucket per user, per IP. - * 3. Scope — resolved from the SessionUser via ScopeResolver. - * 4. Validate — typed DTO validators reject anything out of range. - * 5. Query — typed builders only, then BackendClient with the - * AccountID/ProjectID headers. Response goes through redaction - * before reaching JSON. - * - * The resources are intentionally small — most of the security logic - * lives in the dtos / builders / scope objects so it's unit-testable - * without a Dropwizard fixture. - */ + * Dropwizard resources for the observability gateway. + * + * Every endpoint runs the same five-step skeleton (see DESIGN.md): + * 1. Auth — handled by @Auth + @RolesAllowed. + * 2. Rate limit — token bucket per user, per IP. + * 3. Scope — resolved from the SessionUser via ScopeResolver. + * 4. Validate — typed DTO validators reject anything out of range. + * 5. Query — typed builders only, then BackendClient with the + * AccountID/ProjectID headers. Response goes through redaction + * before reaching JSON. + * + * The resources are intentionally small — most of the security logic + * lives in the dtos / builders / scope objects so it's unit-testable + * without a Dropwizard fixture. + */ /** Tiny helper that turns a [[GatewayError]] into a Dropwizard - * Response. Kept here so every resource uses the same shape. */ + * Response. Kept here so every resource uses the same shape. + */ private[gateway] object Respond extends LazyLogging { def err(e: GatewayError): Response = { // One breadcrumb per rejected request, at a level keyed to severity: @@ -111,7 +112,9 @@ class ObservabilityHealthResource(ctx: GatewayContext) extends LazyLogging { ) val unreachable = checks.collect { case (signal, false) => signal }.toSeq.sorted if (unreachable.nonEmpty) - logger.warn(s"observability health check: unreachable backend(s): ${unreachable.mkString(", ")}") + logger.warn( + s"observability health check: unreachable backend(s): ${unreachable.mkString(", ")}" + ) else logger.debug(s"observability health check: all backends reachable") Respond.json(Map("status" -> "ok", "checks" -> checks)) diff --git a/amber/src/main/scala/org/apache/texera/web/observability/gateway/RateLimiter.scala b/amber/src/main/scala/org/apache/texera/web/observability/gateway/RateLimiter.scala index 7bbe9b2185b..e08f07160ae 100644 --- a/amber/src/main/scala/org/apache/texera/web/observability/gateway/RateLimiter.scala +++ b/amber/src/main/scala/org/apache/texera/web/observability/gateway/RateLimiter.scala @@ -23,21 +23,21 @@ import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicReference /** - * Simple token-bucket rate limiter, keyed by an arbitrary string - * (typically userId or remote IP). - * - * Trade-offs vs. a fancier approach: - * - In-memory only. A single instance per JVM. A rolling-restart - * deploy would clear the buckets — acceptable for the - * observability gateway's traffic shape (low QPS, low - * consequences for a single missed limit at restart). - * - No background thread. Refill happens lazily on the next - * [[tryAcquire]] call against the same key. Simpler reasoning, - * no scheduler involved. - * - Keys never expire from the map. The map is bounded informally - * by the number of distinct (user, IP) tuples a deploy sees; - * a future PR can add a periodic clean-up if cardinality grows. - */ + * Simple token-bucket rate limiter, keyed by an arbitrary string + * (typically userId or remote IP). + * + * Trade-offs vs. a fancier approach: + * - In-memory only. A single instance per JVM. A rolling-restart + * deploy would clear the buckets — acceptable for the + * observability gateway's traffic shape (low QPS, low + * consequences for a single missed limit at restart). + * - No background thread. Refill happens lazily on the next + * [[tryAcquire]] call against the same key. Simpler reasoning, + * no scheduler involved. + * - Keys never expire from the map. The map is bounded informally + * by the number of distinct (user, IP) tuples a deploy sees; + * a future PR can add a periodic clean-up if cardinality grows. + */ class RateLimiter(capacity: Long, refillPerSecond: Double) { require(capacity > 0, "capacity must be positive") @@ -47,8 +47,9 @@ class RateLimiter(capacity: Long, refillPerSecond: Double) { new ConcurrentHashMap[String, AtomicReference[Bucket]]() /** Try to consume one token. Returns true on success, false on - * rate-limit. Time-aware: the bucket refills based on wall clock - * elapsed since the last call. */ + * rate-limit. Time-aware: the bucket refills based on wall clock + * elapsed since the last call. + */ def tryAcquire(key: String, nowMillis: Long = System.currentTimeMillis()): Boolean = { val ref = buckets.computeIfAbsent( key, @@ -86,12 +87,15 @@ class RateLimiter(capacity: Long, refillPerSecond: Double) { } object RateLimiter { + /** Default per-user limit per the PR plan: 20 req/s with a 20- - * token burst capacity. */ + * token burst capacity. + */ def defaultPerUser(): RateLimiter = new RateLimiter(capacity = 20, refillPerSecond = 20.0) /** Default per-IP limit: looser per-user limit, tighter per-IP - * to defend against an attacker burning through a fleet of - * accounts. */ + * to defend against an attacker burning through a fleet of + * accounts. + */ def defaultPerIp(): RateLimiter = new RateLimiter(capacity = 100, refillPerSecond = 50.0) } diff --git a/amber/src/main/scala/org/apache/texera/web/observability/gateway/ScopeResolver.scala b/amber/src/main/scala/org/apache/texera/web/observability/gateway/ScopeResolver.scala index 254dd5999cb..1fbaace1ede 100644 --- a/amber/src/main/scala/org/apache/texera/web/observability/gateway/ScopeResolver.scala +++ b/amber/src/main/scala/org/apache/texera/web/observability/gateway/ScopeResolver.scala @@ -31,40 +31,40 @@ import org.apache.texera.dao.jooq.generated.Tables.{ import scala.jdk.CollectionConverters._ /** - * Resolves a [[GatewayScope]] for a session user. - * - * The single security invariant: there is no public path through - * which a caller can widen the scope. The resolver reads - * authoritative state (jOOQ tables) and returns a closed set of - * allowed workflow / project ids — every backend builder consumes - * this set, and any request that names a workflow id outside it is - * rejected at the resource layer. - * - * Provided in two flavours: - * - [[ScopeResolver.Jooq]] — the production path, queries the - * existing access-control tables. - * - [[ScopeResolver.Stub]] — used by tests so they don't need to - * stand up a real database. - */ + * Resolves a [[GatewayScope]] for a session user. + * + * The single security invariant: there is no public path through + * which a caller can widen the scope. The resolver reads + * authoritative state (jOOQ tables) and returns a closed set of + * allowed workflow / project ids — every backend builder consumes + * this set, and any request that names a workflow id outside it is + * rejected at the resource layer. + * + * Provided in two flavours: + * - [[ScopeResolver.Jooq]] — the production path, queries the + * existing access-control tables. + * - [[ScopeResolver.Stub]] — used by tests so they don't need to + * stand up a real database. + */ trait ScopeResolver { def resolve(user: SessionUser): GatewayScope /** Membership check: does the caller actually have access to the - * named workflow id? Returns true if no workflowId was supplied - * (defaults to caller's full scope) or if the id is in the - * resolved allow-set. - * - * Implementation note: Jackson Scala module deserializes a JSON - * number that fits in 32 bits as java.lang.Integer regardless of - * the declared `Option[Long]` type (type parameters are erased on - * the JVM). A typed `id: Long` closure then triggers a runtime - * Integer→Long unbox via BoxesRunTime.unboxToLong, which throws a - * ClassCastException. We normalise to a primitive long up front - * via `Number.longValue()` so the contains-check is type-safe. - */ + * named workflow id? Returns true if no workflowId was supplied + * (defaults to caller's full scope) or if the id is in the + * resolved allow-set. + * + * Implementation note: Jackson Scala module deserializes a JSON + * number that fits in 32 bits as java.lang.Integer regardless of + * the declared `Option[Long]` type (type parameters are erased on + * the JVM). A typed `id: Long` closure then triggers a runtime + * Integer→Long unbox via BoxesRunTime.unboxToLong, which throws a + * ClassCastException. We normalise to a primitive long up front + * via `Number.longValue()` so the contains-check is type-safe. + */ def assertWorkflowAllowed(scope: GatewayScope, workflowId: Option[Long]): Boolean = workflowId match { - case None => true + case None => true case Some(id) => // The `id` here may actually be a java.lang.Integer at runtime; // route through Number.longValue() so the unbox cannot fail. @@ -79,9 +79,10 @@ trait ScopeResolver { object ScopeResolver { /** Production implementation backed by jOOQ. Queries - * WORKFLOW_OF_USER (owned workflows) ∪ WORKFLOW_USER_ACCESS - * (shared workflows) for the user, and PROJECT_USER_ACCESS for - * the set of projects the user can see. */ + * WORKFLOW_OF_USER (owned workflows) ∪ WORKFLOW_USER_ACCESS + * (shared workflows) for the user, and PROJECT_USER_ACCESS for + * the set of projects the user can see. + */ class Jooq extends ScopeResolver with LazyLogging { override def resolve(user: SessionUser): GatewayScope = { val ctx = SqlServer.getInstance().createDSLContext() @@ -128,7 +129,8 @@ object ScopeResolver { } /** Test double. Constructed with a static scope; ignores the - * caller's SessionUser. */ + * caller's SessionUser. + */ class Stub(scope: GatewayScope) extends ScopeResolver { override def resolve(user: SessionUser): GatewayScope = scope } diff --git a/amber/src/main/scala/org/apache/texera/web/observability/gateway/builders.scala b/amber/src/main/scala/org/apache/texera/web/observability/gateway/builders.scala index ea2cd08d4c1..77bc4af7efe 100644 --- a/amber/src/main/scala/org/apache/texera/web/observability/gateway/builders.scala +++ b/amber/src/main/scala/org/apache/texera/web/observability/gateway/builders.scala @@ -22,21 +22,22 @@ package org.apache.texera.web.observability.gateway import org.apache.texera.web.observability.gateway.dtos._ /** - * Typed query builders. Each takes a validated DTO + the caller's - * resolved scope and returns a backend-specific query string + the - * query parameters that should accompany it. - * - * Security invariant: no field of the input DTO is concatenated into - * the output query without first passing through a typed accessor. - * Even free-text fields are emitted only as escaped *values* in the - * query DSL — never as DSL syntax. - * - * Each builder is pure (no side effects, no I/O). Exhaustive - * injection tests live in BuildersSpec. - */ + * Typed query builders. Each takes a validated DTO + the caller's + * resolved scope and returns a backend-specific query string + the + * query parameters that should accompany it. + * + * Security invariant: no field of the input DTO is concatenated into + * the output query without first passing through a typed accessor. + * Even free-text fields are emitted only as escaped *values* in the + * query DSL — never as DSL syntax. + * + * Each builder is pure (no side effects, no I/O). Exhaustive + * injection tests live in BuildersSpec. + */ /** Tenancy / scope envelope. Computed by [[ObservabilityScope]]; - * every builder consumes it so the caller cannot widen scope. */ + * every builder consumes it so the caller cannot widen scope. + */ case class GatewayScope( userId: Long, allowedWorkflowIds: Set[Long], @@ -44,17 +45,19 @@ case class GatewayScope( ) { /** Allowed list joined as the typed parameter to a backend query. - * Empty allowed-set yields "0" (a workflow id that cannot exist), - * which produces a zero-result query without breaking syntax. */ + * Empty allowed-set yields "0" (a workflow id that cannot exist), + * which produces a zero-result query without breaking syntax. + */ def workflowIdsCsv: String = { if (allowedWorkflowIds.isEmpty) "0" else allowedWorkflowIds.toSeq.sorted.mkString(",") } /** Allowed list joined as a regex-alternation body (no anchors, no - * parens). For use inside a LogsQL stream filter as - * ``field=~"^()$"``. Empty allow-set yields "0" — a numeric - * literal that matches nothing real and keeps regex syntax valid. */ + * parens). For use inside a LogsQL stream filter as + * ``field=~"^()$"``. Empty allow-set yields "0" — a numeric + * literal that matches nothing real and keeps regex syntax valid. + */ def workflowIdsRegexAlt: String = { if (allowedWorkflowIds.isEmpty) "0" else allowedWorkflowIds.toSeq.sorted.mkString("|") diff --git a/amber/src/main/scala/org/apache/texera/web/observability/gateway/dtos.scala b/amber/src/main/scala/org/apache/texera/web/observability/gateway/dtos.scala index 733834daedd..be28a521a04 100644 --- a/amber/src/main/scala/org/apache/texera/web/observability/gateway/dtos.scala +++ b/amber/src/main/scala/org/apache/texera/web/observability/gateway/dtos.scala @@ -22,22 +22,22 @@ package org.apache.texera.web.observability.gateway import java.time.{Duration, Instant} /** - * Strongly-typed request / response DTOs for the gateway. - * - * Every field is either a typed primitive (Long, Instant, enum) or - * a length-/range-validated wrapper. There is no public field that - * accepts an arbitrary string and lets it through to a backend - * query language verbatim — the typed builders in [[builders]] - * receive only validated values from these DTOs. - * - * Time-window caps differ per signal, per the PR plan: - * logs ≤ 7 days - * metrics ≤ 90 days - * traces ≤ 24 hours - * profiles ≤ 7 days - * - * Page size is server-clamped at [[MaxPageSize]] for every signal. - */ + * Strongly-typed request / response DTOs for the gateway. + * + * Every field is either a typed primitive (Long, Instant, enum) or + * a length-/range-validated wrapper. There is no public field that + * accepts an arbitrary string and lets it through to a backend + * query language verbatim — the typed builders in [[builders]] + * receive only validated values from these DTOs. + * + * Time-window caps differ per signal, per the PR plan: + * logs ≤ 7 days + * metrics ≤ 90 days + * traces ≤ 24 hours + * profiles ≤ 7 days + * + * Page size is server-clamped at [[MaxPageSize]] for every signal. + */ object dtos { val MaxPageSize: Int = 1000 @@ -45,7 +45,8 @@ object dtos { val MaxResponseBytes: Long = 10L * 1024L * 1024L // 10 MiB hard cap /** Log severity, exposed to the UI as a closed enum. The wire - * value is the standard OTel severity-text string. */ + * value is the standard OTel severity-text string. + */ sealed abstract class LogLevel(val name: String) object LogLevel { case object TRACE extends LogLevel("TRACE") @@ -69,25 +70,28 @@ object dtos { } /** Maximum time window per signal, in seconds. */ - def maxWindowSeconds(signal: Signal): Long = signal match { - case Signal.Logs => 7L * 24L * 3600L - case Signal.Metrics => 90L * 24L * 3600L - case Signal.Traces => 24L * 3600L - case Signal.Profiles => 7L * 24L * 3600L - } + def maxWindowSeconds(signal: Signal): Long = + signal match { + case Signal.Logs => 7L * 24L * 3600L + case Signal.Metrics => 90L * 24L * 3600L + case Signal.Traces => 24L * 3600L + case Signal.Profiles => 7L * 24L * 3600L + } // ---- typed unions for validator results ------------------------------- /** Result of validating an inbound request. Either a clean - * typed value, or a GatewayError with a stable code + message - * shape suitable for serializing to JSON. */ + * typed value, or a GatewayError with a stable code + message + * shape suitable for serializing to JSON. + */ sealed trait ValidationResult[+T] case class Valid[T](value: T) extends ValidationResult[T] case class Invalid(error: GatewayError) extends ValidationResult[Nothing] /** Stable error shape returned to the UI. ``code`` is a short - * machine-readable token, ``message`` is human-readable and - * redaction-safe (we never echo raw user input here). */ + * machine-readable token, ``message`` is human-readable and + * redaction-safe (we never echo raw user input here). + */ case class GatewayError(code: String, message: String, status: Int) object GatewayError { @@ -122,7 +126,8 @@ object dtos { // ---- shared bits ------------------------------------------------------ /** Validated time window. Both ends are Instants, range bounded - * per signal, ``to > from`` strictly. */ + * per signal, ``to > from`` strictly. + */ case class TimeWindow(from: Instant, to: Instant) object TimeWindow { @@ -141,8 +146,9 @@ object dtos { } /** Free text used as a *value* (never as syntax) in a backend - * query. Length-capped and CRLF-stripped before reaching a - * builder. ``None`` for absent input. */ + * query. Length-capped and CRLF-stripped before reaching a + * builder. ``None`` for absent input. + */ case class FreeText(value: String) object FreeText { @@ -152,7 +158,7 @@ object dtos { case Some(s) => if (s.length > MaxFreeTextLen) Invalid(GatewayError.FreeTextTooLong) else { - val stripped = s.filter(c => c >= 0x20 && c != 0x7F) + val stripped = s.filter(c => c >= 0x20 && c != 0x7f) if (stripped.isEmpty) Valid(None) else Valid(Some(FreeText(stripped))) } } @@ -171,7 +177,8 @@ object dtos { // ---- per-signal request DTOs (validated) ----------------------------- /** Inbound logs search request before validation. Strings/longs - * only — never reaches a query builder unvalidated. */ + * only — never reaches a query builder unvalidated. + */ case class RawLogsSearchRequest( workflowId: Option[Long], executionId: Option[Long], @@ -207,8 +214,9 @@ object dtos { ) /** Closed enum of sort orders. Backed by a LogsQL `| sort by (...)` - * clause; the LogsQL fragment is in [[LogsQLBuilder]] so this DTO - * stays storage-agnostic. */ + * clause; the LogsQL fragment is in [[LogsQLBuilder]] so this DTO + * stays storage-agnostic. + */ sealed abstract class LogSort(val name: String) object LogSort { case object NewestFirst extends LogSort("newest") @@ -222,10 +230,11 @@ object dtos { } /** Validated service name. Texera service names are emitted by the - * OTel resource attribute `service.name` — we know they match the - * pattern `texera-?[a-z0-9-]+` because the JVM bootstrap controls - * them. We enforce that pattern here so a forged value cannot - * inject LogsQL syntax via the service filter. */ + * OTel resource attribute `service.name` — we know they match the + * pattern `texera-?[a-z0-9-]+` because the JVM bootstrap controls + * them. We enforce that pattern here so a forged value cannot + * inject LogsQL syntax via the service filter. + */ case class ServiceName(value: String) object ServiceName { @@ -269,10 +278,11 @@ object dtos { ) /** Distinct filter values currently present in the logs store. - * Powers the UI's autofill dropdowns for service / workflow id / - * CU id / user id. Service names are returned as raw strings - * (the UI doesn't need typed parsing — it just renders the chip - * and passes the value back). */ + * Powers the UI's autofill dropdowns for service / workflow id / + * CU id / user id. Service names are returned as raw strings + * (the UI doesn't need typed parsing — it just renders the chip + * and passes the value back). + */ case class LogSourcesResponse( services: Seq[String], workflowIds: Seq[Long], @@ -283,7 +293,8 @@ object dtos { // ---- metrics --------------------------------------------------------- /** Named server-side metric query. We do not let the client send - * raw MetricsQL — they pick from a fixed enum. */ + * raw MetricsQL — they pick from a fixed enum. + */ sealed abstract class NamedMetric(val name: String) object NamedMetric { case object RunsPerDay extends NamedMetric("runsPerDay") @@ -297,8 +308,17 @@ object dtos { case object P99Duration extends NamedMetric("p99Duration") val all: Seq[NamedMetric] = - Seq(RunsPerDay, TotalRuns, ActiveWorkflows, SuccessRate, FailureRate, - AvgDuration, P50Duration, P95Duration, P99Duration) + Seq( + RunsPerDay, + TotalRuns, + ActiveWorkflows, + SuccessRate, + FailureRate, + AvgDuration, + P50Duration, + P95Duration, + P99Duration + ) def parse(raw: String): Option[NamedMetric] = Option(raw).flatMap(s => all.find(_.name == s)) } @@ -326,7 +346,8 @@ object dtos { // ---- traces ---------------------------------------------------------- /** Inbound trace lookup. ``traceId`` must match the regex - * ``^[0-9a-f]{32}$`` (same as W3C trace-id). */ + * ``^[0-9a-f]{32}$`` (same as W3C trace-id). + */ case class RawTracesGetRequest(traceId: String) case class ValidatedTracesGetRequest(traceId: String) @@ -368,7 +389,8 @@ object dtos { ) /** Profiles are returned as a tree of frames. We render the tree - * in the UI; the gateway is only responsible for shape + size. */ + * in the UI; the gateway is only responsible for shape + size. + */ case class FlameFrame(name: String, value: Long, children: Seq[FlameFrame]) case class ProfilesQueryResponse(root: Option[FlameFrame], totalSamples: Long) diff --git a/amber/src/test/scala/org/apache/texera/web/observability/RequestContextMdcFilterSpec.scala b/amber/src/test/scala/org/apache/texera/web/observability/RequestContextMdcFilterSpec.scala index fadb8ac948f..8db39308c81 100644 --- a/amber/src/test/scala/org/apache/texera/web/observability/RequestContextMdcFilterSpec.scala +++ b/amber/src/test/scala/org/apache/texera/web/observability/RequestContextMdcFilterSpec.scala @@ -47,16 +47,19 @@ class RequestContextMdcFilterSpec extends AnyFlatSpec with Matchers with BeforeA } /** Captures the MDC state from inside the chain so we can assert on - * the per-request context, not the post-finally cleared state. */ + * the per-request context, not the post-finally cleared state. + */ private class CapturingChain extends FilterChain { var captured: Map[String, String] = Map.empty override def doFilter(request: ServletRequest, response: ServletResponse): Unit = { val ctx = MDC.getCopyOfContextMap - captured = if (ctx == null) Map.empty else { - val sb = scala.collection.mutable.Map.empty[String, String] - ctx.forEach((k, v) => sb.put(k, v)) - sb.toMap - } + captured = + if (ctx == null) Map.empty + else { + val sb = scala.collection.mutable.Map.empty[String, String] + ctx.forEach((k, v) => sb.put(k, v)) + sb.toMap + } } } @@ -97,7 +100,8 @@ class RequestContextMdcFilterSpec extends AnyFlatSpec with Matchers with BeforeA "/api/workflow/999", headers = Map("X-Texera-Workflow-Id" -> "123") ), - null, chain + null, + chain ) // Header wins. chain.captured("texera.workflow.id") shouldBe "123" @@ -107,7 +111,8 @@ class RequestContextMdcFilterSpec extends AnyFlatSpec with Matchers with BeforeA val chain = new CapturingChain filter.doFilter( new StubReq("/", headers = Map("X-Texera-Workflow-Id" -> "evil\r\nINJECT")), - null, chain + null, + chain ) chain.captured.get("texera.workflow.id") shouldBe None } @@ -117,7 +122,8 @@ class RequestContextMdcFilterSpec extends AnyFlatSpec with Matchers with BeforeA val twentyDigits = "1" * 20 filter.doFilter( new StubReq("/", headers = Map("X-Texera-Computing-Unit-Id" -> twentyDigits)), - null, chain + null, + chain ) chain.captured.get("texera.computing_unit.id") shouldBe None } @@ -157,7 +163,8 @@ class RequestContextMdcFilterSpec extends AnyFlatSpec with Matchers with BeforeA "/wsapi/workflow-websocket", params = Map("cuid" -> "8", "wid" -> "441", "eid" -> "9001") ), - null, chain + null, + chain ) chain.captured should contain allOf ( "texera.computing_unit.id" -> "8", @@ -170,7 +177,8 @@ class RequestContextMdcFilterSpec extends AnyFlatSpec with Matchers with BeforeA val chain = new CapturingChain filter.doFilter( new StubReq("/x", params = Map("cuid" -> "abc")), - null, chain + null, + chain ) chain.captured.get("texera.computing_unit.id") shouldBe None } @@ -179,7 +187,8 @@ class RequestContextMdcFilterSpec extends AnyFlatSpec with Matchers with BeforeA val chain = new CapturingChain filter.doFilter( new StubReq("/api/workflow/441", params = Map("wid" -> "999")), - null, chain + null, + chain ) // Path is checked first; param check skips when MDC already set. chain.captured("texera.workflow.id") shouldBe "441" @@ -195,13 +204,16 @@ class RequestContextMdcFilterSpec extends AnyFlatSpec with Matchers with BeforeA } /** Local trait that gives the unimplemented HttpServletRequest a - * no-op default for every other method — keeps test classes small - * without pulling in a mocking library. */ + * no-op default for every other method — keeps test classes small + * without pulling in a mocking library. + */ private abstract class HttpServletRequestStub extends HttpServletRequest { // Unused methods throw — surfaces an unexpected dependency before it // silently returns null and produces a flaky test. - private def stub(): Nothing = throw new UnsupportedOperationException( - "method not stubbed; add to StubReq if a test requires it") + private def stub(): Nothing = + throw new UnsupportedOperationException( + "method not stubbed; add to StubReq if a test requires it" + ) override def getAuthType: String = stub() override def getCookies: Array[javax.servlet.http.Cookie] = stub() override def getDateHeader(name: String): Long = stub() @@ -230,7 +242,8 @@ private abstract class HttpServletRequestStub extends HttpServletRequest { override def logout(): Unit = stub() override def getParts: java.util.Collection[javax.servlet.http.Part] = stub() override def getPart(name: String): javax.servlet.http.Part = stub() - override def upgrade[T <: javax.servlet.http.HttpUpgradeHandler](handlerClass: Class[T]): T = stub() + override def upgrade[T <: javax.servlet.http.HttpUpgradeHandler](handlerClass: Class[T]): T = + stub() override def getRequestURL: StringBuffer = new StringBuffer(getRequestURI) override def getAttribute(name: String): AnyRef = null override def getAttributeNames: java.util.Enumeration[String] = stub() @@ -264,9 +277,11 @@ private abstract class HttpServletRequestStub extends HttpServletRequest { override def getLocalPort: Int = 8080 override def getServletContext: javax.servlet.ServletContext = null override def startAsync(): javax.servlet.AsyncContext = stub() - override def startAsync(req: ServletRequest, resp: ServletResponse): javax.servlet.AsyncContext = stub() + override def startAsync(req: ServletRequest, resp: ServletResponse): javax.servlet.AsyncContext = + stub() override def isAsyncStarted: Boolean = false override def isAsyncSupported: Boolean = false override def getAsyncContext: javax.servlet.AsyncContext = stub() - override def getDispatcherType: javax.servlet.DispatcherType = javax.servlet.DispatcherType.REQUEST + override def getDispatcherType: javax.servlet.DispatcherType = + javax.servlet.DispatcherType.REQUEST } diff --git a/amber/src/test/scala/org/apache/texera/web/observability/UserContextMdcFilterSpec.scala b/amber/src/test/scala/org/apache/texera/web/observability/UserContextMdcFilterSpec.scala index 4a998d3a944..e204ac97b11 100644 --- a/amber/src/test/scala/org/apache/texera/web/observability/UserContextMdcFilterSpec.scala +++ b/amber/src/test/scala/org/apache/texera/web/observability/UserContextMdcFilterSpec.scala @@ -43,18 +43,19 @@ class UserContextMdcFilterSpec extends AnyFlatSpec with Matchers with BeforeAndA new SessionUser(u) } - private def secCtxWith(principal: Principal): SecurityContext = new SecurityContext { - override def getUserPrincipal: Principal = principal - override def isUserInRole(role: String): Boolean = false - override def isSecure: Boolean = false - override def getAuthenticationScheme: String = "STUB" - } + private def secCtxWith(principal: Principal): SecurityContext = + new SecurityContext { + override def getUserPrincipal: Principal = principal + override def isUserInRole(role: String): Boolean = false + override def isSecure: Boolean = false + override def getAuthenticationScheme: String = "STUB" + } /** Minimal ContainerRequestContext that only implements the - * surface the filter actually touches. Avoids pulling Mockito for - * what is effectively a one-method stub. */ - private class StubRequest(secCtx: SecurityContext) - extends StubRequestContextBase { + * surface the filter actually touches. Avoids pulling Mockito for + * what is effectively a one-method stub. + */ + private class StubRequest(secCtx: SecurityContext) extends StubRequestContextBase { override def getSecurityContext: SecurityContext = secCtx } @@ -106,9 +107,11 @@ class UserContextMdcFilterSpec extends AnyFlatSpec with Matchers with BeforeAndA } /** Base stub that throws on every ContainerRequestContext method; - * subclasses override only what they need. Keeps the spec narrow on - * the actual filter dependencies (just `getSecurityContext`). */ -private abstract class StubRequestContextBase extends javax.ws.rs.container.ContainerRequestContext { + * subclasses override only what they need. Keeps the spec narrow on + * the actual filter dependencies (just `getSecurityContext`). + */ +private abstract class StubRequestContextBase + extends javax.ws.rs.container.ContainerRequestContext { private def stub(): Nothing = throw new UnsupportedOperationException("not stubbed; override if a test needs it") override def getProperty(name: String): AnyRef = null diff --git a/amber/src/test/scala/org/apache/texera/web/observability/gateway/BackendClientSpec.scala b/amber/src/test/scala/org/apache/texera/web/observability/gateway/BackendClientSpec.scala index 55d3b1aba29..8f8580795c2 100644 --- a/amber/src/test/scala/org/apache/texera/web/observability/gateway/BackendClientSpec.scala +++ b/amber/src/test/scala/org/apache/texera/web/observability/gateway/BackendClientSpec.scala @@ -29,16 +29,16 @@ import java.net.InetSocketAddress import java.nio.charset.StandardCharsets /** - * Tests for [[BackendClient]] using an in-process JDK HttpServer. - * - * We avoid mocking the HttpClient — that would only exercise our - * adapter glue. A real socket on the loopback exercises: - * - URL composition (path concatenation with the base URL) - * - AccountID / ProjectID headers (multi-tenancy guards) - * - Status code propagation into [[BackendResponse.isOk]] - * - Body decoding under UTF-8 - * - The 10 MiB response cap that protects against runaway backends - */ + * Tests for [[BackendClient]] using an in-process JDK HttpServer. + * + * We avoid mocking the HttpClient — that would only exercise our + * adapter glue. A real socket on the loopback exercises: + * - URL composition (path concatenation with the base URL) + * - AccountID / ProjectID headers (multi-tenancy guards) + * - Status code propagation into [[BackendResponse.isOk]] + * - Body decoding under UTF-8 + * - The 10 MiB response cap that protects against runaway backends + */ class BackendClientSpec extends AnyFlatSpec with Matchers with BeforeAndAfterAll with OptionValues { private var server: HttpServer = _ @@ -56,21 +56,24 @@ class BackendClientSpec extends AnyFlatSpec with Matchers with BeforeAndAfterAll override def beforeAll(): Unit = { server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0) - server.createContext("/", new HttpHandler { - override def handle(ex: HttpExchange): Unit = { - lastHeaders.clear() - ex.getRequestHeaders.keySet().forEach { k => - lastHeaders.put(k, ex.getRequestHeaders.getFirst(k)) + server.createContext( + "/", + new HttpHandler { + override def handle(ex: HttpExchange): Unit = { + lastHeaders.clear() + ex.getRequestHeaders.keySet().forEach { k => + lastHeaders.put(k, ex.getRequestHeaders.getFirst(k)) + } + lastPath = ex.getRequestURI.toString + lastMethod = ex.getRequestMethod + lastBody = ex.getRequestBody.readAllBytes() + ex.getResponseHeaders.set("Content-Type", responseContentType) + ex.sendResponseHeaders(responseStatus, responseBody.length) + ex.getResponseBody.write(responseBody) + ex.getResponseBody.close() } - lastPath = ex.getRequestURI.toString - lastMethod = ex.getRequestMethod - lastBody = ex.getRequestBody.readAllBytes() - ex.getResponseHeaders.set("Content-Type", responseContentType) - ex.sendResponseHeaders(responseStatus, responseBody.length) - ex.getResponseBody.write(responseBody) - ex.getResponseBody.close() } - }) + ) server.setExecutor(null) server.start() baseUrl = s"http://127.0.0.1:${server.getAddress.getPort}" @@ -112,7 +115,8 @@ class BackendClientSpec extends AnyFlatSpec with Matchers with BeforeAndAfterAll } it should "default ProjectID to '0' when the allow-set is empty" in { - val emptyProject = GatewayScope(userId = 1L, allowedWorkflowIds = Set.empty, allowedProjectIds = Set.empty) + val emptyProject = + GatewayScope(userId = 1L, allowedWorkflowIds = Set.empty, allowedProjectIds = Set.empty) responseStatus = 200; responseBody = "{}".getBytes val client = new BackendClient(baseUrl) client.get("/x", emptyProject, "logs") @@ -166,6 +170,9 @@ class BackendClientSpec extends AnyFlatSpec with Matchers with BeforeAndAfterAll client.post("/insert", sent, "application/x-ndjson", scope, "logs") lastMethod shouldBe "POST" new String(lastBody, StandardCharsets.UTF_8) shouldBe "logs-payload" - lastHeaders.find { case (k, _) => k.equalsIgnoreCase("Content-Type") }.value._2 shouldBe "application/x-ndjson" + lastHeaders + .find { case (k, _) => k.equalsIgnoreCase("Content-Type") } + .value + ._2 shouldBe "application/x-ndjson" } } diff --git a/common/config/src/main/scala/org/apache/texera/config/ObservabilityGatewayConfig.scala b/common/config/src/main/scala/org/apache/texera/config/ObservabilityGatewayConfig.scala index cf5976cd552..9191143f9d8 100644 --- a/common/config/src/main/scala/org/apache/texera/config/ObservabilityGatewayConfig.scala +++ b/common/config/src/main/scala/org/apache/texera/config/ObservabilityGatewayConfig.scala @@ -21,14 +21,14 @@ package org.apache.texera.config import com.typesafe.config.{Config, ConfigFactory} /** - * Query endpoints for the observability dashboard's backend gateway. - * - * Source of truth is `observability-gateway.conf`, which defaults to the - * host-local stack and lets each URL be overridden per deployment via the - * `TEXERA_OBS_*_URL` env vars (docker-compose sets these to the bridge-network - * service names). This is the query side; the OTLP export endpoint that - * services push to is configured separately in [[org.apache.texera.observability.OtelInit]]. - */ + * Query endpoints for the observability dashboard's backend gateway. + * + * Source of truth is `observability-gateway.conf`, which defaults to the + * host-local stack and lets each URL be overridden per deployment via the + * `TEXERA_OBS_*_URL` env vars (docker-compose sets these to the bridge-network + * service names). This is the query side; the OTLP export endpoint that + * services push to is configured separately in [[org.apache.texera.observability.OtelInit]]. + */ object ObservabilityGatewayConfig { private val conf: Config = ConfigFactory.parseResources("observability-gateway.conf").resolve() diff --git a/frontend/src/app/dashboard/component/user/observability/observability.component.spec.ts b/frontend/src/app/dashboard/component/user/observability/observability.component.spec.ts index 3cd65cc9418..1fb6e75d9d3 100644 --- a/frontend/src/app/dashboard/component/user/observability/observability.component.spec.ts +++ b/frontend/src/app/dashboard/component/user/observability/observability.component.spec.ts @@ -93,10 +93,12 @@ describe("ObservabilityComponent", () => { }); it("starts on the Logs tab (index 0)", () => { - mockService.health.mockReturnValue(of({ - status: "ok", - checks: { logs: true, metrics: true, traces: true, profiles: true }, - } as ObservabilityHealth)); + mockService.health.mockReturnValue( + of({ + status: "ok", + checks: { logs: true, metrics: true, traces: true, profiles: true }, + } as ObservabilityHealth) + ); fixture.detectChanges(); expect(component.activeTab).toBe(0); }); diff --git a/frontend/src/app/dashboard/component/user/observability/observability.component.ts b/frontend/src/app/dashboard/component/user/observability/observability.component.ts index 9c5b56fd586..2fff503833e 100644 --- a/frontend/src/app/dashboard/component/user/observability/observability.component.ts +++ b/frontend/src/app/dashboard/component/user/observability/observability.component.ts @@ -44,14 +44,7 @@ import { Subject, takeUntil } from "rxjs"; selector: "texera-observability", templateUrl: "./observability.component.html", styleUrls: ["./observability.component.scss"], - imports: [ - NgIf, - FormsModule, - NzTabsComponent, - NzTabComponent, - NzAlertComponent, - NzEmptyComponent, - ], + imports: [NgIf, FormsModule, NzTabsComponent, NzTabComponent, NzAlertComponent, NzEmptyComponent], }) export class ObservabilityComponent implements OnInit, OnDestroy { /** Reachability state. ``null`` means "still loading". A failed @@ -79,24 +72,27 @@ export class ObservabilityComponent implements OnInit, OnDestroy { ) {} ngOnInit(): void { - this.observabilityService.health().subscribe({ - next: h => { - this.health = h; - this.healthError = false; - }, - error: err => { - // eslint-disable-next-line no-console - console.error( - "[observability] health check failed — gateway unreachable; rendering all signals as degraded", - err - ); - this.health = { - status: "degraded", - checks: { logs: false, metrics: false, traces: false, profiles: false }, - }; - this.healthError = true; - }, - }); + this.observabilityService + .health() + .pipe(takeUntil(this.destroy$)) + .subscribe({ + next: h => { + this.health = h; + this.healthError = false; + }, + error: (err: unknown) => { + // eslint-disable-next-line no-console + console.error( + "[observability] health check failed — gateway unreachable; rendering all signals as degraded", + err + ); + this.health = { + status: "degraded", + checks: { logs: false, metrics: false, traces: false, profiles: false }, + }; + this.healthError = true; + }, + }); this.tracesPivot.onPivot.pipe(takeUntil(this.destroy$)).subscribe(traceId => { this.pivotedTraceId = traceId; diff --git a/frontend/src/app/dashboard/service/user/observability/observability.service.spec.ts b/frontend/src/app/dashboard/service/user/observability/observability.service.spec.ts index 19ccd5f2b9f..04510e286ef 100644 --- a/frontend/src/app/dashboard/service/user/observability/observability.service.spec.ts +++ b/frontend/src/app/dashboard/service/user/observability/observability.service.spec.ts @@ -19,13 +19,9 @@ import { TestBed } from "@angular/core/testing"; import { HttpClientTestingModule, HttpTestingController } from "@angular/common/http/testing"; +import { HttpErrorResponse } from "@angular/common/http"; import { ObservabilityService, ValidationError } from "./observability.service"; -import { - LogsSearchRequest, - MAX_FREE_TEXT_LEN, - MAX_PAGE_SIZE, - MetricsQueryRequest, -} from "./observability.types"; +import { LogsSearchRequest, MAX_FREE_TEXT_LEN, MAX_PAGE_SIZE, MetricsQueryRequest } from "./observability.types"; describe("ObservabilityService", () => { let service: ObservabilityService; @@ -121,15 +117,18 @@ describe("ObservabilityService", () => { let observedStatus = 0; service.searchLogs(req).subscribe({ next: () => {}, - error: err => { - observedStatus = err.status; + error: (err: unknown) => { + observedStatus = (err as HttpErrorResponse).status; }, }); const http = httpMock.expectOne(r => r.url.endsWith("/observability/logs/search")); - http.flush({ code: "forbidden", message: "no access to that scope" }, { - status: 403, - statusText: "Forbidden", - }); + http.flush( + { code: "forbidden", message: "no access to that scope" }, + { + status: 403, + statusText: "Forbidden", + } + ); expect(observedStatus).toBe(403); }); @@ -186,7 +185,7 @@ describe("ObservabilityService", () => { expect(() => service.getTrace("not-a-trace-id")).toThrow(ValidationError); expect(() => service.getTrace("../../etc/passwd")).toThrow(ValidationError); expect(() => service.getTrace("0AF7651916CD43DD8448EB211C80319C")).toThrow(); // uppercase - expect(() => service.getTrace("0af7651916cd43dd8448eb211c80319")).toThrow(); // too short + expect(() => service.getTrace("0af7651916cd43dd8448eb211c80319")).toThrow(); // too short httpMock.expectNone(r => r.url.includes("/observability/traces/")); }); @@ -203,9 +202,7 @@ describe("ObservabilityService", () => { // ----- queryProfiles (PR 11) ---------------------------------------- it("queryProfiles rejects an inverted time window client-side", () => { - expect(() => - service.queryProfiles({ fromMs: 100, toMs: 50 }) - ).toThrow(ValidationError); + expect(() => service.queryProfiles({ fromMs: 100, toMs: 50 })).toThrow(ValidationError); httpMock.expectNone(r => r.url.endsWith("/observability/profiles/query")); }); diff --git a/frontend/src/app/dashboard/service/user/observability/observability.service.ts b/frontend/src/app/dashboard/service/user/observability/observability.service.ts index dd47bd527e6..9dd2c843c25 100644 --- a/frontend/src/app/dashboard/service/user/observability/observability.service.ts +++ b/frontend/src/app/dashboard/service/user/observability/observability.service.ts @@ -103,9 +103,7 @@ export class ObservabilityService { */ getTrace(traceId: string): Observable { assertValidTraceId(traceId); - return this.http.get( - `${BASE_URL}/traces/${encodeURIComponent(traceId)}` - ); + return this.http.get(`${BASE_URL}/traces/${encodeURIComponent(traceId)}`); } /** @@ -129,10 +127,7 @@ function assertValid(req: LogsSearchRequest): void { throw new ValidationError("bad_page_size", `pageSize must be between 1 and ${MAX_PAGE_SIZE}.`); } if (req.query !== undefined && req.query.length > MAX_FREE_TEXT_LEN) { - throw new ValidationError( - "free_text_too_long", - `Query text must be ${MAX_FREE_TEXT_LEN} characters or fewer.` - ); + throw new ValidationError("free_text_too_long", `Query text must be ${MAX_FREE_TEXT_LEN} characters or fewer.`); } } @@ -156,10 +151,7 @@ function assertValidProfiles(req: ProfilesQueryRequest): void { function assertValidTraceId(traceId: string): void { if (typeof traceId !== "string" || !TRACE_ID_RE.test(traceId)) { - throw new ValidationError( - "bad_trace_id", - "Trace id must be 32 lowercase hex characters." - ); + throw new ValidationError("bad_trace_id", "Trace id must be 32 lowercase hex characters."); } } diff --git a/frontend/src/app/dashboard/service/user/observability/traces-pivot.service.spec.ts b/frontend/src/app/dashboard/service/user/observability/traces-pivot.service.spec.ts index 68b531c21f7..567933266a9 100644 --- a/frontend/src/app/dashboard/service/user/observability/traces-pivot.service.spec.ts +++ b/frontend/src/app/dashboard/service/user/observability/traces-pivot.service.spec.ts @@ -43,7 +43,7 @@ describe("TracesPivotService", () => { service.pivot("not-a-trace-id"); service.pivot("../../etc/passwd"); service.pivot("0AF7651916CD43DD8448EB211C80319C"); // uppercase - service.pivot("0af7651916cd43dd8448eb211c8031"); // too short + service.pivot("0af7651916cd43dd8448eb211c8031"); // too short service.pivot(""); service.pivot(undefined as unknown as string); expect(observed).toEqual([]); From 9e48e0d03ddf20374d4d7fe6fc8d9d3a71172cf9 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 5 Jun 2026 10:54:33 -0700 Subject: [PATCH 09/26] adding licenses for pr1 --- access-control-service/LICENSE-binary | 12 ++++++++++++ amber/LICENSE-binary-java | 12 ++++++++++++ computing-unit-managing-service/LICENSE-binary | 12 ++++++++++++ config-service/LICENSE-binary | 12 ++++++++++++ file-service/LICENSE-binary | 12 ++++++++++++ workflow-compiling-service/LICENSE-binary | 12 ++++++++++++ 6 files changed, 72 insertions(+) diff --git a/access-control-service/LICENSE-binary b/access-control-service/LICENSE-binary index 3abc86ea44c..48cd986319e 100644 --- a/access-control-service/LICENSE-binary +++ b/access-control-service/LICENSE-binary @@ -274,6 +274,18 @@ Scala/Java jars: - io.dropwizard.metrics.metrics-json-4.2.25.jar - io.dropwizard.metrics.metrics-jvm-4.2.25.jar - io.dropwizard.metrics.metrics-logback-4.2.25.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.r2dbc.r2dbc-spi-0.9.0.RELEASE.jar - jakarta.inject.jakarta.inject-api-2.0.1.jar - jakarta.validation.jakarta.validation-api-3.0.2.jar diff --git a/amber/LICENSE-binary-java b/amber/LICENSE-binary-java index fba8dd9cd29..f1911ef6a79 100644 --- a/amber/LICENSE-binary-java +++ b/amber/LICENSE-binary-java @@ -362,6 +362,18 @@ Scala/Java jars: - io.netty.netty-transport-native-unix-common-4.1.96.Final.jar - io.opencensus.opencensus-api-0.31.1.jar - io.opencensus.opencensus-contrib-http-util-0.31.1.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.perfmark.perfmark-api-0.26.0.jar - io.r2dbc.r2dbc-spi-0.9.0.RELEASE.jar - io.reactivex.rxjava3.rxjava-3.1.6.jar diff --git a/computing-unit-managing-service/LICENSE-binary b/computing-unit-managing-service/LICENSE-binary index 37d78afb47c..de8d4699977 100644 --- a/computing-unit-managing-service/LICENSE-binary +++ b/computing-unit-managing-service/LICENSE-binary @@ -369,6 +369,18 @@ Scala/Java jars: - io.netty.netty-tcnative-classes-2.0.61.Final.jar - io.netty.netty-transport-4.1.104.Final.jar - io.netty.netty-transport-native-unix-common-4.1.104.Final.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.perfmark.perfmark-api-0.26.0.jar - io.r2dbc.r2dbc-spi-0.9.0.RELEASE.jar - io.swagger.swagger-annotations-1.6.14.jar diff --git a/config-service/LICENSE-binary b/config-service/LICENSE-binary index 78c3b7878f1..3b5736fa130 100644 --- a/config-service/LICENSE-binary +++ b/config-service/LICENSE-binary @@ -274,6 +274,18 @@ Scala/Java jars: - io.dropwizard.metrics.metrics-json-4.2.25.jar - io.dropwizard.metrics.metrics-jvm-4.2.25.jar - io.dropwizard.metrics.metrics-logback-4.2.25.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.r2dbc.r2dbc-spi-0.9.0.RELEASE.jar - jakarta.inject.jakarta.inject-api-2.0.1.jar - jakarta.validation.jakarta.validation-api-3.0.2.jar diff --git a/file-service/LICENSE-binary b/file-service/LICENSE-binary index 138fd6cad09..393f82e948e 100644 --- a/file-service/LICENSE-binary +++ b/file-service/LICENSE-binary @@ -334,6 +334,18 @@ Scala/Java jars: - io.netty.netty-tcnative-classes-2.0.61.Final.jar - io.netty.netty-transport-4.1.104.Final.jar - io.netty.netty-transport-native-unix-common-4.1.104.Final.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.perfmark.perfmark-api-0.26.0.jar - io.r2dbc.r2dbc-spi-0.9.0.RELEASE.jar - jakarta.inject.jakarta.inject-api-2.0.1.jar diff --git a/workflow-compiling-service/LICENSE-binary b/workflow-compiling-service/LICENSE-binary index ed6a9e1d266..2ad42d6a62f 100644 --- a/workflow-compiling-service/LICENSE-binary +++ b/workflow-compiling-service/LICENSE-binary @@ -336,6 +336,18 @@ Scala/Java jars: - io.netty.netty-tcnative-classes-2.0.61.Final.jar - io.netty.netty-transport-4.1.104.Final.jar - io.netty.netty-transport-native-unix-common-4.1.104.Final.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.perfmark.perfmark-api-0.26.0.jar - io.r2dbc.r2dbc-spi-0.9.0.RELEASE.jar - jakarta.inject.jakarta.inject-api-2.0.1.jar From 8335eb4a9e723b0f803e490910b9c94b6d9d1292 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 5 Jun 2026 11:02:15 -0700 Subject: [PATCH 10/26] changing spec file values --- .../src/app/dashboard/component/dashboard.component.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/dashboard/component/dashboard.component.spec.ts b/frontend/src/app/dashboard/component/dashboard.component.spec.ts index a53244b3cdd..889069590f0 100644 --- a/frontend/src/app/dashboard/component/dashboard.component.spec.ts +++ b/frontend/src/app/dashboard/component/dashboard.component.spec.ts @@ -283,7 +283,7 @@ describe("DashboardComponent", () => { }; fixture.detectChanges(); - // 6 "Your Work" links + 4 admin links + 1 about link = 11 - expect(fixture.debugElement.queryAll(By.directive(RouterLink)).length).toBe(11); + // 6 "Your Work" links + 4 admin links + 1 observability link + 1 about link = 12 + expect(fixture.debugElement.queryAll(By.directive(RouterLink)).length).toBe(12); }); }); From dca5b7e0bc301b0cfa962038718b0d4f56ba0902 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 5 Jun 2026 11:20:36 -0700 Subject: [PATCH 11/26] adding licenses for pr1 --- access-control-service/LICENSE-binary | 8 ++++++++ config-service/LICENSE-binary | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/access-control-service/LICENSE-binary b/access-control-service/LICENSE-binary index 48cd986319e..4a66e653c11 100644 --- a/access-control-service/LICENSE-binary +++ b/access-control-service/LICENSE-binary @@ -242,6 +242,9 @@ Scala/Java jars: - com.google.guava.listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar - com.google.j2objc.j2objc-annotations-2.8.jar - com.helger.profiler-1.1.1.jar + - com.squareup.okhttp3.okhttp-4.12.0.jar + - com.squareup.okio.okio-3.6.0.jar + - com.squareup.okio.okio-jvm-3.6.0.jar - com.thesamet.scalapb.lenses_2.13-0.11.20.jar - com.thesamet.scalapb.scalapb-json4s_2.13-0.12.0.jar - com.thesamet.scalapb.scalapb-runtime_2.13-0.11.20.jar @@ -304,6 +307,11 @@ Scala/Java jars: - org.hibernate.validator.hibernate-validator-7.0.5.Final.jar - org.javassist.javassist-3.30.2-GA.jar - org.jboss.logging.jboss-logging-3.5.3.Final.jar + - org.jetbrains.annotations-13.0.jar + - org.jetbrains.kotlin.kotlin-stdlib-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-common-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-jdk7-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-jdk8-1.9.10.jar - org.jooq.jooq-3.16.23.jar - org.json4s.json4s-ast_2.13-4.0.1.jar - org.json4s.json4s-jackson-core_2.13-4.0.1.jar diff --git a/config-service/LICENSE-binary b/config-service/LICENSE-binary index 3b5736fa130..3b5a6db4d3e 100644 --- a/config-service/LICENSE-binary +++ b/config-service/LICENSE-binary @@ -242,6 +242,9 @@ Scala/Java jars: - com.google.guava.listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar - com.google.j2objc.j2objc-annotations-2.8.jar - com.helger.profiler-1.1.1.jar + - com.squareup.okhttp3.okhttp-4.12.0.jar + - com.squareup.okio.okio-3.6.0.jar + - com.squareup.okio.okio-jvm-3.6.0.jar - com.thesamet.scalapb.lenses_2.13-0.11.20.jar - com.thesamet.scalapb.scalapb-json4s_2.13-0.12.0.jar - com.thesamet.scalapb.scalapb-runtime_2.13-0.11.20.jar @@ -304,6 +307,11 @@ Scala/Java jars: - org.hibernate.validator.hibernate-validator-7.0.5.Final.jar - org.javassist.javassist-3.30.2-GA.jar - org.jboss.logging.jboss-logging-3.5.3.Final.jar + - org.jetbrains.annotations-13.0.jar + - org.jetbrains.kotlin.kotlin-stdlib-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-common-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-jdk7-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-jdk8-1.9.10.jar - org.jooq.jooq-3.16.23.jar - org.json4s.json4s-ast_2.13-4.0.1.jar - org.json4s.json4s-jackson-core_2.13-4.0.1.jar From 5d9f6f71e2f5b4652187a8d55febb63f2fbb4b20 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 5 Jun 2026 12:30:21 -0700 Subject: [PATCH 12/26] added missing libraries for ci --- .../web/observability/gateway/ObservabilityResources.scala | 3 +-- .../org/apache/texera/web/observability/gateway/builders.scala | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/web/observability/gateway/ObservabilityResources.scala b/amber/src/main/scala/org/apache/texera/web/observability/gateway/ObservabilityResources.scala index ddab2445989..0036e2bf04d 100644 --- a/amber/src/main/scala/org/apache/texera/web/observability/gateway/ObservabilityResources.scala +++ b/amber/src/main/scala/org/apache/texera/web/observability/gateway/ObservabilityResources.scala @@ -21,9 +21,8 @@ package org.apache.texera.web.observability.gateway import com.typesafe.scalalogging.LazyLogging import io.dropwizard.auth.Auth -import javax.annotation.security.RolesAllowed import javax.ws.rs._ -import javax.ws.rs.core.{Context, MediaType, Response} +import javax.ws.rs.core.{MediaType, Response} import javax.servlet.http.HttpServletRequest import org.apache.texera.auth.SessionUser import org.apache.texera.web.observability.gateway.dtos._ diff --git a/amber/src/main/scala/org/apache/texera/web/observability/gateway/builders.scala b/amber/src/main/scala/org/apache/texera/web/observability/gateway/builders.scala index 77bc4af7efe..7a6dcf37155 100644 --- a/amber/src/main/scala/org/apache/texera/web/observability/gateway/builders.scala +++ b/amber/src/main/scala/org/apache/texera/web/observability/gateway/builders.scala @@ -19,8 +19,6 @@ package org.apache.texera.web.observability.gateway -import org.apache.texera.web.observability.gateway.dtos._ - /** * Typed query builders. Each takes a validated DTO + the caller's * resolved scope and returns a backend-specific query string + the From 7b8a9e2835b22b4f93ace5d2544df1333006f9a4 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Sun, 14 Jun 2026 03:42:38 -0700 Subject: [PATCH 13/26] fixed the log sanitizer format and allow callers outside sanitize to apply cap --- .../org/apache/texera/observability/LogSanitizer.scala | 6 ++++-- .../texera/observability/TexeraOtelLogAppender.scala | 9 +++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala b/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala index aa92c558c27..cab34fcf58b 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala @@ -101,8 +101,10 @@ object LogSanitizer { truncate(scrubbed) } - /** Truncate to MaxBodyBytes, appending the marker if cut. */ - private def truncate(body: String): String = { + /** Truncate to MaxBodyBytes, appending the marker if cut. Public so + * callers building a body outside `sanitize` can enforce the cap. + */ + def truncate(body: String): String = { if (body.length <= MaxBodyBytes) body else body.substring(0, MaxBodyBytes - TruncatedMarker.length) + TruncatedMarker } diff --git a/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala b/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala index 2dbf160bcfb..f41e88dd569 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala @@ -86,12 +86,9 @@ class TexeraOtelLogAppender extends UnsynchronizedAppenderBase[ILoggingEvent] { val baseBody = LogSanitizer.sanitize(event.getFormattedMessage) val body = Option(event.getThrowableProxy) match { case Some(proxy) => - // JVM-generated stack frames are trusted (not user input), so - // we skip the C0 strip that would collapse newlines and ruin - // readability. We do still cap the total length implicitly - // via the OTel SDK's per-record body limit, and the body - // stays valid UTF-8 because Logback emits ASCII frame text. - baseBody + "\n" + formatThrowable(proxy) + // Trusted JVM frames: skip the C0 strip so newlines survive, + // but still cap length (the OTel SDK does not bound the body). + LogSanitizer.truncate(baseBody + "\n" + formatThrowable(proxy)) case None => baseBody } val builder = logger From 9c45fd531b7638d02fcd9879a508d264484fb76c Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Sun, 14 Jun 2026 03:58:11 -0700 Subject: [PATCH 14/26] fixed incorrect comment in parca-agent.env --- bin/observability/parca/parca-agent.env | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bin/observability/parca/parca-agent.env b/bin/observability/parca/parca-agent.env index 5ff164a20da..2e72ea94b44 100644 --- a/bin/observability/parca/parca-agent.env +++ b/bin/observability/parca/parca-agent.env @@ -36,9 +36,8 @@ # * node: host identifier — set by the compose layer to the docker # host's hostname. Overridable. # * metadata-external-labels: static labels attached to every -# profile. We attach service.name so the Parca query layer can -# group profiles by Texera service the same way logs/traces are -# grouped (matches the OTel resource attr). +# profile (deployment=texera;cluster=local). Coarse fields only; +# no workflow/execution id (cardinality DoS in Parca storage). PARCA_AGENT_REMOTE_STORE_ADDRESS=parca:7070 PARCA_AGENT_REMOTE_STORE_INSECURE=true From 012ee006a1e47472a70f5994b521d7ceb25c1bf9 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Tue, 23 Jun 2026 03:00:48 -0700 Subject: [PATCH 15/26] feat(observability): bootstrap OTel log bridge + framework log caps Call OtelInit.init() in each service main so its logs bridge to the OTel collector under its own service.name; cap noisy framework loggers (pekko/iceberg/hadoop/kafka/jetty/jersey/grpc/ netty/hikari/awssdk) at WARN in each service config. Services: access-control, config, file, computing-unit-managing, workflow-compiling, computing-unit-master, texera-web, amber. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../access-control-service-web-config.yaml | 13 +++++++++++++ .../texera/service/AccessControlService.scala | 2 ++ .../resources/computing-unit-master-config.yml | 12 ++++++++++++ .../texera-compiling-service-web-config.yml | 12 ++++++++++++ amber/src/main/resources/web-config.yml | 12 ++++++++++++ .../apache/texera/web/ComputingUnitMaster.scala | 1 + .../computing-unit-managing-service-config.yaml | 14 +++++++++++++- .../service/ComputingUnitManagingService.scala | 2 ++ .../main/resources/config-service-web-config.yaml | 13 +++++++++++++ .../org/apache/texera/service/ConfigService.scala | 2 ++ .../main/resources/file-service-web-config.yaml | 12 ++++++++++++ .../org/apache/texera/service/FileService.scala | 2 ++ .../workflow-compiling-service-config.yaml | 12 ++++++++++++ .../texera/service/WorkflowCompilingService.scala | 2 ++ 14 files changed, 110 insertions(+), 1 deletion(-) diff --git a/access-control-service/src/main/resources/access-control-service-web-config.yaml b/access-control-service/src/main/resources/access-control-service-web-config.yaml index 8c7895e9858..bd78ecaa82a 100644 --- a/access-control-service/src/main/resources/access-control-service-web-config.yaml +++ b/access-control-service/src/main/resources/access-control-service-web-config.yaml @@ -26,6 +26,19 @@ server: logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + loggers: + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console threshold: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} diff --git a/access-control-service/src/main/scala/org/apache/texera/service/AccessControlService.scala b/access-control-service/src/main/scala/org/apache/texera/service/AccessControlService.scala index f01d06f9417..c1f5114c09f 100644 --- a/access-control-service/src/main/scala/org/apache/texera/service/AccessControlService.scala +++ b/access-control-service/src/main/scala/org/apache/texera/service/AccessControlService.scala @@ -65,6 +65,8 @@ class AccessControlService extends Application[AccessControlServiceConfiguration configuration: AccessControlServiceConfiguration, environment: Environment ): Unit = { + // Bridge this service's logs to the OTel collector under its own service.name. + org.apache.texera.observability.OtelInit.init("access-control-service") // Serve backend at /api environment.jersey.setUrlPattern("/api/*") diff --git a/amber/src/main/resources/computing-unit-master-config.yml b/amber/src/main/resources/computing-unit-master-config.yml index 0dba594b8ae..ee578c3cf90 100644 --- a/amber/src/main/resources/computing-unit-master-config.yml +++ b/amber/src/main/resources/computing-unit-master-config.yml @@ -34,6 +34,18 @@ logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console logFormat: "[%date{ISO8601}] [%level] [%logger] [%thread] - %msg %n" diff --git a/amber/src/main/resources/texera-compiling-service-web-config.yml b/amber/src/main/resources/texera-compiling-service-web-config.yml index ea2c1b9c1e9..c0b6e8aa762 100644 --- a/amber/src/main/resources/texera-compiling-service-web-config.yml +++ b/amber/src/main/resources/texera-compiling-service-web-config.yml @@ -43,6 +43,18 @@ logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console logFormat: "[%date{ISO8601}] [%level] [%logger] [%thread] - %msg %n" diff --git a/amber/src/main/resources/web-config.yml b/amber/src/main/resources/web-config.yml index 9fde1d078e8..9b3c743c89c 100644 --- a/amber/src/main/resources/web-config.yml +++ b/amber/src/main/resources/web-config.yml @@ -43,6 +43,18 @@ logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console logFormat: "[%date{ISO8601}] [%level] [%logger] [%thread] - %msg %n" diff --git a/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala b/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala index d4a061781c9..8d6213e8514 100644 --- a/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala +++ b/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala @@ -139,6 +139,7 @@ class ComputingUnitMaster extends io.dropwizard.Application[Configuration] with } override def run(configuration: Configuration, environment: Environment): Unit = { + org.apache.texera.observability.OtelInit.init("computing-unit-master") ObjectMapperUtils.warmupObjectMapperForOperatorsSerde() SqlServer.initConnection( diff --git a/computing-unit-managing-service/src/main/resources/computing-unit-managing-service-config.yaml b/computing-unit-managing-service/src/main/resources/computing-unit-managing-service-config.yaml index 523b4197989..ea428fcadcb 100644 --- a/computing-unit-managing-service/src/main/resources/computing-unit-managing-service-config.yaml +++ b/computing-unit-managing-service/src/main/resources/computing-unit-managing-service-config.yaml @@ -30,4 +30,16 @@ server: logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: - "com.example": ${TEXERA_SERVICE_LOG_LEVEL:-DEBUG} \ No newline at end of file + "com.example": ${TEXERA_SERVICE_LOG_LEVEL:-DEBUG} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN \ No newline at end of file diff --git a/computing-unit-managing-service/src/main/scala/org/apache/texera/service/ComputingUnitManagingService.scala b/computing-unit-managing-service/src/main/scala/org/apache/texera/service/ComputingUnitManagingService.scala index 0650990264d..dba67a12c3e 100644 --- a/computing-unit-managing-service/src/main/scala/org/apache/texera/service/ComputingUnitManagingService.scala +++ b/computing-unit-managing-service/src/main/scala/org/apache/texera/service/ComputingUnitManagingService.scala @@ -59,6 +59,8 @@ class ComputingUnitManagingService extends Application[ComputingUnitManagingServ configuration: ComputingUnitManagingServiceConfiguration, environment: Environment ): Unit = { + // Bridge this service's logs to the OTel collector under its own service.name. + org.apache.texera.observability.OtelInit.init("computing-unit-managing-service") // Register http resources environment.jersey.setUrlPattern("/api/*") environment.jersey.register(classOf[HealthCheckResource]) diff --git a/config-service/src/main/resources/config-service-web-config.yaml b/config-service/src/main/resources/config-service-web-config.yaml index 4aa67af82e1..8559e1fd507 100644 --- a/config-service/src/main/resources/config-service-web-config.yaml +++ b/config-service/src/main/resources/config-service-web-config.yaml @@ -26,6 +26,19 @@ server: logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + loggers: + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console threshold: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} diff --git a/config-service/src/main/scala/org/apache/texera/service/ConfigService.scala b/config-service/src/main/scala/org/apache/texera/service/ConfigService.scala index e4736cf2511..43fdf5e4840 100644 --- a/config-service/src/main/scala/org/apache/texera/service/ConfigService.scala +++ b/config-service/src/main/scala/org/apache/texera/service/ConfigService.scala @@ -60,6 +60,8 @@ class ConfigService extends Application[ConfigServiceConfiguration] with LazyLog } override def run(configuration: ConfigServiceConfiguration, environment: Environment): Unit = { + // Bridge this service's logs to the OTel collector under its own service.name. + org.apache.texera.observability.OtelInit.init("config-service") // Serve backend at /api environment.jersey.setUrlPattern("/api/*") diff --git a/file-service/src/main/resources/file-service-web-config.yaml b/file-service/src/main/resources/file-service-web-config.yaml index 41f8d1b1748..db5a7ec6645 100644 --- a/file-service/src/main/resources/file-service-web-config.yaml +++ b/file-service/src/main/resources/file-service-web-config.yaml @@ -28,6 +28,18 @@ logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console - type: file diff --git a/file-service/src/main/scala/org/apache/texera/service/FileService.scala b/file-service/src/main/scala/org/apache/texera/service/FileService.scala index 76d78dfef86..d984c3c72b6 100644 --- a/file-service/src/main/scala/org/apache/texera/service/FileService.scala +++ b/file-service/src/main/scala/org/apache/texera/service/FileService.scala @@ -67,6 +67,8 @@ class FileService extends Application[FileServiceConfiguration] with LazyLogging } override def run(configuration: FileServiceConfiguration, environment: Environment): Unit = { + // Bridge this service's logs to the OTel collector under its own service.name. + org.apache.texera.observability.OtelInit.init("file-service") // Serve backend at /api environment.jersey.setUrlPattern("/api/*") SqlServer.initConnection( diff --git a/workflow-compiling-service/src/main/resources/workflow-compiling-service-config.yaml b/workflow-compiling-service/src/main/resources/workflow-compiling-service-config.yaml index 5b9016af1b6..37e413c15b6 100644 --- a/workflow-compiling-service/src/main/resources/workflow-compiling-service-config.yaml +++ b/workflow-compiling-service/src/main/resources/workflow-compiling-service-config.yaml @@ -28,6 +28,18 @@ logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console - type: file diff --git a/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala b/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala index c278b21b4d3..94dcff217cb 100644 --- a/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala +++ b/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala @@ -51,6 +51,8 @@ class WorkflowCompilingService extends Application[WorkflowCompilingServiceConfi configuration: WorkflowCompilingServiceConfiguration, environment: Environment ): Unit = { + // Bridge this service's logs to the OTel collector under its own service.name. + org.apache.texera.observability.OtelInit.init("workflow-compiling-service") ObjectMapperUtils.warmupObjectMapperForOperatorsSerde() // serve backend at /api From 5ee44a567563a302ac31c7cbd6e933c376222939 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Tue, 23 Jun 2026 03:06:58 -0700 Subject: [PATCH 16/26] feat(observability): emit workflow lifecycle metrics + run-level trace span - WorkflowMetricsRecorder: emit workflow lifecycle metrics keyed by execution, driven from the ExecutionStateStore state-transition chokepoint; registered via WorkflowMetricsRecorder.init() in ComputingUnitMaster - WorkflowService: wrap initExecutionService in a run-level TexeraTracer span so setup-path logs carry the trace id Co-Authored-By: Claude Opus 4.8 (1M context) --- .../texera/web/ComputingUnitMaster.scala | 1 + .../WorkflowMetricsRecorder.scala | 107 ++++++++++++++++++ .../texera/web/service/WorkflowService.scala | 24 ++++ .../web/storage/ExecutionStateStore.scala | 4 + 4 files changed, 136 insertions(+) create mode 100644 amber/src/main/scala/org/apache/texera/web/observability/WorkflowMetricsRecorder.scala diff --git a/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala b/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala index 8d6213e8514..97393747ae8 100644 --- a/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala +++ b/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala @@ -140,6 +140,7 @@ class ComputingUnitMaster extends io.dropwizard.Application[Configuration] with override def run(configuration: Configuration, environment: Environment): Unit = { org.apache.texera.observability.OtelInit.init("computing-unit-master") + org.apache.texera.web.observability.WorkflowMetricsRecorder.init() ObjectMapperUtils.warmupObjectMapperForOperatorsSerde() SqlServer.initConnection( diff --git a/amber/src/main/scala/org/apache/texera/web/observability/WorkflowMetricsRecorder.scala b/amber/src/main/scala/org/apache/texera/web/observability/WorkflowMetricsRecorder.scala new file mode 100644 index 00000000000..572aa74e3c2 --- /dev/null +++ b/amber/src/main/scala/org/apache/texera/web/observability/WorkflowMetricsRecorder.scala @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.web.observability + +import com.typesafe.scalalogging.LazyLogging +import org.apache.texera.amber.core.virtualidentity.ExecutionIdentity +import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAggregatedState +import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAggregatedState.{ + COMPLETED, + FAILED, + KILLED, + PAUSED, + PAUSING, + RESUMING, + RUNNING +} +import org.apache.texera.observability.TexeraMetrics +import org.apache.texera.observability.TexeraMetrics.WorkflowKind +import org.apache.texera.web.service.WorkflowService + +import java.util.concurrent.ConcurrentHashMap + +/** + * Drives [[TexeraMetrics]] from amber's execution lifecycle. The metric + * instruments live in common/config; this object is the single place + * that records them, so the lifecycle code only needs one-line calls. + * + * - start / terminal counters and the duration histogram are recorded + * from [[onStart]] and [[onStateChange]]; + * - the always-polled `texera.workflow.active` gauge is sourced from the + * live WorkflowService registry via the supplier registered in [[init]]. + */ +object WorkflowMetricsRecorder extends LazyLogging { + + // Start time + kind per in-flight run, so a terminal transition can emit + // a duration and attribute the outcome. Keyed by the execution identity. + private val inFlight = new ConcurrentHashMap[ExecutionIdentity, (Long, WorkflowKind)]() + + private val ActiveStates: Set[WorkflowAggregatedState] = Set(RUNNING, PAUSING, PAUSED, RESUMING) + private val TerminalStates: Set[WorkflowAggregatedState] = Set(COMPLETED, FAILED, KILLED) + + /** Register the active-executions gauge supplier and bind the instruments. + * Call once at startup, after OtelInit.init. The supplier is polled on + * every metric collection, so it must never throw. + */ + def init(): Unit = { + TexeraMetrics.setActiveExecutionsSupplier(() => + try { + WorkflowService.getAllWorkflowServices.iterator + .flatMap(s => Option(s.executionService.getValue)) + .map(_.executionStateStore.metadataStore.getState.state) + .count(ActiveStates.contains) + .toLong + } catch { + case _: Throwable => 0L + } + ) + TexeraMetrics.ensureBound() + } + + /** Record that a run started. */ + def onStart( + executionId: ExecutionIdentity, + kind: WorkflowKind = WorkflowKind.Interactive + ): Unit = { + inFlight.put(executionId, (System.currentTimeMillis(), kind)) + TexeraMetrics.recordStart(kind) + } + + /** Record terminal counters + duration exactly once, on the first + * transition from a non-terminal into a terminal state. Safe to call on + * every state change; non-terminal and repeat-terminal calls are no-ops. + */ + def onStateChange( + executionId: ExecutionIdentity, + oldState: WorkflowAggregatedState, + newState: WorkflowAggregatedState + ): Unit = { + if (!TerminalStates.contains(newState) || TerminalStates.contains(oldState)) return + val entry = Option(inFlight.remove(executionId)) + val kind = entry.map(_._2).getOrElse(WorkflowKind.Interactive) + val durationSec = entry.map(e => (System.currentTimeMillis() - e._1) / 1000.0).getOrElse(0.0) + newState match { + case COMPLETED => TexeraMetrics.recordCompletion(kind, durationSec) + case FAILED => TexeraMetrics.recordFailure(kind, durationSec) + case KILLED => TexeraMetrics.recordCancellation(kind) + case _ => () + } + } +} diff --git a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala index a241121da20..a4dcd945643 100644 --- a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala +++ b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala @@ -21,6 +21,7 @@ package org.apache.texera.web.service import com.google.protobuf.timestamp.Timestamp import com.typesafe.scalalogging.LazyLogging +import io.opentelemetry.api.trace.Span import io.reactivex.rxjava3.disposables.{CompositeDisposable, Disposable} import io.reactivex.rxjava3.subjects.BehaviorSubject import org.apache.texera.common.config.ApplicationConfig @@ -49,6 +50,7 @@ import org.apache.texera.amber.error.ErrorUtils.{ getStackTraceWithAllCauses } import org.apache.texera.dao.jooq.generated.tables.pojos.User +import org.apache.texera.observability.TexeraTracer import org.apache.texera.service.util.LargeBinaryManager import org.apache.texera.web.model.websocket.event.TexeraWebSocketEvent import org.apache.texera.web.model.websocket.request.WorkflowExecuteRequest @@ -185,6 +187,25 @@ class WorkflowService( userOpt: Option[User], sessionUri: URI ): Unit = { + TexeraTracer.withSpan( + "workflow.execute", + _.setAttribute("texera.workflow.id", workflowId.id.toString) + ) { span => + initExecutionServiceSpanned(req, userOpt, sessionUri, span) + } + } + + /** Body of [[initExecutionService]], run inside the run-level span so + * logs on the setup path carry its trace id. The span covers the + * synchronous setup and the handoff to async execution via + * `executeWorkflow()`; it does not span the full async run. + */ + private def initExecutionServiceSpanned( + req: WorkflowExecuteRequest, + userOpt: Option[User], + sessionUri: URI, + span: Span + ): Unit = { if (executionService.hasValue) { executionService.getValue.unsubscribeAll() @@ -216,6 +237,9 @@ class WorkflowService( convertToJson(req.engineVersion), req.computingUnitId ) + span.setAttribute("texera.execution.id", workflowContext.executionId.id.toString) + // A run has started: record the start counter and stamp its start time. + org.apache.texera.web.observability.WorkflowMetricsRecorder.onStart(workflowContext.executionId) if (ApplicationConfig.faultToleranceLogRootFolder.isDefined) { val writeLocation = ApplicationConfig.faultToleranceLogRootFolder.get.resolve( diff --git a/amber/src/main/scala/org/apache/texera/web/storage/ExecutionStateStore.scala b/amber/src/main/scala/org/apache/texera/web/storage/ExecutionStateStore.scala index 654acbbefd1..bc97b39d185 100644 --- a/amber/src/main/scala/org/apache/texera/web/storage/ExecutionStateStore.scala +++ b/amber/src/main/scala/org/apache/texera/web/storage/ExecutionStateStore.scala @@ -27,6 +27,7 @@ import org.apache.texera.amber.engine.common.executionruntimestate.{ ExecutionMetadataStore, ExecutionStatsStore } +import org.apache.texera.web.observability.WorkflowMetricsRecorder import org.apache.texera.web.service.ExecutionsMetadataPersistService import java.sql.Timestamp @@ -44,6 +45,9 @@ object ExecutionStateStore { execution.setStatus(maptoStatusCode(state)) execution.setLastUpdateTime(new Timestamp(System.currentTimeMillis())) } + // Single chokepoint for every state transition: emit lifecycle metrics + // once on the first transition into a terminal state. + WorkflowMetricsRecorder.onStateChange(metadataStore.executionId, metadataStore.state, state) metadataStore.withState(state) } } From c47a7d0e1b1a7d9cf5a706ed12639215dfedff27 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Tue, 23 Jun 2026 03:08:48 -0700 Subject: [PATCH 17/26] chore(observability): single-node compose wiring for the collector stack - bin/observability/docker-compose.yml: collector + parca-agent stack - bin/single-node/docker-compose.yml: mount the otel-collector and parca configs and run the parca-agent sidecar Co-Authored-By: Claude Opus 4.8 (1M context) --- bin/observability/docker-compose.yml | 39 ++++++++++++++++++++++++++++ bin/single-node/docker-compose.yml | 6 ++--- 2 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 bin/observability/docker-compose.yml diff --git a/bin/observability/docker-compose.yml b/bin/observability/docker-compose.yml new file mode 100644 index 00000000000..adce75a4a40 --- /dev/null +++ b/bin/observability/docker-compose.yml @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Dedicated entry point for the observability stack, so it can be started like +# the other dev targets (e.g. file-service's own compose) instead of reaching +# into the single-node deployment file. +# +# It does NOT redefine the services: it `include`s the single-node compose, the +# single source of truth for every container definition. The observability +# services there are gated behind compose profiles, so they are selected via +# COMPOSE_PROFILES + the service list (see start_app.sh `up_obs`). The other +# single-node services are not profile-gated, so they are simply not named and +# stay down. +# +# Relative volume paths inside the included file (e.g. the collector/parca +# configs under ../observability/) are resolved relative to the included file's +# own directory, so they keep working unchanged. +# +# The project name matches single-node so this manages the SAME containers +# whether obs is brought up here or as part of the full single-node deployment +# (the containers use fixed container_names, so a different project would clash). +name: texera-single-node + +include: + - ../single-node/docker-compose.yml diff --git a/bin/single-node/docker-compose.yml b/bin/single-node/docker-compose.yml index e796d1baf4c..370744ea7ed 100644 --- a/bin/single-node/docker-compose.yml +++ b/bin/single-node/docker-compose.yml @@ -515,7 +515,7 @@ services: security_opt: - no-new-privileges:true volumes: - - ../observability/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml:ro + - ../observability/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml:ro,z command: ["--config=/etc/otelcol-contrib/config.yaml"] # Local dev override: publish OTLP receiver ports on the host # loopback so a Scala backend running outside docker (sbt / @@ -615,7 +615,7 @@ services: security_opt: - no-new-privileges:true volumes: - - ../observability/parca/parca.yaml:/parca.yaml:ro + - ../observability/parca/parca.yaml:/parca.yaml:ro,z - parca_data:/var/lib/parca command: - "/parca" @@ -634,7 +634,7 @@ services: # observability service that requires elevated permissions; the # surface is documented and reviewed. parca-agent: - image: ghcr.io/parca-dev/parca-agent:v0.47.1 + image: ghcr.io/parca-dev/parca-agent:v0.48.0 container_name: texera-parca-agent profiles: [observability-profiles] restart: always From 31bbdabc7c4cee96966070813da2efe3564078d9 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Tue, 23 Jun 2026 03:17:41 -0700 Subject: [PATCH 18/26] feat(observability): gateway-core window-cap removal + UI shell nav - dtos: drop the per-signal Signal/maxWindowSeconds enum and the upper bound on TimeWindow.validate -- DB-backed counts have no retention limit and the backends just return what they retain; BadTimeWindow becomes a plain value (only empty/inverted windows are rejected) - DtoValidationSpec: cover the new unbounded-window behavior - UI shell: observability route + dashboard navigation entry Co-Authored-By: Claude Opus 4.8 (1M context) --- .../web/observability/gateway/dtos.scala | 31 +++++-------------- .../gateway/DtoValidationSpec.scala | 28 ++++++----------- frontend/src/app/app-routing.constant.ts | 5 +-- frontend/src/app/app-routing.module.ts | 8 ++--- .../component/dashboard.component.html | 22 ++++++------- .../component/dashboard.component.scss | 10 +++--- .../component/dashboard.component.spec.ts | 2 +- .../component/dashboard.component.ts | 4 +-- 8 files changed, 43 insertions(+), 67 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/web/observability/gateway/dtos.scala b/amber/src/main/scala/org/apache/texera/web/observability/gateway/dtos.scala index be28a521a04..93c9898b7df 100644 --- a/amber/src/main/scala/org/apache/texera/web/observability/gateway/dtos.scala +++ b/amber/src/main/scala/org/apache/texera/web/observability/gateway/dtos.scala @@ -60,24 +60,6 @@ object dtos { Option(raw).flatMap(s => all.find(_.name.equalsIgnoreCase(s.trim))) } - /** Signal kind, used by metric/audit code to discriminate. */ - sealed trait Signal - object Signal { - case object Logs extends Signal - case object Metrics extends Signal - case object Traces extends Signal - case object Profiles extends Signal - } - - /** Maximum time window per signal, in seconds. */ - def maxWindowSeconds(signal: Signal): Long = - signal match { - case Signal.Logs => 7L * 24L * 3600L - case Signal.Metrics => 90L * 24L * 3600L - case Signal.Traces => 24L * 3600L - case Signal.Profiles => 7L * 24L * 3600L - } - // ---- typed unions for validator results ------------------------------- /** Result of validating an inbound request. Either a clean @@ -95,8 +77,12 @@ object dtos { case class GatewayError(code: String, message: String, status: Int) object GatewayError { - val BadTimeWindow: (Long => GatewayError) = (maxSec: Long) => - GatewayError("bad_time_window", s"time window must be > 0 and <= ${maxSec}s", 400) + // No upper bound on the window: the DB-backed count has no retention + // limit, and the metrics/logs/traces/profiles backends simply return + // whatever they still retain for the requested range. The only invalid + // window is an empty or inverted one (to must be after from). + val BadTimeWindow: GatewayError = + GatewayError("bad_time_window", "time window must be > 0 (end must be after start)", 400) val BadPageSize: GatewayError = GatewayError("bad_page_size", s"pageSize must be in [1, ${MaxPageSize}]", 400) val BadLevel: GatewayError = @@ -132,15 +118,14 @@ object dtos { object TimeWindow { def validate( - signal: Signal, fromMs: Long, toMs: Long ): ValidationResult[TimeWindow] = { val from = Instant.ofEpochMilli(fromMs) val to = Instant.ofEpochMilli(toMs) val seconds = Duration.between(from, to).toSeconds - if (seconds <= 0 || seconds > maxWindowSeconds(signal)) - Invalid(GatewayError.BadTimeWindow(maxWindowSeconds(signal))) + // No maximum: only reject an empty or inverted window. + if (seconds <= 0) Invalid(GatewayError.BadTimeWindow) else Valid(TimeWindow(from, to)) } } diff --git a/amber/src/test/scala/org/apache/texera/web/observability/gateway/DtoValidationSpec.scala b/amber/src/test/scala/org/apache/texera/web/observability/gateway/DtoValidationSpec.scala index 15f3c48167f..b5e33747a43 100644 --- a/amber/src/test/scala/org/apache/texera/web/observability/gateway/DtoValidationSpec.scala +++ b/amber/src/test/scala/org/apache/texera/web/observability/gateway/DtoValidationSpec.scala @@ -27,30 +27,22 @@ class DtoValidationSpec extends AnyFlatSpec with Matchers { // ----- TimeWindow ---------------------------------------------------- - "TimeWindow.validate" should "accept a 1-hour window for logs" in { + "TimeWindow.validate" should "accept a 1-hour window" in { val hourMs = 3_600_000L - TimeWindow.validate(Signal.Logs, 0L, hourMs) shouldBe a[Valid[_]] + TimeWindow.validate(0L, hourMs) shouldBe a[Valid[_]] } - it should "reject a window > 7 days for logs" in { - val eightDaysMs = 8L * 24L * 3_600_000L - val result = TimeWindow.validate(Signal.Logs, 0L, eightDaysMs) - result shouldBe an[Invalid] - } - - it should "accept a 30-day window for metrics" in { - val thirtyDaysMs = 30L * 24L * 3_600_000L - TimeWindow.validate(Signal.Metrics, 0L, thirtyDaysMs) shouldBe a[Valid[_]] - } - - it should "reject a 25-hour window for traces" in { - val twentyFiveHrMs = 25L * 3_600_000L - TimeWindow.validate(Signal.Traces, 0L, twentyFiveHrMs) shouldBe an[Invalid] + it should "accept arbitrarily large windows (no maximum)" in { + // A 10-year span: there is no upper bound on the window any more. The + // backends return whatever they retain; the DB-backed count is exact + // over any range. + val tenYearsMs = 10L * 365L * 24L * 3_600_000L + TimeWindow.validate(0L, tenYearsMs) shouldBe a[Valid[_]] } it should "reject zero and negative windows" in { - TimeWindow.validate(Signal.Logs, 0L, 0L) shouldBe an[Invalid] - TimeWindow.validate(Signal.Logs, 100L, 50L) shouldBe an[Invalid] + TimeWindow.validate(0L, 0L) shouldBe an[Invalid] + TimeWindow.validate(100L, 50L) shouldBe an[Invalid] } // ----- PageSize ------------------------------------------------------ diff --git a/frontend/src/app/app-routing.constant.ts b/frontend/src/app/app-routing.constant.ts index 0a78d30b871..affab28ddca 100644 --- a/frontend/src/app/app-routing.constant.ts +++ b/frontend/src/app/app-routing.constant.ts @@ -38,12 +38,13 @@ export const USER_COMPUTING_UNIT = `${USER}/compute`; export const USER_PYTHON_VENV = `${USER}/python-venv`; export const USER_QUOTA = `${USER}/quota`; export const USER_DISCUSSION = `${USER}/discussion`; -export const USER_OBSERVABILITY = `${USER}/observability`; - export const ADMIN = "/admin"; export const ADMIN_USER = `${ADMIN}/user`; export const ADMIN_GMAIL = `${ADMIN}/gmail`; export const ADMIN_EXECUTION = `${ADMIN}/execution`; export const ADMIN_SETTINGS = `${ADMIN}/settings`; +// Observability is an admin-only, system-wide view (logs, metrics, traces, +// profiles), not a per-user feature. +export const ADMIN_OBSERVABILITY = `${ADMIN}/observability`; export const SEARCH = "/search"; diff --git a/frontend/src/app/app-routing.module.ts b/frontend/src/app/app-routing.module.ts index 0c46fbe24b3..d366fdc64f5 100644 --- a/frontend/src/app/app-routing.module.ts +++ b/frontend/src/app/app-routing.module.ts @@ -142,10 +142,6 @@ routes.push({ path: "discussion", component: FlarumComponent, }, - { - path: "observability", - component: ObservabilityComponent, - }, ], }, { @@ -168,6 +164,10 @@ routes.push({ path: "settings", component: AdminSettingsComponent, }, + { + path: "observability", + component: ObservabilityComponent, + }, ], }, { diff --git a/frontend/src/app/dashboard/component/dashboard.component.html b/frontend/src/app/dashboard/component/dashboard.component.html index 268b8e4f560..6724aaee2f0 100644 --- a/frontend/src/app/dashboard/component/dashboard.component.html +++ b/frontend/src/app/dashboard/component/dashboard.component.html @@ -132,17 +132,6 @@ nzType="dashboard"> Quota
    • -
  • - - Observability -
  • Settings
    • +
  • + + Observability +
    • diff --git a/frontend/src/app/dashboard/component/dashboard.component.scss b/frontend/src/app/dashboard/component/dashboard.component.scss index 5df9388d67b..ad0bcdb0c34 100644 --- a/frontend/src/app/dashboard/component/dashboard.component.scss +++ b/frontend/src/app/dashboard/component/dashboard.component.scss @@ -19,6 +19,10 @@ #nav { height: 70px; + // Pin the header height: it is a flex child of the page-content column, so + // without this it gets compressed when the content area is tall. + flex-shrink: 0; + max-width: 100%; padding: 10px 15px; display: flex; align-items: center; @@ -138,9 +142,3 @@ nz-content { .hidden { display: none; } - -#nav { - max-width: 100%; - max-height: 100%; - overflow: hidden; -} diff --git a/frontend/src/app/dashboard/component/dashboard.component.spec.ts b/frontend/src/app/dashboard/component/dashboard.component.spec.ts index 148bae7145f..f20c1ef582d 100644 --- a/frontend/src/app/dashboard/component/dashboard.component.spec.ts +++ b/frontend/src/app/dashboard/component/dashboard.component.spec.ts @@ -283,7 +283,7 @@ describe("DashboardComponent", () => { }; fixture.detectChanges(); - // 7 "Your Work" links (incl. Python Venvs) + 4 admin links + 1 observability link + 1 about link = 13 + // 7 "Your Work" links (incl. Python Venvs) + 5 admin links (incl. Observability) + 1 about link = 13 expect(fixture.debugElement.queryAll(By.directive(RouterLink)).length).toBe(13); }); }); diff --git a/frontend/src/app/dashboard/component/dashboard.component.ts b/frontend/src/app/dashboard/component/dashboard.component.ts index efb5b770381..8af792ec524 100644 --- a/frontend/src/app/dashboard/component/dashboard.component.ts +++ b/frontend/src/app/dashboard/component/dashboard.component.ts @@ -32,12 +32,12 @@ import { ABOUT, ADMIN_EXECUTION, ADMIN_GMAIL, + ADMIN_OBSERVABILITY, ADMIN_SETTINGS, ADMIN_USER, USER_COMPUTING_UNIT, USER_DATASET, USER_DISCUSSION, - USER_OBSERVABILITY, USER_PROJECT, USER_PYTHON_VENV, USER_QUOTA, @@ -114,11 +114,11 @@ export class DashboardComponent implements OnInit { protected readonly USER_PYTHON_VENV = USER_PYTHON_VENV; protected readonly USER_QUOTA = USER_QUOTA; protected readonly USER_DISCUSSION = USER_DISCUSSION; - protected readonly USER_OBSERVABILITY = USER_OBSERVABILITY; protected readonly ADMIN_USER = ADMIN_USER; protected readonly ADMIN_GMAIL = ADMIN_GMAIL; protected readonly ADMIN_EXECUTION = ADMIN_EXECUTION; protected readonly ADMIN_SETTINGS = ADMIN_SETTINGS; + protected readonly ADMIN_OBSERVABILITY = ADMIN_OBSERVABILITY; protected readonly ABOUT = ABOUT; protected readonly String = String; From 1c57ed4b87be62a2aa037ae60b93d38483802fa8 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 26 Jun 2026 16:56:08 -0700 Subject: [PATCH 19/26] added license binaries to appropriate sections --- notebook-migration-service/LICENSE-binary | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/notebook-migration-service/LICENSE-binary b/notebook-migration-service/LICENSE-binary index 3abc86ea44c..4a66e653c11 100644 --- a/notebook-migration-service/LICENSE-binary +++ b/notebook-migration-service/LICENSE-binary @@ -242,6 +242,9 @@ Scala/Java jars: - com.google.guava.listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar - com.google.j2objc.j2objc-annotations-2.8.jar - com.helger.profiler-1.1.1.jar + - com.squareup.okhttp3.okhttp-4.12.0.jar + - com.squareup.okio.okio-3.6.0.jar + - com.squareup.okio.okio-jvm-3.6.0.jar - com.thesamet.scalapb.lenses_2.13-0.11.20.jar - com.thesamet.scalapb.scalapb-json4s_2.13-0.12.0.jar - com.thesamet.scalapb.scalapb-runtime_2.13-0.11.20.jar @@ -274,6 +277,18 @@ Scala/Java jars: - io.dropwizard.metrics.metrics-json-4.2.25.jar - io.dropwizard.metrics.metrics-jvm-4.2.25.jar - io.dropwizard.metrics.metrics-logback-4.2.25.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.r2dbc.r2dbc-spi-0.9.0.RELEASE.jar - jakarta.inject.jakarta.inject-api-2.0.1.jar - jakarta.validation.jakarta.validation-api-3.0.2.jar @@ -292,6 +307,11 @@ Scala/Java jars: - org.hibernate.validator.hibernate-validator-7.0.5.Final.jar - org.javassist.javassist-3.30.2-GA.jar - org.jboss.logging.jboss-logging-3.5.3.Final.jar + - org.jetbrains.annotations-13.0.jar + - org.jetbrains.kotlin.kotlin-stdlib-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-common-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-jdk7-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-jdk8-1.9.10.jar - org.jooq.jooq-3.16.23.jar - org.json4s.json4s-ast_2.13-4.0.1.jar - org.json4s.json4s-jackson-core_2.13-4.0.1.jar From d927cff7994363d6f7f7f502b1cdcae56cf372fe Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 26 Jun 2026 17:04:02 -0700 Subject: [PATCH 20/26] changed parac-agent to compatible version --- bin/single-node/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/single-node/docker-compose.yml b/bin/single-node/docker-compose.yml index 370744ea7ed..b1d827511a6 100644 --- a/bin/single-node/docker-compose.yml +++ b/bin/single-node/docker-compose.yml @@ -634,7 +634,7 @@ services: # observability service that requires elevated permissions; the # surface is documented and reviewed. parca-agent: - image: ghcr.io/parca-dev/parca-agent:v0.48.0 + image: ghcr.io/parca-dev/parca-agent:v0.47.1 container_name: texera-parca-agent profiles: [observability-profiles] restart: always From 2f467ace1902a5dde3f7c2eb0308dbf8e9e18cc6 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 26 Jun 2026 20:45:17 -0700 Subject: [PATCH 21/26] fixed spec submitted as bytes --- .../observability/LogSanitizerSpec.scala | Bin 4667 -> 4687 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/common/config/src/test/scala/org/apache/texera/observability/LogSanitizerSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/LogSanitizerSpec.scala index c5957b65e4b163a89ae7681b69a223339f568d9e..ba6ebc37b80d9fcc5a8c6fa7cf284c58e18b5a68 100644 GIT binary patch delta 69 zcmdn3a$aS_0#@yqQUf6H3-tjr%$;0)VoE|Boj^Rpq{Ot6)FKemJS`BO{>)S~*7)SSs5SY$RoXKi3&e$=pRa~8)_ FP5`qJ5^?|l From 8deddfdb0def592c070a41672e0b8e6c90730afa Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Sat, 27 Jun 2026 11:18:16 -0700 Subject: [PATCH 22/26] fix(comments): remove excess details and old information reduced the comments to include less design details and information not needed in the codebase. --- .../texera/observability/LogSanitizer.scala | 60 ++------ .../texera/observability/OtelInit.scala | 136 +++++------------- .../observability/TexeraOtelLogAppender.scala | 46 ++---- .../texera/observability/OtelInitSpec.scala | 14 +- 4 files changed, 66 insertions(+), 190 deletions(-) diff --git a/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala b/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala index cab34fcf58b..f73fcdf88bc 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/LogSanitizer.scala @@ -22,76 +22,46 @@ package org.apache.texera.observability import scala.jdk.CollectionConverters._ /** - * Pure functions that sanitize log records before they leave the - * process via the OTel logs bridge. Lives in its own object so the - * security-critical behaviour can be unit-tested without a Logback - * fixture. - * - * Three invariants: - * 1. No control characters in the body (prevents log forging via - * CR/LF injection in user-supplied strings). - * 2. No oversized bodies (a 1 GiB log line must never reach the - * exporter). - * 3. No secrets in plain text (Bearer tokens, password=, AWS keys). - * - * Plus an MDC allowlist so accidental MDC pollution from a downstream - * library cannot leak unintended fields into the exporter. + * Pure functions that sanitize log bodies and MDC before export: + * strip control characters, redact secrets, cap body size, and + * filter MDC to an allowlist. */ object LogSanitizer { - /** Per-record body cap. The OTel SDK and OTLP have higher limits, - * but 16 KiB is plenty for a useful log line and protects the - * collector from a runaway log. - */ + /** Per-record body byte cap. */ val MaxBodyBytes: Int = 16 * 1024 - /** Suffix appended to truncated bodies. Chosen to be visually - * obvious in a UI but short enough not to dominate the cap. - */ + /** Suffix appended to truncated bodies. */ val TruncatedMarker: String = "...[truncated]" - /** C0 control characters except TAB (0x09). Stripping CR/LF here - * prevents log forging via newline injection in user-supplied - * message bodies. DEL (0x7F) included for the same reason. - */ + /** C0 control characters except TAB (0x09), plus DEL (0x7F). */ private val C0ControlRegex = "[\\x00-\\x08\\x0A-\\x1F\\x7F]".r - /** Secret patterns. Order is significant: most specific first so a - * partial match doesn't shadow a tighter pattern. - */ + /** Secret patterns, redacted from bodies. Most specific first. */ private val SecretPatterns: Seq[scala.util.matching.Regex] = Seq( - // Authorization: Bearer — bearer token in header form. + // Bearer token """(?i)Bearer\s+[A-Za-z0-9._\-/+=]{8,}""".r, - // password=…, password: … — generic credential keyvalue. + // password=... or password: ... """(?i)password\s*[=:]\s*[^\s,;"']+""".r, - // AWS access key ID (canonical AKIA…16-char format). + // AWS access key ID """AKIA[0-9A-Z]{16}""".r, - // AWS secret access key, when explicitly labelled. + // labelled AWS secret access key """(?i)aws_secret_access_key\s*[=:]\s*[A-Za-z0-9/+=]{20,}""".r ) - /** MDC keys we will forward to OTel log attributes. Anything else - * is dropped — additions require a code change + reviewer - * acknowledgement of the privacy implications. - */ + /** MDC keys forwarded to OTel log attributes; others are dropped. */ val AllowedMdcKeys: Set[String] = Set( "trace_id", "span_id", "texera.user.id", "texera.workflow.id", "texera.execution.id", - // Computing-unit id identifies the dev process / k8s pod that - // emitted the record. Required for the dashboard's CU-scoped - // log filter — without this key in the allowlist, the OTel - // appender silently strips it and the CU filter matches nothing. "texera.computing_unit.id", "texera.project.id", "texera.operator.id" ) - /** Apply all three body-side transformations. Idempotent — running - * sanitize on already-sanitized output is a no-op. - */ + /** Strip control chars, redact secrets, then truncate. Idempotent. */ def sanitize(body: String): String = { if (body == null || body.isEmpty) return "" val stripped = C0ControlRegex.replaceAllIn(body, "") @@ -101,9 +71,7 @@ object LogSanitizer { truncate(scrubbed) } - /** Truncate to MaxBodyBytes, appending the marker if cut. Public so - * callers building a body outside `sanitize` can enforce the cap. - */ + /** Truncate to MaxBodyBytes, appending the marker if cut. */ def truncate(body: String): String = { if (body.length <= MaxBodyBytes) body else body.substring(0, MaxBodyBytes - TruncatedMarker.length) + TruncatedMarker diff --git a/common/config/src/main/scala/org/apache/texera/observability/OtelInit.scala b/common/config/src/main/scala/org/apache/texera/observability/OtelInit.scala index d7c803fc79e..5302362bc3a 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/OtelInit.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/OtelInit.scala @@ -41,32 +41,15 @@ import scala.util.{Failure, Success, Try} /** * Bootstraps the OpenTelemetry SDK for a Texera service. * - * Design notes: - * - Default-disabled. Sets up nothing unless `OTEL_SDK_DISABLED=false`. - * - We deliberately do not use the autoconfigure SPI: the security model - * requires endpoint + resource-attribute filtering to happen BEFORE - * any exporter is constructed. Autoconfigure would parse env vars - * behind our back. - * - Validation is a single pure function so it can be unit-tested - * without spinning the SDK. - * - On any validation failure we log one WARN and return None. We - * do NOT throw — observability is opt-in plumbing; misconfiguration - * must never crash the service. - * - This is the only place in Texera that reads `OTEL_*` environment - * variables. Other modules consume the returned `OpenTelemetry` - * instance directly. + * Enabled by default; set OTEL_SDK_DISABLED=true to turn it off. Reads + * OTEL_* env vars, validates the endpoint against an allowlist, builds + * tracer/log/metric providers, and attaches a Logback appender. + * Returns None when disabled or misconfigured; never throws. */ object OtelInit extends LazyLogging { - /** Resource attribute keys we accept from OTEL_RESOURCE_ATTRIBUTES. - * Resource attrs ride on every record this JVM emits (logs, - * metrics, traces) — so for a per-CU JVM (ComputingUnitMaster / - * ComputingUnitWorker) setting `texera.computing_unit.id=N` at - * boot is enough to tag every record without per-request MDC - * plumbing. Workflow/execution ids vary per task and still need - * MDC at the message boundary, but exposing them in the allowlist - * lets test harnesses + future per-task code populate them via - * the same mechanism. + /** Resource attribute keys accepted from OTEL_RESOURCE_ATTRIBUTES; + * applied to every record this JVM emits. Others are dropped. */ private[observability] val AllowedResourceKeys: Set[String] = Set( "service.name", @@ -88,35 +71,25 @@ object OtelInit extends LazyLogging { "[::1]" ) - /** Default endpoint when SDK is enabled but no endpoint set explicitly. - * Uses 127.0.0.1 (not "localhost") so a natively-run service reaches the - * IPv4-only collector port published by docker-compose — on dual-stack - * hosts "localhost" resolves to ::1 first and the OTLP export silently - * fails. Inside docker the endpoint is overridden to otel-collector:4317. + /** Default endpoint. 127.0.0.1 (not "localhost") to force IPv4 so a + * natively-run service reaches the collector on dual-stack hosts. */ private val DefaultEndpoint = "http://127.0.0.1:4317" - /** Metric export interval bounds. Values outside this range get - * clamped to the default with a one-shot WARN. The lower bound - * prevents an attacker tipping the exporter into busy-loop mode; - * the upper bound keeps metrics useful for human operators. + /** Metric export interval bounds; out-of-range values clamp to the + * default (see clampIntervalMs). */ private[observability] val MinMetricIntervalMs: Long = 1000L private[observability] val MaxMetricIntervalMs: Long = 10L * 60L * 1000L private[observability] val DefaultMetricIntervalMs: Long = 60L * 1000L - // Idempotency guard. The SDK installs global handlers and a shutdown - // hook; calling init() repeatedly must be a no-op after the first call. + // Idempotency guard: init() is a no-op after the first call. @volatile private var initialized: Option[OpenTelemetry] = None /** - * Initialize the SDK for the given service name. - * Returns Some(sdk) on success, None on disabled / invalid config. - * - * Side effect when enabled: attaches a [[TexeraOtelLogAppender]] to - * the Logback ROOT logger so application logs are mirrored to the - * OTel collector, with the security guards in [[LogSanitizer]] - * applied to every record. + * Initialize the SDK for the given service name. Returns Some on + * success, None when disabled or misconfigured. When enabled, also + * attaches a [[TexeraOtelLogAppender]] to the Logback ROOT logger. */ def init(serviceName: String): Option[OpenTelemetry] = synchronized { @@ -131,13 +104,9 @@ object OtelInit extends LazyLogging { metricExporterFactory = endpoint => Some(buildOtlpMetricExporter(endpoint)), logbackAttacher = LogbackBinder.attach ) - // Register globally so [[TexeraTracer]] and any other OTel-aware - // code can call ``GlobalOpenTelemetry.getTracer(...)`` without - // threading the SDK through every callsite. set() throws on a - // second call within the same JVM — our outer ``initialized`` - // guard makes that unreachable, but wrap defensively. The test - // path deliberately skips this so multiple isolated SDKs can be - // built within one JVM. + // Register globally so OTel-aware code can use GlobalOpenTelemetry + // without threading the SDK through callsites. set() throws on a + // second call; wrap defensively. result.foreach { sdk => Try(GlobalOpenTelemetry.set(sdk)).failed.foreach { t => logger.warn( @@ -149,11 +118,8 @@ object OtelInit extends LazyLogging { } /** - * Test-only entry point. Allows the test to inject an env-var map - * and a span exporter so the SDK does not attempt a real network - * connection. The Logback appender is NOT attached in tests — - * appender tests construct it directly with an in-memory log - * exporter. + * Test-only entry point: injects an env-var map and exporters so the + * SDK makes no network connection. Does not attach the Logback appender. */ private[observability] def initForTest( serviceName: String, @@ -197,14 +163,8 @@ object OtelInit extends LazyLogging { ): Option[OpenTelemetry] = { if (initialized.isDefined) return initialized - // Default to ENABLED so an `sbt run` of any Texera service emits - // telemetry without per-JVM env-var configuration. Operators who - // need to silence telemetry (CI, embedded-tests, security-locked - // deployments) set OTEL_SDK_DISABLED=true explicitly. If the - // configured endpoint isn't reachable, the OTel SDK's - // BatchProcessor logs a single error and drops records — it - // does NOT crash the host service, so a missing collector at - // dev time is a quiet no-op rather than a startup failure. + // Enabled by default; OTEL_SDK_DISABLED=true opts out. An + // unreachable endpoint drops records without crashing the service. val disabled = envProvider("OTEL_SDK_DISABLED").getOrElse("false") if (disabled.equalsIgnoreCase("true")) { logger.info( @@ -221,8 +181,7 @@ object OtelInit extends LazyLogging { validateEndpoint(endpoint, allowedHosts) match { case Left(reason) => - // One WARN, no further detail (endpoint is not echoed beyond what - // the operator already knows). No spans will be emitted. + // One WARN; no telemetry is emitted. logger.warn( s"OpenTelemetry SDK disabled: invalid OTEL_EXPORTER_OTLP_ENDPOINT — $reason. " + "Set TEXERA_OTEL_ALLOWED_HOSTS to extend the allowlist." @@ -243,8 +202,7 @@ object OtelInit extends LazyLogging { val sdkBuilder = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider) - // Logger provider is optional — controlled by the factory. Skipped - // in tests so the appender path can be exercised independently. + // Logger provider is optional; the factory returns None in tests. val loggerProviderOpt = logExporterFactory(endpoint).map { logExporter => val lp = SdkLoggerProvider .builder() @@ -255,11 +213,7 @@ object OtelInit extends LazyLogging { lp } - // Meter provider is optional too. Export interval is clamped to - // [MinMetricIntervalMs, MaxMetricIntervalMs]; an out-of-range - // value gets reset to the default with one WARN — keeps an - // attacker from coaxing the reader into busy-loop mode by - // setting OTEL_METRIC_EXPORT_INTERVAL to a tiny value. + // Meter provider is optional too; export interval is clamped. val intervalMs = clampIntervalMs(envProvider("OTEL_METRIC_EXPORT_INTERVAL")) val meterProviderOpt = metricExporterFactory(endpoint).map { metricExporter => val reader = PeriodicMetricReader @@ -277,22 +231,17 @@ object OtelInit extends LazyLogging { val sdk = sdkBuilder.build() - // One startup span. Carries only service.name (no env, host, or - // version data beyond the allowlisted resource attrs). + // One startup span carrying only service.name. val span = sdk.getTracer("texera.bootstrap").spanBuilder("service.start").startSpan() Try(span.setAttribute("service.name", serviceName)) span.end() - // Wire the Logback appender so subsequent application logs flow to - // the collector with sanitisation applied. Failure here must never - // crash the service — observability is opt-in. + // Wire the Logback appender; failure here must not crash the service. Try(logbackAttacher(serviceName, sdk)).failed.foreach { t => logger.warn(s"Failed to attach OTel Logback appender (logs not exported): ${t.getMessage}") } - // Make sure providers flush on shutdown. We add the hook only after - // the SDK has been fully built so a panic during init doesn't leave - // a dangling hook pointing at a half-constructed provider. + // Flush providers on shutdown. Added after the SDK is fully built. Runtime.getRuntime.addShutdownHook( new Thread( () => { @@ -311,9 +260,8 @@ object OtelInit extends LazyLogging { } /** - * Validate that the endpoint is parseable, uses an allowlisted scheme, - * and resolves to an allowlisted host. Pure function — safe to test - * without standing up the SDK. + * Validate the endpoint is parseable and uses an allowlisted scheme + * and host. Pure function. */ private[observability] def validateEndpoint( endpoint: String, @@ -344,9 +292,8 @@ object OtelInit extends LazyLogging { } /** - * Build a Resource from the service name plus the allowlisted subset - * of OTEL_RESOURCE_ATTRIBUTES. Unknown keys are dropped silently; - * service.name from env is ignored in favour of the argument. + * Build a Resource from the service name and the allowlisted subset + * of OTEL_RESOURCE_ATTRIBUTES. service.name from env is ignored. */ private[observability] def buildResource(serviceName: String, rawAttrs: String): Resource = { val builder = Attributes.builder() @@ -355,7 +302,7 @@ object OtelInit extends LazyLogging { parseAttrs(rawAttrs).foreach { case (key, value) if AllowedResourceKeys.contains(key) && key != "service.name" => builder.put(AttributeKey.stringKey(key), value) - case _ => // dropped — not in allowlist or overrides service.name + case _ => // not in allowlist, or overrides service.name } Resource.create(builder.build()) @@ -387,10 +334,8 @@ object OtelInit extends LazyLogging { OtlpGrpcMetricExporter.builder().setEndpoint(endpoint).build() /** - * Parse and clamp OTEL_METRIC_EXPORT_INTERVAL (milliseconds). - * Out-of-range or unparseable input falls back to the default and - * emits a single WARN. Pure-ish — easy to test without standing up - * the meter SDK. + * Parse and clamp OTEL_METRIC_EXPORT_INTERVAL (ms). Out-of-range or + * unparseable input falls back to the default with one WARN. */ private[observability] def clampIntervalMs(raw: Option[String]): Long = { raw match { @@ -417,17 +362,14 @@ object OtelInit extends LazyLogging { } /** - * Hides the Logback attach step behind a small object so [[OtelInit]] - * doesn't import Logback types directly (keeps the SDK init testable - * without a Logback dependency on the classpath in test runs that - * inject a mock attacher). + * Isolates the Logback attach step so [[OtelInit]] does not import + * Logback types directly, keeping SDK init testable with a mock attacher. */ private[observability] object LogbackBinder extends LazyLogging { - /** Attempts to find the Logback ROOT logger, attach a fresh - * [[TexeraOtelLogAppender]] bound to `otel`, and start it. If - * Logback is not the active SLF4J binding (or for any other - * classpath issue), emits one WARN and returns — never throws. + /** Attach a [[TexeraOtelLogAppender]] bound to `otel` to the Logback + * ROOT logger. Emits one WARN and returns if Logback is not the + * active SLF4J binding. */ def attach(serviceName: String, otel: OpenTelemetry): Unit = { val factory = org.slf4j.LoggerFactory.getILoggerFactory diff --git a/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala b/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala index f41e88dd569..5105031ad92 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala @@ -31,25 +31,13 @@ import io.opentelemetry.context.Context import java.util.concurrent.TimeUnit /** - * Logback appender that forwards every event through [[LogSanitizer]] - * before emitting it as an OTel LogRecord. - * - * Lifecycle: - * - Construct with no args (Logback / programmatic instantiation). - * - Call [[bind]] once with the active [[OpenTelemetry]] instance - * (done by [[OtelInit]] after the SDK is built). Until then, - * [[append]] is a silent no-op — log events keep flowing to - * stdout/file unimpeded. - * - Stopping the appender unbinds; subsequent events drop. - * - * This is intentionally a thin shim. All security-critical logic - * lives in [[LogSanitizer]] so it can be tested without a Logback - * fixture. + * Logback appender that sanitizes each event via [[LogSanitizer]] and + * emits it as an OTel LogRecord. [[append]] is a no-op until [[bind]] + * is called and after [[stop]]. */ class TexeraOtelLogAppender extends UnsynchronizedAppenderBase[ILoggingEvent] { - // @volatile so a late [[bind]] is visible to appender threads - // without taking a lock on the hot path. + // @volatile so a late bind() is visible to appender threads. @volatile private var otelLogger: Option[Logger] = None def bind(otel: OpenTelemetry): Unit = { @@ -63,13 +51,12 @@ class TexeraOtelLogAppender extends UnsynchronizedAppenderBase[ILoggingEvent] { override def append(event: ILoggingEvent): Unit = { otelLogger match { - case None => () // disabled or not yet wired + case None => () // not bound case Some(logger) => try { emit(logger, event) } catch { - // Logback's addStatus contract: errors from inside an - // appender must not throw out into the calling thread. + // An appender must not throw into the calling thread. case t: Throwable => addError("OTel log emission failed", t) } @@ -77,17 +64,11 @@ class TexeraOtelLogAppender extends UnsynchronizedAppenderBase[ILoggingEvent] { } private def emit(logger: Logger, event: ILoggingEvent): Unit = { - // Append the throwable's full stack trace to the body when one is - // attached. Without this, Dropwizard's LoggingExceptionMapper logs - // "Error handling a request: " and the exception itself never - // reaches the observability backend — making 500s impossible to - // diagnose from the dashboard. ThrowableProxyUtil emits a Logback- - // formatted trace that fits inside a single log record. + // Append the stack trace to the body when a throwable is attached. val baseBody = LogSanitizer.sanitize(event.getFormattedMessage) val body = Option(event.getThrowableProxy) match { case Some(proxy) => - // Trusted JVM frames: skip the C0 strip so newlines survive, - // but still cap length (the OTel SDK does not bound the body). + // Skip the C0 strip so trace newlines survive, but still cap. LogSanitizer.truncate(baseBody + "\n" + formatThrowable(proxy)) case None => baseBody } @@ -98,8 +79,7 @@ class TexeraOtelLogAppender extends UnsynchronizedAppenderBase[ILoggingEvent] { .setSeverityText(event.getLevel.toString) .setTimestamp(event.getTimeStamp, TimeUnit.MILLISECONDS) - // MDC subset: typed AttributeKeys only, so no string injection - // path exists for downstream consumers. + // Allowlisted MDC keys as typed attributes. LogSanitizer.filterMdc(event.getMDCPropertyMap).foreach { case (k, v) => builder.setAttribute(AttributeKey.stringKey(k), v) } @@ -107,8 +87,7 @@ class TexeraOtelLogAppender extends UnsynchronizedAppenderBase[ILoggingEvent] { builder.setAttribute(AttributeKey.stringKey("logger.name"), event.getLoggerName) builder.setAttribute(AttributeKey.stringKey("thread.name"), event.getThreadName) - // Attach the current trace context so the SDK populates trace_id / - // span_id on the LogRecord automatically when a span is active. + // Attach trace context so the SDK sets trace_id / span_id. val span = Span.current() if (span.getSpanContext.isValid) { builder.setContext(Context.current()) @@ -117,10 +96,7 @@ class TexeraOtelLogAppender extends UnsynchronizedAppenderBase[ILoggingEvent] { builder.emit() } - /** Pretty-print a Logback throwable proxy. Matches what Logback's - * default pattern layout would produce for `%ex` — class name, - * message, full stack frames, then walks the cause chain. - */ + /** Format a throwable proxy as a Logback-style stack trace. */ private def formatThrowable(proxy: IThrowableProxy): String = ThrowableProxyUtil.asString(proxy) diff --git a/common/config/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala index 45bbac119af..4124c16e7f4 100644 --- a/common/config/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala +++ b/common/config/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala @@ -134,11 +134,6 @@ class OtelInitSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { } it should "carry the texera.* resource attrs so a CU JVM auto-tags every emitted record" in { - // Setting the CU id at JVM boot is the only sane place to attach - // it for ComputingUnitMaster / ComputingUnitWorker — those JVMs - // are CU-scoped by deployment, and there is no HTTP request to - // hang an MDC value off. The dashboard's CU filter relies on - // this key being present on every record. val r = OtelInit.buildResource( "texera-computing-unit-master", "texera.computing_unit.id=8,texera.workflow.id=441,texera.execution.id=1234" @@ -175,16 +170,11 @@ class OtelInitSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { } it should "initialize by default (no OTEL_SDK_DISABLED set) so `sbt run` services emit without per-JVM config" in { - // The previous default was `true` (opt-in), which forced every - // service Run Configuration to set OTEL_SDK_DISABLED=false - // explicitly. New default is `false` so a fresh sbt run is - // immediately tagged in the dashboard. Operators who need to - // silence telemetry still set the env var explicitly. val exporter = InMemorySpanExporter.create() val result = OtelInit.initForTest( "svc", Map( - // OTEL_SDK_DISABLED deliberately omitted — defaults to false. + // OTEL_SDK_DISABLED omitted; defaults to false. "OTEL_EXPORTER_OTLP_ENDPOINT" -> "http://localhost:4317" ), exporter @@ -204,7 +194,7 @@ class OtelInitSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { ) result.isDefined shouldBe true - // BatchSpanProcessor is async — flush before reading. + // BatchSpanProcessor is async; flush before reading. result.get .asInstanceOf[io.opentelemetry.sdk.OpenTelemetrySdk] .getSdkTracerProvider From 3d7ae02879641771b8b8acc7347aed41328f5f75 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Sat, 11 Jul 2026 15:17:37 -0700 Subject: [PATCH 23/26] wiring up observability into the helm and docker install --- bin/k8s/Chart.lock | 21 ++++ bin/k8s/install.sh | 56 +++++++++ bin/k8s/templates/base/_helpers.tpl | 20 ++++ .../access-control-service-deployment.yaml | 1 + .../config-service-deployment.yaml | 1 + .../file-service/file-service-deployment.yaml | 3 +- .../templates/base/observability/jaeger.yaml | 81 +++++++++++++ .../otel-collector-configmap.yaml | 108 +++++++++++++++++ .../base/observability/otel-collector.yaml | 95 +++++++++++++++ .../base/observability/victorialogs.yaml | 111 +++++++++++++++++ .../base/observability/victoriametrics.yaml | 112 ++++++++++++++++++ .../base/webserver/webserver-deployment.yaml | 3 +- ...workflow-compiling-service-deployment.yaml | 3 +- ...low-computing-unit-manager-deployment.yaml | 3 +- bin/k8s/values.yaml | 62 ++++++++++ bin/single-node/.env | 11 ++ 16 files changed, 687 insertions(+), 4 deletions(-) create mode 100644 bin/k8s/Chart.lock create mode 100755 bin/k8s/install.sh create mode 100644 bin/k8s/templates/base/observability/jaeger.yaml create mode 100644 bin/k8s/templates/base/observability/otel-collector-configmap.yaml create mode 100644 bin/k8s/templates/base/observability/otel-collector.yaml create mode 100644 bin/k8s/templates/base/observability/victorialogs.yaml create mode 100644 bin/k8s/templates/base/observability/victoriametrics.yaml diff --git a/bin/k8s/Chart.lock b/bin/k8s/Chart.lock new file mode 100644 index 00000000000..af90dd0b29b --- /dev/null +++ b/bin/k8s/Chart.lock @@ -0,0 +1,21 @@ +dependencies: +- name: postgresql + repository: https://charts.bitnami.com/bitnami + version: 16.5.6 +- name: minio + repository: https://charts.bitnami.com/bitnami + version: 15.0.7 +- name: lakefs + repository: https://charts.lakefs.io + version: 1.8.1 +- name: gateway-helm + repository: oci://docker.io/envoyproxy + version: 1.6.3 +- name: lakekeeper + repository: https://lakekeeper.github.io/lakekeeper-charts/ + version: 0.9.0 +- name: metrics-server + repository: https://kubernetes-sigs.github.io/metrics-server/ + version: 3.12.2 +digest: sha256:031184824e3bd3e03883a3debe64459f462610e4f5bd7c147f03fd05333b169e +generated: "2026-07-11T11:09:44.800500493-07:00" diff --git a/bin/k8s/install.sh b/bin/k8s/install.sh new file mode 100755 index 00000000000..d1ed9b55e6e --- /dev/null +++ b/bin/k8s/install.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# One-command install/upgrade of the Texera Helm chart. +# +# `helm install` on its own fails on a fresh checkout because the subchart +# dependencies (postgresql, minio, lakefs, envoy-gateway, lakekeeper, +# metrics-server) are declared in Chart.yaml but not vendored into charts/. +# This wrapper fetches them first (idempotent), then upgrades-or-installs. +# +# Usage (run from anywhere): +# bin/k8s/install.sh # app only +# bin/k8s/install.sh --set observability.enabled=true # app + observability +# RELEASE=texera NAMESPACE=texera bin/k8s/install.sh --set observability.enabled=true +# +# Any extra args are forwarded verbatim to `helm upgrade --install`, so +# --set / -f values.override.yaml / --namespace etc. all work. + +set -euo pipefail + +CHART_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RELEASE="${RELEASE:-texera}" +NAMESPACE="${NAMESPACE:-texera}" + +command -v helm >/dev/null 2>&1 || { echo "helm not found on PATH" >&2; exit 1; } + +# Fetch/refresh subcharts into charts/. `dependency build` uses Chart.lock when +# present (reproducible); it falls back to `dependency update` if there is no +# lock yet. Both are safe to re-run. +echo "==> Resolving chart dependencies" +if [ -f "${CHART_DIR}/Chart.lock" ]; then + helm dependency build "${CHART_DIR}" +else + helm dependency update "${CHART_DIR}" +fi + +echo "==> helm upgrade --install ${RELEASE} (namespace: ${NAMESPACE})" +exec helm upgrade --install "${RELEASE}" "${CHART_DIR}" \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + "$@" diff --git a/bin/k8s/templates/base/_helpers.tpl b/bin/k8s/templates/base/_helpers.tpl index e044b7285a8..e90e98cabee 100644 --- a/bin/k8s/templates/base/_helpers.tpl +++ b/bin/k8s/templates/base/_helpers.tpl @@ -54,3 +54,23 @@ services fall back to the in-cluster MinIO Service and its auto-generated {{- define "texera.s3.secretAccessKeyKey" -}} {{- if .Values.storage.s3.endpoint -}}secret-access-key{{- else -}}root-password{{- end -}} {{- end -}} + +{{/* +Observability emission env for a Scala service pod. + +Renders OTEL_EXPORTER_OTLP_ENDPOINT (pointing at the in-cluster OTel +Collector Service) plus TEXERA_OTEL_ALLOWED_HOSTS. OtelInit validates the +endpoint host against an allowlist whose default is localhost only, so the +collector's Service name must be added explicitly or the SDK rejects it and +emits nothing. Renders nothing unless both the observability stack and the +collector are enabled, so the default install is unchanged. Include inside a +container's `env:` list, e.g. `{{- include "texera.observability.env" . | nindent 12 }}`. +*/}} +{{- define "texera.observability.env" -}} +{{- if and .Values.observability.enabled .Values.observability.collector.enabled }} +- name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://{{ .Release.Name }}-otel-collector:{{ .Values.observability.collector.grpcPort }}" +- name: TEXERA_OTEL_ALLOWED_HOSTS + value: "{{ .Release.Name }}-otel-collector" +{{- end }} +{{- end -}} diff --git a/bin/k8s/templates/base/access-control-service/access-control-service-deployment.yaml b/bin/k8s/templates/base/access-control-service/access-control-service-deployment.yaml index 99713e70713..4056d637638 100644 --- a/bin/k8s/templates/base/access-control-service/access-control-service-deployment.yaml +++ b/bin/k8s/templates/base/access-control-service/access-control-service-deployment.yaml @@ -64,6 +64,7 @@ spec: - name: {{ .name }} value: "{{ .value }}" {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} livenessProbe: httpGet: path: /api/healthcheck diff --git a/bin/k8s/templates/base/config-service/config-service-deployment.yaml b/bin/k8s/templates/base/config-service/config-service-deployment.yaml index f0748785c3a..3817be8a68a 100644 --- a/bin/k8s/templates/base/config-service/config-service-deployment.yaml +++ b/bin/k8s/templates/base/config-service/config-service-deployment.yaml @@ -51,6 +51,7 @@ spec: - name: {{ .name }} value: "{{ .value }}" {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} livenessProbe: httpGet: path: /api/healthcheck diff --git a/bin/k8s/templates/base/file-service/file-service-deployment.yaml b/bin/k8s/templates/base/file-service/file-service-deployment.yaml index 6a9190bc6cf..37568ef2254 100644 --- a/bin/k8s/templates/base/file-service/file-service-deployment.yaml +++ b/bin/k8s/templates/base/file-service/file-service-deployment.yaml @@ -79,4 +79,5 @@ spec: {{- range .Values.texeraEnvVars }} - name: {{ .name }} value: "{{ .value }}" - {{- end }} \ No newline at end of file + {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} \ No newline at end of file diff --git a/bin/k8s/templates/base/observability/jaeger.yaml b/bin/k8s/templates/base/observability/jaeger.yaml new file mode 100644 index 00000000000..2821a977736 --- /dev/null +++ b/bin/k8s/templates/base/observability/jaeger.yaml @@ -0,0 +1,81 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.traces.enabled }} +# Jaeger v2: traces backend + UI, k8s counterpart of the docker-compose +# jaeger service. v2 accepts OTLP natively (the collector exports here on +# otlpPort). In-memory storage: a restart wipes traces, same as the +# single-node compose. Swap in a persistent storage backend for +# multi-node / production use. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-jaeger + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-jaeger +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-jaeger + template: + metadata: + labels: + app: {{ .Release.Name }}-jaeger + spec: + securityContext: + runAsNonRoot: true + containers: + - name: jaeger + image: {{ .Values.observability.traces.image }} + imagePullPolicy: {{ .Values.texeraImages.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + ports: + - name: query + containerPort: {{ .Values.observability.traces.queryPort }} + - name: otlp-grpc + containerPort: {{ .Values.observability.traces.otlpPort }} + resources: + {{- toYaml .Values.observability.traces.resources | nindent 12 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-jaeger + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-jaeger +spec: + type: ClusterIP + selector: + app: {{ .Release.Name }}-jaeger + ports: + - name: query + protocol: TCP + port: {{ .Values.observability.traces.queryPort }} + targetPort: {{ .Values.observability.traces.queryPort }} + - name: otlp-grpc + protocol: TCP + port: {{ .Values.observability.traces.otlpPort }} + targetPort: {{ .Values.observability.traces.otlpPort }} +{{- end }} diff --git a/bin/k8s/templates/base/observability/otel-collector-configmap.yaml b/bin/k8s/templates/base/observability/otel-collector-configmap.yaml new file mode 100644 index 00000000000..4a12dc8a791 --- /dev/null +++ b/bin/k8s/templates/base/observability/otel-collector-configmap.yaml @@ -0,0 +1,108 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.collector.enabled }} +# OpenTelemetry Collector config, k8s counterpart of +# bin/observability/otel-collector/config.yaml. Same pipeline topology; +# exporters point at the in-cluster backend Services instead of the +# docker-compose service names. Only the pipelines for enabled signals +# are rendered, so disabling a signal drops its exporter cleanly. +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-otel-collector-config + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-otel-collector +data: + config.yaml: | + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:{{ .Values.observability.collector.grpcPort }} + max_recv_msg_size_mib: 4 + http: + endpoint: 0.0.0.0:{{ .Values.observability.collector.httpPort }} + + processors: + batch: + send_batch_size: 1024 + timeout: 5s + memory_limiter: + check_interval: 1s + limit_mib: 512 + spike_limit_mib: 128 + + exporters: + {{- if .Values.observability.logs.enabled }} + otlphttp/victorialogs: + endpoint: http://{{ .Release.Name }}-victorialogs:{{ .Values.observability.logs.port }}/insert/opentelemetry + compression: gzip + timeout: 10s + tls: + insecure: true + {{- end }} + {{- if .Values.observability.metrics.enabled }} + prometheusremotewrite: + endpoint: http://{{ .Release.Name }}-victoriametrics:{{ .Values.observability.metrics.port }}/api/v1/write + tls: + insecure: true + remote_write_queue: + queue_size: 1000 + num_consumers: 4 + {{- end }} + {{- if .Values.observability.traces.enabled }} + otlp/jaeger: + endpoint: {{ .Release.Name }}-jaeger:{{ .Values.observability.traces.otlpPort }} + tls: + insecure: true + {{- end }} + + service: + pipelines: + {{- if .Values.observability.logs.enabled }} + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/victorialogs] + {{- end }} + {{- if .Values.observability.metrics.enabled }} + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheusremotewrite] + {{- end }} + {{- if .Values.observability.traces.enabled }} + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp/jaeger] + {{- end }} + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 127.0.0.1 + port: 8888 + logs: + level: info +{{- end }} diff --git a/bin/k8s/templates/base/observability/otel-collector.yaml b/bin/k8s/templates/base/observability/otel-collector.yaml new file mode 100644 index 00000000000..89c537317d8 --- /dev/null +++ b/bin/k8s/templates/base/observability/otel-collector.yaml @@ -0,0 +1,95 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.collector.enabled }} +# OpenTelemetry Collector: the single OTLP ingress for all signals, +# k8s counterpart of the docker-compose otel-collector service. The +# receiver Ports are exposed only on the ClusterIP Service (no Ingress), +# mirroring the loopback-only posture of the compose deployment. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-otel-collector + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-otel-collector +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-otel-collector + template: + metadata: + labels: + app: {{ .Release.Name }}-otel-collector + annotations: + # Roll the pod when the rendered config changes. + checksum/config: {{ include (print $.Template.BasePath "/base/observability/otel-collector-configmap.yaml") . | sha256sum }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 10001 + fsGroup: 10001 + containers: + - name: otel-collector + image: {{ .Values.observability.collector.image }} + imagePullPolicy: {{ .Values.texeraImages.pullPolicy }} + args: ["--config=/etc/otelcol-contrib/config.yaml"] + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + ports: + - name: otlp-grpc + containerPort: {{ .Values.observability.collector.grpcPort }} + - name: otlp-http + containerPort: {{ .Values.observability.collector.httpPort }} + volumeMounts: + - name: config + mountPath: /etc/otelcol-contrib + readOnly: true + resources: + {{- toYaml .Values.observability.collector.resources | nindent 12 }} + volumes: + - name: config + configMap: + name: {{ .Release.Name }}-otel-collector-config +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-otel-collector + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-otel-collector +spec: + type: ClusterIP + selector: + app: {{ .Release.Name }}-otel-collector + ports: + - name: otlp-grpc + protocol: TCP + port: {{ .Values.observability.collector.grpcPort }} + targetPort: {{ .Values.observability.collector.grpcPort }} + - name: otlp-http + protocol: TCP + port: {{ .Values.observability.collector.httpPort }} + targetPort: {{ .Values.observability.collector.httpPort }} +{{- end }} diff --git a/bin/k8s/templates/base/observability/victorialogs.yaml b/bin/k8s/templates/base/observability/victorialogs.yaml new file mode 100644 index 00000000000..6896ffd7b2e --- /dev/null +++ b/bin/k8s/templates/base/observability/victorialogs.yaml @@ -0,0 +1,111 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.logs.enabled }} +# VictoriaLogs: log store, k8s counterpart of the docker-compose +# victorialogs service. OTLP ingest + LogsQL query API on one port. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Release.Name }}-victorialogs-data + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-victorialogs +spec: + accessModes: ["ReadWriteOnce"] + {{- with .Values.observability.logs.storageClass }} + storageClassName: {{ . }} + {{- end }} + resources: + requests: + storage: {{ .Values.observability.logs.storage }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-victorialogs + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-victorialogs +spec: + replicas: 1 + # Single writer bound to one RWO volume; never run two at once. + strategy: + type: Recreate + selector: + matchLabels: + app: {{ .Release.Name }}-victorialogs + template: + metadata: + labels: + app: {{ .Release.Name }}-victorialogs + spec: + securityContext: + runAsNonRoot: true + runAsUser: 10002 + fsGroup: 10002 + containers: + - name: victorialogs + image: {{ .Values.observability.logs.image }} + imagePullPolicy: {{ .Values.texeraImages.pullPolicy }} + args: + - "-storageDataPath=/data" + - "-retentionPeriod={{ .Values.observability.logs.retentionPeriod }}" + - "-httpListenAddr=:{{ .Values.observability.logs.port }}" + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + ports: + - name: http + containerPort: {{ .Values.observability.logs.port }} + volumeMounts: + - name: data + mountPath: /data + livenessProbe: + httpGet: + path: /health + port: {{ .Values.observability.logs.port }} + initialDelaySeconds: 10 + periodSeconds: 15 + resources: + {{- toYaml .Values.observability.logs.resources | nindent 12 }} + volumes: + - name: data + persistentVolumeClaim: + claimName: {{ .Release.Name }}-victorialogs-data +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-victorialogs + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-victorialogs +spec: + type: ClusterIP + selector: + app: {{ .Release.Name }}-victorialogs + ports: + - name: http + protocol: TCP + port: {{ .Values.observability.logs.port }} + targetPort: {{ .Values.observability.logs.port }} +{{- end }} diff --git a/bin/k8s/templates/base/observability/victoriametrics.yaml b/bin/k8s/templates/base/observability/victoriametrics.yaml new file mode 100644 index 00000000000..9e793de5a80 --- /dev/null +++ b/bin/k8s/templates/base/observability/victoriametrics.yaml @@ -0,0 +1,112 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.metrics.enabled }} +# VictoriaMetrics: metrics store, k8s counterpart of the docker-compose +# victoriametrics service. Accepts Prometheus remote-write from the +# collector; MetricsQL query API on the same port. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Release.Name }}-victoriametrics-data + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-victoriametrics +spec: + accessModes: ["ReadWriteOnce"] + {{- with .Values.observability.metrics.storageClass }} + storageClassName: {{ . }} + {{- end }} + resources: + requests: + storage: {{ .Values.observability.metrics.storage }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-victoriametrics + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-victoriametrics +spec: + replicas: 1 + # Single writer bound to one RWO volume; never run two at once. + strategy: + type: Recreate + selector: + matchLabels: + app: {{ .Release.Name }}-victoriametrics + template: + metadata: + labels: + app: {{ .Release.Name }}-victoriametrics + spec: + securityContext: + runAsNonRoot: true + runAsUser: 10003 + fsGroup: 10003 + containers: + - name: victoriametrics + image: {{ .Values.observability.metrics.image }} + imagePullPolicy: {{ .Values.texeraImages.pullPolicy }} + args: + - "-storageDataPath=/data" + - "-retentionPeriod={{ .Values.observability.metrics.retentionPeriod }}" + - "-httpListenAddr=:{{ .Values.observability.metrics.port }}" + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + ports: + - name: http + containerPort: {{ .Values.observability.metrics.port }} + volumeMounts: + - name: data + mountPath: /data + livenessProbe: + httpGet: + path: /health + port: {{ .Values.observability.metrics.port }} + initialDelaySeconds: 10 + periodSeconds: 15 + resources: + {{- toYaml .Values.observability.metrics.resources | nindent 12 }} + volumes: + - name: data + persistentVolumeClaim: + claimName: {{ .Release.Name }}-victoriametrics-data +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-victoriametrics + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-victoriametrics +spec: + type: ClusterIP + selector: + app: {{ .Release.Name }}-victoriametrics + ports: + - name: http + protocol: TCP + port: {{ .Values.observability.metrics.port }} + targetPort: {{ .Values.observability.metrics.port }} +{{- end }} diff --git a/bin/k8s/templates/base/webserver/webserver-deployment.yaml b/bin/k8s/templates/base/webserver/webserver-deployment.yaml index 983c6269947..ca0fe4a39af 100644 --- a/bin/k8s/templates/base/webserver/webserver-deployment.yaml +++ b/bin/k8s/templates/base/webserver/webserver-deployment.yaml @@ -74,4 +74,5 @@ spec: {{- range .Values.texeraEnvVars }} - name: {{ .name }} value: "{{ .value }}" - {{- end }} \ No newline at end of file + {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} \ No newline at end of file diff --git a/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml b/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml index 50a0a04e1b7..b12bb68266b 100644 --- a/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml +++ b/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml @@ -66,4 +66,5 @@ spec: {{- range .Values.texeraEnvVars }} - name: {{ .name }} value: "{{ .value }}" - {{- end }} \ No newline at end of file + {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} \ No newline at end of file diff --git a/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml b/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml index a9118450412..2057444e5e9 100644 --- a/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml +++ b/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml @@ -123,4 +123,5 @@ spec: {{- range .Values.texeraEnvVars }} - name: {{ .name }} value: "{{ .value }}" - {{- end }} \ No newline at end of file + {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} \ No newline at end of file diff --git a/bin/k8s/values.yaml b/bin/k8s/values.yaml index 64642b517a9..3b4c3ac2592 100644 --- a/bin/k8s/values.yaml +++ b/bin/k8s/values.yaml @@ -408,3 +408,65 @@ envoy-gateway: extensionApis: enableBackend: true enableEnvoyPatchPolicy: true + +# Observability stack (hand-rolled, mirrors bin/observability + the +# single-node docker-compose profiles). Everything is gated behind +# `enabled` (default off) so the standard install is unchanged. When +# enabled, the Scala services emit OTLP to the in-cluster collector, +# which fans logs/metrics/traces to the backends below. All Services are +# ClusterIP only (no Ingress), matching the loopback-only posture of the +# compose deployment. Profiling (Parca) is intentionally not included: it +# needs a privileged eBPF DaemonSet and is a separate change. +observability: + # Master switch for the whole stack. + enabled: false + + # OpenTelemetry Collector: the single OTLP ingress. Required whenever + # any signal below is enabled (it is what the services emit to). + collector: + enabled: true + image: otel/opentelemetry-collector-contrib:0.153.0 + grpcPort: 4317 + httpPort: 4318 + resources: + limits: + memory: 768Mi + cpu: "1" + + # VictoriaLogs: log store (OTLP ingest + LogsQL). + logs: + enabled: true + image: victoriametrics/victoria-logs:v1.50.0 + port: 9428 + retentionPeriod: 30d + storage: 10Gi + storageClass: local-path + resources: + limits: + memory: 1Gi + cpu: "1" + + # VictoriaMetrics: metrics store (Prometheus remote-write + MetricsQL). + metrics: + enabled: true + image: victoriametrics/victoria-metrics:v1.144.0 + port: 8428 + retentionPeriod: 90d + storage: 10Gi + storageClass: local-path + resources: + limits: + memory: 1Gi + cpu: "1" + + # Jaeger v2: traces backend + UI. In-memory storage (restart wipes + # traces); swap in persistent storage for production. + traces: + enabled: true + image: jaegertracing/jaeger:2.18.0 + queryPort: 16686 + otlpPort: 4317 + resources: + limits: + memory: 1Gi + cpu: "1" diff --git a/bin/single-node/.env b/bin/single-node/.env index 8f282ec8fc4..be9b2f88aa1 100644 --- a/bin/single-node/.env +++ b/bin/single-node/.env @@ -130,3 +130,14 @@ TEXERA_OBS_LOGS_URL=http://victorialogs:9428 TEXERA_OBS_METRICS_URL=http://victoriametrics:8428 TEXERA_OBS_TRACES_URL=http://jaeger:16686 TEXERA_OBS_PROFILES_URL=http://parca:7070 + +# Emission side: the Scala services (via OtelInit) push OTLP to the bundled +# collector over the bridge network. OtelInit validates the endpoint host +# against an allowlist (localhost only by default), so the collector's compose +# service name has to be allowlisted explicitly, or the SDK rejects it and +# emits nothing. These are read by every service that loads this .env; only the +# OtelInit-enabled Scala services act on them. When the observability-collector +# profile is disabled the collector is absent and OtelInit fails quietly +# (telemetry dropped, one warning, service unaffected). +OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 +TEXERA_OTEL_ALLOWED_HOSTS=otel-collector From 82dbb217bdafdb947ca4e0dba9c9c3c755b48359 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Wed, 15 Jul 2026 23:24:11 -0700 Subject: [PATCH 24/26] feat(observability): address #5376 review - direct OTel API + WorkflowMetrics rename Follow the OTel Java demo patterns instead of custom wrappers (zuozhiw review): - WorkflowService: start the run-level span with the standard OTel API, name it WorkflowService.initExecutionService, and drop the initExecutionServiceSpanned split so no span is passed as an argument. The real execution failure is now recorded onto the span from errorHandler, where it is actually caught. - TexeraTracer: drop the withSpan wrapper, keeping only the tracer accessor (single instrumentation scope) and currentContext. - SpanAttrs: drop the awkward with*/set* setter helpers, keep the standard label keys and sanitizeFreeText; callers set attributes via the OTel API. - Rename TexeraMetrics to WorkflowMetrics, document that this facade is only for the workflow-execution cluster, and add an example of calling the OTel meter API directly at a call site. Update SpanAttrsSpec and WorkflowMetricsSpec to match. --- .../WorkflowMetricsRecorder.scala | 18 +- .../texera/web/service/WorkflowService.scala | 255 +++++++++--------- .../texera/observability/SpanAttrs.scala | 70 +---- .../texera/observability/TexeraTracer.scala | 63 ++--- ...eraMetrics.scala => WorkflowMetrics.scala} | 24 +- .../texera/observability/SpanAttrsSpec.scala | Bin 5460 -> 3870 bytes ...csSpec.scala => WorkflowMetricsSpec.scala} | 40 +-- 7 files changed, 207 insertions(+), 263 deletions(-) rename common/config/src/main/scala/org/apache/texera/observability/{TexeraMetrics.scala => WorkflowMetrics.scala} (90%) rename common/config/src/test/scala/org/apache/texera/observability/{TexeraMetricsSpec.scala => WorkflowMetricsSpec.scala} (82%) diff --git a/amber/src/main/scala/org/apache/texera/web/observability/WorkflowMetricsRecorder.scala b/amber/src/main/scala/org/apache/texera/web/observability/WorkflowMetricsRecorder.scala index 572aa74e3c2..0f9ab2401aa 100644 --- a/amber/src/main/scala/org/apache/texera/web/observability/WorkflowMetricsRecorder.scala +++ b/amber/src/main/scala/org/apache/texera/web/observability/WorkflowMetricsRecorder.scala @@ -31,14 +31,14 @@ import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAg RESUMING, RUNNING } -import org.apache.texera.observability.TexeraMetrics -import org.apache.texera.observability.TexeraMetrics.WorkflowKind +import org.apache.texera.observability.WorkflowMetrics +import org.apache.texera.observability.WorkflowMetrics.WorkflowKind import org.apache.texera.web.service.WorkflowService import java.util.concurrent.ConcurrentHashMap /** - * Drives [[TexeraMetrics]] from amber's execution lifecycle. The metric + * Drives [[WorkflowMetrics]] from amber's execution lifecycle. The metric * instruments live in common/config; this object is the single place * that records them, so the lifecycle code only needs one-line calls. * @@ -61,7 +61,7 @@ object WorkflowMetricsRecorder extends LazyLogging { * every metric collection, so it must never throw. */ def init(): Unit = { - TexeraMetrics.setActiveExecutionsSupplier(() => + WorkflowMetrics.setActiveExecutionsSupplier(() => try { WorkflowService.getAllWorkflowServices.iterator .flatMap(s => Option(s.executionService.getValue)) @@ -72,7 +72,7 @@ object WorkflowMetricsRecorder extends LazyLogging { case _: Throwable => 0L } ) - TexeraMetrics.ensureBound() + WorkflowMetrics.ensureBound() } /** Record that a run started. */ @@ -81,7 +81,7 @@ object WorkflowMetricsRecorder extends LazyLogging { kind: WorkflowKind = WorkflowKind.Interactive ): Unit = { inFlight.put(executionId, (System.currentTimeMillis(), kind)) - TexeraMetrics.recordStart(kind) + WorkflowMetrics.recordStart(kind) } /** Record terminal counters + duration exactly once, on the first @@ -98,9 +98,9 @@ object WorkflowMetricsRecorder extends LazyLogging { val kind = entry.map(_._2).getOrElse(WorkflowKind.Interactive) val durationSec = entry.map(e => (System.currentTimeMillis() - e._1) / 1000.0).getOrElse(0.0) newState match { - case COMPLETED => TexeraMetrics.recordCompletion(kind, durationSec) - case FAILED => TexeraMetrics.recordFailure(kind, durationSec) - case KILLED => TexeraMetrics.recordCancellation(kind) + case COMPLETED => WorkflowMetrics.recordCompletion(kind, durationSec) + case FAILED => WorkflowMetrics.recordFailure(kind, durationSec) + case KILLED => WorkflowMetrics.recordCancellation(kind) case _ => () } } diff --git a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala index 3d3f79d10cd..bf591ef282c 100644 --- a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala +++ b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala @@ -21,7 +21,7 @@ package org.apache.texera.web.service import com.google.protobuf.timestamp.Timestamp import com.typesafe.scalalogging.LazyLogging -import io.opentelemetry.api.trace.Span +import io.opentelemetry.api.trace.StatusCode import io.reactivex.rxjava3.disposables.{CompositeDisposable, Disposable} import io.reactivex.rxjava3.subjects.BehaviorSubject import org.apache.texera.common.config.ApplicationConfig @@ -180,148 +180,157 @@ class WorkflowService( new WorkflowContext(workflowId = workflowId, cuid = Some(computingUnitId)) } + /** Sets up and launches a workflow execution inside a run-level span so + * setup-path logs carry its trace id. The span covers the synchronous + * setup and the handoff to async execution via `executeWorkflow()`; it + * does not span the full async run. The real execution failure is + * recorded onto the current span from `errorHandler`. + */ def initExecutionService( req: WorkflowExecuteRequest, userOpt: Option[User], sessionUri: URI ): Unit = { - TexeraTracer.withSpan( - "workflow.execute", - _.setAttribute("texera.workflow.id", workflowId.id.toString) - ) { span => - initExecutionServiceSpanned(req, userOpt, sessionUri, span) - } - } - - /** Body of [[initExecutionService]], run inside the run-level span so - * logs on the setup path carry its trace id. The span covers the - * synchronous setup and the handoff to async execution via - * `executeWorkflow()`; it does not span the full async run. - */ - private def initExecutionServiceSpanned( - req: WorkflowExecuteRequest, - userOpt: Option[User], - sessionUri: URI, - span: Span - ): Unit = { + val span = TexeraTracer.tracer + .spanBuilder("WorkflowService.initExecutionService") + .setAttribute("texera.workflow.id", workflowId.id.toString) + .startSpan() + val scope = span.makeCurrent() + try { - if (executionService.hasValue) { - executionService.getValue.unsubscribeAll() - } + if (executionService.hasValue) { + executionService.getValue.unsubscribeAll() + } - val (uidOpt, userEmailOpt) = userOpt.map(user => (user.getUid, user.getEmail)).unzip + val (uidOpt, userEmailOpt) = userOpt.map(user => (user.getUid, user.getEmail)).unzip - // uid is NOT NULL in the DB; fail early here rather than letting the insert fail downstream. - val uid = uidOpt.getOrElse( - throw new IllegalArgumentException( - "Cannot start execution: a user id (uid) is required but none was provided." + // uid is NOT NULL in the DB; fail early here rather than letting the insert fail downstream. + val uid = uidOpt.getOrElse( + throw new IllegalArgumentException( + "Cannot start execution: a user id (uid) is required but none was provided." + ) ) - ) - - val workflowContext: WorkflowContext = createWorkflowContext() - var controllerConf = ControllerConfig.default - - // clean up results from previous run - val previousExecutionId = - WorkflowExecutionService.getLatestExecutionId(workflowId, req.computingUnitId) - previousExecutionId.foreach(eid => { - clearExecutionResources(eid) - }) // TODO: change this behavior after enabling cache. - - workflowContext.executionId = ExecutionsMetadataPersistService.insertNewExecution( - workflowContext.workflowId, - uid, - req.executionName, - convertToJson(req.engineVersion), - req.computingUnitId - ) - span.setAttribute("texera.execution.id", workflowContext.executionId.id.toString) - // A run has started: record the start counter and stamp its start time. - org.apache.texera.web.observability.WorkflowMetricsRecorder.onStart(workflowContext.executionId) - if (ApplicationConfig.faultToleranceLogRootFolder.isDefined) { - val writeLocation = ApplicationConfig.faultToleranceLogRootFolder.get.resolve( - s"${workflowContext.workflowId}/${workflowContext.executionId}/" + val workflowContext: WorkflowContext = createWorkflowContext() + var controllerConf = ControllerConfig.default + + // clean up results from previous run + val previousExecutionId = + WorkflowExecutionService.getLatestExecutionId(workflowId, req.computingUnitId) + previousExecutionId.foreach(eid => { + clearExecutionResources(eid) + }) // TODO: change this behavior after enabling cache. + + workflowContext.executionId = ExecutionsMetadataPersistService.insertNewExecution( + workflowContext.workflowId, + uid, + req.executionName, + convertToJson(req.engineVersion), + req.computingUnitId ) - ExecutionsMetadataPersistService.tryUpdateExistingExecution(workflowContext.executionId) { - execution => execution.setLogLocation(writeLocation.toString) + span.setAttribute("texera.execution.id", workflowContext.executionId.id.toString) + // A run has started: record the start counter and stamp its start time. + org.apache.texera.web.observability.WorkflowMetricsRecorder + .onStart(workflowContext.executionId) + + if (ApplicationConfig.faultToleranceLogRootFolder.isDefined) { + val writeLocation = ApplicationConfig.faultToleranceLogRootFolder.get.resolve( + s"${workflowContext.workflowId}/${workflowContext.executionId}/" + ) + ExecutionsMetadataPersistService.tryUpdateExistingExecution(workflowContext.executionId) { + execution => execution.setLogLocation(writeLocation.toString) + } + controllerConf = controllerConf.copy(faultToleranceConfOpt = + Some(FaultToleranceConfig(writeTo = writeLocation)) + ) } - controllerConf = controllerConf.copy(faultToleranceConfOpt = - Some(FaultToleranceConfig(writeTo = writeLocation)) - ) - } - if (req.replayFromExecution.isDefined) { - val replayInfo = req.replayFromExecution.get - ExecutionsMetadataPersistService - .tryGetExistingExecution(ExecutionIdentity(replayInfo.eid)) - .foreach { execution => - val readLocation = new URI(execution.getLogLocation) - controllerConf = controllerConf.copy(stateRestoreConfOpt = - Some( - StateRestoreConfig( - readFrom = readLocation, - replayDestination = EmbeddedControlMessageIdentity(replayInfo.interaction) + if (req.replayFromExecution.isDefined) { + val replayInfo = req.replayFromExecution.get + ExecutionsMetadataPersistService + .tryGetExistingExecution(ExecutionIdentity(replayInfo.eid)) + .foreach { execution => + val readLocation = new URI(execution.getLogLocation) + controllerConf = controllerConf.copy(stateRestoreConfOpt = + Some( + StateRestoreConfig( + readFrom = readLocation, + replayDestination = EmbeddedControlMessageIdentity(replayInfo.interaction) + ) ) ) - ) - } - } + } + } - val executionStateStore = new ExecutionStateStore() - // assign execution id to find the execution from DB in case the constructor fails. - executionStateStore.metadataStore.updateState(state => - state.withExecutionId(workflowContext.executionId) - ) - val errorHandler: Throwable => Unit = { t => - { - val fromActorOpt = t match { - case ex: WorkflowRuntimeException => - ex.relatedWorkerId - case other => - None - } - val (operatorId, workerId) = getOperatorFromActorIdOpt(fromActorOpt) - logger.error("error during execution", t) - executionStateStore.statsStore.updateState(stats => - stats.withEndTimeStamp(System.currentTimeMillis()) - ) - executionStateStore.metadataStore.updateState { metadataStore => - updateWorkflowState(FAILED, metadataStore).addFatalErrors( - WorkflowFatalError( - EXECUTION_FAILURE, - Timestamp(Instant.now), - t.toString, - getStackTraceWithAllCauses(t), - operatorId, - workerId - ) + val executionStateStore = new ExecutionStateStore() + // assign execution id to find the execution from DB in case the constructor fails. + executionStateStore.metadataStore.updateState(state => + state.withExecutionId(workflowContext.executionId) + ) + val errorHandler: Throwable => Unit = { t => + { + val fromActorOpt = t match { + case ex: WorkflowRuntimeException => + ex.relatedWorkerId + case other => + None + } + val (operatorId, workerId) = getOperatorFromActorIdOpt(fromActorOpt) + logger.error("error during execution", t) + // Record the real execution failure on the run-level span. Handled + // here rather than in initExecutionService's catch because this is + // where the failure is actually caught (it does not propagate up). + span.recordException(t) + span.setStatus(StatusCode.ERROR) + executionStateStore.statsStore.updateState(stats => + stats.withEndTimeStamp(System.currentTimeMillis()) ) + executionStateStore.metadataStore.updateState { metadataStore => + updateWorkflowState(FAILED, metadataStore).addFatalErrors( + WorkflowFatalError( + EXECUTION_FAILURE, + Timestamp(Instant.now), + t.toString, + getStackTraceWithAllCauses(t), + operatorId, + workerId + ) + ) + } } } - } - // WorkflowExecutionService construction does no external work and cannot - // throw; it registers its error/state diff handler up front. Once published - // via `executionService.onNext`, any failure in `executeWorkflow()` is - // recorded by `errorHandler` into the metadata store, whose handler emits a - // WorkflowErrorEvent that `connectToExecution` forwards. - try { - val execution = new WorkflowExecutionService( - controllerConf, - workflowContext, - resultService, - req, - executionStateStore, - errorHandler, - userEmailOpt, - sessionUri - ) - lifeCycleManager.registerCleanUpOnStateChange(executionStateStore) - executionService.onNext(execution) - execution.executeWorkflow() + // WorkflowExecutionService construction does no external work and cannot + // throw; it registers its error/state diff handler up front. Once published + // via `executionService.onNext`, any failure in `executeWorkflow()` is + // recorded by `errorHandler` into the metadata store, whose handler emits a + // WorkflowErrorEvent that `connectToExecution` forwards. + try { + val execution = new WorkflowExecutionService( + controllerConf, + workflowContext, + resultService, + req, + executionStateStore, + errorHandler, + userEmailOpt, + sessionUri + ) + lifeCycleManager.registerCleanUpOnStateChange(executionStateStore) + executionService.onNext(execution) + execution.executeWorkflow() + } catch { + case e: Throwable => errorHandler(e) + } + } catch { - case e: Throwable => errorHandler(e) + case t: Throwable => + // Synchronous setup failure (before the run's own errorHandler is wired). + span.recordException(t) + span.setStatus(StatusCode.ERROR) + throw t + } finally { + scope.close() + span.end() } - } def convertToJson(frontendVersion: String): String = { diff --git a/common/config/src/main/scala/org/apache/texera/observability/SpanAttrs.scala b/common/config/src/main/scala/org/apache/texera/observability/SpanAttrs.scala index a1ac0bbc18b..c818f69bc16 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/SpanAttrs.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/SpanAttrs.scala @@ -20,33 +20,21 @@ package org.apache.texera.observability import io.opentelemetry.api.common.AttributeKey -import io.opentelemetry.api.trace.{Span, SpanBuilder} /** - * Thin helper for setting span attributes safely. + * Standard Texera span-attribute keys plus a free-text sanitizer. * - * Three rules: - * 1. Typed setters only — no public escape hatch for arbitrary - * untyped strings to land on a span as untrusted free text. - * 2. Free-text values are CRLF-stripped + capped at - * [[FreeTextMaxLen]] to prevent log/span forging via embedded - * newlines. - * 3. Operator IDs and workflow/execution IDs must match a strict - * character set — otherwise dropped silently (the operator - * identifier should be a stable internal value, not user free - * text). + * These are the shared label keys so every callsite tags spans with the + * same names. Set them with the standard OTel API at the callsite, e.g. + * ``spanBuilder.setAttribute(SpanAttrs.WorkflowId, id)`` or + * ``span.setAttribute(SpanAttrs.WorkflowId, id)``. Run any free-text value + * through [[sanitizeFreeText]] first to strip CRLF and cap its length. */ object SpanAttrs { /** Maximum length for free-text span attribute values. */ val FreeTextMaxLen: Int = 256 - /** Validates the shape we accept for operator IDs: alnum + `_.-`, - * 1–64 chars. Anything else is dropped (not coerced — we'd rather - * miss a label than leak an unbounded string into a span). - */ - private val OperatorIdPattern = "^[A-Za-z0-9_.\\-]{1,64}$".r.pattern - // ---- Standard Texera correlation labels ------------------------------ val WorkflowId: AttributeKey[java.lang.Long] = AttributeKey.longKey("texera.workflow.id") @@ -57,52 +45,6 @@ object SpanAttrs { val OperatorName: AttributeKey[String] = AttributeKey.stringKey("texera.operator.name") val Outcome: AttributeKey[String] = AttributeKey.stringKey("texera.outcome") - // ---- Typed setters for SpanBuilder (used at span-start time) --------- - - def withWorkflowId(b: SpanBuilder, id: Long): SpanBuilder = - b.setAttribute(WorkflowId, java.lang.Long.valueOf(id)) - - def withExecutionId(b: SpanBuilder, id: Long): SpanBuilder = - b.setAttribute(ExecutionId, java.lang.Long.valueOf(id)) - - def withProjectId(b: SpanBuilder, id: Long): SpanBuilder = - b.setAttribute(ProjectId, java.lang.Long.valueOf(id)) - - def withUserId(b: SpanBuilder, id: Long): SpanBuilder = - b.setAttribute(UserId, java.lang.Long.valueOf(id)) - - /** Sets the operator id only if it passes the strict character - * check; otherwise the attribute is omitted. Returns the same - * builder either way for fluent chaining. - */ - def withOperatorId(b: SpanBuilder, id: String): SpanBuilder = { - if (id != null && OperatorIdPattern.matcher(id).matches()) { - b.setAttribute(OperatorId, id) - } - b - } - - /** Sets a free-text label after stripping CRLF and capping length. */ - def withOperatorName(b: SpanBuilder, name: String): SpanBuilder = { - val safe = sanitizeFreeText(name) - if (safe != null) b.setAttribute(OperatorName, safe) else b - } - - // ---- Typed setters for Span (used after a span is active) ------------ - - def setWorkflowId(s: Span, id: Long): Span = - s.setAttribute(WorkflowId, java.lang.Long.valueOf(id)) - def setExecutionId(s: Span, id: Long): Span = - s.setAttribute(ExecutionId, java.lang.Long.valueOf(id)) - def setOperatorId(s: Span, id: String): Span = { - if (id != null && OperatorIdPattern.matcher(id).matches()) s.setAttribute(OperatorId, id) - else s - } - def setOutcome(s: Span, outcome: String): Span = { - val safe = sanitizeFreeText(outcome) - if (safe != null) s.setAttribute(Outcome, safe) else s - } - // ---- Pure helpers (exposed for testing) ------------------------------ /** diff --git a/common/config/src/main/scala/org/apache/texera/observability/TexeraTracer.scala b/common/config/src/main/scala/org/apache/texera/observability/TexeraTracer.scala index ae8f71c82e6..38e1c36b620 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/TexeraTracer.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/TexeraTracer.scala @@ -20,25 +20,30 @@ package org.apache.texera.observability import io.opentelemetry.api.GlobalOpenTelemetry -import io.opentelemetry.api.trace.{Span, SpanBuilder, StatusCode, Tracer} -import io.opentelemetry.context.{Context, Scope} +import io.opentelemetry.api.trace.Tracer +import io.opentelemetry.context.Context /** - * Thin convenience wrapper around the global OTel tracer. + * Accessor for the Texera OTel tracer. * - * Two reasons to go through this rather than calling - * ``GlobalOpenTelemetry.getTracer`` directly at every callsite: + * The only thing this adds over calling ``GlobalOpenTelemetry.getTracer`` + * directly is a single instrumentation scope name (``org.apache.texera``), + * so every Texera-produced span shows up under one logical scope in the + * backend, separable from anything emitted by transitive libraries. * - * 1. Single instrumentation scope name (``org.apache.texera``) — so - * every Texera-produced span shows up under one logical scope in - * the backend, separable from anything emitted by transitive - * libraries. - * 2. One ergonomic ``withSpan`` API that handles exception → status, - * scope cleanup, and span end in a single try/finally. Callers - * don't have to remember the ceremony at every site. + * Start and end spans with the standard OTel API at the callsite (see the + * OpenTelemetry Java demo for the recommended pattern): * - * When the SDK is disabled, ``GlobalOpenTelemetry.getTracer`` returns - * a no-op tracer, so calling these methods is safe at any time. + * {{{ + * val span = TexeraTracer.tracer.spanBuilder("MyClass.myMethod").startSpan() + * val scope = span.makeCurrent() + * try { ... } catch { + * case t: Throwable => span.recordException(t); span.setStatus(ERROR); throw t + * } finally { scope.close(); span.end() } + * }}} + * + * When the SDK is disabled, ``GlobalOpenTelemetry.getTracer`` returns a + * no-op tracer, so calling this is safe at any time. */ object TexeraTracer { @@ -46,38 +51,10 @@ object TexeraTracer { def tracer: Tracer = GlobalOpenTelemetry.getTracer(InstrumentationScope) - def spanBuilder(name: String): SpanBuilder = tracer.spanBuilder(name) - - /** - * Run ``block`` inside a fresh span; record exceptions, propagate - * the right span status, and ensure the span is ended exactly once. - * - * Use this for synchronous critical sections. For async (Future- - * returning) code paths use ``withAsyncSpan`` so the span doesn't - * close before the async work completes. - */ - def withSpan[T](name: String, configure: SpanBuilder => SpanBuilder = identity)( - block: Span => T - ): T = { - val span = configure(spanBuilder(name)).startSpan() - val scope: Scope = span.makeCurrent() - try { - block(span) - } catch { - case t: Throwable => - span.recordException(t) - span.setStatus(StatusCode.ERROR) - throw t - } finally { - scope.close() - span.end() - } - } - /** * Snapshot the current OTel ``Context`` so async callbacks can * re-attach it via ``Context.makeCurrent`` later. Useful at the - * Scala↔Python boundary where the calling thread is not the + * Scala to Python boundary where the calling thread is not the * receiving thread. */ def currentContext: Context = Context.current() diff --git a/common/config/src/main/scala/org/apache/texera/observability/TexeraMetrics.scala b/common/config/src/main/scala/org/apache/texera/observability/WorkflowMetrics.scala similarity index 90% rename from common/config/src/main/scala/org/apache/texera/observability/TexeraMetrics.scala rename to common/config/src/main/scala/org/apache/texera/observability/WorkflowMetrics.scala index ef7ad78aeef..1035e8a52ec 100644 --- a/common/config/src/main/scala/org/apache/texera/observability/TexeraMetrics.scala +++ b/common/config/src/main/scala/org/apache/texera/observability/WorkflowMetrics.scala @@ -25,21 +25,37 @@ import io.opentelemetry.api.common.{AttributeKey, Attributes} import io.opentelemetry.api.metrics.Meter /** - * Strongly-typed façade for Texera-emitted metrics. + * Strongly-typed facade for the workflow-execution metrics cluster. * - * Cardinality safety is enforced by the API surface, not by + * This facade pattern is deliberate here because these instruments are a + * small, fixed, correlated set with strict cardinality rules, and it is + * worth centralizing that control in one place. It is NOT the default + * pattern for metrics in general: most call sites should just call the OTel + * meter API directly next to the business logic, e.g. + * + * {{{ + * private val meter = GlobalOpenTelemetry.getMeter("org.apache.texera") + * private val requests = meter.counterBuilder("myfeature.requests").build() + * // in the handler: + * requests.add(1, Attributes.of(AttributeKey.stringKey("route"), route)) + * }}} + * + * Only reach for a facade like this one when the metrics are complex or + * need centralized, standardized control. Do not copy it by default. + * + * Cardinality safety here is enforced by the API surface, not by * documentation: there is no public method that accepts an arbitrary * string as a label key or value. The only labels that ever land on * an instrument are the two enums [[Outcome]] and [[WorkflowKind]], * each restricted to a fixed set. ``workflow.id`` / ``execution.id`` - * are deliberately NOT metric labels — per-execution detail belongs + * are deliberately NOT metric labels: per-execution detail belongs * in traces and logs, joined on ``trace_id`` at query time. * * Histogram bucket bounds are hard-coded constants so they can't be * coerced by request input. The OTel SDK applies its own default * attribute-value-length cap to anything that does slip through. */ -object TexeraMetrics extends LazyLogging { +object WorkflowMetrics extends LazyLogging { /** Outcome enum, the only mutable label on lifecycle counters. */ sealed abstract class Outcome(val name: String) diff --git a/common/config/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala index 1afca195533073f1d8ff2b0b8c8b4dd524da77e2..af98d327b6de267ae6bbefae18d05a52d97410c5 100644 GIT binary patch delta 357 zcmcbjHBWBCL=J=ElEl1}#G(|1oW!Km9EI%E%3>fhFGZm&Gf|-=BUPawCowZm!9N5j z=@{UtF!>;dh&MD6;?xp_qB;Y9eY9(z7z!ZLYi+6LhB2!B12Qbf)wlcme|!5mGW z^RPHBI~CJ*B1weQCoJ1R36^an7-Qlib&DbwI}5o$JON!TCrYab9XLPhKdbf`_PYMEt+iLFD-?(D#HgA7ewC%5afU5UK71i!$6Y=^F^)$D4Bb7au=;)Pi zLEEb@rz)xJW%g7lYtM%9K;xN#_Rhl60@hcm^PuaA_xo-WxHvuW`2@$a1lqpF8FOPD>-0KFwhzHAq!W2zR)lU1n|k1vD`JUfMg4uK gRyvn0k`zf&cDJKO##a^ReqDv1uiku#^PjH#1{~k0!~g&Q diff --git a/common/config/src/test/scala/org/apache/texera/observability/TexeraMetricsSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/WorkflowMetricsSpec.scala similarity index 82% rename from common/config/src/test/scala/org/apache/texera/observability/TexeraMetricsSpec.scala rename to common/config/src/test/scala/org/apache/texera/observability/WorkflowMetricsSpec.scala index 8bed93d6b01..17c9c4c74dd 100644 --- a/common/config/src/test/scala/org/apache/texera/observability/TexeraMetricsSpec.scala +++ b/common/config/src/test/scala/org/apache/texera/observability/WorkflowMetricsSpec.scala @@ -28,7 +28,7 @@ import org.scalatest.matchers.should.Matchers import scala.jdk.CollectionConverters._ -class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { +class WorkflowMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { private var reader: InMemoryMetricReader = _ private var provider: SdkMeterProvider = _ @@ -36,12 +36,12 @@ class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEac override def beforeEach(): Unit = { reader = InMemoryMetricReader.create() provider = SdkMeterProvider.builder().registerMetricReader(reader).build() - TexeraMetrics.resetForTest() - TexeraMetrics.bindForTest(provider.get("org.apache.texera")) + WorkflowMetrics.resetForTest() + WorkflowMetrics.bindForTest(provider.get("org.apache.texera")) } override def afterEach(): Unit = { - TexeraMetrics.resetForTest() + WorkflowMetrics.resetForTest() provider.close() } @@ -51,8 +51,8 @@ class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEac // ----- positive: lifecycle emissions ---------------------------------- - "TexeraMetrics" should "increment workflow.starts on recordStart" in { - TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) + "WorkflowMetrics" should "increment workflow.starts on recordStart" in { + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) val metrics = collectAll() metrics.keySet should contain("texera.workflow.starts") @@ -69,9 +69,9 @@ class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEac // confirm the gauge reports exactly what the supplier returns — regardless // of how many starts/completions were recorded. @volatile var live = 3L - TexeraMetrics.setActiveExecutionsSupplier(() => live) - TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) - TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) + WorkflowMetrics.setActiveExecutionsSupplier(() => live) + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) val first = collectAll()("texera.workflow.active").getLongGaugeData.getPoints.asScala.head first.getValue shouldBe 3L @@ -84,8 +84,8 @@ class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEac } it should "record a completion and duration sample on recordCompletion" in { - TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) - TexeraMetrics.recordCompletion(TexeraMetrics.WorkflowKind.Interactive, durationSec = 2.5) + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + WorkflowMetrics.recordCompletion(WorkflowMetrics.WorkflowKind.Interactive, durationSec = 2.5) val metrics = collectAll() metrics.keySet should contain allOf ( @@ -103,8 +103,8 @@ class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEac } it should "record a failure as a non-success completion (so failure-rate queries work)" in { - TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Scheduled) - TexeraMetrics.recordFailure(TexeraMetrics.WorkflowKind.Scheduled, durationSec = 12.0) + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Scheduled) + WorkflowMetrics.recordFailure(WorkflowMetrics.WorkflowKind.Scheduled, durationSec = 12.0) val metrics = collectAll() // A failure shares the completions counter with successes — the @@ -124,8 +124,8 @@ class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEac } it should "record a cancellation that is not a completion" in { - TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) - TexeraMetrics.recordCancellation(TexeraMetrics.WorkflowKind.Interactive) + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + WorkflowMetrics.recordCancellation(WorkflowMetrics.WorkflowKind.Interactive) val metrics = collectAll() metrics( @@ -140,8 +140,8 @@ class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEac // ----- security: cardinality safety ----------------------------------- it should "only emit the texera.outcome and texera.workflow.kind labels" in { - TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) - TexeraMetrics.recordCompletion(TexeraMetrics.WorkflowKind.Interactive, durationSec = 1.0) + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + WorkflowMetrics.recordCompletion(WorkflowMetrics.WorkflowKind.Interactive, durationSec = 1.0) val attrKeys = collectAll().values.flatMap { md => val pointSet = md.getType.name() match { @@ -166,7 +166,7 @@ class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEac // untyped public method like recordStart(attrs: Attributes), // this assertion still passes but the design intent is broken. // Make the intent explicit: - val methodNames = classOf[TexeraMetrics.type].getDeclaredMethods + val methodNames = classOf[WorkflowMetrics.type].getDeclaredMethods .map(_.getName) .toSet methodNames should contain allOf ("recordStart", "recordCompletion", "recordFailure") @@ -176,10 +176,10 @@ class TexeraMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEac // ----- histogram buckets are constants -------------------------------- it should "use the hard-coded explicit bucket boundaries for duration" in { - TexeraMetrics.recordStart(TexeraMetrics.WorkflowKind.Interactive) + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) // Hit a few bucket bounds. Seq(0.05, 0.6, 7.0, 65.0, 400.0).foreach { d => - TexeraMetrics.recordCompletion(TexeraMetrics.WorkflowKind.Interactive, durationSec = d) + WorkflowMetrics.recordCompletion(WorkflowMetrics.WorkflowKind.Interactive, durationSec = d) } val histogram = collectAll()("texera.workflow.duration").getHistogramData From 30b841c1b84709713e7a0c051522d72c1c1ecd81 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Tue, 4 Aug 2026 12:30:02 -0700 Subject: [PATCH 25/26] use SpanAttrs keys and stop recording on ended span in WorkflowService --- .../texera/web/service/WorkflowService.scala | 27 ++++++++++-------- .../texera/observability/SpanAttrsSpec.scala | Bin 3870 -> 4084 bytes 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala index b37d9d821b1..2e1fee4c8ba 100644 --- a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala +++ b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala @@ -50,7 +50,7 @@ import org.apache.texera.amber.error.ErrorUtils.{ getStackTraceWithAllCauses } import org.apache.texera.dao.jooq.generated.tables.pojos.User -import org.apache.texera.observability.TexeraTracer +import org.apache.texera.observability.{SpanAttrs, TexeraTracer} import org.apache.texera.service.util.LargeBinaryManager import org.apache.texera.web.model.websocket.event.TexeraWebSocketEvent import org.apache.texera.web.model.websocket.request.WorkflowExecuteRequest @@ -181,10 +181,12 @@ class WorkflowService( } /** Sets up and launches a workflow execution inside a run-level span so - * setup-path logs carry its trace id. The span covers the synchronous - * setup and the handoff to async execution via `executeWorkflow()`; it - * does not span the full async run. The real execution failure is - * recorded onto the current span from `errorHandler`. + * setup-path logs carry its trace id. The span covers only the synchronous + * setup and the handoff to async execution via `executeWorkflow()`; it does + * not span the full async run. Only synchronous setup failures are recorded + * on the span (in the catch below); async execution failures arrive via + * `errorHandler` after the span has ended and are surfaced through the + * metadata store instead. */ def initExecutionService( req: WorkflowExecuteRequest, @@ -193,7 +195,7 @@ class WorkflowService( ): Unit = { val span = TexeraTracer.tracer .spanBuilder("WorkflowService.initExecutionService") - .setAttribute("texera.workflow.id", workflowId.id.toString) + .setAttribute(SpanAttrs.WorkflowId, Long.box(workflowId.id)) .startSpan() val scope = span.makeCurrent() try { @@ -228,7 +230,7 @@ class WorkflowService( convertToJson(req.engineVersion), req.computingUnitId ) - span.setAttribute("texera.execution.id", workflowContext.executionId.id.toString) + span.setAttribute(SpanAttrs.ExecutionId, Long.box(workflowContext.executionId.id)) // A run has started: record the start counter and stamp its start time. org.apache.texera.web.observability.WorkflowMetricsRecorder .onStart(workflowContext.executionId) @@ -276,11 +278,12 @@ class WorkflowService( } val (operatorId, workerId) = getOperatorFromActorIdOpt(fromActorOpt) logger.error("error during execution", t) - // Record the real execution failure on the run-level span. Handled - // here rather than in initExecutionService's catch because this is - // where the failure is actually caught (it does not propagate up). - span.recordException(t) - span.setStatus(StatusCode.ERROR) + // Do NOT touch `span` here: this handler is passed into + // WorkflowExecutionService and invoked asynchronously (runtime, + // websocket, startWorkflow callbacks) after initExecutionService has + // returned and the setup span has already ended, so recording onto it + // would be a silent no-op. The failure is surfaced via the metadata + // store below; setup-span errors are recorded in the catch block. executionStateStore.statsStore.updateState(stats => stats.withEndTimeStamp(System.currentTimeMillis()) ) diff --git a/common/config/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala index af98d327b6de267ae6bbefae18d05a52d97410c5..5553a36e997809029167db2558a2c36b0c8f6243 100644 GIT binary patch delta 180 zcmZ{cu?@m75Jd|lilm}QOHPq!C_DruOmR*QV&(9G?T~~u+_M1^5?fI-07lgO?n&?c z+wxsLuKCsk@U8URKt;n)3Kj{Df delta 12 Tcmew&KTmE$75C- Date: Wed, 12 Aug 2026 14:01:16 -0700 Subject: [PATCH 26/26] feat(observability): logging foundations (bootstrap, bridge, sanitizer) --- .github/workflows/build.yml | 2 +- access-control-service/LICENSE-binary | 20 + .../access-control-service-web-config.yaml | 13 + .../texera/service/AccessControlService.scala | 2 + amber/LICENSE-binary-java | 12 + .../computing-unit-master-config.yml | 12 + .../texera-compiling-service-web-config.yml | 12 + amber/src/main/resources/web-config.yml | 12 + .../texera/web/ComputingUnitMaster.scala | 1 + bin/k8s/values.yaml | 15 + bin/single-node/.env | 12 + build.sbt | 18 +- .../src/main/resources/observability.conf | 45 ++ .../common/config/EnvironmentalVariable.scala | 7 + .../common/config/ObservabilityConfig.scala | 38 ++ .../config/ObservabilityConfigSpec.scala | 65 +++ common/observability/build.sbt | 73 ++++ .../texera/observability/LogSanitizer.scala | 122 ++++++ .../texera/observability/OtelInit.scala | 403 ++++++++++++++++++ .../observability/TexeraOtelLogAppender.scala | 127 ++++++ .../observability/LogSanitizerSpec.scala | 140 ++++++ .../texera/observability/OtelInitSpec.scala | 263 ++++++++++++ .../TexeraOtelLogAppenderSpec.scala | 214 ++++++++++ .../LICENSE-binary | 12 + ...omputing-unit-managing-service-config.yaml | 14 +- .../ComputingUnitManagingService.scala | 2 + config-service/LICENSE-binary | 20 + .../resources/config-service-web-config.yaml | 13 + .../apache/texera/service/ConfigService.scala | 2 + file-service/LICENSE-binary | 12 + .../resources/file-service-web-config.yaml | 12 + .../apache/texera/service/FileService.scala | 2 + workflow-compiling-service/LICENSE-binary | 12 + .../workflow-compiling-service-config.yaml | 12 + .../service/WorkflowCompilingService.scala | 2 + 35 files changed, 1735 insertions(+), 8 deletions(-) create mode 100644 common/config/src/main/resources/observability.conf create mode 100644 common/config/src/main/scala/org/apache/texera/common/config/ObservabilityConfig.scala create mode 100644 common/config/src/test/scala/org/apache/texera/common/config/ObservabilityConfigSpec.scala create mode 100644 common/observability/build.sbt create mode 100644 common/observability/src/main/scala/org/apache/texera/observability/LogSanitizer.scala create mode 100644 common/observability/src/main/scala/org/apache/texera/observability/OtelInit.scala create mode 100644 common/observability/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala create mode 100644 common/observability/src/test/scala/org/apache/texera/observability/LogSanitizerSpec.scala create mode 100644 common/observability/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala create mode 100644 common/observability/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c28457b1986..d826606b310 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -307,7 +307,7 @@ jobs: BACKPORT_TARGET_BRANCH: ${{ inputs.backport_target_branch }} run: | # Backport builds filter by the checked-out build.sbt. - want=(DAO Auth Config Resource Util PyBuilder WorkflowCore + want=(DAO Auth Config Observability Resource Util PyBuilder WorkflowCore WorkflowOperator WorkflowCompiler WorkflowExecutionService) tasks=() if [ -n "${BACKPORT_TARGET_BRANCH}" ]; then diff --git a/access-control-service/LICENSE-binary b/access-control-service/LICENSE-binary index 78f1df46a94..a4ceb120c84 100644 --- a/access-control-service/LICENSE-binary +++ b/access-control-service/LICENSE-binary @@ -242,6 +242,9 @@ Scala/Java jars: - com.google.guava.listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar - com.google.j2objc.j2objc-annotations-2.8.jar - com.helger.profiler-1.1.1.jar + - com.squareup.okhttp3.okhttp-4.12.0.jar + - com.squareup.okio.okio-3.6.0.jar + - com.squareup.okio.okio-jvm-3.6.0.jar - com.thesamet.scalapb.lenses_2.13-0.11.20.jar - com.thesamet.scalapb.scalapb-json4s_2.13-0.12.0.jar - com.thesamet.scalapb.scalapb-runtime_2.13-0.11.20.jar @@ -274,6 +277,18 @@ Scala/Java jars: - io.dropwizard.metrics.metrics-json-4.2.25.jar - io.dropwizard.metrics.metrics-jvm-4.2.25.jar - io.dropwizard.metrics.metrics-logback-4.2.25.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.r2dbc.r2dbc-spi-1.0.0.RELEASE.jar - jakarta.inject.jakarta.inject-api-2.0.1.jar - jakarta.validation.jakarta.validation-api-3.0.2.jar @@ -292,6 +307,11 @@ Scala/Java jars: - org.hibernate.validator.hibernate-validator-7.0.5.Final.jar - org.javassist.javassist-3.30.2-GA.jar - org.jboss.logging.jboss-logging-3.5.3.Final.jar + - org.jetbrains.annotations-13.0.jar + - org.jetbrains.kotlin.kotlin-stdlib-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-common-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-jdk7-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-jdk8-1.9.10.jar - org.jooq.jooq-3.19.36.jar - org.json4s.json4s-ast_2.13-4.0.1.jar - org.json4s.json4s-jackson-core_2.13-4.0.1.jar diff --git a/access-control-service/src/main/resources/access-control-service-web-config.yaml b/access-control-service/src/main/resources/access-control-service-web-config.yaml index 8c7895e9858..bd78ecaa82a 100644 --- a/access-control-service/src/main/resources/access-control-service-web-config.yaml +++ b/access-control-service/src/main/resources/access-control-service-web-config.yaml @@ -26,6 +26,19 @@ server: logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + loggers: + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console threshold: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} diff --git a/access-control-service/src/main/scala/org/apache/texera/service/AccessControlService.scala b/access-control-service/src/main/scala/org/apache/texera/service/AccessControlService.scala index 1f50c86c9f5..a7942d34b53 100644 --- a/access-control-service/src/main/scala/org/apache/texera/service/AccessControlService.scala +++ b/access-control-service/src/main/scala/org/apache/texera/service/AccessControlService.scala @@ -58,6 +58,8 @@ class AccessControlService extends Application[AccessControlServiceConfiguration configuration: AccessControlServiceConfiguration, environment: Environment ): Unit = { + // Bridge this service's logs to the OTel collector under its own service.name. + org.apache.texera.observability.OtelInit.init("access-control-service") // Serve backend at /api environment.jersey.setUrlPattern("/api/*") diff --git a/amber/LICENSE-binary-java b/amber/LICENSE-binary-java index 90d27b4447c..424194d8e32 100644 --- a/amber/LICENSE-binary-java +++ b/amber/LICENSE-binary-java @@ -363,6 +363,18 @@ Scala/Java jars: - org.jspecify.jspecify-1.0.0.jar - io.opencensus.opencensus-api-0.31.1.jar - io.opencensus.opencensus-contrib-http-util-0.31.1.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.perfmark.perfmark-api-0.27.0.jar - io.r2dbc.r2dbc-spi-1.0.0.RELEASE.jar - io.reactivex.rxjava3.rxjava-3.1.12.jar diff --git a/amber/src/main/resources/computing-unit-master-config.yml b/amber/src/main/resources/computing-unit-master-config.yml index 0dba594b8ae..ee578c3cf90 100644 --- a/amber/src/main/resources/computing-unit-master-config.yml +++ b/amber/src/main/resources/computing-unit-master-config.yml @@ -34,6 +34,18 @@ logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console logFormat: "[%date{ISO8601}] [%level] [%logger] [%thread] - %msg %n" diff --git a/amber/src/main/resources/texera-compiling-service-web-config.yml b/amber/src/main/resources/texera-compiling-service-web-config.yml index ea2c1b9c1e9..c0b6e8aa762 100644 --- a/amber/src/main/resources/texera-compiling-service-web-config.yml +++ b/amber/src/main/resources/texera-compiling-service-web-config.yml @@ -43,6 +43,18 @@ logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console logFormat: "[%date{ISO8601}] [%level] [%logger] [%thread] - %msg %n" diff --git a/amber/src/main/resources/web-config.yml b/amber/src/main/resources/web-config.yml index 9fde1d078e8..9b3c743c89c 100644 --- a/amber/src/main/resources/web-config.yml +++ b/amber/src/main/resources/web-config.yml @@ -43,6 +43,18 @@ logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console logFormat: "[%date{ISO8601}] [%level] [%logger] [%thread] - %msg %n" diff --git a/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala b/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala index 6616ff47d1e..b31a906c5d0 100644 --- a/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala +++ b/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala @@ -139,6 +139,7 @@ class ComputingUnitMaster extends io.dropwizard.Application[Configuration] with } override def run(configuration: Configuration, environment: Environment): Unit = { + org.apache.texera.observability.OtelInit.init("computing-unit-master") ObjectMapperUtils.warmupObjectMapperForOperatorsSerde() SqlServer.initConnection( diff --git a/bin/k8s/values.yaml b/bin/k8s/values.yaml index 4651bae6947..8085b975eb3 100644 --- a/bin/k8s/values.yaml +++ b/bin/k8s/values.yaml @@ -358,6 +358,21 @@ texeraEnvVars: - name: AUTH_JWT_SECRET # Development-only default (256-bit HS256 secret). Production environments MUST override this with a different, securely generated secret. value: "a7f3c8e9b14d2e6f5a0b9c3d8e1f4a6b2c5d7e9f0a3b6c8d1e4f7a9b2c5d8e1f" + # OpenTelemetry (observability). Disabled by default; set OTEL_SDK_DISABLED to + # "false" and point the endpoint at a reachable OTLP collector (http/https only) to enable. + - name: OTEL_SDK_DISABLED + value: "true" + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://127.0.0.1:4317" + # Comma-separated resource attributes (k1=v1,k2=v2); service.name is set per service. + - name: OTEL_RESOURCE_ATTRIBUTES + value: "" + # Comma-separated extra hosts allowed for the OTLP endpoint (loopback allowed by default). + - name: TEXERA_OTEL_ALLOWED_HOSTS + value: "" + # Metric export interval in ms. + - name: OTEL_METRIC_EXPORT_INTERVAL + value: "30000" yWebsocketServer: name: y-websocket-server diff --git a/bin/single-node/.env b/bin/single-node/.env index 555e14db7df..cf5552c2224 100644 --- a/bin/single-node/.env +++ b/bin/single-node/.env @@ -96,3 +96,15 @@ LLM_ENDPOINT=http://nginx:8080 TEXERA_DASHBOARD_SERVICE_ENDPOINT=http://dashboard-service:8080 WORKFLOW_COMPILING_SERVICE_ENDPOINT=http://workflow-compiling-service:9090 WORKFLOW_EXECUTION_SERVICE_ENDPOINT=http://workflow-runtime-coordinator-service:8085 + +# OpenTelemetry (observability). Disabled by default; the SDK stays inert until +# OTEL_SDK_DISABLED=false. Point the endpoint at a reachable OTLP collector +# (http/https only) before enabling. +OTEL_SDK_DISABLED=true +OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4317 +# Comma-separated resource attributes (k1=v1,k2=v2); service.name is set per service. +OTEL_RESOURCE_ATTRIBUTES= +# Comma-separated extra hosts allowed for the OTLP endpoint (loopback allowed by default). +TEXERA_OTEL_ALLOWED_HOSTS= +# Metric export interval in ms. +OTEL_METRIC_EXPORT_INTERVAL=30000 diff --git a/build.sbt b/build.sbt index 3779413e041..64504508e98 100644 --- a/build.sbt +++ b/build.sbt @@ -122,6 +122,11 @@ ThisBuild / excludeDependencies += ExclusionRule("log4j", "log4j") lazy val Util = (project in file("common/util")).settings(commonModuleSettings) lazy val DAO = (project in file("common/dao")).settings(commonModuleSettings) lazy val Config = (project in file("common/config")).settings(commonModuleSettings) +// OpenTelemetry bootstrap (OtelInit, log appender, sanitizer) shared by every +// service entry point; pins the OTel dependency versions in one place. Depends +// on Config to read OTEL_* settings from observability.conf. +lazy val Observability = + (project in file("common/observability")).settings(commonModuleSettings).dependsOn(Config) lazy val Resource = (project in file("common/resource")).settings(commonModuleSettings) lazy val Auth = (project in file("common/auth")) .settings(commonModuleSettings) @@ -129,7 +134,7 @@ lazy val Auth = (project in file("common/auth")) .dependsOn(DAO, Config) .dependsOn(DAO % "test->test") // reuse MockTexeraDB embedded Postgres in tests lazy val ConfigService = (project in file("config-service")) - .dependsOn(Auth, Config, DAO, Resource) + .dependsOn(Auth, Config, DAO, Resource, Observability) .dependsOn(DAO % "test->test") // reuse MockTexeraDB embedded Postgres in tests .settings(commonModuleSettings) .settings( @@ -139,7 +144,7 @@ lazy val ConfigService = (project in file("config-service")) ) ) lazy val AccessControlService = (project in file("access-control-service")) - .dependsOn(Auth, Config, DAO, Resource) + .dependsOn(Auth, Config, DAO, Resource, Observability) .settings(commonModuleSettings) .settings( dependencyOverrides ++= Seq( @@ -163,7 +168,7 @@ lazy val WorkflowCore = (project in file("common/workflow-core")) .configs(Test) .dependsOn(DAO % "test->test") // test scope dependency lazy val ComputingUnitManagingService = (project in file("computing-unit-managing-service")) - .dependsOn(WorkflowCore, Auth, Config, Resource) + .dependsOn(WorkflowCore, Auth, Config, Resource, Observability) .configs(Test) .dependsOn(DAO % "test->test") // reuse MockTexeraDB embedded Postgres in tests .settings(commonModuleSettings) @@ -210,7 +215,7 @@ lazy val ComputingUnitManagingService = (project in file("computing-unit-managin ) lazy val FileService = (project in file("file-service")) .settings(commonModuleSettings) - .dependsOn(WorkflowCore, Auth, Config, Resource, Util) + .dependsOn(WorkflowCore, Auth, Config, Resource, Util, Observability) .configs(Test) .dependsOn(DAO % "test->test") // test scope dependency .settings( @@ -238,7 +243,7 @@ lazy val WorkflowCompiler = (project in file("common/workflow-compiler")) .configs(Test) .dependsOn(WorkflowOperator) lazy val WorkflowCompilingService = (project in file("workflow-compiling-service")) - .dependsOn(WorkflowCompiler, Auth, Config, Resource) + .dependsOn(WorkflowCompiler, Auth, Config, Resource, Observability) .settings(commonModuleSettings) .settings( dependencyOverrides ++= Seq( @@ -250,7 +255,7 @@ lazy val WorkflowCompilingService = (project in file("workflow-compiling-service ) lazy val WorkflowExecutionService = (project in file("amber")) - .dependsOn(WorkflowCompiler, Auth, Config) + .dependsOn(WorkflowCompiler, Auth, Config, Observability) .settings(commonModuleSettings) .settings( dependencyOverrides ++= Seq( @@ -285,6 +290,7 @@ lazy val TexeraProject = (project in file(".")) // common libraries Auth, Config, + Observability, Resource, Util, DAO, diff --git a/common/config/src/main/resources/observability.conf b/common/config/src/main/resources/observability.conf new file mode 100644 index 00000000000..2bf7a4e10d6 --- /dev/null +++ b/common/config/src/main/resources/observability.conf @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# OpenTelemetry SDK bootstrap settings, consumed by OtelInit. Each value has a +# safe default and an optional environment override, so an operator can find and +# tune every knob here instead of in code. +observability { + # Master switch. Disabled by default: no exporters start and no telemetry is + # emitted until this is set to false (i.e. OTEL_SDK_DISABLED=false). + sdk-disabled = "true" + sdk-disabled = ${?OTEL_SDK_DISABLED} + + # OTLP collector endpoint. http/https only; loopback-only host allowlist by + # default (extend via allowed-hosts below). + endpoint = "http://127.0.0.1:4317" + endpoint = ${?OTEL_EXPORTER_OTLP_ENDPOINT} + + # Comma-separated resource attributes (k1=v1,k2=v2). service.name is set by + # the service and cannot be overridden here. + resource-attributes = "" + resource-attributes = ${?OTEL_RESOURCE_ATTRIBUTES} + + # Comma-separated extra hosts added to the endpoint allowlist. + allowed-hosts = "" + allowed-hosts = ${?TEXERA_OTEL_ALLOWED_HOSTS} + + # Metric export interval in milliseconds; out-of-range values fall back to + # the default. + metric-export-interval-ms = "30000" + metric-export-interval-ms = ${?OTEL_METRIC_EXPORT_INTERVAL} +} diff --git a/common/config/src/main/scala/org/apache/texera/common/config/EnvironmentalVariable.scala b/common/config/src/main/scala/org/apache/texera/common/config/EnvironmentalVariable.scala index a335ddeff6c..2876770df57 100644 --- a/common/config/src/main/scala/org/apache/texera/common/config/EnvironmentalVariable.scala +++ b/common/config/src/main/scala/org/apache/texera/common/config/EnvironmentalVariable.scala @@ -45,6 +45,13 @@ object EnvironmentalVariable { val ENV_USER_JWT_TOKEN = "USER_JWT_TOKEN" val ENV_AUTH_JWT_SECRET = "AUTH_JWT_SECRET" + // OpenTelemetry observability (see observability.conf) + val ENV_OTEL_SDK_DISABLED = "OTEL_SDK_DISABLED" + val ENV_OTEL_EXPORTER_OTLP_ENDPOINT = "OTEL_EXPORTER_OTLP_ENDPOINT" + val ENV_OTEL_RESOURCE_ATTRIBUTES = "OTEL_RESOURCE_ATTRIBUTES" + val ENV_OTEL_METRIC_EXPORT_INTERVAL = "OTEL_METRIC_EXPORT_INTERVAL" + val ENV_TEXERA_OTEL_ALLOWED_HOSTS = "TEXERA_OTEL_ALLOWED_HOSTS" + // JDBC val ENV_JDBC_URL = "STORAGE_JDBC_URL" val ENV_JDBC_USERNAME = "STORAGE_JDBC_USERNAME" diff --git a/common/config/src/main/scala/org/apache/texera/common/config/ObservabilityConfig.scala b/common/config/src/main/scala/org/apache/texera/common/config/ObservabilityConfig.scala new file mode 100644 index 00000000000..29d121dbd4d --- /dev/null +++ b/common/config/src/main/scala/org/apache/texera/common/config/ObservabilityConfig.scala @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.texera.common.config + +import com.typesafe.config.{Config, ConfigFactory} + +/** + * Typed view over observability.conf. Each field carries the HOCON default + * already merged with its OTEL_* environment override, so OtelInit reads its + * settings from one place instead of calling System.getenv directly. Values + * are kept as strings and interpreted by OtelInit, which tolerates malformed + * input without throwing. + */ +object ObservabilityConfig { + private val conf: Config = ConfigFactory.parseResources("observability.conf").resolve() + + val sdkDisabled: String = conf.getString("observability.sdk-disabled") + val endpoint: String = conf.getString("observability.endpoint") + val resourceAttributes: String = conf.getString("observability.resource-attributes") + val allowedHosts: String = conf.getString("observability.allowed-hosts") + val metricExportIntervalMs: String = conf.getString("observability.metric-export-interval-ms") +} diff --git a/common/config/src/test/scala/org/apache/texera/common/config/ObservabilityConfigSpec.scala b/common/config/src/test/scala/org/apache/texera/common/config/ObservabilityConfigSpec.scala new file mode 100644 index 00000000000..659485bbc9f --- /dev/null +++ b/common/config/src/test/scala/org/apache/texera/common/config/ObservabilityConfigSpec.scala @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.common.config + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** + * Spec for [[ObservabilityConfig]]. Reading each value forces resolution from + * observability.conf, so a renamed key surfaces here as a ConfigException. + * Exact-value assertions are guarded on the OTEL_* override being unset. + */ +class ObservabilityConfigSpec extends AnyFlatSpec with Matchers { + + // `${?VAR}` in HOCON can be satisfied by an OS env var or a JVM system property. + private def isOverridden(name: String): Boolean = + sys.env.contains(name) || sys.props.contains(name) + + "ObservabilityConfig" should "default to disabled per issue #5367" in { + if (!isOverridden(EnvironmentalVariable.ENV_OTEL_SDK_DISABLED)) { + ObservabilityConfig.sdkDisabled shouldBe "true" + } else { + ObservabilityConfig.sdkDisabled should not be empty + } + } + + it should "default to a loopback OTLP endpoint" in { + if (!isOverridden(EnvironmentalVariable.ENV_OTEL_EXPORTER_OTLP_ENDPOINT)) { + ObservabilityConfig.endpoint shouldBe "http://127.0.0.1:4317" + } else { + ObservabilityConfig.endpoint should not be empty + } + } + + it should "default the metric export interval to 30s" in { + if (!isOverridden(EnvironmentalVariable.ENV_OTEL_METRIC_EXPORT_INTERVAL)) { + ObservabilityConfig.metricExportIntervalMs shouldBe "30000" + } else { + ObservabilityConfig.metricExportIntervalMs should not be empty + } + } + + it should "resolve resource-attributes and allowed-hosts without error" in { + // Empty by default; the point is that the keys exist and resolve. + noException should be thrownBy ObservabilityConfig.resourceAttributes + noException should be thrownBy ObservabilityConfig.allowedHosts + } +} diff --git a/common/observability/build.sbt b/common/observability/build.sbt new file mode 100644 index 00000000000..ac33d09e417 --- /dev/null +++ b/common/observability/build.sbt @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import scala.collection.Seq + +name := "observability" + + +enablePlugins(JavaAppPackaging) + +// Enable semanticdb for Scalafix +ThisBuild / semanticdbEnabled := true +ThisBuild / semanticdbVersion := scalafixSemanticdb.revision + +// Manage dependency conflicts by always using the latest revision +ThisBuild / conflictManager := ConflictManager.latestRevision + +// Restrict parallel execution of tests to avoid conflicts +Global / concurrentRestrictions += Tags.limit(Tags.Test, 1) + +///////////////////////////////////////////////////////////////////////////// +// Compiler Options +///////////////////////////////////////////////////////////////////////////// + +// Scala compiler options +Compile / scalacOptions ++= Seq( + "-Xelide-below", "WARNING", // Turn on optimizations with "WARNING" as the threshold + "-feature", // Check feature warnings + "-deprecation", // Check deprecation warnings + "-Ywarn-unused:imports" // Check for unused imports +) + +///////////////////////////////////////////////////////////////////////////// +// Dependencies +///////////////////////////////////////////////////////////////////////////// + +// OpenTelemetry version is pinned here as the single source of truth; every +// service picks it up via dependsOn(Observability). Bump deliberately. +val openTelemetryVersion = "1.50.0" + +libraryDependencies ++= Seq( + "com.typesafe.scala-logging" %% "scala-logging" % "3.9.5", // for LazyLogging in OtelInit + // OpenTelemetry SDK bootstrap (Apache-2.0). We deliberately do NOT use + // sdk-extension-autoconfigure: the security model requires that endpoint + // + resource-attribute filtering run before any exporter is configured. + "io.opentelemetry" % "opentelemetry-api" % openTelemetryVersion, + "io.opentelemetry" % "opentelemetry-sdk" % openTelemetryVersion, + "io.opentelemetry" % "opentelemetry-exporter-otlp" % openTelemetryVersion, + // Logback Classic is needed at compile time to write the OTel log + // appender. Marked `provided` because every service already brings + // Logback in transitively (via Dropwizard / SLF4J), so we don't + // bundle a second copy. + "ch.qos.logback" % "logback-classic" % "1.2.13" % "provided", + // Test-only: in-memory exporter for OtelInitSpec; avoids hitting a real + // collector during unit tests. + "io.opentelemetry" % "opentelemetry-sdk-testing" % openTelemetryVersion % Test, + "ch.qos.logback" % "logback-classic" % "1.2.13" % Test, + "org.scalatest" %% "scalatest" % "3.2.15" % Test // ScalaTest (for unit tests) +) diff --git a/common/observability/src/main/scala/org/apache/texera/observability/LogSanitizer.scala b/common/observability/src/main/scala/org/apache/texera/observability/LogSanitizer.scala new file mode 100644 index 00000000000..ffb35cebb8e --- /dev/null +++ b/common/observability/src/main/scala/org/apache/texera/observability/LogSanitizer.scala @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import scala.jdk.CollectionConverters._ + +/** + * Pure functions that sanitize log bodies and MDC before export: + * strip control characters, redact secrets, cap body size, and + * filter MDC down by dropping denied keys. + */ +object LogSanitizer { + + /** Per-record body length cap, in chars. */ + val MaxBodyBytes: Int = 16 * 1024 + + /** Suffix appended to truncated bodies. */ + val TruncatedMarker: String = "...[truncated]" + + /** C0 control characters except TAB (0x09), plus DEL (0x7F). */ + private val C0ControlRegex = "[\\x00-\\x08\\x0A-\\x1F\\x7F]".r + + /** Secret patterns, redacted from bodies. Most specific first. */ + private val SecretPatterns: Seq[scala.util.matching.Regex] = Seq( + // Bearer token + """(?i)Bearer\s+[A-Za-z0-9._\-/+=]{8,}""".r, + // password=... or password: ... + """(?i)password\s*[=:]\s*[^\s,;"']+""".r, + // AWS access key ID + """AKIA[0-9A-Z]{16}""".r, + // labelled AWS secret access key + """(?i)aws_secret_access_key\s*[=:]\s*[A-Za-z0-9/+=]{20,}""".r + ) + + /** MDC keys never forwarded to OTel log attributes. Default-allow: any key + * our instrumentation sets is exported, so adding a new correlation field + * needs no edit here. Only the noisy keys Pekko's SLF4J bridge injects are + * dropped, since they are redundant with the log body and would bloat every + * exported record. Values that pass through are still run through + * [[sanitize]], so secret-shaped content is redacted regardless of key; a + * key whose name looks credential-bearing (see [[isSecretKey]]) has its + * value redacted wholesale. + */ + val DeniedMdcKeys: Set[String] = Set( + "sourceThread", + "pekkoSource", + "pekkoAddress", + "pekkoTimestamp", + "sourceActorSystem" + ) + + /** Substrings marking an MDC key as credential-bearing. A matching key has + * its value redacted whole, since the value alone (e.g. a bare password) + * need not match any [[SecretPatterns]] regex to be a secret. + */ + private val SecretKeySubstrings: Seq[String] = + Seq( + "password", + "passwd", + "pwd", + "secret", + "token", + "apikey", + "api_key", + "authorization", + "credential" + ) + + private def isSecretKey(key: String): Boolean = { + val k = key.toLowerCase + SecretKeySubstrings.exists(k.contains) + } + + /** Strip C0 control characters (except TAB) and DEL. Null-safe. */ + def stripControlChars(body: String): String = + if (body == null) "" else C0ControlRegex.replaceAllIn(body, "") + + /** Redact secret-shaped substrings. Null-safe, idempotent. */ + def redactSecrets(body: String): String = + if (body == null) "" + else SecretPatterns.foldLeft(body)((acc, p) => p.replaceAllIn(acc, "[REDACTED]")) + + /** Strip control chars, redact secrets, then truncate. Idempotent. */ + def sanitize(body: String): String = { + if (body == null || body.isEmpty) return "" + truncate(redactSecrets(stripControlChars(body))) + } + + /** Truncate to MaxBodyBytes, appending the marker if cut. */ + def truncate(body: String): String = { + if (body.length <= MaxBodyBytes) body + else body.substring(0, MaxBodyBytes - TruncatedMarker.length) + TruncatedMarker + } + + /** Drop denied MDC keys, sanitize the surviving values. A key whose name is + * credential-bearing has its value redacted whole. Null-safe. + */ + def filterMdc(mdc: java.util.Map[String, String]): Map[String, String] = { + if (mdc == null) return Map.empty + mdc.asScala.iterator.collect { + case (k, v) if k != null && v != null && !DeniedMdcKeys.contains(k) => + if (isSecretKey(k)) k -> "[REDACTED]" else k -> sanitize(v) + }.toMap + } +} diff --git a/common/observability/src/main/scala/org/apache/texera/observability/OtelInit.scala b/common/observability/src/main/scala/org/apache/texera/observability/OtelInit.scala new file mode 100644 index 00000000000..50390986773 --- /dev/null +++ b/common/observability/src/main/scala/org/apache/texera/observability/OtelInit.scala @@ -0,0 +1,403 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import com.typesafe.scalalogging.LazyLogging +import org.apache.texera.common.config.{EnvironmentalVariable, ObservabilityConfig} +import io.opentelemetry.api.{GlobalOpenTelemetry, OpenTelemetry} +import io.opentelemetry.api.common.{AttributeKey, Attributes} +import io.opentelemetry.exporter.otlp.logs.OtlpGrpcLogRecordExporter +import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter +import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.logs.SdkLoggerProvider +import io.opentelemetry.sdk.logs.`export`.{BatchLogRecordProcessor, LogRecordExporter} +import io.opentelemetry.sdk.metrics.SdkMeterProvider +import io.opentelemetry.sdk.metrics.`export`.{MetricExporter, PeriodicMetricReader} +import io.opentelemetry.sdk.resources.Resource +import io.opentelemetry.sdk.trace.SdkTracerProvider +import io.opentelemetry.sdk.trace.`export`.{BatchSpanProcessor, SpanExporter} + +import java.net.URI +import java.time.Duration +import scala.util.{Failure, Success, Try} + +/** + * Bootstraps the OpenTelemetry SDK for a Texera service. + * + * Disabled by default; set OTEL_SDK_DISABLED=false to enable it. Reads its + * settings from observability.conf (each defaulted, each OTEL_*-overridable), + * validates the endpoint against an allowlist, builds tracer/log/metric + * providers, and attaches a Logback appender. Returns None when disabled or + * misconfigured; never throws. + */ +object OtelInit extends LazyLogging { + + /** Endpoint schemes we accept. OTLP-over-gRPC uses http/https endpoints; + * the exporter rejects a `grpc://` scheme outright, so it is not allowed. + */ + private[observability] val AllowedSchemes: Set[String] = Set("http", "https") + + /** Hosts we accept for the OTLP endpoint by default. */ + private[observability] val DefaultAllowedHosts: Set[String] = Set( + "localhost", + "127.0.0.1", + "[::1]" + ) + + /** Default endpoint. 127.0.0.1 (not "localhost") to force IPv4 so a + * natively-run service reaches the collector on dual-stack hosts. + */ + private val DefaultEndpoint = "http://127.0.0.1:4317" + + /** Metric export interval bounds; out-of-range values fall back to the + * default (see clampIntervalMs). + */ + private[observability] val MinMetricIntervalMs: Long = 1000L + private[observability] val MaxMetricIntervalMs: Long = 10L * 60L * 1000L + private[observability] val DefaultMetricIntervalMs: Long = 30L * 1000L + + // Idempotency guard: init() is a no-op after the first call. + @volatile private var initialized: Option[OpenTelemetry] = None + + /** + * Initialize the SDK for the given service name. Returns Some on + * success, None when disabled or misconfigured. When enabled, also + * attaches a [[TexeraOtelLogAppender]] to the Logback ROOT logger. + */ + def init(serviceName: String): Option[OpenTelemetry] = + synchronized { + if (initialized.isDefined) return initialized + + // Source the OTEL_* settings from observability.conf (HOCON defaults + // already merged with any env override); fall back to the raw environment + // for anything else. + val env = (key: String) => + key match { + case EnvironmentalVariable.ENV_OTEL_SDK_DISABLED => Some(ObservabilityConfig.sdkDisabled) + case EnvironmentalVariable.ENV_OTEL_EXPORTER_OTLP_ENDPOINT => + Some(ObservabilityConfig.endpoint) + case EnvironmentalVariable.ENV_OTEL_RESOURCE_ATTRIBUTES => + Some(ObservabilityConfig.resourceAttributes) + case EnvironmentalVariable.ENV_TEXERA_OTEL_ALLOWED_HOSTS => + Some(ObservabilityConfig.allowedHosts) + case EnvironmentalVariable.ENV_OTEL_METRIC_EXPORT_INTERVAL => + Some(ObservabilityConfig.metricExportIntervalMs) + case other => Option(System.getenv(other)) + } + val result = initInternal( + serviceName = serviceName, + envProvider = env, + spanExporterFactory = buildOtlpSpanExporter, + logExporterFactory = endpoint => Some(buildOtlpLogExporter(endpoint)), + metricExporterFactory = endpoint => Some(buildOtlpMetricExporter(endpoint)), + logbackAttacher = LogbackBinder.attach + ) + // Register globally so OTel-aware code can use GlobalOpenTelemetry + // without threading the SDK through callsites. set() throws on a + // second call; wrap defensively. + result.foreach { sdk => + Try(GlobalOpenTelemetry.set(sdk)).failed.foreach { t => + logger.warn( + s"GlobalOpenTelemetry already set; using the existing instance: ${t.getMessage}" + ) + } + } + result + } + + /** + * Test-only entry point: injects an env-var map and exporters so the + * SDK makes no network connection. Does not attach the Logback appender. + */ + private[observability] def initForTest( + serviceName: String, + envOverride: Map[String, String], + exporter: SpanExporter, + metricExporter: Option[MetricExporter] = None + ): Option[OpenTelemetry] = + synchronized { + initInternal( + serviceName = serviceName, + envProvider = envOverride.get, + spanExporterFactory = _ => exporter, + logExporterFactory = _ => None, + metricExporterFactory = _ => metricExporter, + logbackAttacher = (_, _) => () // no-op in tests + ) + } + + /** Test-only: forget any previously-installed SDK. Does not unregister + * shutdown hooks (the previous SDK is closed instead). + */ + private[observability] def resetForTest(): Unit = + synchronized { + initialized.foreach { + case sdk: OpenTelemetrySdk => + Try(sdk.getSdkTracerProvider.close()) + Try(sdk.getSdkLoggerProvider.close()) + Try(sdk.getSdkMeterProvider.close()) + case _ => () + } + initialized = None + } + + private def initInternal( + serviceName: String, + envProvider: String => Option[String], + spanExporterFactory: String => SpanExporter, + logExporterFactory: String => Option[LogRecordExporter], + metricExporterFactory: String => Option[MetricExporter], + logbackAttacher: (String, OpenTelemetry) => Unit + ): Option[OpenTelemetry] = { + if (initialized.isDefined) return initialized + + // Disabled by default (issue #5367): stay inert unless OTEL_SDK_DISABLED is + // explicitly false. An unreachable endpoint drops records without crashing. + val disabled = envProvider(EnvironmentalVariable.ENV_OTEL_SDK_DISABLED).getOrElse("true") + if (!disabled.equalsIgnoreCase("false")) { + logger.info( + "OpenTelemetry SDK disabled (OTEL_SDK_DISABLED not false). No telemetry will be emitted." + ) + return None + } + + val endpoint = + envProvider(EnvironmentalVariable.ENV_OTEL_EXPORTER_OTLP_ENDPOINT).getOrElse(DefaultEndpoint) + val extraAllowed = envProvider(EnvironmentalVariable.ENV_TEXERA_OTEL_ALLOWED_HOSTS) + .map(_.split(',').iterator.map(_.trim.toLowerCase).filter(_.nonEmpty).toSet) + .getOrElse(Set.empty) + val allowedHosts = DefaultAllowedHosts ++ extraAllowed + + validateEndpoint(endpoint, allowedHosts) match { + case Left(reason) => + // One WARN; no telemetry is emitted. + logger.warn( + s"OpenTelemetry SDK disabled: invalid OTEL_EXPORTER_OTLP_ENDPOINT — $reason. " + + "Set TEXERA_OTEL_ALLOWED_HOSTS to extend the allowlist." + ) + return None + case Right(_) => // ok + } + + val rawAttrs = envProvider(EnvironmentalVariable.ENV_OTEL_RESOURCE_ATTRIBUTES).getOrElse("") + val resource = buildResource(serviceName, rawAttrs) + + val spanExporter = spanExporterFactory(endpoint) + val tracerProvider = SdkTracerProvider + .builder() + .setResource(resource) + .addSpanProcessor(BatchSpanProcessor.builder(spanExporter).build()) + .build() + + val sdkBuilder = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider) + + // Logger provider is optional; the factory returns None in tests. + val loggerProviderOpt = logExporterFactory(endpoint).map { logExporter => + val lp = SdkLoggerProvider + .builder() + .setResource(resource) + .addLogRecordProcessor(BatchLogRecordProcessor.builder(logExporter).build()) + .build() + sdkBuilder.setLoggerProvider(lp) + lp + } + + // Meter provider is optional too; interval falls back to the default + // when out of range. + val intervalMs = + clampIntervalMs(envProvider(EnvironmentalVariable.ENV_OTEL_METRIC_EXPORT_INTERVAL)) + val meterProviderOpt = metricExporterFactory(endpoint).map { metricExporter => + val reader = PeriodicMetricReader + .builder(metricExporter) + .setInterval(Duration.ofMillis(intervalMs)) + .build() + val mp = SdkMeterProvider + .builder() + .setResource(resource) + .registerMetricReader(reader) + .build() + sdkBuilder.setMeterProvider(mp) + mp + } + + val sdk = sdkBuilder.build() + + // One startup span carrying only service.name. + val span = sdk.getTracer("texera.bootstrap").spanBuilder("service.start").startSpan() + Try(span.setAttribute("service.name", serviceName)) + span.end() + + // Wire the Logback appender; failure here must not crash the service. + Try(logbackAttacher(serviceName, sdk)).failed.foreach { t => + logger.warn(s"Failed to attach OTel Logback appender (logs not exported): ${t.getMessage}") + } + + // Flush providers on shutdown. Added after the SDK is fully built. + Runtime.getRuntime.addShutdownHook( + new Thread( + () => { + Try(tracerProvider.close()) + loggerProviderOpt.foreach(lp => Try(lp.close())) + meterProviderOpt.foreach(mp => Try(mp.close())) + () + }, + "otel-shutdown" + ) + ) + + initialized = Some(sdk) + logger.info(s"OpenTelemetry SDK initialized for service '$serviceName' (endpoint=$endpoint).") + initialized + } + + /** + * Validate the endpoint is parseable and uses an allowlisted scheme + * and host. Pure function. + */ + private[observability] def validateEndpoint( + endpoint: String, + allowedHosts: Set[String] + ): Either[String, Unit] = { + Try(URI.create(endpoint)) match { + case Failure(e) => + Left(s"unparseable URI (${e.getClass.getSimpleName})") + case Success(uri) => + val scheme = Option(uri.getScheme).map(_.toLowerCase).getOrElse("") + if (scheme.isEmpty) { + Left("missing scheme") + } else if (!AllowedSchemes.contains(scheme)) { + Left( + s"scheme '$scheme' not in allowlist ${AllowedSchemes.toSeq.sorted.mkString("{", ",", "}")}" + ) + } else { + val host = Option(uri.getHost).map(_.toLowerCase).getOrElse("") + if (host.isEmpty) { + Left("missing host") + } else if (!allowedHosts.contains(host)) { + Left(s"host '$host' not in allowlist") + } else { + Right(()) + } + } + } + } + + /** + * Build a Resource from the service name and OTEL_RESOURCE_ATTRIBUTES. + * Every parsed attribute is applied so new resource fields need no edit + * here; the one exception is service.name, which the argument controls + * and env cannot override. + */ + private[observability] def buildResource(serviceName: String, rawAttrs: String): Resource = { + val builder = Attributes.builder() + builder.put(AttributeKey.stringKey("service.name"), serviceName) + + parseAttrs(rawAttrs).foreach { + case (key, value) if key != "service.name" => + builder.put(AttributeKey.stringKey(key), value) + case _ => // env cannot override service.name + } + + Resource.create(builder.build()) + } + + /** Parse a `k1=v1,k2=v2` string. Malformed entries are skipped. */ + private[observability] def parseAttrs(raw: String): Seq[(String, String)] = { + if (raw == null || raw.isEmpty) return Seq.empty + raw + .split(',') + .iterator + .map(_.trim) + .filter(_.nonEmpty) + .flatMap { entry => + val idx = entry.indexOf('=') + if (idx <= 0 || idx == entry.length - 1) None + else Some(entry.substring(0, idx).trim -> entry.substring(idx + 1).trim) + } + .toSeq + } + + private def buildOtlpSpanExporter(endpoint: String): SpanExporter = + OtlpGrpcSpanExporter.builder().setEndpoint(endpoint).build() + + private def buildOtlpLogExporter(endpoint: String): LogRecordExporter = + OtlpGrpcLogRecordExporter.builder().setEndpoint(endpoint).build() + + private def buildOtlpMetricExporter(endpoint: String): MetricExporter = + OtlpGrpcMetricExporter.builder().setEndpoint(endpoint).build() + + /** + * Parse and clamp OTEL_METRIC_EXPORT_INTERVAL (ms). Out-of-range or + * unparseable input falls back to the default with one WARN. + */ + private[observability] def clampIntervalMs(raw: Option[String]): Long = { + raw match { + case None => DefaultMetricIntervalMs + case Some(value) => + Try(value.trim.toLong) match { + case Failure(_) => + logger.warn( + s"OTEL_METRIC_EXPORT_INTERVAL '$value' is not a number; " + + s"using default ${DefaultMetricIntervalMs}ms." + ) + DefaultMetricIntervalMs + case Success(ms) if ms < MinMetricIntervalMs || ms > MaxMetricIntervalMs => + logger.warn( + s"OTEL_METRIC_EXPORT_INTERVAL=${ms}ms out of range " + + s"[${MinMetricIntervalMs}, ${MaxMetricIntervalMs}]; " + + s"using default ${DefaultMetricIntervalMs}ms." + ) + DefaultMetricIntervalMs + case Success(ms) => ms + } + } + } +} + +/** + * Isolates the Logback attach step so [[OtelInit]] does not import + * Logback types directly, keeping SDK init testable with a mock attacher. + */ +private[observability] object LogbackBinder extends LazyLogging { + + /** Attach a [[TexeraOtelLogAppender]] bound to `otel` to the Logback + * ROOT logger. Emits one WARN and returns if Logback is not the + * active SLF4J binding. + */ + def attach(serviceName: String, otel: OpenTelemetry): Unit = { + val factory = org.slf4j.LoggerFactory.getILoggerFactory + factory match { + case ctx: ch.qos.logback.classic.LoggerContext => + val root = ctx.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME) + val appender = new TexeraOtelLogAppender() + appender.setContext(ctx) + appender.setName(s"texera-otel-$serviceName") + appender.bind(otel) + appender.start() + root.addAppender(appender) + case other => + logger.warn( + s"SLF4J binding is not Logback (${other.getClass.getName}); " + + "OTel log export is not wired. Application logs to stdout/file are unaffected." + ) + } + } +} diff --git a/common/observability/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala b/common/observability/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala new file mode 100644 index 00000000000..306236d3318 --- /dev/null +++ b/common/observability/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import ch.qos.logback.classic.Level +import ch.qos.logback.classic.spi.{ILoggingEvent, IThrowableProxy, ThrowableProxyUtil} +import ch.qos.logback.core.UnsynchronizedAppenderBase +import io.opentelemetry.api.OpenTelemetry +import io.opentelemetry.api.common.AttributeKey +import io.opentelemetry.api.logs.{Logger, Severity} +import io.opentelemetry.api.trace.Span +import io.opentelemetry.context.Context + +import java.util.concurrent.TimeUnit + +/** + * Logback appender that sanitizes each event via [[LogSanitizer]] and + * emits it as an OTel LogRecord. [[append]] is a no-op until [[bind]] + * is called and after [[stop]]. + * + * This is internal plumbing, not the developer logging API. Code logs + * through the normal SLF4J / scala-logging interface and adds correlation + * ids via MDC; [[OtelInit.init]] attaches this appender to the ROOT logger + * so those records also reach OTel: + * + * {{{ + * class Foo extends LazyLogging { + * MDC.put("workflowId", id) // forwarded as an OTel log attribute + * try logger.info("started") // body + severity + trace context + * finally MDC.remove("workflowId") + * } + * }}} + */ +class TexeraOtelLogAppender extends UnsynchronizedAppenderBase[ILoggingEvent] { + + // @volatile so a late bind() is visible to appender threads. + @volatile private var otelLogger: Option[Logger] = None + + def bind(otel: OpenTelemetry): Unit = { + otelLogger = Some(otel.getLogsBridge.get("texera.logback")) + } + + override def stop(): Unit = { + otelLogger = None + super.stop() + } + + override def append(event: ILoggingEvent): Unit = { + otelLogger match { + case None => () // not bound + case Some(logger) => + try { + emit(logger, event) + } catch { + // An appender must not throw into the calling thread. + case t: Throwable => + addError("OTel log emission failed", t) + } + } + } + + private def emit(logger: Logger, event: ILoggingEvent): Unit = { + // Control-strip the message only (trace newlines must survive), then append + // the stack trace, redact secrets across the whole body, and cap length. + val message = LogSanitizer.stripControlChars(event.getFormattedMessage) + val combined = Option(event.getThrowableProxy) match { + case Some(proxy) => message + "\n" + formatThrowable(proxy) + case None => message + } + val body = LogSanitizer.truncate(LogSanitizer.redactSecrets(combined)) + val builder = logger + .logRecordBuilder() + .setBody(body) + .setSeverity(severityFromLevel(event.getLevel)) + .setSeverityText(event.getLevel.toString) + .setTimestamp(event.getTimeStamp, TimeUnit.MILLISECONDS) + + // Surviving (deny-list filtered) MDC keys as typed attributes. + LogSanitizer.filterMdc(event.getMDCPropertyMap).foreach { + case (k, v) => builder.setAttribute(AttributeKey.stringKey(k), v) + } + + builder.setAttribute(AttributeKey.stringKey("logger.name"), event.getLoggerName) + builder.setAttribute(AttributeKey.stringKey("thread.name"), event.getThreadName) + + // Attach trace context so the SDK sets trace_id / span_id. + val span = Span.current() + if (span.getSpanContext.isValid) { + builder.setContext(Context.current()) + } + + builder.emit() + } + + /** Format a throwable proxy as a Logback-style stack trace. */ + private def formatThrowable(proxy: IThrowableProxy): String = + ThrowableProxyUtil.asString(proxy) + + private def severityFromLevel(level: Level): Severity = { + if (level == null) return Severity.UNDEFINED_SEVERITY_NUMBER + level.toInt match { + case Level.TRACE_INT => Severity.TRACE + case Level.DEBUG_INT => Severity.DEBUG + case Level.INFO_INT => Severity.INFO + case Level.WARN_INT => Severity.WARN + case Level.ERROR_INT => Severity.ERROR + case _ => Severity.UNDEFINED_SEVERITY_NUMBER + } + } +} diff --git a/common/observability/src/test/scala/org/apache/texera/observability/LogSanitizerSpec.scala b/common/observability/src/test/scala/org/apache/texera/observability/LogSanitizerSpec.scala new file mode 100644 index 00000000000..022b52aa68a --- /dev/null +++ b/common/observability/src/test/scala/org/apache/texera/observability/LogSanitizerSpec.scala @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import scala.jdk.CollectionConverters._ + +class LogSanitizerSpec extends AnyFlatSpec with Matchers { + + // ----- sanitize: control characters ---------------------------------- + + "sanitize" should "strip CR/LF so a user-supplied message cannot forge a new log line" in { + val crlfPayload = "hello\r\nFAKE LOG LINE\r\nworld" + LogSanitizer.sanitize(crlfPayload) shouldBe "helloFAKE LOG LINEworld" + } + + it should "strip other C0 control characters but preserve TAB" in { + val payload = "before\u0000NUL\u0007BEL\tTAB\u001bafter\u007fdel" + LogSanitizer.sanitize(payload) shouldBe "beforeNULBEL\tTABafterdel" + } + + it should "handle empty / null bodies cleanly" in { + LogSanitizer.sanitize("") shouldBe "" + LogSanitizer.sanitize(null) shouldBe "" + } + + // ----- sanitize: secret scrubbing ------------------------------------ + + it should "redact Bearer tokens regardless of case" in { + LogSanitizer.sanitize("Authorization: Bearer abc123XYZ.foo") should include("[REDACTED]") + LogSanitizer.sanitize("Authorization: Bearer abc123XYZ.foo") should not include "abc123XYZ" + LogSanitizer.sanitize("auth = bearer eyJhbGci.tok") should include("[REDACTED]") + } + + it should "redact password=... key/value forms" in { + val out = LogSanitizer.sanitize("connecting: user=alice password=hunter2 host=db") + out should include("[REDACTED]") + out should not include "hunter2" + // surrounding context preserved + out should include("user=alice") + out should include("host=db") + } + + it should "redact AWS access key IDs" in { + val out = LogSanitizer.sanitize("found key AKIAIOSFODNN7EXAMPLE in env") + out should include("[REDACTED]") + out should not include "AKIAIOSFODNN7EXAMPLE" + } + + it should "redact AWS secret access keys when explicitly labelled" in { + val out = LogSanitizer.sanitize( + "aws_secret_access_key=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY some text" + ) + out should include("[REDACTED]") + out should not include "wJalrXUtnFEMI" + } + + it should "leave already-redacted content alone (idempotent)" in { + val once = LogSanitizer.sanitize("Authorization: Bearer abc12345.deadbeef") + val twice = LogSanitizer.sanitize(once) + twice shouldBe once + } + + // ----- sanitize: size cap -------------------------------------------- + + it should "truncate bodies larger than MaxBodyBytes and append the marker" in { + val oversize = "a" * (LogSanitizer.MaxBodyBytes * 4) // ~64 KiB + val out = LogSanitizer.sanitize(oversize) + out.length shouldBe LogSanitizer.MaxBodyBytes + out should endWith(LogSanitizer.TruncatedMarker) + } + + it should "leave bodies at or below the cap unchanged in length" in { + val rightAtCap = "x" * LogSanitizer.MaxBodyBytes + LogSanitizer.sanitize(rightAtCap).length shouldBe LogSanitizer.MaxBodyBytes + } + + // ----- filterMdc ----------------------------------------------------- + + "filterMdc" should "drop denied Pekko keys and pass every other key through" in { + val mdc = Map( + "trace_id" -> "abc", + "span_id" -> "def", + "texera.workflow.id" -> "42", + "app.new.tag" -> "kept", + "sourceThread" -> "dispatcher-3", + "pekkoSource" -> "akka://sys/user/actor" + ).asJava + val out = LogSanitizer.filterMdc(mdc) + out.keySet shouldBe Set("trace_id", "span_id", "texera.workflow.id", "app.new.tag") + } + + it should "scrub secret-shaped values on keys that pass through" in { + val mdc = Map("note" -> "password=p4ssw0rd").asJava + LogSanitizer.filterMdc(mdc)("note") should not include "p4ssw0rd" + } + + it should "redact the value of a credential-named key even when the value itself is benign" in { + val mdc = Map( + "password" -> "hunter2", + "user.api_key" -> "abcdef", + "authToken" -> "xyz", + "trace_id" -> "keep-me" + ).asJava + val out = LogSanitizer.filterMdc(mdc) + out("password") shouldBe "[REDACTED]" + out("user.api_key") shouldBe "[REDACTED]" + out("authToken") shouldBe "[REDACTED]" + out("trace_id") shouldBe "keep-me" + } + + it should "tolerate null map and null values" in { + LogSanitizer.filterMdc(null) shouldBe empty + + val javaMap = new java.util.HashMap[String, String]() + javaMap.put("trace_id", null) + javaMap.put("texera.user.id", "7") + val out = LogSanitizer.filterMdc(javaMap) + out shouldBe Map("texera.user.id" -> "7") + } +} diff --git a/common/observability/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala b/common/observability/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala new file mode 100644 index 00000000000..6b8c9bcb437 --- /dev/null +++ b/common/observability/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala @@ -0,0 +1,263 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter +import org.scalatest.BeforeAndAfterEach +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import scala.jdk.CollectionConverters._ + +class OtelInitSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { + + override def beforeEach(): Unit = { + OtelInit.resetForTest() + } + + override def afterEach(): Unit = { + OtelInit.resetForTest() + } + + // ----- validateEndpoint: pure function, exhaustive cases ------------- + + "validateEndpoint" should "accept loopback OTLP http(s) URLs" in { + OtelInit.validateEndpoint("http://localhost:4317", OtelInit.DefaultAllowedHosts) shouldBe Right( + () + ) + OtelInit.validateEndpoint("http://127.0.0.1:4317", OtelInit.DefaultAllowedHosts) shouldBe Right( + () + ) + OtelInit.validateEndpoint( + "https://localhost:4318", + OtelInit.DefaultAllowedHosts + ) shouldBe Right(()) + } + + it should "reject a grpc:// endpoint (the OTLP exporter accepts only http/https)" in { + val result = OtelInit.validateEndpoint("grpc://127.0.0.1:4317", OtelInit.DefaultAllowedHosts) + result.isLeft shouldBe true + result.left.toOption.get should include("scheme") + } + + it should "reject file:// schemes (path traversal style attack)" in { + val result = OtelInit.validateEndpoint("file:///etc/passwd", OtelInit.DefaultAllowedHosts) + result.isLeft shouldBe true + result.left.toOption.get should include("scheme") + } + + it should "reject arbitrary remote hosts not in the allowlist" in { + val result = OtelInit.validateEndpoint( + "http://attacker.example.com:4317", + OtelInit.DefaultAllowedHosts + ) + result.isLeft shouldBe true + result.left.toOption.get should include("host") + } + + it should "accept hosts added to the allowlist" in { + val widened = OtelInit.DefaultAllowedHosts + "collector.internal" + OtelInit.validateEndpoint("http://collector.internal:4317", widened) shouldBe Right(()) + } + + it should "reject endpoints with no scheme" in { + val result = OtelInit.validateEndpoint("localhost:4317", OtelInit.DefaultAllowedHosts) + result.isLeft shouldBe true + } + + it should "reject endpoints with no host" in { + val result = OtelInit.validateEndpoint("http:///path", OtelInit.DefaultAllowedHosts) + result.isLeft shouldBe true + } + + it should "reject completely malformed input" in { + val result = OtelInit.validateEndpoint("not a uri at all :: bad", OtelInit.DefaultAllowedHosts) + result.isLeft shouldBe true + } + + // ----- buildResource: passthrough with service.name protected --------- + + "buildResource" should "always include the service.name from the argument" in { + val r = OtelInit.buildResource("my-service", "") + Option( + r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.name")) + ) shouldBe Some( + "my-service" + ) + } + + it should "honor keys from OTEL_RESOURCE_ATTRIBUTES" in { + val r = OtelInit.buildResource("svc", "service.version=1.2.3,deployment.environment=staging") + Option( + r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.version")) + ) shouldBe Some( + "1.2.3" + ) + Option( + r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("deployment.environment")) + ) shouldBe Some("staging") + } + + it should "pass custom keys through so new resource fields need no code edit" in { + val r = OtelInit.buildResource( + "svc", + "service.version=1.0,custom.tag=team-a,texera.region.id=us-west" + ) + val attrs: Map[String, String] = r.getAttributes.asMap.asScala.iterator.map { + case (k, v) => k.getKey -> v.toString + }.toMap + + attrs("service.version") shouldBe "1.0" + attrs("custom.tag") shouldBe "team-a" + attrs("texera.region.id") shouldBe "us-west" + } + + it should "refuse to let OTEL_RESOURCE_ATTRIBUTES override service.name" in { + val r = OtelInit.buildResource("real-svc", "service.name=spoofed") + Option( + r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.name")) + ) shouldBe Some( + "real-svc" + ) + } + + it should "carry the texera.* resource attrs so a CU JVM auto-tags every emitted record" in { + val r = OtelInit.buildResource( + "texera-computing-unit-master", + "texera.computing_unit.id=8,texera.workflow.id=441,texera.execution.id=1234" + ) + val attrs: Map[String, String] = r.getAttributes.asMap.asScala.iterator.map { + case (k, v) => k.getKey -> v.toString + }.toMap + attrs("texera.computing_unit.id") shouldBe "8" + attrs("texera.workflow.id") shouldBe "441" + attrs("texera.execution.id") shouldBe "1234" + } + + it should "ignore malformed pairs without crashing" in { + val r = OtelInit.buildResource("svc", ",,,=,foo,service.version=,=bar,service.version=1.0,") + Option( + r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.version")) + ) shouldBe Some( + "1.0" + ) + } + + it should "handle empty / null input cleanly" in { + OtelInit.parseAttrs("") shouldBe empty + OtelInit.parseAttrs(null) shouldBe empty + } + + // ----- end-to-end init: span emission + disable behaviour ------------- + + "init" should "be a no-op when OTEL_SDK_DISABLED is explicitly set to true" in { + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest("svc", Map("OTEL_SDK_DISABLED" -> "true"), exporter) + result shouldBe None + exporter.getFinishedSpanItems.asScala shouldBe empty + } + + it should "stay inert by default when OTEL_SDK_DISABLED is unset (issue #5367)" in { + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest( + "svc", + Map( + // OTEL_SDK_DISABLED omitted; the SDK stays disabled by default. + "OTEL_EXPORTER_OTLP_ENDPOINT" -> "http://localhost:4317" + ), + exporter + ) + result shouldBe None + exporter.getFinishedSpanItems.asScala shouldBe empty + } + + it should "stay inert for any OTEL_SDK_DISABLED value other than an explicit false" in { + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest( + "svc", + Map("OTEL_SDK_DISABLED" -> "", "OTEL_EXPORTER_OTLP_ENDPOINT" -> "http://localhost:4317"), + exporter + ) + result shouldBe None + } + + it should "emit a single service.start span when enabled with a valid endpoint" in { + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest( + "my-service", + Map( + "OTEL_SDK_DISABLED" -> "false", + "OTEL_EXPORTER_OTLP_ENDPOINT" -> "http://localhost:4317" + ), + exporter + ) + result.isDefined shouldBe true + + // BatchSpanProcessor is async; flush before reading. + result.get + .asInstanceOf[io.opentelemetry.sdk.OpenTelemetrySdk] + .getSdkTracerProvider + .forceFlush() + .join(2, java.util.concurrent.TimeUnit.SECONDS) + + val spans = exporter.getFinishedSpanItems.asScala + spans should have size 1 + spans.head.getName shouldBe "service.start" + } + + it should "refuse to initialize when the endpoint scheme is file://" in { + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest( + "svc", + Map( + "OTEL_SDK_DISABLED" -> "false", + "OTEL_EXPORTER_OTLP_ENDPOINT" -> "file:///etc/passwd" + ), + exporter + ) + result shouldBe None + exporter.getFinishedSpanItems.asScala shouldBe empty + } + + it should "refuse to initialize when the endpoint host is off-allowlist" in { + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest( + "svc", + Map( + "OTEL_SDK_DISABLED" -> "false", + "OTEL_EXPORTER_OTLP_ENDPOINT" -> "http://attacker.example.com:4317" + ), + exporter + ) + result shouldBe None + exporter.getFinishedSpanItems.asScala shouldBe empty + } + + it should "be idempotent — second init returns the same instance" in { + val exporter = InMemorySpanExporter.create() + val env = Map( + "OTEL_SDK_DISABLED" -> "false", + "OTEL_EXPORTER_OTLP_ENDPOINT" -> "http://localhost:4317" + ) + val first = OtelInit.initForTest("svc", env, exporter) + val second = OtelInit.initForTest("svc", env, exporter) + second shouldBe first + } +} diff --git a/common/observability/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala b/common/observability/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala new file mode 100644 index 00000000000..21407f13035 --- /dev/null +++ b/common/observability/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import ch.qos.logback.classic.{Level, Logger, LoggerContext} +import ch.qos.logback.classic.spi.LoggingEvent +import io.opentelemetry.api.OpenTelemetry +import io.opentelemetry.api.logs.Severity +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.logs.SdkLoggerProvider +import io.opentelemetry.sdk.logs.`export`.SimpleLogRecordProcessor +import io.opentelemetry.sdk.testing.exporter.InMemoryLogRecordExporter +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.slf4j.LoggerFactory + +import scala.jdk.CollectionConverters._ + +class TexeraOtelLogAppenderSpec extends AnyFlatSpec with Matchers { + + /** Build an OpenTelemetry SDK whose LoggerProvider drains to the + * given in-memory exporter via the synchronous SimpleLogRecordProcessor, + * so tests don't depend on batch timing. + */ + private def newFixture(): (OpenTelemetry, InMemoryLogRecordExporter, TexeraOtelLogAppender) = { + val exporter = InMemoryLogRecordExporter.create() + val lp = SdkLoggerProvider + .builder() + .addLogRecordProcessor(SimpleLogRecordProcessor.create(exporter)) + .build() + val sdk = OpenTelemetrySdk.builder().setLoggerProvider(lp).build() + val appender = new TexeraOtelLogAppender() + appender.setContext(LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext]) + appender.bind(sdk) + appender.start() + (sdk, exporter, appender) + } + + private def makeEvent( + message: String, + level: Level = Level.INFO, + mdc: Map[String, String] = Map.empty + ): LoggingEvent = { + val ctx = LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext] + val logger = ctx.getLogger("test.logger").asInstanceOf[Logger] + val ev = new LoggingEvent("fqcn", logger, level, message, null, null) + if (mdc.nonEmpty) ev.setMDCPropertyMap(mdc.asJava) + ev + } + + // ----- positive paths ------------------------------------------------- + + "TexeraOtelLogAppender" should "emit an INFO record with body + severity" in { + val (_, exporter, appender) = newFixture() + appender.doAppend(makeEvent("hello world")) + + val records = exporter.getFinishedLogRecordItems.asScala + records should have size 1 + records.head.getBodyValue.asString shouldBe "hello world" + records.head.getSeverity shouldBe Severity.INFO + records.head.getSeverityText shouldBe "INFO" + } + + it should "map every log level to a distinct OTel severity" in { + val (_, exporter, appender) = newFixture() + Seq(Level.TRACE, Level.DEBUG, Level.INFO, Level.WARN, Level.ERROR).foreach { lvl => + appender.doAppend(makeEvent(s"msg-$lvl", lvl)) + } + val severities = exporter.getFinishedLogRecordItems.asScala.map(_.getSeverity).toSet + severities shouldBe Set( + Severity.TRACE, + Severity.DEBUG, + Severity.INFO, + Severity.WARN, + Severity.ERROR + ) + } + + // ----- security: sanitisation happens at the boundary ----------------- + + it should "strip CRLF from a forged log-injection payload before emission" in { + val (_, exporter, appender) = newFixture() + appender.doAppend(makeEvent("hello\r\nFAKE LOG LINE\r\nworld")) + + val body = exporter.getFinishedLogRecordItems.asScala.head.getBodyValue.asString + body shouldBe "helloFAKE LOG LINEworld" + body should not include "\n" + body should not include "\r" + } + + it should "redact Bearer tokens at emission time" in { + val (_, exporter, appender) = newFixture() + appender.doAppend(makeEvent("Authorization: Bearer abc123XYZ.foo")) + + val body = exporter.getFinishedLogRecordItems.asScala.head.getBodyValue.asString + body should include("[REDACTED]") + body should not include "abc123XYZ" + } + + it should "redact secrets inside an attached stack trace, not just the message" in { + val (_, exporter, appender) = newFixture() + val ctx = LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext] + val logger = ctx.getLogger("test.logger").asInstanceOf[Logger] + val boom = new RuntimeException("db connect failed for password=hunter2") + val ev = new LoggingEvent("fqcn", logger, Level.ERROR, "operation failed", boom, null) + appender.doAppend(ev) + + val body = exporter.getFinishedLogRecordItems.asScala.head.getBodyValue.asString + body should include("[REDACTED]") + body should not include "hunter2" + // The stack trace's newlines are preserved (only the message is C0-stripped). + body should include("\n") + } + + it should "truncate a 1 MiB body to MaxBodyBytes with the marker" in { + val (_, exporter, appender) = newFixture() + val oversize = "x" * (1024 * 1024) + appender.doAppend(makeEvent(oversize)) + + val body = exporter.getFinishedLogRecordItems.asScala.head.getBodyValue.asString + body.length shouldBe LogSanitizer.MaxBodyBytes + body should endWith(LogSanitizer.TruncatedMarker) + } + + // ----- security: MDC deny-list ---------------------------------------- + + private def attrsOf(record: io.opentelemetry.sdk.logs.data.LogRecordData): Map[String, String] = + record.getAttributes.asMap.asScala.iterator.map { case (k, v) => k.getKey -> v.toString }.toMap + + it should "forward arbitrary correlation MDC keys (deny-list, not allow-list)" in { + val (_, exporter, appender) = newFixture() + appender.doAppend( + makeEvent( + "msg", + mdc = Map( + "trace_id" -> "abc", + "texera.workflow.id" -> "42", + "some.new.key" -> "kept" + ) + ) + ) + + val attrs = attrsOf(exporter.getFinishedLogRecordItems.asScala.head) + attrs.keySet should contain allOf ("trace_id", "texera.workflow.id", "some.new.key") + attrs("some.new.key") shouldBe "kept" + } + + it should "drop the noisy Pekko bridge MDC keys" in { + val (_, exporter, appender) = newFixture() + val denied = LogSanitizer.DeniedMdcKeys.iterator.map(_ -> "noise").toMap + appender.doAppend(makeEvent("msg", mdc = denied + ("trace_id" -> "abc"))) + + val attrs = attrsOf(exporter.getFinishedLogRecordItems.asScala.head) + attrs.keySet should contain("trace_id") + attrs.keySet should contain noElementsOf LogSanitizer.DeniedMdcKeys + } + + it should "redact a secret-shaped MDC value while keeping its key" in { + val (_, exporter, appender) = newFixture() + appender.doAppend(makeEvent("msg", mdc = Map("authz" -> "Bearer abc123XYZ.foo"))) + + val attrs = attrsOf(exporter.getFinishedLogRecordItems.asScala.head) + attrs.keySet should contain("authz") + attrs("authz") should include("[REDACTED]") + attrs("authz") should not include "abc123XYZ" + } + + it should "redact the value of a credential-named MDC key even when the value looks benign" in { + val (_, exporter, appender) = newFixture() + appender.doAppend(makeEvent("msg", mdc = Map("password" -> "p4ssw0rd", "api_key" -> "plain"))) + + val attrs = attrsOf(exporter.getFinishedLogRecordItems.asScala.head) + attrs("password") shouldBe "[REDACTED]" + attrs("api_key") shouldBe "[REDACTED]" + attrs.values should contain noElementsOf Seq("p4ssw0rd", "plain") + } + + // ----- lifecycle ------------------------------------------------------ + + it should "be a silent no-op when not yet bound to an OpenTelemetry instance" in { + val unbound = new TexeraOtelLogAppender() + unbound.setContext(LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext]) + unbound.start() + // Should not throw, even though no SDK is wired. + noException should be thrownBy unbound.doAppend(makeEvent("hello")) + } + + it should "stop emitting after stop() is called" in { + val (_, exporter, appender) = newFixture() + appender.doAppend(makeEvent("first")) + appender.stop() + appender.doAppend(makeEvent("second")) + + val bodies = exporter.getFinishedLogRecordItems.asScala.map(_.getBodyValue.asString) + bodies should contain only "first" + } +} diff --git a/computing-unit-managing-service/LICENSE-binary b/computing-unit-managing-service/LICENSE-binary index 0b8e4c1286b..bf057b8c40e 100644 --- a/computing-unit-managing-service/LICENSE-binary +++ b/computing-unit-managing-service/LICENSE-binary @@ -369,6 +369,18 @@ Scala/Java jars: - io.netty.netty-transport-native-epoll-4.2.15.Final.jar - io.netty.netty-transport-native-unix-common-4.2.15.Final.jar - org.jspecify.jspecify-1.0.0.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.perfmark.perfmark-api-0.27.0.jar - io.r2dbc.r2dbc-spi-1.0.0.RELEASE.jar - io.swagger.swagger-annotations-1.6.14.jar diff --git a/computing-unit-managing-service/src/main/resources/computing-unit-managing-service-config.yaml b/computing-unit-managing-service/src/main/resources/computing-unit-managing-service-config.yaml index 523b4197989..ea428fcadcb 100644 --- a/computing-unit-managing-service/src/main/resources/computing-unit-managing-service-config.yaml +++ b/computing-unit-managing-service/src/main/resources/computing-unit-managing-service-config.yaml @@ -30,4 +30,16 @@ server: logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: - "com.example": ${TEXERA_SERVICE_LOG_LEVEL:-DEBUG} \ No newline at end of file + "com.example": ${TEXERA_SERVICE_LOG_LEVEL:-DEBUG} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN \ No newline at end of file diff --git a/computing-unit-managing-service/src/main/scala/org/apache/texera/service/ComputingUnitManagingService.scala b/computing-unit-managing-service/src/main/scala/org/apache/texera/service/ComputingUnitManagingService.scala index f0dffc89e11..cd43b888997 100644 --- a/computing-unit-managing-service/src/main/scala/org/apache/texera/service/ComputingUnitManagingService.scala +++ b/computing-unit-managing-service/src/main/scala/org/apache/texera/service/ComputingUnitManagingService.scala @@ -53,6 +53,8 @@ class ComputingUnitManagingService extends Application[ComputingUnitManagingServ configuration: ComputingUnitManagingServiceConfiguration, environment: Environment ): Unit = { + // Bridge this service's logs to the OTel collector under its own service.name. + org.apache.texera.observability.OtelInit.init("computing-unit-managing-service") // Register http resources environment.jersey.setUrlPattern("/api/*") environment.jersey.register(classOf[HealthCheckResource]) diff --git a/config-service/LICENSE-binary b/config-service/LICENSE-binary index 93348da932e..5f93f2b6780 100644 --- a/config-service/LICENSE-binary +++ b/config-service/LICENSE-binary @@ -242,6 +242,9 @@ Scala/Java jars: - com.google.guava.listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar - com.google.j2objc.j2objc-annotations-2.8.jar - com.helger.profiler-1.1.1.jar + - com.squareup.okhttp3.okhttp-4.12.0.jar + - com.squareup.okio.okio-3.6.0.jar + - com.squareup.okio.okio-jvm-3.6.0.jar - com.thesamet.scalapb.lenses_2.13-0.11.20.jar - com.thesamet.scalapb.scalapb-json4s_2.13-0.12.0.jar - com.thesamet.scalapb.scalapb-runtime_2.13-0.11.20.jar @@ -274,6 +277,18 @@ Scala/Java jars: - io.dropwizard.metrics.metrics-json-4.2.25.jar - io.dropwizard.metrics.metrics-jvm-4.2.25.jar - io.dropwizard.metrics.metrics-logback-4.2.25.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.r2dbc.r2dbc-spi-1.0.0.RELEASE.jar - jakarta.inject.jakarta.inject-api-2.0.1.jar - jakarta.validation.jakarta.validation-api-3.0.2.jar @@ -292,6 +307,11 @@ Scala/Java jars: - org.hibernate.validator.hibernate-validator-7.0.5.Final.jar - org.javassist.javassist-3.30.2-GA.jar - org.jboss.logging.jboss-logging-3.5.3.Final.jar + - org.jetbrains.annotations-13.0.jar + - org.jetbrains.kotlin.kotlin-stdlib-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-common-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-jdk7-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-jdk8-1.9.10.jar - org.jooq.jooq-3.19.36.jar - org.json4s.json4s-ast_2.13-4.0.1.jar - org.json4s.json4s-jackson-core_2.13-4.0.1.jar diff --git a/config-service/src/main/resources/config-service-web-config.yaml b/config-service/src/main/resources/config-service-web-config.yaml index 4aa67af82e1..8559e1fd507 100644 --- a/config-service/src/main/resources/config-service-web-config.yaml +++ b/config-service/src/main/resources/config-service-web-config.yaml @@ -26,6 +26,19 @@ server: logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + loggers: + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console threshold: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} diff --git a/config-service/src/main/scala/org/apache/texera/service/ConfigService.scala b/config-service/src/main/scala/org/apache/texera/service/ConfigService.scala index a7e9b61d994..df7335d21f7 100644 --- a/config-service/src/main/scala/org/apache/texera/service/ConfigService.scala +++ b/config-service/src/main/scala/org/apache/texera/service/ConfigService.scala @@ -53,6 +53,8 @@ class ConfigService extends Application[ConfigServiceConfiguration] with LazyLog } override def run(configuration: ConfigServiceConfiguration, environment: Environment): Unit = { + // Bridge this service's logs to the OTel collector under its own service.name. + org.apache.texera.observability.OtelInit.init("config-service") // Serve backend at /api environment.jersey.setUrlPattern("/api/*") diff --git a/file-service/LICENSE-binary b/file-service/LICENSE-binary index b84e5912d67..681bf5444a9 100644 --- a/file-service/LICENSE-binary +++ b/file-service/LICENSE-binary @@ -334,6 +334,18 @@ Scala/Java jars: - io.netty.netty-transport-native-epoll-4.2.15.Final.jar - io.netty.netty-transport-native-unix-common-4.2.15.Final.jar - org.jspecify.jspecify-1.0.0.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.perfmark.perfmark-api-0.27.0.jar - io.r2dbc.r2dbc-spi-1.0.0.RELEASE.jar - jakarta.inject.jakarta.inject-api-2.0.1.jar diff --git a/file-service/src/main/resources/file-service-web-config.yaml b/file-service/src/main/resources/file-service-web-config.yaml index 41f8d1b1748..db5a7ec6645 100644 --- a/file-service/src/main/resources/file-service-web-config.yaml +++ b/file-service/src/main/resources/file-service-web-config.yaml @@ -28,6 +28,18 @@ logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console - type: file diff --git a/file-service/src/main/scala/org/apache/texera/service/FileService.scala b/file-service/src/main/scala/org/apache/texera/service/FileService.scala index 1bb29f5dab3..e0e40573659 100644 --- a/file-service/src/main/scala/org/apache/texera/service/FileService.scala +++ b/file-service/src/main/scala/org/apache/texera/service/FileService.scala @@ -62,6 +62,8 @@ class FileService extends Application[FileServiceConfiguration] with LazyLogging } override def run(configuration: FileServiceConfiguration, environment: Environment): Unit = { + // Bridge this service's logs to the OTel collector under its own service.name. + org.apache.texera.observability.OtelInit.init("file-service") // Serve backend at /api environment.jersey.setUrlPattern("/api/*") SqlServer.initConnection( diff --git a/workflow-compiling-service/LICENSE-binary b/workflow-compiling-service/LICENSE-binary index 96c723694ae..90befd23b9a 100644 --- a/workflow-compiling-service/LICENSE-binary +++ b/workflow-compiling-service/LICENSE-binary @@ -336,6 +336,18 @@ Scala/Java jars: - io.netty.netty-transport-native-epoll-4.2.15.Final.jar - io.netty.netty-transport-native-unix-common-4.2.15.Final.jar - org.jspecify.jspecify-1.0.0.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.perfmark.perfmark-api-0.27.0.jar - io.r2dbc.r2dbc-spi-1.0.0.RELEASE.jar - jakarta.inject.jakarta.inject-api-2.0.1.jar diff --git a/workflow-compiling-service/src/main/resources/workflow-compiling-service-config.yaml b/workflow-compiling-service/src/main/resources/workflow-compiling-service-config.yaml index 5b9016af1b6..37e413c15b6 100644 --- a/workflow-compiling-service/src/main/resources/workflow-compiling-service-config.yaml +++ b/workflow-compiling-service/src/main/resources/workflow-compiling-service-config.yaml @@ -28,6 +28,18 @@ logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console - type: file diff --git a/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala b/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala index a69ef545246..9938694d452 100644 --- a/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala +++ b/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala @@ -49,6 +49,8 @@ class WorkflowCompilingService extends Application[WorkflowCompilingServiceConfi configuration: WorkflowCompilingServiceConfiguration, environment: Environment ): Unit = { + // Bridge this service's logs to the OTel collector under its own service.name. + org.apache.texera.observability.OtelInit.init("workflow-compiling-service") ObjectMapperUtils.warmupObjectMapperForOperatorsSerde() // serve backend at /api