feat(observability): traces end-to-end (gateway + UI panel)

.github/workflows/build.yml

@@ -329,7 +329,7 @@ jobs:
           BACKPORT_TARGET_BRANCH: ${{ inputs.backport_target_branch }}
         run: |
           # Backport builds filter by the checked-out build.sbt.
-          want=(DAO Auth Config Resource Util PyBuilder WorkflowCore
+          want=(DAO Auth Config Observability Resource Util PyBuilder WorkflowCore
                 WorkflowOperator WorkflowCompiler WorkflowExecutionService)
           tasks=()
           if [ -n "${BACKPORT_TARGET_BRANCH}" ]; then

.github/workflows/create-release-candidate.yml

@@ -174,7 +174,7 @@ jobs:
 
           # Export the single-node directory from the tagged source
           mkdir -p "$TEMP_DIR/_raw"
-          git archive --format=tar "$TAG_NAME" -- bin/single-node/ sql/ NOTICE licenses/LICENSE-CC-BY-4.0.txt | tar -x -C "$TEMP_DIR/_raw"
+          git archive --format=tar "$TAG_NAME" -- bin/single-node/ bin/observability/ sql/ NOTICE licenses/LICENSE-CC-BY-4.0.txt | tar -x -C "$TEMP_DIR/_raw"
 
           # Copy deployment files
           cp "$TEMP_DIR/_raw/bin/single-node/docker-compose.yml" "$BUNDLE_DIR/"
@@ -187,10 +187,19 @@ jobs:
           cp "$TEMP_DIR/_raw/licenses/LICENSE-CC-BY-4.0.txt" "$BUNDLE_DIR/licenses/"
           cp -r "$TEMP_DIR/_raw/sql" "$BUNDLE_DIR/"
 
+          # Observability configs (OTel collector + Parca) referenced by the
+          # opt-in observability compose profiles. Shipped next to the compose
+          # file so the ../observability mounts resolve in the flattened bundle.
+          cp -r "$TEMP_DIR/_raw/bin/observability" "$BUNDLE_DIR/observability"
+
           # Patch the SQL mount path for the self-contained bundle layout
           # In the repo it's ../../sql (relative to bin/single-node/), in the bundle it's ./sql
           sed -i 's|../../sql|./sql|g' "$BUNDLE_DIR/docker-compose.yml"
 
+          # Patch the observability mount paths: ../observability (relative to
+          # bin/single-node/ in the repo) becomes ./observability in the bundle.
+          sed -i 's|../observability|./observability|g' "$BUNDLE_DIR/docker-compose.yml"
+
           # Generate a release-pinned .env file with the version tag
           # Start from the source .env and ensure IMAGE_REGISTRY and IMAGE_TAG are set
           cp "$TEMP_DIR/_raw/bin/single-node/.env" "$BUNDLE_DIR/.env"

.licenserc.yaml

@@ -37,6 +37,8 @@ header:
     - '**/.env.example'
     - '**/.prettierrc'
     - '**/bun.lock'
+    # Helm's generated dependency lock file (no header; like yarn.lock/bun.lock)
+    - '**/Chart.lock'
     # Third-party code with MIT license - see LICENSE file for attribution
     - 'common/workflow-operator/src/main/scala/com/kjetland/**'
     # TypeFox monaco-languageclient derived files (MIT License)

access-control-service/LICENSE-binary

@@ -242,6 +242,9 @@ Scala/Java jars:
   - com.google.guava.listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
   - com.google.j2objc.j2objc-annotations-2.8.jar
   - com.helger.profiler-1.1.1.jar
+  - com.squareup.okhttp3.okhttp-4.12.0.jar
+  - com.squareup.okio.okio-3.6.0.jar
+  - com.squareup.okio.okio-jvm-3.6.0.jar
   - com.thesamet.scalapb.lenses_2.13-0.11.20.jar
   - com.thesamet.scalapb.scalapb-json4s_2.13-0.12.0.jar
   - com.thesamet.scalapb.scalapb-runtime_2.13-0.11.20.jar
@@ -300,6 +303,18 @@ Scala/Java jars:
   - io.fabric8.kubernetes-model-scheduling-6.12.1.jar
   - io.fabric8.kubernetes-model-storageclass-6.12.1.jar
   - io.fabric8.zjsonpatch-0.3.0.jar
+  - io.opentelemetry.opentelemetry-api-1.50.0.jar
+  - io.opentelemetry.opentelemetry-context-1.50.0.jar
+  - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar
+  - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar
+  - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar
+  - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar
+  - io.opentelemetry.opentelemetry-sdk-1.50.0.jar
+  - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar
+  - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar
+  - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar
+  - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar
+  - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar
   - io.r2dbc.r2dbc-spi-1.0.0.RELEASE.jar
   - jakarta.inject.jakarta.inject-api-2.0.1.jar
   - jakarta.validation.jakarta.validation-api-3.0.2.jar
@@ -318,6 +333,11 @@ Scala/Java jars:
   - org.hibernate.validator.hibernate-validator-7.0.5.Final.jar
   - org.javassist.javassist-3.30.2-GA.jar
   - org.jboss.logging.jboss-logging-3.5.3.Final.jar
+  - org.jetbrains.annotations-13.0.jar
+  - org.jetbrains.kotlin.kotlin-stdlib-1.9.10.jar
+  - org.jetbrains.kotlin.kotlin-stdlib-common-1.9.10.jar
+  - org.jetbrains.kotlin.kotlin-stdlib-jdk7-1.9.10.jar
+  - org.jetbrains.kotlin.kotlin-stdlib-jdk8-1.9.10.jar
   - org.jooq.jooq-3.19.36.jar
   - org.json4s.json4s-ast_2.13-4.0.1.jar
   - org.json4s.json4s-jackson-core_2.13-4.0.1.jar

access-control-service/src/main/resources/access-control-service-web-config.yaml

@@ -26,6 +26,19 @@ server:
 
 logging:
   level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO}
+  loggers:
+    # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code
+    # (org.apache.texera) without the framework firehose.
+    "org.apache.pekko": WARN
+    "org.apache.iceberg": WARN
+    "org.apache.hadoop": WARN
+    "org.apache.kafka": WARN
+    "org.eclipse.jetty": WARN
+    "org.glassfish.jersey": WARN
+    "io.grpc": WARN
+    "io.netty": WARN
+    "com.zaxxer.hikari": WARN
+    "software.amazon.awssdk": WARN
   appenders:
     - type: console
       threshold: ${TEXERA_SERVICE_LOG_LEVEL:-INFO}

access-control-service/src/main/scala/org/apache/texera/service/AccessControlService.scala

@@ -59,6 +59,8 @@ class AccessControlService extends Application[AccessControlServiceConfiguration
       configuration: AccessControlServiceConfiguration,
       environment: Environment
   ): Unit = {
+    // Bridge this service's logs to the OTel collector under its own service.name.
+    org.apache.texera.observability.OtelInit.init("access-control-service")
     // Serve backend at /api
     environment.jersey.setUrlPattern("/api/*")
 

amber/LICENSE-binary-java

@@ -363,6 +363,18 @@ Scala/Java jars:
   - org.jspecify.jspecify-1.0.0.jar
   - io.opencensus.opencensus-api-0.31.1.jar
   - io.opencensus.opencensus-contrib-http-util-0.31.1.jar
+  - io.opentelemetry.opentelemetry-api-1.50.0.jar
+  - io.opentelemetry.opentelemetry-context-1.50.0.jar
+  - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar
+  - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar
+  - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar
+  - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar
+  - io.opentelemetry.opentelemetry-sdk-1.50.0.jar
+  - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar
+  - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar
+  - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar
+  - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar
+  - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar
   - io.perfmark.perfmark-api-0.27.0.jar
   - io.r2dbc.r2dbc-spi-1.0.0.RELEASE.jar
   - io.reactivex.rxjava3.rxjava-3.1.12.jar

amber/src/main/resources/computing-unit-master-config.yml

@@ -34,6 +34,18 @@ logging:
   level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO}
   loggers:
     "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO}
+    # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code
+    # (org.apache.texera) without the framework firehose.
+    "org.apache.pekko": WARN
+    "org.apache.iceberg": WARN
+    "org.apache.hadoop": WARN
+    "org.apache.kafka": WARN
+    "org.eclipse.jetty": WARN
+    "org.glassfish.jersey": WARN
+    "io.grpc": WARN
+    "io.netty": WARN
+    "com.zaxxer.hikari": WARN
+    "software.amazon.awssdk": WARN
   appenders:
     - type: console
       logFormat: "[%date{ISO8601}] [%level] [%logger] [%thread] - %msg %n"

amber/src/main/resources/texera-compiling-service-web-config.yml

@@ -43,6 +43,18 @@ logging:
   level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO}
   loggers:
     "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO}
+    # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code
+    # (org.apache.texera) without the framework firehose.
+    "org.apache.pekko": WARN
+    "org.apache.iceberg": WARN
+    "org.apache.hadoop": WARN
+    "org.apache.kafka": WARN
+    "org.eclipse.jetty": WARN
+    "org.glassfish.jersey": WARN
+    "io.grpc": WARN
+    "io.netty": WARN
+    "com.zaxxer.hikari": WARN
+    "software.amazon.awssdk": WARN
   appenders:
     - type: console
       logFormat: "[%date{ISO8601}] [%level] [%logger] [%thread] - %msg %n"

amber/src/main/resources/web-config.yml

@@ -43,6 +43,18 @@ logging:
   level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO}
   loggers:
     "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO}
+    # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code
+    # (org.apache.texera) without the framework firehose.
+    "org.apache.pekko": WARN
+    "org.apache.iceberg": WARN
+    "org.apache.hadoop": WARN
+    "org.apache.kafka": WARN
+    "org.eclipse.jetty": WARN
+    "org.glassfish.jersey": WARN
+    "io.grpc": WARN
+    "io.netty": WARN
+    "com.zaxxer.hikari": WARN
+    "software.amazon.awssdk": WARN
   appenders:
     - type: console
       logFormat: "[%date{ISO8601}] [%level] [%logger] [%thread] - %msg %n"

amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala

@@ -138,6 +138,8 @@ class ComputingUnitMaster extends io.dropwizard.Application[Configuration] with
   }
 
   override def run(configuration: Configuration, environment: Environment): Unit = {
+    org.apache.texera.observability.OtelInit.init("computing-unit-master")
+    org.apache.texera.web.observability.WorkflowMetricsRecorder.init()
     ObjectMapperUtils.warmupObjectMapperForOperatorsSerde()
 
     SqlServer.initConnection(

amber/src/main/scala/org/apache/texera/web/TexeraWebApplication.scala

@@ -31,7 +31,15 @@ import org.apache.texera.amber.engine.common.Utils
 import org.apache.texera.amber.util.ObjectMapperUtils
 import org.apache.texera.auth.SessionUser
 import org.apache.texera.dao.SqlServer
+import org.apache.texera.observability.OtelInit
 import org.apache.texera.web.auth.JwtAuth.setupJwtAuth
+import org.apache.texera.web.observability.gateway.{
+  GatewayContext,
+  LogsResource,
+  MetricsResource,
+  ObservabilityHealthResource,
+  TracesResource
+}
 import org.apache.texera.web.resource._
 import org.apache.texera.web.resource.auth.{
   AppleAuthResource,
@@ -59,14 +67,15 @@ import org.glassfish.jersey.server.filter.RolesAllowedDynamicFeature
 
 import java.time.Duration
 
-object TexeraWebApplication {
+object TexeraWebApplication extends LazyLogging {
 
   def main(args: Array[String]): Unit = {
 
     // TODO: figure out a safety way of calling discardUncommittedChangesOfAllDatasets
     // Currently in kubernetes, multiple pods calling this function can result into thread competition
     // discardUncommittedChangesOfAllDatasets()
 
+    logger.info("Starting TexeraWebApplication")
     // start web server
     new TexeraWebApplication().run(
       "server",
@@ -101,11 +110,13 @@ class TexeraWebApplication
   }
 
   override def run(configuration: TexeraWebConfiguration, environment: Environment): Unit = {
+    org.apache.texera.observability.OtelInit.init("texera-web-application")
     ObjectMapperUtils.warmupObjectMapperForOperatorsSerde()
 
     // serve backend at /api
     environment.jersey.setUrlPattern("/api/*")
 
+    logger.debug(s"Connecting to database at ${StorageConfig.jdbcUrl}")
     SqlServer.initConnection(
       StorageConfig.jdbcUrl,
       StorageConfig.jdbcUsername,
@@ -141,6 +152,11 @@ class TexeraWebApplication
     )
     environment.jersey.register(classOf[RolesAllowedDynamicFeature])
 
+    // Tag every log record emitted while handling a request with the
+    // authenticated user's id (MDC key `texera.user.id`). Registered
+    // here so it runs AFTER the JWT auth filter set up by setupJwtAuth.
+    environment.jersey.register(classOf[org.apache.texera.web.observability.UserContextMdcFilter])
+
     environment.jersey.register(classOf[AuthResource])
     environment.jersey.register(classOf[GoogleAuthResource])
     environment.jersey.register(classOf[OrcidAuthResource])
@@ -162,8 +178,31 @@ class TexeraWebApplication
     environment.jersey.register(classOf[HuggingFaceModelResource])
     environment.jersey.register(classOf[WarehouseResource])
 
+    // Observability gateway. A single GatewayContext is shared by all
+    // resources so the rate-limiter buckets and the backend HTTP clients
+    // are reused across requests. The health resource lands here in
+    // gateway-core; the per-signal resources are registered by their
+    // respective PRs (obs/pr10..13).
+    logger.debug("Registering observability gateway resources")
+    val obsCtx = GatewayContext.default()
+    environment.jersey.register(new LogsResource(obsCtx))
+    environment.jersey.register(new MetricsResource(obsCtx))
+    environment.jersey.register(new TracesResource(obsCtx))
+    environment.jersey.register(new ObservabilityHealthResource(obsCtx))
+
     AuthResource.createAdminUser()
 
+    // Tag every log record emitted during request handling with the
+    // workflow / execution / computing-unit id when one is visible in
+    // the URL or headers. Required so the observability dashboard's
+    // CU/workflow filters match live records (the JVM otherwise emits
+    // them without any per-request context).
+    environment.getApplicationContext.addFilter(
+      new FilterHolder(new org.apache.texera.web.observability.RequestContextMdcFilter()),
+      "/*",
+      java.util.EnumSet.allOf(classOf[javax.servlet.DispatcherType])
+    )
+
     // Set Cache-Control on static frontend asset responses.
     environment.getApplicationContext.addFilter(
       new FilterHolder(new StaticAssetCacheFilter),
@@ -196,5 +235,7 @@ class TexeraWebApplication
       "/*",
       java.util.EnumSet.allOf(classOf[javax.servlet.DispatcherType])
     )
+
+    logger.info("TexeraWebApplication web server initialized and ready")
   }
 }