From 45ed6c796a2e7aadda0d26bdd133825329884f46 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Wed, 7 Oct 2026 02:37:01 -0700 Subject: [PATCH 1/2] feat(observability): logging foundations (bootstrap, bridge, sanitizer) Introduce the OpenTelemetry foundation for Texera: an SDK bootstrap, a Logback-to-OTel log bridge, and a log sanitizer, wired into every service entry point. This is PR 1 of the observability stack; it ships logging only, with the trace and metric exporters wired but not yet emitted (those arrive in follow-up PRs). New module (common/observability): - OtelInit: one-call SDK bootstrap per service. Reads OTEL_* settings from observability.conf, validates the OTLP endpoint (scheme + host allowlist) before any exporter is built, wires the span/log/metric providers explicitly (no sdk-extension-autoconfigure), clamps the metric export interval, and attaches the log appender to the Logback ROOT logger. The whole init body is guarded so a missing or malformed config returns None with one WARN instead of throwing into a service's run(), even when telemetry is disabled. - TexeraOtelLogAppender: bridges Logback events to OTel log records. Maps severity, forwards MDC, sets exception.type/message/stacktrace semantic attributes, and drops records from io.opentelemetry loggers so export-failure diagnostics are not fed back to the collector that just failed. - LogSanitizer: strips C0 control characters, redacts secrets, and caps body length (MaxBodyChars) to keep individual records bounded. Config (common/config): - observability.conf with the OTEL_* defaults, ObservabilityConfig to read it, and ENV_OTEL_* entries in EnvironmentalVariable. Wiring: - build.sbt defines the Observability module and adds dependsOn(Observability) to the eight Dropwizard services. - Each service entry point calls OtelInit.init with its own service name, and each service config gains a logging block. Deployment: - OTEL_* env entries in bin/single-node/.env, bin/k8s/values.yaml, and bin/k8s/values-development.yaml (kept a name-for-name mirror). - LICENSE-binary manifests updated with the pinned OTel 1.50.0 jars. Tests: OtelInitSpec, TexeraOtelLogAppenderSpec, LogSanitizerSpec, and ObservabilityConfigSpec cover endpoint validation, interval clamping, severity/MDC/exception mapping, self-diagnostic filtering, redaction, and truncation. Rebased onto current main. --- .github/workflows/build.yml | 2 +- access-control-service/LICENSE-binary | 20 + .../access-control-service-web-config.yaml | 13 + .../texera/service/AccessControlService.scala | 2 + amber/LICENSE-binary-java | 12 + .../computing-unit-master-config.yml | 12 + .../texera-compiling-service-web-config.yml | 12 + amber/src/main/resources/web-config.yml | 12 + .../texera/web/ComputingUnitMaster.scala | 1 + .../texera/web/TexeraWebApplication.scala | 1 + bin/k8s/values-development.yaml | 15 + bin/k8s/values.yaml | 15 + bin/single-node/.env | 12 + build.sbt | 20 +- .../src/main/resources/observability.conf | 45 ++ .../common/config/EnvironmentalVariable.scala | 7 + .../common/config/ObservabilityConfig.scala | 38 ++ .../config/ObservabilityConfigSpec.scala | 65 +++ common/observability/build.sbt | 75 ++++ .../texera/observability/LogSanitizer.scala | 122 +++++ .../texera/observability/OtelInit.scala | 418 ++++++++++++++++++ .../observability/TexeraOtelLogAppender.scala | 154 +++++++ .../observability/LogSanitizerSpec.scala | 140 ++++++ .../texera/observability/OtelInitSpec.scala | 295 ++++++++++++ .../TexeraOtelLogAppenderSpec.scala | 243 ++++++++++ .../LICENSE-binary | 12 + ...omputing-unit-managing-service-config.yaml | 14 +- .../ComputingUnitManagingService.scala | 2 + config-service/LICENSE-binary | 20 + .../resources/config-service-web-config.yaml | 13 + .../apache/texera/service/ConfigService.scala | 2 + file-service/LICENSE-binary | 12 + .../resources/file-service-web-config.yaml | 12 + .../apache/texera/service/FileService.scala | 2 + notebook-migration-service/LICENSE-binary | 22 +- .../service/NotebookMigrationService.scala | 1 + workflow-compiling-service/LICENSE-binary | 12 + .../workflow-compiling-service-config.yaml | 12 + .../service/WorkflowCompilingService.scala | 2 + 39 files changed, 1878 insertions(+), 11 deletions(-) create mode 100644 common/config/src/main/resources/observability.conf create mode 100644 common/config/src/main/scala/org/apache/texera/common/config/ObservabilityConfig.scala create mode 100644 common/config/src/test/scala/org/apache/texera/common/config/ObservabilityConfigSpec.scala create mode 100644 common/observability/build.sbt create mode 100644 common/observability/src/main/scala/org/apache/texera/observability/LogSanitizer.scala create mode 100644 common/observability/src/main/scala/org/apache/texera/observability/OtelInit.scala create mode 100644 common/observability/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala create mode 100644 common/observability/src/test/scala/org/apache/texera/observability/LogSanitizerSpec.scala create mode 100644 common/observability/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala create mode 100644 common/observability/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c3cfd13be3f..590f28fdd18 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -329,7 +329,7 @@ jobs: BACKPORT_TARGET_BRANCH: ${{ inputs.backport_target_branch }} run: | # Backport builds filter by the checked-out build.sbt. - want=(DAO Auth Config Resource Util PyBuilder WorkflowCore + want=(DAO Auth Config Observability Resource Util PyBuilder WorkflowCore WorkflowOperator WorkflowCompiler WorkflowExecutionService) tasks=() if [ -n "${BACKPORT_TARGET_BRANCH}" ]; then diff --git a/access-control-service/LICENSE-binary b/access-control-service/LICENSE-binary index 4f40d12738d..abf08120e60 100644 --- a/access-control-service/LICENSE-binary +++ b/access-control-service/LICENSE-binary @@ -242,6 +242,9 @@ Scala/Java jars: - com.google.guava.listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar - com.google.j2objc.j2objc-annotations-2.8.jar - com.helger.profiler-1.1.1.jar + - com.squareup.okhttp3.okhttp-4.12.0.jar + - com.squareup.okio.okio-3.6.0.jar + - com.squareup.okio.okio-jvm-3.6.0.jar - com.thesamet.scalapb.lenses_2.13-0.11.20.jar - com.thesamet.scalapb.scalapb-json4s_2.13-0.12.0.jar - com.thesamet.scalapb.scalapb-runtime_2.13-0.11.20.jar @@ -300,6 +303,18 @@ Scala/Java jars: - io.fabric8.kubernetes-model-scheduling-6.12.1.jar - io.fabric8.kubernetes-model-storageclass-6.12.1.jar - io.fabric8.zjsonpatch-0.3.0.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.r2dbc.r2dbc-spi-1.0.0.RELEASE.jar - jakarta.inject.jakarta.inject-api-2.0.1.jar - jakarta.validation.jakarta.validation-api-3.0.2.jar @@ -318,6 +333,11 @@ Scala/Java jars: - org.hibernate.validator.hibernate-validator-7.0.5.Final.jar - org.javassist.javassist-3.30.2-GA.jar - org.jboss.logging.jboss-logging-3.5.3.Final.jar + - org.jetbrains.annotations-13.0.jar + - org.jetbrains.kotlin.kotlin-stdlib-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-common-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-jdk7-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-jdk8-1.9.10.jar - org.jooq.jooq-3.19.36.jar - org.json4s.json4s-ast_2.13-4.0.1.jar - org.json4s.json4s-jackson-core_2.13-4.0.1.jar diff --git a/access-control-service/src/main/resources/access-control-service-web-config.yaml b/access-control-service/src/main/resources/access-control-service-web-config.yaml index 8c7895e9858..bd78ecaa82a 100644 --- a/access-control-service/src/main/resources/access-control-service-web-config.yaml +++ b/access-control-service/src/main/resources/access-control-service-web-config.yaml @@ -26,6 +26,19 @@ server: logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + loggers: + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console threshold: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} diff --git a/access-control-service/src/main/scala/org/apache/texera/service/AccessControlService.scala b/access-control-service/src/main/scala/org/apache/texera/service/AccessControlService.scala index ca2c797ab98..4a48efa7df0 100644 --- a/access-control-service/src/main/scala/org/apache/texera/service/AccessControlService.scala +++ b/access-control-service/src/main/scala/org/apache/texera/service/AccessControlService.scala @@ -59,6 +59,8 @@ class AccessControlService extends Application[AccessControlServiceConfiguration configuration: AccessControlServiceConfiguration, environment: Environment ): Unit = { + // Bridge this service's logs to the OTel collector under its own service.name. + org.apache.texera.observability.OtelInit.init("access-control-service") // Serve backend at /api environment.jersey.setUrlPattern("/api/*") diff --git a/amber/LICENSE-binary-java b/amber/LICENSE-binary-java index a1eb3812702..ef4f7b0a1fa 100644 --- a/amber/LICENSE-binary-java +++ b/amber/LICENSE-binary-java @@ -363,6 +363,18 @@ Scala/Java jars: - org.jspecify.jspecify-1.0.0.jar - io.opencensus.opencensus-api-0.31.1.jar - io.opencensus.opencensus-contrib-http-util-0.31.1.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.perfmark.perfmark-api-0.27.0.jar - io.r2dbc.r2dbc-spi-1.0.0.RELEASE.jar - io.reactivex.rxjava3.rxjava-3.1.12.jar diff --git a/amber/src/main/resources/computing-unit-master-config.yml b/amber/src/main/resources/computing-unit-master-config.yml index 0dba594b8ae..ee578c3cf90 100644 --- a/amber/src/main/resources/computing-unit-master-config.yml +++ b/amber/src/main/resources/computing-unit-master-config.yml @@ -34,6 +34,18 @@ logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console logFormat: "[%date{ISO8601}] [%level] [%logger] [%thread] - %msg %n" diff --git a/amber/src/main/resources/texera-compiling-service-web-config.yml b/amber/src/main/resources/texera-compiling-service-web-config.yml index ea2c1b9c1e9..c0b6e8aa762 100644 --- a/amber/src/main/resources/texera-compiling-service-web-config.yml +++ b/amber/src/main/resources/texera-compiling-service-web-config.yml @@ -43,6 +43,18 @@ logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console logFormat: "[%date{ISO8601}] [%level] [%logger] [%thread] - %msg %n" diff --git a/amber/src/main/resources/web-config.yml b/amber/src/main/resources/web-config.yml index 9fde1d078e8..9b3c743c89c 100644 --- a/amber/src/main/resources/web-config.yml +++ b/amber/src/main/resources/web-config.yml @@ -43,6 +43,18 @@ logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console logFormat: "[%date{ISO8601}] [%level] [%logger] [%thread] - %msg %n" diff --git a/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala b/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala index 6f49fe1c3d0..b5bdcbf5573 100644 --- a/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala +++ b/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala @@ -138,6 +138,7 @@ class ComputingUnitMaster extends io.dropwizard.Application[Configuration] with } override def run(configuration: Configuration, environment: Environment): Unit = { + org.apache.texera.observability.OtelInit.init("computing-unit-master") ObjectMapperUtils.warmupObjectMapperForOperatorsSerde() SqlServer.initConnection( diff --git a/amber/src/main/scala/org/apache/texera/web/TexeraWebApplication.scala b/amber/src/main/scala/org/apache/texera/web/TexeraWebApplication.scala index 4a1d33f62e9..07e07856e91 100644 --- a/amber/src/main/scala/org/apache/texera/web/TexeraWebApplication.scala +++ b/amber/src/main/scala/org/apache/texera/web/TexeraWebApplication.scala @@ -101,6 +101,7 @@ class TexeraWebApplication } override def run(configuration: TexeraWebConfiguration, environment: Environment): Unit = { + org.apache.texera.observability.OtelInit.init("texera-web-application") ObjectMapperUtils.warmupObjectMapperForOperatorsSerde() // serve backend at /api diff --git a/bin/k8s/values-development.yaml b/bin/k8s/values-development.yaml index 5cbda8ee0d7..accf9f39c99 100644 --- a/bin/k8s/values-development.yaml +++ b/bin/k8s/values-development.yaml @@ -435,6 +435,21 @@ texeraEnvVars: # same value, so this is a fixed shared string rather than a per-pod random one. # Production environments MUST override this with a securely generated secret. value: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + # OpenTelemetry (observability). Disabled by default; set OTEL_SDK_DISABLED to + # "false" and point the endpoint at a reachable OTLP collector (http/https only) to enable. + - name: OTEL_SDK_DISABLED + value: "true" + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://127.0.0.1:4317" + # Comma-separated resource attributes (k1=v1,k2=v2); service.name is set per service. + - name: OTEL_RESOURCE_ATTRIBUTES + value: "" + # Comma-separated extra hosts allowed for the OTLP endpoint (loopback allowed by default). + - name: TEXERA_OTEL_ALLOWED_HOSTS + value: "" + # Metric export interval in ms. + - name: OTEL_METRIC_EXPORT_INTERVAL + value: "30000" yWebsocketServer: name: y-websocket-server diff --git a/bin/k8s/values.yaml b/bin/k8s/values.yaml index c72120a2e2a..be89f7cabc5 100644 --- a/bin/k8s/values.yaml +++ b/bin/k8s/values.yaml @@ -533,6 +533,21 @@ texeraEnvVars: - name: AUTH_JWT_SECRET # Development-only default (256-bit HS256 secret). Production environments MUST override this with a different, securely generated secret. value: "a7f3c8e9b14d2e6f5a0b9c3d8e1f4a6b2c5d7e9f0a3b6c8d1e4f7a9b2c5d8e1f" + # OpenTelemetry (observability). Disabled by default; set OTEL_SDK_DISABLED to + # "false" and point the endpoint at a reachable OTLP collector (http/https only) to enable. + - name: OTEL_SDK_DISABLED + value: "true" + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://127.0.0.1:4317" + # Comma-separated resource attributes (k1=v1,k2=v2); service.name is set per service. + - name: OTEL_RESOURCE_ATTRIBUTES + value: "" + # Comma-separated extra hosts allowed for the OTLP endpoint (loopback allowed by default). + - name: TEXERA_OTEL_ALLOWED_HOSTS + value: "" + # Metric export interval in ms. + - name: OTEL_METRIC_EXPORT_INTERVAL + value: "30000" yWebsocketServer: name: y-websocket-server diff --git a/bin/single-node/.env b/bin/single-node/.env index 69a9ceeecb8..40098c54c01 100644 --- a/bin/single-node/.env +++ b/bin/single-node/.env @@ -101,6 +101,18 @@ TEXERA_DASHBOARD_SERVICE_ENDPOINT=http://dashboard-service:8080 WORKFLOW_COMPILING_SERVICE_ENDPOINT=http://workflow-compiling-service:9090 WORKFLOW_EXECUTION_SERVICE_ENDPOINT=http://workflow-runtime-coordinator-service:8085 +# OpenTelemetry (observability). Disabled by default; the SDK stays inert until +# OTEL_SDK_DISABLED=false. Point the endpoint at a reachable OTLP collector +# (http/https only) before enabling. +OTEL_SDK_DISABLED=true +OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4317 +# Comma-separated resource attributes (k1=v1,k2=v2); service.name is set per service. +OTEL_RESOURCE_ATTRIBUTES= +# Comma-separated extra hosts allowed for the OTLP endpoint (loopback allowed by default). +TEXERA_OTEL_ALLOWED_HOSTS= +# Metric export interval in ms. +OTEL_METRIC_EXPORT_INTERVAL=30000 + # Notebook migration tool # Toggles the tool in the GUI; the migration itself needs an LLM key above. GUI_WORKFLOW_WORKSPACE_PYTHON_NOTEBOOK_MIGRATION_ENABLED=true diff --git a/build.sbt b/build.sbt index d89f7a75080..2fd6b526fe5 100644 --- a/build.sbt +++ b/build.sbt @@ -122,6 +122,11 @@ ThisBuild / excludeDependencies += ExclusionRule("log4j", "log4j") lazy val Util = (project in file("common/util")).settings(commonModuleSettings) lazy val DAO = (project in file("common/dao")).settings(commonModuleSettings) lazy val Config = (project in file("common/config")).settings(commonModuleSettings) +// OpenTelemetry bootstrap (OtelInit, log appender, sanitizer) shared by every +// service entry point; pins the OTel dependency versions in one place. Depends +// on Config to read OTEL_* settings from observability.conf. +lazy val Observability = + (project in file("common/observability")).settings(commonModuleSettings).dependsOn(Config) lazy val Resource = (project in file("common/resource")).settings(commonModuleSettings).dependsOn(DAO) lazy val Auth = (project in file("common/auth")) .settings(commonModuleSettings) @@ -129,7 +134,7 @@ lazy val Auth = (project in file("common/auth")) .dependsOn(DAO, Config) .dependsOn(DAO % "test->test") // reuse MockTexeraDB embedded Postgres in tests lazy val ConfigService = (project in file("config-service")) - .dependsOn(Auth, Config, DAO, Resource) + .dependsOn(Auth, Config, DAO, Resource, Observability) .dependsOn(DAO % "test->test") // reuse MockTexeraDB embedded Postgres in tests .settings(commonModuleSettings) .settings( @@ -139,7 +144,7 @@ lazy val ConfigService = (project in file("config-service")) ) ) lazy val AccessControlService = (project in file("access-control-service")) - .dependsOn(Auth, Config, DAO, Resource) + .dependsOn(Auth, Config, DAO, Resource, Observability) .settings(commonModuleSettings) .settings( dependencyOverrides ++= Seq( @@ -163,7 +168,7 @@ lazy val WorkflowCore = (project in file("common/workflow-core")) .configs(Test) .dependsOn(DAO % "test->test") // test scope dependency lazy val ComputingUnitManagingService = (project in file("computing-unit-managing-service")) - .dependsOn(WorkflowCore, Auth, Config, Resource) + .dependsOn(WorkflowCore, Auth, Config, Resource, Observability) .configs(Test) .dependsOn(DAO % "test->test") // reuse MockTexeraDB embedded Postgres in tests .settings(commonModuleSettings) @@ -210,7 +215,7 @@ lazy val ComputingUnitManagingService = (project in file("computing-unit-managin ) lazy val FileService = (project in file("file-service")) .settings(commonModuleSettings) - .dependsOn(WorkflowCore, Auth, Config, Resource, Util) + .dependsOn(WorkflowCore, Auth, Config, Resource, Util, Observability) .configs(Test) .dependsOn(DAO % "test->test") // test scope dependency .dependsOn(WorkflowCore % "test->test") // reuse RustFSContainer in MockLakeFS @@ -241,7 +246,7 @@ lazy val WorkflowCompiler = (project in file("common/workflow-compiler")) .configs(Test) .dependsOn(WorkflowOperator) lazy val WorkflowCompilingService = (project in file("workflow-compiling-service")) - .dependsOn(WorkflowCompiler, Auth, Config, Resource) + .dependsOn(WorkflowCompiler, Auth, Config, Resource, Observability) .settings(commonModuleSettings) .settings( dependencyOverrides ++= Seq( @@ -253,7 +258,7 @@ lazy val WorkflowCompilingService = (project in file("workflow-compiling-service ) lazy val WorkflowExecutionService = (project in file("amber")) - .dependsOn(WorkflowCompiler, Auth, Config) + .dependsOn(WorkflowCompiler, Auth, Config, Observability) .settings(commonModuleSettings) .settings( dependencyOverrides ++= Seq( @@ -272,7 +277,7 @@ lazy val WorkflowExecutionService = (project in file("amber")) .configs(Test) .dependsOn(DAO % "test->test", Auth % "test->test") // test scope dependency lazy val NotebookMigrationService = (project in file("notebook-migration-service")) - .dependsOn(Auth, Config, DAO, Resource) + .dependsOn(Auth, Config, DAO, Resource, Observability) .settings(commonModuleSettings) .settings( dependencyOverrides ++= Seq( @@ -288,6 +293,7 @@ lazy val TexeraProject = (project in file(".")) // common libraries Auth, Config, + Observability, Resource, Util, DAO, diff --git a/common/config/src/main/resources/observability.conf b/common/config/src/main/resources/observability.conf new file mode 100644 index 00000000000..2bf7a4e10d6 --- /dev/null +++ b/common/config/src/main/resources/observability.conf @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# OpenTelemetry SDK bootstrap settings, consumed by OtelInit. Each value has a +# safe default and an optional environment override, so an operator can find and +# tune every knob here instead of in code. +observability { + # Master switch. Disabled by default: no exporters start and no telemetry is + # emitted until this is set to false (i.e. OTEL_SDK_DISABLED=false). + sdk-disabled = "true" + sdk-disabled = ${?OTEL_SDK_DISABLED} + + # OTLP collector endpoint. http/https only; loopback-only host allowlist by + # default (extend via allowed-hosts below). + endpoint = "http://127.0.0.1:4317" + endpoint = ${?OTEL_EXPORTER_OTLP_ENDPOINT} + + # Comma-separated resource attributes (k1=v1,k2=v2). service.name is set by + # the service and cannot be overridden here. + resource-attributes = "" + resource-attributes = ${?OTEL_RESOURCE_ATTRIBUTES} + + # Comma-separated extra hosts added to the endpoint allowlist. + allowed-hosts = "" + allowed-hosts = ${?TEXERA_OTEL_ALLOWED_HOSTS} + + # Metric export interval in milliseconds; out-of-range values fall back to + # the default. + metric-export-interval-ms = "30000" + metric-export-interval-ms = ${?OTEL_METRIC_EXPORT_INTERVAL} +} diff --git a/common/config/src/main/scala/org/apache/texera/common/config/EnvironmentalVariable.scala b/common/config/src/main/scala/org/apache/texera/common/config/EnvironmentalVariable.scala index e1006c3fb51..6d82d666f1c 100644 --- a/common/config/src/main/scala/org/apache/texera/common/config/EnvironmentalVariable.scala +++ b/common/config/src/main/scala/org/apache/texera/common/config/EnvironmentalVariable.scala @@ -48,6 +48,13 @@ object EnvironmentalVariable { val ENV_USER_JWT_TOKEN = "USER_JWT_TOKEN" val ENV_AUTH_JWT_SECRET = "AUTH_JWT_SECRET" + // OpenTelemetry observability (see observability.conf) + val ENV_OTEL_SDK_DISABLED = "OTEL_SDK_DISABLED" + val ENV_OTEL_EXPORTER_OTLP_ENDPOINT = "OTEL_EXPORTER_OTLP_ENDPOINT" + val ENV_OTEL_RESOURCE_ATTRIBUTES = "OTEL_RESOURCE_ATTRIBUTES" + val ENV_OTEL_METRIC_EXPORT_INTERVAL = "OTEL_METRIC_EXPORT_INTERVAL" + val ENV_TEXERA_OTEL_ALLOWED_HOSTS = "TEXERA_OTEL_ALLOWED_HOSTS" + /** * Dataset-mount vars injected into the CU pod. The mount is performed by the per-node * mounter and reaches the pod through mount propagation, so the pod only needs to know diff --git a/common/config/src/main/scala/org/apache/texera/common/config/ObservabilityConfig.scala b/common/config/src/main/scala/org/apache/texera/common/config/ObservabilityConfig.scala new file mode 100644 index 00000000000..29d121dbd4d --- /dev/null +++ b/common/config/src/main/scala/org/apache/texera/common/config/ObservabilityConfig.scala @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.texera.common.config + +import com.typesafe.config.{Config, ConfigFactory} + +/** + * Typed view over observability.conf. Each field carries the HOCON default + * already merged with its OTEL_* environment override, so OtelInit reads its + * settings from one place instead of calling System.getenv directly. Values + * are kept as strings and interpreted by OtelInit, which tolerates malformed + * input without throwing. + */ +object ObservabilityConfig { + private val conf: Config = ConfigFactory.parseResources("observability.conf").resolve() + + val sdkDisabled: String = conf.getString("observability.sdk-disabled") + val endpoint: String = conf.getString("observability.endpoint") + val resourceAttributes: String = conf.getString("observability.resource-attributes") + val allowedHosts: String = conf.getString("observability.allowed-hosts") + val metricExportIntervalMs: String = conf.getString("observability.metric-export-interval-ms") +} diff --git a/common/config/src/test/scala/org/apache/texera/common/config/ObservabilityConfigSpec.scala b/common/config/src/test/scala/org/apache/texera/common/config/ObservabilityConfigSpec.scala new file mode 100644 index 00000000000..659485bbc9f --- /dev/null +++ b/common/config/src/test/scala/org/apache/texera/common/config/ObservabilityConfigSpec.scala @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.common.config + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** + * Spec for [[ObservabilityConfig]]. Reading each value forces resolution from + * observability.conf, so a renamed key surfaces here as a ConfigException. + * Exact-value assertions are guarded on the OTEL_* override being unset. + */ +class ObservabilityConfigSpec extends AnyFlatSpec with Matchers { + + // `${?VAR}` in HOCON can be satisfied by an OS env var or a JVM system property. + private def isOverridden(name: String): Boolean = + sys.env.contains(name) || sys.props.contains(name) + + "ObservabilityConfig" should "default to disabled per issue #5367" in { + if (!isOverridden(EnvironmentalVariable.ENV_OTEL_SDK_DISABLED)) { + ObservabilityConfig.sdkDisabled shouldBe "true" + } else { + ObservabilityConfig.sdkDisabled should not be empty + } + } + + it should "default to a loopback OTLP endpoint" in { + if (!isOverridden(EnvironmentalVariable.ENV_OTEL_EXPORTER_OTLP_ENDPOINT)) { + ObservabilityConfig.endpoint shouldBe "http://127.0.0.1:4317" + } else { + ObservabilityConfig.endpoint should not be empty + } + } + + it should "default the metric export interval to 30s" in { + if (!isOverridden(EnvironmentalVariable.ENV_OTEL_METRIC_EXPORT_INTERVAL)) { + ObservabilityConfig.metricExportIntervalMs shouldBe "30000" + } else { + ObservabilityConfig.metricExportIntervalMs should not be empty + } + } + + it should "resolve resource-attributes and allowed-hosts without error" in { + // Empty by default; the point is that the keys exist and resolve. + noException should be thrownBy ObservabilityConfig.resourceAttributes + noException should be thrownBy ObservabilityConfig.allowedHosts + } +} diff --git a/common/observability/build.sbt b/common/observability/build.sbt new file mode 100644 index 00000000000..6b6f71f6528 --- /dev/null +++ b/common/observability/build.sbt @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import scala.collection.Seq + +name := "observability" + + +enablePlugins(JavaAppPackaging) + +// Enable semanticdb for Scalafix +ThisBuild / semanticdbEnabled := true +ThisBuild / semanticdbVersion := scalafixSemanticdb.revision + +// Manage dependency conflicts by always using the latest revision +ThisBuild / conflictManager := ConflictManager.latestRevision + +// Restrict parallel execution of tests to avoid conflicts +Global / concurrentRestrictions += Tags.limit(Tags.Test, 1) + +///////////////////////////////////////////////////////////////////////////// +// Compiler Options +///////////////////////////////////////////////////////////////////////////// + +// Scala compiler options +Compile / scalacOptions ++= Seq( + "-Xelide-below", "WARNING", // Turn on optimizations with "WARNING" as the threshold + "-feature", // Check feature warnings + "-deprecation", // Check deprecation warnings + "-Ywarn-unused:imports" // Check for unused imports +) + +///////////////////////////////////////////////////////////////////////////// +// Dependencies +///////////////////////////////////////////////////////////////////////////// + +// OpenTelemetry version is pinned here as the single source of truth; every +// service picks it up via dependsOn(Observability). Bump deliberately. +val openTelemetryVersion = "1.50.0" + +libraryDependencies ++= Seq( + "com.typesafe.scala-logging" %% "scala-logging" % "3.9.5", // for LazyLogging in OtelInit + // OpenTelemetry SDK bootstrap (Apache-2.0). We deliberately do NOT use + // sdk-extension-autoconfigure: endpoint validation (scheme + host allowlist) + // must run before any exporter is configured, and we build the providers + // explicitly rather than let anything be wired from the environment behind + // our back. + "io.opentelemetry" % "opentelemetry-api" % openTelemetryVersion, + "io.opentelemetry" % "opentelemetry-sdk" % openTelemetryVersion, + "io.opentelemetry" % "opentelemetry-exporter-otlp" % openTelemetryVersion, + // Logback Classic is needed at compile time to write the OTel log + // appender. Marked `provided` because every service already brings + // Logback in transitively (via Dropwizard / SLF4J), so we don't + // bundle a second copy. + "ch.qos.logback" % "logback-classic" % "1.2.13" % "provided", + // Test-only: in-memory exporter for OtelInitSpec; avoids hitting a real + // collector during unit tests. + "io.opentelemetry" % "opentelemetry-sdk-testing" % openTelemetryVersion % Test, + "ch.qos.logback" % "logback-classic" % "1.2.13" % Test, + "org.scalatest" %% "scalatest" % "3.2.15" % Test // ScalaTest (for unit tests) +) diff --git a/common/observability/src/main/scala/org/apache/texera/observability/LogSanitizer.scala b/common/observability/src/main/scala/org/apache/texera/observability/LogSanitizer.scala new file mode 100644 index 00000000000..3cc3713c239 --- /dev/null +++ b/common/observability/src/main/scala/org/apache/texera/observability/LogSanitizer.scala @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import scala.jdk.CollectionConverters._ + +/** + * Pure functions that sanitize log bodies and MDC before export: + * strip control characters, redact secrets, cap body size, and + * filter MDC down by dropping denied keys. + */ +object LogSanitizer { + + /** Per-record body length cap, in chars. */ + val MaxBodyChars: Int = 16 * 1024 + + /** Suffix appended to truncated bodies. */ + val TruncatedMarker: String = "...[truncated]" + + /** C0 control characters except TAB (0x09), plus DEL (0x7F). */ + private val C0ControlRegex = "[\\x00-\\x08\\x0A-\\x1F\\x7F]".r + + /** Secret patterns, redacted from bodies. Most specific first. */ + private val SecretPatterns: Seq[scala.util.matching.Regex] = Seq( + // Bearer token + """(?i)Bearer\s+[A-Za-z0-9._\-/+=]{8,}""".r, + // password=... or password: ... + """(?i)password\s*[=:]\s*[^\s,;"']+""".r, + // AWS access key ID + """AKIA[0-9A-Z]{16}""".r, + // labelled AWS secret access key + """(?i)aws_secret_access_key\s*[=:]\s*[A-Za-z0-9/+=]{20,}""".r + ) + + /** MDC keys never forwarded to OTel log attributes. Default-allow: any key + * our instrumentation sets is exported, so adding a new correlation field + * needs no edit here. Only the noisy keys Pekko's SLF4J bridge injects are + * dropped, since they are redundant with the log body and would bloat every + * exported record. Values that pass through are still run through + * [[sanitize]], so secret-shaped content is redacted regardless of key; a + * key whose name looks credential-bearing (see [[isSecretKey]]) has its + * value redacted wholesale. + */ + val DeniedMdcKeys: Set[String] = Set( + "sourceThread", + "pekkoSource", + "pekkoAddress", + "pekkoTimestamp", + "sourceActorSystem" + ) + + /** Substrings marking an MDC key as credential-bearing. A matching key has + * its value redacted whole, since the value alone (e.g. a bare password) + * need not match any [[SecretPatterns]] regex to be a secret. + */ + private val SecretKeySubstrings: Seq[String] = + Seq( + "password", + "passwd", + "pwd", + "secret", + "token", + "apikey", + "api_key", + "authorization", + "credential" + ) + + private def isSecretKey(key: String): Boolean = { + val k = key.toLowerCase + SecretKeySubstrings.exists(k.contains) + } + + /** Strip C0 control characters (except TAB) and DEL. Null-safe. */ + def stripControlChars(body: String): String = + if (body == null) "" else C0ControlRegex.replaceAllIn(body, "") + + /** Redact secret-shaped substrings. Null-safe, idempotent. */ + def redactSecrets(body: String): String = + if (body == null) "" + else SecretPatterns.foldLeft(body)((acc, p) => p.replaceAllIn(acc, "[REDACTED]")) + + /** Strip control chars, redact secrets, then truncate. Idempotent. */ + def sanitize(body: String): String = { + if (body == null || body.isEmpty) return "" + truncate(redactSecrets(stripControlChars(body))) + } + + /** Truncate to MaxBodyChars, appending the marker if cut. */ + def truncate(body: String): String = { + if (body.length <= MaxBodyChars) body + else body.substring(0, MaxBodyChars - TruncatedMarker.length) + TruncatedMarker + } + + /** Drop denied MDC keys, sanitize the surviving values. A key whose name is + * credential-bearing has its value redacted whole. Null-safe. + */ + def filterMdc(mdc: java.util.Map[String, String]): Map[String, String] = { + if (mdc == null) return Map.empty + mdc.asScala.iterator.collect { + case (k, v) if k != null && v != null && !DeniedMdcKeys.contains(k) => + if (isSecretKey(k)) k -> "[REDACTED]" else k -> sanitize(v) + }.toMap + } +} diff --git a/common/observability/src/main/scala/org/apache/texera/observability/OtelInit.scala b/common/observability/src/main/scala/org/apache/texera/observability/OtelInit.scala new file mode 100644 index 00000000000..df12c5a614c --- /dev/null +++ b/common/observability/src/main/scala/org/apache/texera/observability/OtelInit.scala @@ -0,0 +1,418 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import com.typesafe.scalalogging.LazyLogging +import org.apache.texera.common.config.{EnvironmentalVariable, ObservabilityConfig} +import io.opentelemetry.api.{GlobalOpenTelemetry, OpenTelemetry} +import io.opentelemetry.api.common.{AttributeKey, Attributes} +import io.opentelemetry.exporter.otlp.logs.OtlpGrpcLogRecordExporter +import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter +import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.logs.SdkLoggerProvider +import io.opentelemetry.sdk.logs.`export`.{BatchLogRecordProcessor, LogRecordExporter} +import io.opentelemetry.sdk.metrics.SdkMeterProvider +import io.opentelemetry.sdk.metrics.`export`.{MetricExporter, PeriodicMetricReader} +import io.opentelemetry.sdk.resources.Resource +import io.opentelemetry.sdk.trace.SdkTracerProvider +import io.opentelemetry.sdk.trace.`export`.{BatchSpanProcessor, SpanExporter} + +import java.net.URI +import java.time.Duration +import scala.util.{Failure, Success, Try} + +/** + * Bootstraps the OpenTelemetry SDK for a Texera service. + * + * Disabled by default; set OTEL_SDK_DISABLED=false to enable it. Reads its + * settings from observability.conf (each defaulted, each OTEL_*-overridable), + * validates the endpoint against an allowlist, builds tracer/log/metric + * providers, and attaches a Logback appender. Returns None when disabled or + * misconfigured; never throws. + */ +object OtelInit extends LazyLogging { + + /** Endpoint schemes we accept. OTLP-over-gRPC uses http/https endpoints; + * the exporter rejects a `grpc://` scheme outright, so it is not allowed. + */ + private[observability] val AllowedSchemes: Set[String] = Set("http", "https") + + /** Hosts we accept for the OTLP endpoint by default. */ + private[observability] val DefaultAllowedHosts: Set[String] = Set( + "localhost", + "127.0.0.1", + "[::1]" + ) + + /** Default endpoint. 127.0.0.1 (not "localhost") to force IPv4 so a + * natively-run service reaches the collector on dual-stack hosts. + */ + private val DefaultEndpoint = "http://127.0.0.1:4317" + + /** Metric export interval bounds; out-of-range values fall back to the + * default (see clampIntervalMs). + */ + private[observability] val MinMetricIntervalMs: Long = 1000L + private[observability] val MaxMetricIntervalMs: Long = 10L * 60L * 1000L + private[observability] val DefaultMetricIntervalMs: Long = 30L * 1000L + + // Idempotency guard: init() is a no-op after the first call. + @volatile private var initialized: Option[OpenTelemetry] = None + + /** + * Initialize the SDK for the given service name. Returns Some on + * success, None when disabled or misconfigured. When enabled, also + * attaches a [[TexeraOtelLogAppender]] to the Logback ROOT logger. + */ + def init(serviceName: String): Option[OpenTelemetry] = + synchronized { + if (initialized.isDefined) return initialized + + // Guard the whole body: reading observability.conf (ObservabilityConfig's + // eager vals) and building the SDK are the fallible steps. A missing or + // malformed config would otherwise escape as ExceptionInInitializerError, + // even when telemetry is disabled, breaking init()'s "never throws" + // contract. On any failure, disable telemetry with one WARN and return None. + Try { + // Source the OTEL_* settings from observability.conf (HOCON defaults + // already merged with any env override); fall back to the raw environment + // for anything else. + val env = (key: String) => + key match { + case EnvironmentalVariable.ENV_OTEL_SDK_DISABLED => + Some(ObservabilityConfig.sdkDisabled) + case EnvironmentalVariable.ENV_OTEL_EXPORTER_OTLP_ENDPOINT => + Some(ObservabilityConfig.endpoint) + case EnvironmentalVariable.ENV_OTEL_RESOURCE_ATTRIBUTES => + Some(ObservabilityConfig.resourceAttributes) + case EnvironmentalVariable.ENV_TEXERA_OTEL_ALLOWED_HOSTS => + Some(ObservabilityConfig.allowedHosts) + case EnvironmentalVariable.ENV_OTEL_METRIC_EXPORT_INTERVAL => + Some(ObservabilityConfig.metricExportIntervalMs) + case other => Option(System.getenv(other)) + } + val result = initInternal( + serviceName = serviceName, + envProvider = env, + spanExporterFactory = buildOtlpSpanExporter, + logExporterFactory = endpoint => Some(buildOtlpLogExporter(endpoint)), + metricExporterFactory = endpoint => Some(buildOtlpMetricExporter(endpoint)), + logbackAttacher = LogbackBinder.attach + ) + // Register globally so OTel-aware code can use GlobalOpenTelemetry + // without threading the SDK through callsites. set() throws on a + // second call; wrap defensively. + result.foreach { sdk => + Try(GlobalOpenTelemetry.set(sdk)).failed.foreach { t => + logger.warn( + s"GlobalOpenTelemetry already set; using the existing instance: ${t.getMessage}" + ) + } + } + result + } match { + case Success(result) => result + case Failure(t) => + logger.warn( + s"OpenTelemetry SDK initialization failed; continuing without telemetry: ${t.getMessage}" + ) + None + } + } + + /** + * Test-only entry point: injects an env-var map and exporters so the + * SDK makes no network connection. Does not attach the Logback appender. + */ + private[observability] def initForTest( + serviceName: String, + envOverride: Map[String, String], + exporter: SpanExporter, + metricExporter: Option[MetricExporter] = None + ): Option[OpenTelemetry] = + synchronized { + initInternal( + serviceName = serviceName, + envProvider = envOverride.get, + spanExporterFactory = _ => exporter, + logExporterFactory = _ => None, + metricExporterFactory = _ => metricExporter, + logbackAttacher = (_, _) => () // no-op in tests + ) + } + + /** Test-only: forget any previously-installed SDK. Does not unregister + * shutdown hooks (the previous SDK is closed instead). + */ + private[observability] def resetForTest(): Unit = + synchronized { + initialized.foreach { + case sdk: OpenTelemetrySdk => + Try(sdk.getSdkTracerProvider.close()) + Try(sdk.getSdkLoggerProvider.close()) + Try(sdk.getSdkMeterProvider.close()) + case _ => () + } + initialized = None + } + + private def initInternal( + serviceName: String, + envProvider: String => Option[String], + spanExporterFactory: String => SpanExporter, + logExporterFactory: String => Option[LogRecordExporter], + metricExporterFactory: String => Option[MetricExporter], + logbackAttacher: (String, OpenTelemetry) => Unit + ): Option[OpenTelemetry] = { + if (initialized.isDefined) return initialized + + // Disabled by default (issue #5367): stay inert unless OTEL_SDK_DISABLED is + // explicitly false. An unreachable endpoint drops records without crashing. + val disabled = envProvider(EnvironmentalVariable.ENV_OTEL_SDK_DISABLED).getOrElse("true") + if (!disabled.equalsIgnoreCase("false")) { + logger.info( + "OpenTelemetry SDK disabled (OTEL_SDK_DISABLED not false). No telemetry will be emitted." + ) + return None + } + + val endpoint = + envProvider(EnvironmentalVariable.ENV_OTEL_EXPORTER_OTLP_ENDPOINT).getOrElse(DefaultEndpoint) + val extraAllowed = envProvider(EnvironmentalVariable.ENV_TEXERA_OTEL_ALLOWED_HOSTS) + .map(_.split(',').iterator.map(_.trim.toLowerCase).filter(_.nonEmpty).toSet) + .getOrElse(Set.empty) + val allowedHosts = DefaultAllowedHosts ++ extraAllowed + + validateEndpoint(endpoint, allowedHosts) match { + case Left(reason) => + // One WARN; no telemetry is emitted. + logger.warn( + s"OpenTelemetry SDK disabled: invalid OTEL_EXPORTER_OTLP_ENDPOINT — $reason. " + + "Set TEXERA_OTEL_ALLOWED_HOSTS to extend the allowlist." + ) + return None + case Right(_) => // ok + } + + val rawAttrs = envProvider(EnvironmentalVariable.ENV_OTEL_RESOURCE_ATTRIBUTES).getOrElse("") + val resource = buildResource(serviceName, rawAttrs) + + val spanExporter = spanExporterFactory(endpoint) + val tracerProvider = SdkTracerProvider + .builder() + .setResource(resource) + .addSpanProcessor(BatchSpanProcessor.builder(spanExporter).build()) + .build() + + val sdkBuilder = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider) + + // Logger provider is optional; the factory returns None in tests. + val loggerProviderOpt = logExporterFactory(endpoint).map { logExporter => + val lp = SdkLoggerProvider + .builder() + .setResource(resource) + .addLogRecordProcessor(BatchLogRecordProcessor.builder(logExporter).build()) + .build() + sdkBuilder.setLoggerProvider(lp) + lp + } + + // Meter provider is optional too; interval falls back to the default + // when out of range. + val intervalMs = + clampIntervalMs(envProvider(EnvironmentalVariable.ENV_OTEL_METRIC_EXPORT_INTERVAL)) + val meterProviderOpt = metricExporterFactory(endpoint).map { metricExporter => + val reader = PeriodicMetricReader + .builder(metricExporter) + .setInterval(Duration.ofMillis(intervalMs)) + .build() + val mp = SdkMeterProvider + .builder() + .setResource(resource) + .registerMetricReader(reader) + .build() + sdkBuilder.setMeterProvider(mp) + mp + } + + val sdk = sdkBuilder.build() + + // One startup span carrying only service.name. + val span = sdk.getTracer("texera.bootstrap").spanBuilder("service.start").startSpan() + Try(span.setAttribute("service.name", serviceName)) + span.end() + + // Wire the Logback appender; failure here must not crash the service. + Try(logbackAttacher(serviceName, sdk)).failed.foreach { t => + logger.warn(s"Failed to attach OTel Logback appender (logs not exported): ${t.getMessage}") + } + + // Flush providers on shutdown. Added after the SDK is fully built. + Runtime.getRuntime.addShutdownHook( + new Thread( + () => { + Try(tracerProvider.close()) + loggerProviderOpt.foreach(lp => Try(lp.close())) + meterProviderOpt.foreach(mp => Try(mp.close())) + () + }, + "otel-shutdown" + ) + ) + + initialized = Some(sdk) + logger.info(s"OpenTelemetry SDK initialized for service '$serviceName' (endpoint=$endpoint).") + initialized + } + + /** + * Validate the endpoint is parseable and uses an allowlisted scheme + * and host. Pure function. + */ + private[observability] def validateEndpoint( + endpoint: String, + allowedHosts: Set[String] + ): Either[String, Unit] = { + Try(URI.create(endpoint)) match { + case Failure(e) => + Left(s"unparseable URI (${e.getClass.getSimpleName})") + case Success(uri) => + val scheme = Option(uri.getScheme).map(_.toLowerCase).getOrElse("") + if (scheme.isEmpty) { + Left("missing scheme") + } else if (!AllowedSchemes.contains(scheme)) { + Left( + s"scheme '$scheme' not in allowlist ${AllowedSchemes.toSeq.sorted.mkString("{", ",", "}")}" + ) + } else { + val host = Option(uri.getHost).map(_.toLowerCase).getOrElse("") + if (host.isEmpty) { + Left("missing host") + } else if (!allowedHosts.contains(host)) { + Left(s"host '$host' not in allowlist") + } else { + Right(()) + } + } + } + } + + /** + * Build a Resource from the service name and OTEL_RESOURCE_ATTRIBUTES. + * Every parsed attribute is applied so new resource fields need no edit + * here; the one exception is service.name, which the argument controls + * and env cannot override. + */ + private[observability] def buildResource(serviceName: String, rawAttrs: String): Resource = { + val builder = Attributes.builder() + builder.put(AttributeKey.stringKey("service.name"), serviceName) + + parseAttrs(rawAttrs).foreach { + case (key, value) if key != "service.name" => + builder.put(AttributeKey.stringKey(key), value) + case _ => // env cannot override service.name + } + + Resource.create(builder.build()) + } + + /** Parse a `k1=v1,k2=v2` string. Malformed entries are skipped. */ + private[observability] def parseAttrs(raw: String): Seq[(String, String)] = { + if (raw == null || raw.isEmpty) return Seq.empty + raw + .split(',') + .iterator + .map(_.trim) + .filter(_.nonEmpty) + .flatMap { entry => + val idx = entry.indexOf('=') + if (idx <= 0 || idx == entry.length - 1) None + else Some(entry.substring(0, idx).trim -> entry.substring(idx + 1).trim) + } + .toSeq + } + + private def buildOtlpSpanExporter(endpoint: String): SpanExporter = + OtlpGrpcSpanExporter.builder().setEndpoint(endpoint).build() + + private def buildOtlpLogExporter(endpoint: String): LogRecordExporter = + OtlpGrpcLogRecordExporter.builder().setEndpoint(endpoint).build() + + private def buildOtlpMetricExporter(endpoint: String): MetricExporter = + OtlpGrpcMetricExporter.builder().setEndpoint(endpoint).build() + + /** + * Parse and clamp OTEL_METRIC_EXPORT_INTERVAL (ms). Out-of-range or + * unparseable input falls back to the default with one WARN. + */ + private[observability] def clampIntervalMs(raw: Option[String]): Long = { + raw match { + case None => DefaultMetricIntervalMs + case Some(value) => + Try(value.trim.toLong) match { + case Failure(_) => + logger.warn( + s"OTEL_METRIC_EXPORT_INTERVAL '$value' is not a number; " + + s"using default ${DefaultMetricIntervalMs}ms." + ) + DefaultMetricIntervalMs + case Success(ms) if ms < MinMetricIntervalMs || ms > MaxMetricIntervalMs => + logger.warn( + s"OTEL_METRIC_EXPORT_INTERVAL=${ms}ms out of range " + + s"[${MinMetricIntervalMs}, ${MaxMetricIntervalMs}]; " + + s"using default ${DefaultMetricIntervalMs}ms." + ) + DefaultMetricIntervalMs + case Success(ms) => ms + } + } + } +} + +/** + * Isolates the Logback attach step so [[OtelInit]] does not import + * Logback types directly, keeping SDK init testable with a mock attacher. + */ +private[observability] object LogbackBinder extends LazyLogging { + + /** Attach a [[TexeraOtelLogAppender]] bound to `otel` to the Logback + * ROOT logger. Emits one WARN and returns if Logback is not the + * active SLF4J binding. + */ + def attach(serviceName: String, otel: OpenTelemetry): Unit = { + val factory = org.slf4j.LoggerFactory.getILoggerFactory + factory match { + case ctx: ch.qos.logback.classic.LoggerContext => + val root = ctx.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME) + val appender = new TexeraOtelLogAppender() + appender.setContext(ctx) + appender.setName(s"texera-otel-$serviceName") + appender.bind(otel) + appender.start() + root.addAppender(appender) + case other => + logger.warn( + s"SLF4J binding is not Logback (${other.getClass.getName}); " + + "OTel log export is not wired. Application logs to stdout/file are unaffected." + ) + } + } +} diff --git a/common/observability/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala b/common/observability/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala new file mode 100644 index 00000000000..276923b655c --- /dev/null +++ b/common/observability/src/main/scala/org/apache/texera/observability/TexeraOtelLogAppender.scala @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import ch.qos.logback.classic.Level +import ch.qos.logback.classic.spi.{ILoggingEvent, IThrowableProxy, ThrowableProxyUtil} +import ch.qos.logback.core.UnsynchronizedAppenderBase +import io.opentelemetry.api.OpenTelemetry +import io.opentelemetry.api.common.AttributeKey +import io.opentelemetry.api.logs.{Logger, Severity} +import io.opentelemetry.api.trace.Span +import io.opentelemetry.context.Context + +import java.util.concurrent.TimeUnit + +/** + * Logback appender that sanitizes each event via [[LogSanitizer]] and + * emits it as an OTel LogRecord. [[append]] is a no-op until [[bind]] + * is called and after [[stop]]. + * + * This is internal plumbing, not the developer logging API. Code logs + * through the normal SLF4J / scala-logging interface and adds correlation + * ids via MDC; [[OtelInit.init]] attaches this appender to the ROOT logger + * so those records also reach OTel: + * + * {{{ + * class Foo extends LazyLogging { + * MDC.put("workflowId", id) // forwarded as an OTel log attribute + * try logger.info("started") // body + severity + trace context + * finally MDC.remove("workflowId") + * } + * }}} + */ +class TexeraOtelLogAppender extends UnsynchronizedAppenderBase[ILoggingEvent] { + + // @volatile so a late bind() is visible to appender threads. + @volatile private var otelLogger: Option[Logger] = None + + def bind(otel: OpenTelemetry): Unit = { + otelLogger = Some(otel.getLogsBridge.get("texera.logback")) + } + + override def stop(): Unit = { + otelLogger = None + super.stop() + } + + override def append(event: ILoggingEvent): Unit = { + otelLogger match { + case None => () // not bound + case Some(_) if isSelfDiagnostic(event) => + () // drop the exporter's own diagnostics to avoid a feedback loop + case Some(logger) => + try { + emit(logger, event) + } catch { + // An appender must not throw into the calling thread. + case t: Throwable => + addError("OTel log emission failed", t) + } + } + } + + /** OTel's own components report export failures through JUL, which the + * jul-to-slf4j bridge routes to Logback ROOT and back into this appender. + * Re-exporting them to the collector that just failed would feed the failure + * back on itself, so drop any record from an io.opentelemetry logger. + */ + private def isSelfDiagnostic(event: ILoggingEvent): Boolean = { + val name = event.getLoggerName + name != null && name.startsWith("io.opentelemetry") + } + + private def emit(logger: Logger, event: ILoggingEvent): Unit = { + // Control-strip the message only (trace newlines must survive), then append + // the stack trace, redact secrets across the whole body, and cap length. + val message = LogSanitizer.stripControlChars(event.getFormattedMessage) + val combined = Option(event.getThrowableProxy) match { + case Some(proxy) => message + "\n" + formatThrowable(proxy) + case None => message + } + val body = LogSanitizer.truncate(LogSanitizer.redactSecrets(combined)) + val builder = logger + .logRecordBuilder() + .setBody(body) + .setSeverity(severityFromLevel(event.getLevel)) + .setSeverityText(event.getLevel.toString) + .setTimestamp(event.getTimeStamp, TimeUnit.MILLISECONDS) + + // Surviving (deny-list filtered) MDC keys as typed attributes. + LogSanitizer.filterMdc(event.getMDCPropertyMap).foreach { + case (k, v) => builder.setAttribute(AttributeKey.stringKey(k), v) + } + + builder.setAttribute(AttributeKey.stringKey("logger.name"), event.getLoggerName) + builder.setAttribute(AttributeKey.stringKey("thread.name"), event.getThreadName) + + // Exception semantic-convention attributes so backends key error identity + // off exception.type / exception.message instead of parsing the body. The + // message and stack trace are redacted like the body. + Option(event.getThrowableProxy).foreach { proxy => + builder.setAttribute(AttributeKey.stringKey("exception.type"), proxy.getClassName) + builder.setAttribute( + AttributeKey.stringKey("exception.message"), + LogSanitizer.redactSecrets(proxy.getMessage) + ) + builder.setAttribute( + AttributeKey.stringKey("exception.stacktrace"), + LogSanitizer.truncate(LogSanitizer.redactSecrets(formatThrowable(proxy))) + ) + } + + // Attach trace context so the SDK sets trace_id / span_id. + val span = Span.current() + if (span.getSpanContext.isValid) { + builder.setContext(Context.current()) + } + + builder.emit() + } + + /** Format a throwable proxy as a Logback-style stack trace. */ + private def formatThrowable(proxy: IThrowableProxy): String = + ThrowableProxyUtil.asString(proxy) + + private def severityFromLevel(level: Level): Severity = { + if (level == null) return Severity.UNDEFINED_SEVERITY_NUMBER + level.toInt match { + case Level.TRACE_INT => Severity.TRACE + case Level.DEBUG_INT => Severity.DEBUG + case Level.INFO_INT => Severity.INFO + case Level.WARN_INT => Severity.WARN + case Level.ERROR_INT => Severity.ERROR + case _ => Severity.UNDEFINED_SEVERITY_NUMBER + } + } +} diff --git a/common/observability/src/test/scala/org/apache/texera/observability/LogSanitizerSpec.scala b/common/observability/src/test/scala/org/apache/texera/observability/LogSanitizerSpec.scala new file mode 100644 index 00000000000..cade55e99a7 --- /dev/null +++ b/common/observability/src/test/scala/org/apache/texera/observability/LogSanitizerSpec.scala @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import scala.jdk.CollectionConverters._ + +class LogSanitizerSpec extends AnyFlatSpec with Matchers { + + // ----- sanitize: control characters ---------------------------------- + + "sanitize" should "strip CR/LF so a user-supplied message cannot forge a new log line" in { + val crlfPayload = "hello\r\nFAKE LOG LINE\r\nworld" + LogSanitizer.sanitize(crlfPayload) shouldBe "helloFAKE LOG LINEworld" + } + + it should "strip other C0 control characters but preserve TAB" in { + val payload = "before\u0000NUL\u0007BEL\tTAB\u001bafter\u007fdel" + LogSanitizer.sanitize(payload) shouldBe "beforeNULBEL\tTABafterdel" + } + + it should "handle empty / null bodies cleanly" in { + LogSanitizer.sanitize("") shouldBe "" + LogSanitizer.sanitize(null) shouldBe "" + } + + // ----- sanitize: secret scrubbing ------------------------------------ + + it should "redact Bearer tokens regardless of case" in { + LogSanitizer.sanitize("Authorization: Bearer abc123XYZ.foo") should include("[REDACTED]") + LogSanitizer.sanitize("Authorization: Bearer abc123XYZ.foo") should not include "abc123XYZ" + LogSanitizer.sanitize("auth = bearer eyJhbGci.tok") should include("[REDACTED]") + } + + it should "redact password=... key/value forms" in { + val out = LogSanitizer.sanitize("connecting: user=alice password=hunter2 host=db") + out should include("[REDACTED]") + out should not include "hunter2" + // surrounding context preserved + out should include("user=alice") + out should include("host=db") + } + + it should "redact AWS access key IDs" in { + val out = LogSanitizer.sanitize("found key AKIAIOSFODNN7EXAMPLE in env") + out should include("[REDACTED]") + out should not include "AKIAIOSFODNN7EXAMPLE" + } + + it should "redact AWS secret access keys when explicitly labelled" in { + val out = LogSanitizer.sanitize( + "aws_secret_access_key=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY some text" + ) + out should include("[REDACTED]") + out should not include "wJalrXUtnFEMI" + } + + it should "leave already-redacted content alone (idempotent)" in { + val once = LogSanitizer.sanitize("Authorization: Bearer abc12345.deadbeef") + val twice = LogSanitizer.sanitize(once) + twice shouldBe once + } + + // ----- sanitize: size cap -------------------------------------------- + + it should "truncate bodies larger than MaxBodyChars and append the marker" in { + val oversize = "a" * (LogSanitizer.MaxBodyChars * 4) // ~64 KiB + val out = LogSanitizer.sanitize(oversize) + out.length shouldBe LogSanitizer.MaxBodyChars + out should endWith(LogSanitizer.TruncatedMarker) + } + + it should "leave bodies at or below the cap unchanged in length" in { + val rightAtCap = "x" * LogSanitizer.MaxBodyChars + LogSanitizer.sanitize(rightAtCap).length shouldBe LogSanitizer.MaxBodyChars + } + + // ----- filterMdc ----------------------------------------------------- + + "filterMdc" should "drop denied Pekko keys and pass every other key through" in { + val mdc = Map( + "trace_id" -> "abc", + "span_id" -> "def", + "texera.workflow.id" -> "42", + "app.new.tag" -> "kept", + "sourceThread" -> "dispatcher-3", + "pekkoSource" -> "akka://sys/user/actor" + ).asJava + val out = LogSanitizer.filterMdc(mdc) + out.keySet shouldBe Set("trace_id", "span_id", "texera.workflow.id", "app.new.tag") + } + + it should "scrub secret-shaped values on keys that pass through" in { + val mdc = Map("note" -> "password=p4ssw0rd").asJava + LogSanitizer.filterMdc(mdc)("note") should not include "p4ssw0rd" + } + + it should "redact the value of a credential-named key even when the value itself is benign" in { + val mdc = Map( + "password" -> "hunter2", + "user.api_key" -> "abcdef", + "authToken" -> "xyz", + "trace_id" -> "keep-me" + ).asJava + val out = LogSanitizer.filterMdc(mdc) + out("password") shouldBe "[REDACTED]" + out("user.api_key") shouldBe "[REDACTED]" + out("authToken") shouldBe "[REDACTED]" + out("trace_id") shouldBe "keep-me" + } + + it should "tolerate null map and null values" in { + LogSanitizer.filterMdc(null) shouldBe empty + + val javaMap = new java.util.HashMap[String, String]() + javaMap.put("trace_id", null) + javaMap.put("texera.user.id", "7") + val out = LogSanitizer.filterMdc(javaMap) + out shouldBe Map("texera.user.id" -> "7") + } +} diff --git a/common/observability/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala b/common/observability/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala new file mode 100644 index 00000000000..af544695fb5 --- /dev/null +++ b/common/observability/src/test/scala/org/apache/texera/observability/OtelInitSpec.scala @@ -0,0 +1,295 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter +import org.scalatest.BeforeAndAfterEach +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import scala.jdk.CollectionConverters._ + +class OtelInitSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { + + override def beforeEach(): Unit = { + OtelInit.resetForTest() + } + + override def afterEach(): Unit = { + OtelInit.resetForTest() + } + + // ----- validateEndpoint: pure function, exhaustive cases ------------- + + "validateEndpoint" should "accept loopback OTLP http(s) URLs" in { + OtelInit.validateEndpoint("http://localhost:4317", OtelInit.DefaultAllowedHosts) shouldBe Right( + () + ) + OtelInit.validateEndpoint("http://127.0.0.1:4317", OtelInit.DefaultAllowedHosts) shouldBe Right( + () + ) + OtelInit.validateEndpoint( + "https://localhost:4318", + OtelInit.DefaultAllowedHosts + ) shouldBe Right(()) + } + + it should "reject a grpc:// endpoint (the OTLP exporter accepts only http/https)" in { + val result = OtelInit.validateEndpoint("grpc://127.0.0.1:4317", OtelInit.DefaultAllowedHosts) + result.isLeft shouldBe true + result.left.toOption.get should include("scheme") + } + + it should "reject file:// schemes (path traversal style attack)" in { + val result = OtelInit.validateEndpoint("file:///etc/passwd", OtelInit.DefaultAllowedHosts) + result.isLeft shouldBe true + result.left.toOption.get should include("scheme") + } + + it should "reject arbitrary remote hosts not in the allowlist" in { + val result = OtelInit.validateEndpoint( + "http://attacker.example.com:4317", + OtelInit.DefaultAllowedHosts + ) + result.isLeft shouldBe true + result.left.toOption.get should include("host") + } + + it should "accept hosts added to the allowlist" in { + val widened = OtelInit.DefaultAllowedHosts + "collector.internal" + OtelInit.validateEndpoint("http://collector.internal:4317", widened) shouldBe Right(()) + } + + it should "reject endpoints with no scheme" in { + val result = OtelInit.validateEndpoint("localhost:4317", OtelInit.DefaultAllowedHosts) + result.isLeft shouldBe true + } + + it should "reject endpoints with no host" in { + val result = OtelInit.validateEndpoint("http:///path", OtelInit.DefaultAllowedHosts) + result.isLeft shouldBe true + } + + it should "reject completely malformed input" in { + val result = OtelInit.validateEndpoint("not a uri at all :: bad", OtelInit.DefaultAllowedHosts) + result.isLeft shouldBe true + } + + // ----- buildResource: passthrough with service.name protected --------- + + "buildResource" should "always include the service.name from the argument" in { + val r = OtelInit.buildResource("my-service", "") + Option( + r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.name")) + ) shouldBe Some( + "my-service" + ) + } + + it should "honor keys from OTEL_RESOURCE_ATTRIBUTES" in { + val r = OtelInit.buildResource("svc", "service.version=1.2.3,deployment.environment=staging") + Option( + r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.version")) + ) shouldBe Some( + "1.2.3" + ) + Option( + r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("deployment.environment")) + ) shouldBe Some("staging") + } + + it should "pass custom keys through so new resource fields need no code edit" in { + val r = OtelInit.buildResource( + "svc", + "service.version=1.0,custom.tag=team-a,texera.region.id=us-west" + ) + val attrs: Map[String, String] = r.getAttributes.asMap.asScala.iterator.map { + case (k, v) => k.getKey -> v.toString + }.toMap + + attrs("service.version") shouldBe "1.0" + attrs("custom.tag") shouldBe "team-a" + attrs("texera.region.id") shouldBe "us-west" + } + + it should "refuse to let OTEL_RESOURCE_ATTRIBUTES override service.name" in { + val r = OtelInit.buildResource("real-svc", "service.name=spoofed") + Option( + r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.name")) + ) shouldBe Some( + "real-svc" + ) + } + + it should "carry the texera.* resource attrs so a CU JVM auto-tags every emitted record" in { + val r = OtelInit.buildResource( + "texera-computing-unit-master", + "texera.computing_unit.id=8,texera.workflow.id=441,texera.execution.id=1234" + ) + val attrs: Map[String, String] = r.getAttributes.asMap.asScala.iterator.map { + case (k, v) => k.getKey -> v.toString + }.toMap + attrs("texera.computing_unit.id") shouldBe "8" + attrs("texera.workflow.id") shouldBe "441" + attrs("texera.execution.id") shouldBe "1234" + } + + it should "ignore malformed pairs without crashing" in { + val r = OtelInit.buildResource("svc", ",,,=,foo,service.version=,=bar,service.version=1.0,") + Option( + r.getAttribute(io.opentelemetry.api.common.AttributeKey.stringKey("service.version")) + ) shouldBe Some( + "1.0" + ) + } + + it should "handle empty / null input cleanly" in { + OtelInit.parseAttrs("") shouldBe empty + OtelInit.parseAttrs(null) shouldBe empty + } + + // ----- end-to-end init: span emission + disable behaviour ------------- + + "init" should "be a no-op when OTEL_SDK_DISABLED is explicitly set to true" in { + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest("svc", Map("OTEL_SDK_DISABLED" -> "true"), exporter) + result shouldBe None + exporter.getFinishedSpanItems.asScala shouldBe empty + } + + it should "stay inert by default when OTEL_SDK_DISABLED is unset (issue #5367)" in { + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest( + "svc", + Map( + // OTEL_SDK_DISABLED omitted; the SDK stays disabled by default. + "OTEL_EXPORTER_OTLP_ENDPOINT" -> "http://localhost:4317" + ), + exporter + ) + result shouldBe None + exporter.getFinishedSpanItems.asScala shouldBe empty + } + + it should "stay inert for any OTEL_SDK_DISABLED value other than an explicit false" in { + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest( + "svc", + Map("OTEL_SDK_DISABLED" -> "", "OTEL_EXPORTER_OTLP_ENDPOINT" -> "http://localhost:4317"), + exporter + ) + result shouldBe None + } + + it should "emit a single service.start span when enabled with a valid endpoint" in { + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest( + "my-service", + Map( + "OTEL_SDK_DISABLED" -> "false", + "OTEL_EXPORTER_OTLP_ENDPOINT" -> "http://localhost:4317" + ), + exporter + ) + result.isDefined shouldBe true + + // BatchSpanProcessor is async; flush before reading. + result.get + .asInstanceOf[io.opentelemetry.sdk.OpenTelemetrySdk] + .getSdkTracerProvider + .forceFlush() + .join(2, java.util.concurrent.TimeUnit.SECONDS) + + val spans = exporter.getFinishedSpanItems.asScala + spans should have size 1 + spans.head.getName shouldBe "service.start" + } + + it should "refuse to initialize when the endpoint scheme is file://" in { + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest( + "svc", + Map( + "OTEL_SDK_DISABLED" -> "false", + "OTEL_EXPORTER_OTLP_ENDPOINT" -> "file:///etc/passwd" + ), + exporter + ) + result shouldBe None + exporter.getFinishedSpanItems.asScala shouldBe empty + } + + it should "refuse to initialize when the endpoint host is off-allowlist" in { + val exporter = InMemorySpanExporter.create() + val result = OtelInit.initForTest( + "svc", + Map( + "OTEL_SDK_DISABLED" -> "false", + "OTEL_EXPORTER_OTLP_ENDPOINT" -> "http://attacker.example.com:4317" + ), + exporter + ) + result shouldBe None + exporter.getFinishedSpanItems.asScala shouldBe empty + } + + it should "be idempotent — second init returns the same instance" in { + val exporter = InMemorySpanExporter.create() + val env = Map( + "OTEL_SDK_DISABLED" -> "false", + "OTEL_EXPORTER_OTLP_ENDPOINT" -> "http://localhost:4317" + ) + val first = OtelInit.initForTest("svc", env, exporter) + val second = OtelInit.initForTest("svc", env, exporter) + second shouldBe first + } + + // ----- clampIntervalMs: parse + range fallback ------------------------ + + "clampIntervalMs" should "fall back to the default when the value is absent" in { + OtelInit.clampIntervalMs(None) shouldBe OtelInit.DefaultMetricIntervalMs + } + + it should "fall back to the default when the value is not a number" in { + OtelInit.clampIntervalMs(Some("not-a-number")) shouldBe OtelInit.DefaultMetricIntervalMs + } + + it should "fall back to the default when the value is below the minimum" in { + OtelInit.clampIntervalMs(Some((OtelInit.MinMetricIntervalMs - 1).toString)) shouldBe + OtelInit.DefaultMetricIntervalMs + } + + it should "fall back to the default when the value is above the maximum" in { + OtelInit.clampIntervalMs(Some((OtelInit.MaxMetricIntervalMs + 1).toString)) shouldBe + OtelInit.DefaultMetricIntervalMs + } + + it should "keep an in-range value" in { + val inRange = OtelInit.MinMetricIntervalMs + 1234 + OtelInit.clampIntervalMs(Some(inRange.toString)) shouldBe inRange + } + + it should "accept the range boundaries" in { + OtelInit.clampIntervalMs(Some(OtelInit.MinMetricIntervalMs.toString)) shouldBe + OtelInit.MinMetricIntervalMs + OtelInit.clampIntervalMs(Some(OtelInit.MaxMetricIntervalMs.toString)) shouldBe + OtelInit.MaxMetricIntervalMs + } +} diff --git a/common/observability/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala b/common/observability/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala new file mode 100644 index 00000000000..27b6a7ce2c4 --- /dev/null +++ b/common/observability/src/test/scala/org/apache/texera/observability/TexeraOtelLogAppenderSpec.scala @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import ch.qos.logback.classic.{Level, Logger, LoggerContext} +import ch.qos.logback.classic.spi.LoggingEvent +import io.opentelemetry.api.OpenTelemetry +import io.opentelemetry.api.logs.Severity +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.logs.SdkLoggerProvider +import io.opentelemetry.sdk.logs.`export`.SimpleLogRecordProcessor +import io.opentelemetry.sdk.testing.exporter.InMemoryLogRecordExporter +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.slf4j.LoggerFactory + +import scala.jdk.CollectionConverters._ + +class TexeraOtelLogAppenderSpec extends AnyFlatSpec with Matchers { + + /** Build an OpenTelemetry SDK whose LoggerProvider drains to the + * given in-memory exporter via the synchronous SimpleLogRecordProcessor, + * so tests don't depend on batch timing. + */ + private def newFixture(): (OpenTelemetry, InMemoryLogRecordExporter, TexeraOtelLogAppender) = { + val exporter = InMemoryLogRecordExporter.create() + val lp = SdkLoggerProvider + .builder() + .addLogRecordProcessor(SimpleLogRecordProcessor.create(exporter)) + .build() + val sdk = OpenTelemetrySdk.builder().setLoggerProvider(lp).build() + val appender = new TexeraOtelLogAppender() + appender.setContext(LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext]) + appender.bind(sdk) + appender.start() + (sdk, exporter, appender) + } + + private def makeEvent( + message: String, + level: Level = Level.INFO, + mdc: Map[String, String] = Map.empty + ): LoggingEvent = { + val ctx = LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext] + val logger = ctx.getLogger("test.logger").asInstanceOf[Logger] + val ev = new LoggingEvent("fqcn", logger, level, message, null, null) + if (mdc.nonEmpty) ev.setMDCPropertyMap(mdc.asJava) + ev + } + + // ----- positive paths ------------------------------------------------- + + "TexeraOtelLogAppender" should "emit an INFO record with body + severity" in { + val (_, exporter, appender) = newFixture() + appender.doAppend(makeEvent("hello world")) + + val records = exporter.getFinishedLogRecordItems.asScala + records should have size 1 + records.head.getBodyValue.asString shouldBe "hello world" + records.head.getSeverity shouldBe Severity.INFO + records.head.getSeverityText shouldBe "INFO" + } + + it should "map every log level to a distinct OTel severity" in { + val (_, exporter, appender) = newFixture() + Seq(Level.TRACE, Level.DEBUG, Level.INFO, Level.WARN, Level.ERROR).foreach { lvl => + appender.doAppend(makeEvent(s"msg-$lvl", lvl)) + } + val severities = exporter.getFinishedLogRecordItems.asScala.map(_.getSeverity).toSet + severities shouldBe Set( + Severity.TRACE, + Severity.DEBUG, + Severity.INFO, + Severity.WARN, + Severity.ERROR + ) + } + + // ----- security: sanitisation happens at the boundary ----------------- + + it should "strip CRLF from a forged log-injection payload before emission" in { + val (_, exporter, appender) = newFixture() + appender.doAppend(makeEvent("hello\r\nFAKE LOG LINE\r\nworld")) + + val body = exporter.getFinishedLogRecordItems.asScala.head.getBodyValue.asString + body shouldBe "helloFAKE LOG LINEworld" + body should not include "\n" + body should not include "\r" + } + + it should "redact Bearer tokens at emission time" in { + val (_, exporter, appender) = newFixture() + appender.doAppend(makeEvent("Authorization: Bearer abc123XYZ.foo")) + + val body = exporter.getFinishedLogRecordItems.asScala.head.getBodyValue.asString + body should include("[REDACTED]") + body should not include "abc123XYZ" + } + + it should "redact secrets inside an attached stack trace, not just the message" in { + val (_, exporter, appender) = newFixture() + val ctx = LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext] + val logger = ctx.getLogger("test.logger").asInstanceOf[Logger] + val boom = new RuntimeException("db connect failed for password=hunter2") + val ev = new LoggingEvent("fqcn", logger, Level.ERROR, "operation failed", boom, null) + appender.doAppend(ev) + + val body = exporter.getFinishedLogRecordItems.asScala.head.getBodyValue.asString + body should include("[REDACTED]") + body should not include "hunter2" + // The stack trace's newlines are preserved (only the message is C0-stripped). + body should include("\n") + } + + it should "truncate a 1 MiB body to MaxBodyChars with the marker" in { + val (_, exporter, appender) = newFixture() + val oversize = "x" * (1024 * 1024) + appender.doAppend(makeEvent(oversize)) + + val body = exporter.getFinishedLogRecordItems.asScala.head.getBodyValue.asString + body.length shouldBe LogSanitizer.MaxBodyChars + body should endWith(LogSanitizer.TruncatedMarker) + } + + // ----- security: MDC deny-list ---------------------------------------- + + private def attrsOf(record: io.opentelemetry.sdk.logs.data.LogRecordData): Map[String, String] = + record.getAttributes.asMap.asScala.iterator.map { case (k, v) => k.getKey -> v.toString }.toMap + + it should "forward arbitrary correlation MDC keys (deny-list, not allow-list)" in { + val (_, exporter, appender) = newFixture() + appender.doAppend( + makeEvent( + "msg", + mdc = Map( + "trace_id" -> "abc", + "texera.workflow.id" -> "42", + "some.new.key" -> "kept" + ) + ) + ) + + val attrs = attrsOf(exporter.getFinishedLogRecordItems.asScala.head) + attrs.keySet should contain allOf ("trace_id", "texera.workflow.id", "some.new.key") + attrs("some.new.key") shouldBe "kept" + } + + it should "drop the noisy Pekko bridge MDC keys" in { + val (_, exporter, appender) = newFixture() + val denied = LogSanitizer.DeniedMdcKeys.iterator.map(_ -> "noise").toMap + appender.doAppend(makeEvent("msg", mdc = denied + ("trace_id" -> "abc"))) + + val attrs = attrsOf(exporter.getFinishedLogRecordItems.asScala.head) + attrs.keySet should contain("trace_id") + attrs.keySet should contain noElementsOf LogSanitizer.DeniedMdcKeys + } + + it should "redact a secret-shaped MDC value while keeping its key" in { + val (_, exporter, appender) = newFixture() + appender.doAppend(makeEvent("msg", mdc = Map("authz" -> "Bearer abc123XYZ.foo"))) + + val attrs = attrsOf(exporter.getFinishedLogRecordItems.asScala.head) + attrs.keySet should contain("authz") + attrs("authz") should include("[REDACTED]") + attrs("authz") should not include "abc123XYZ" + } + + it should "redact the value of a credential-named MDC key even when the value looks benign" in { + val (_, exporter, appender) = newFixture() + appender.doAppend(makeEvent("msg", mdc = Map("password" -> "p4ssw0rd", "api_key" -> "plain"))) + + val attrs = attrsOf(exporter.getFinishedLogRecordItems.asScala.head) + attrs("password") shouldBe "[REDACTED]" + attrs("api_key") shouldBe "[REDACTED]" + attrs.values should contain noElementsOf Seq("p4ssw0rd", "plain") + } + + // ----- exception attributes + self-diagnostic guard ------------------- + + it should "set exception.* semantic attributes with the message and trace redacted" in { + val (_, exporter, appender) = newFixture() + val ctx = LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext] + val logger = ctx.getLogger("test.logger").asInstanceOf[Logger] + val boom = new IllegalStateException("connect failed for password=hunter2") + val ev = new LoggingEvent("fqcn", logger, Level.ERROR, "operation failed", boom, null) + appender.doAppend(ev) + + val attrs = attrsOf(exporter.getFinishedLogRecordItems.asScala.head) + attrs("exception.type") shouldBe "java.lang.IllegalStateException" + attrs("exception.message") should include("[REDACTED]") + attrs("exception.message") should not include "hunter2" + attrs.keySet should contain("exception.stacktrace") + attrs("exception.stacktrace") should not include "hunter2" + } + + it should "drop records from io.opentelemetry loggers to avoid a feedback loop" in { + val (_, exporter, appender) = newFixture() + val ctx = LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext] + val otelLogger = + ctx.getLogger("io.opentelemetry.exporter.internal.grpc.GrpcExporter").asInstanceOf[Logger] + val ev = new LoggingEvent("fqcn", otelLogger, Level.WARN, "Failed to export spans", null, null) + appender.doAppend(ev) + + exporter.getFinishedLogRecordItems.asScala shouldBe empty + } + + // ----- lifecycle ------------------------------------------------------ + + it should "be a silent no-op when not yet bound to an OpenTelemetry instance" in { + val unbound = new TexeraOtelLogAppender() + unbound.setContext(LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext]) + unbound.start() + // Should not throw, even though no SDK is wired. + noException should be thrownBy unbound.doAppend(makeEvent("hello")) + } + + it should "stop emitting after stop() is called" in { + val (_, exporter, appender) = newFixture() + appender.doAppend(makeEvent("first")) + appender.stop() + appender.doAppend(makeEvent("second")) + + val bodies = exporter.getFinishedLogRecordItems.asScala.map(_.getBodyValue.asString) + bodies should contain only "first" + } +} diff --git a/computing-unit-managing-service/LICENSE-binary b/computing-unit-managing-service/LICENSE-binary index 0b8e4c1286b..bf057b8c40e 100644 --- a/computing-unit-managing-service/LICENSE-binary +++ b/computing-unit-managing-service/LICENSE-binary @@ -369,6 +369,18 @@ Scala/Java jars: - io.netty.netty-transport-native-epoll-4.2.15.Final.jar - io.netty.netty-transport-native-unix-common-4.2.15.Final.jar - org.jspecify.jspecify-1.0.0.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.perfmark.perfmark-api-0.27.0.jar - io.r2dbc.r2dbc-spi-1.0.0.RELEASE.jar - io.swagger.swagger-annotations-1.6.14.jar diff --git a/computing-unit-managing-service/src/main/resources/computing-unit-managing-service-config.yaml b/computing-unit-managing-service/src/main/resources/computing-unit-managing-service-config.yaml index 523b4197989..ea428fcadcb 100644 --- a/computing-unit-managing-service/src/main/resources/computing-unit-managing-service-config.yaml +++ b/computing-unit-managing-service/src/main/resources/computing-unit-managing-service-config.yaml @@ -30,4 +30,16 @@ server: logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: - "com.example": ${TEXERA_SERVICE_LOG_LEVEL:-DEBUG} \ No newline at end of file + "com.example": ${TEXERA_SERVICE_LOG_LEVEL:-DEBUG} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN \ No newline at end of file diff --git a/computing-unit-managing-service/src/main/scala/org/apache/texera/service/ComputingUnitManagingService.scala b/computing-unit-managing-service/src/main/scala/org/apache/texera/service/ComputingUnitManagingService.scala index 785c719fe3b..0d36a99bcb7 100644 --- a/computing-unit-managing-service/src/main/scala/org/apache/texera/service/ComputingUnitManagingService.scala +++ b/computing-unit-managing-service/src/main/scala/org/apache/texera/service/ComputingUnitManagingService.scala @@ -90,6 +90,8 @@ class ComputingUnitManagingService extends Application[ComputingUnitManagingServ configuration: ComputingUnitManagingServiceConfiguration, environment: Environment ): Unit = { + // Bridge this service's logs to the OTel collector under its own service.name. + org.apache.texera.observability.OtelInit.init("computing-unit-managing-service") // Register http resources environment.jersey.setUrlPattern("/api/*") environment.jersey.register(classOf[HealthCheckResource]) diff --git a/config-service/LICENSE-binary b/config-service/LICENSE-binary index 93348da932e..5f93f2b6780 100644 --- a/config-service/LICENSE-binary +++ b/config-service/LICENSE-binary @@ -242,6 +242,9 @@ Scala/Java jars: - com.google.guava.listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar - com.google.j2objc.j2objc-annotations-2.8.jar - com.helger.profiler-1.1.1.jar + - com.squareup.okhttp3.okhttp-4.12.0.jar + - com.squareup.okio.okio-3.6.0.jar + - com.squareup.okio.okio-jvm-3.6.0.jar - com.thesamet.scalapb.lenses_2.13-0.11.20.jar - com.thesamet.scalapb.scalapb-json4s_2.13-0.12.0.jar - com.thesamet.scalapb.scalapb-runtime_2.13-0.11.20.jar @@ -274,6 +277,18 @@ Scala/Java jars: - io.dropwizard.metrics.metrics-json-4.2.25.jar - io.dropwizard.metrics.metrics-jvm-4.2.25.jar - io.dropwizard.metrics.metrics-logback-4.2.25.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.r2dbc.r2dbc-spi-1.0.0.RELEASE.jar - jakarta.inject.jakarta.inject-api-2.0.1.jar - jakarta.validation.jakarta.validation-api-3.0.2.jar @@ -292,6 +307,11 @@ Scala/Java jars: - org.hibernate.validator.hibernate-validator-7.0.5.Final.jar - org.javassist.javassist-3.30.2-GA.jar - org.jboss.logging.jboss-logging-3.5.3.Final.jar + - org.jetbrains.annotations-13.0.jar + - org.jetbrains.kotlin.kotlin-stdlib-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-common-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-jdk7-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-jdk8-1.9.10.jar - org.jooq.jooq-3.19.36.jar - org.json4s.json4s-ast_2.13-4.0.1.jar - org.json4s.json4s-jackson-core_2.13-4.0.1.jar diff --git a/config-service/src/main/resources/config-service-web-config.yaml b/config-service/src/main/resources/config-service-web-config.yaml index 4aa67af82e1..8559e1fd507 100644 --- a/config-service/src/main/resources/config-service-web-config.yaml +++ b/config-service/src/main/resources/config-service-web-config.yaml @@ -26,6 +26,19 @@ server: logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + loggers: + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console threshold: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} diff --git a/config-service/src/main/scala/org/apache/texera/service/ConfigService.scala b/config-service/src/main/scala/org/apache/texera/service/ConfigService.scala index a7e9b61d994..df7335d21f7 100644 --- a/config-service/src/main/scala/org/apache/texera/service/ConfigService.scala +++ b/config-service/src/main/scala/org/apache/texera/service/ConfigService.scala @@ -53,6 +53,8 @@ class ConfigService extends Application[ConfigServiceConfiguration] with LazyLog } override def run(configuration: ConfigServiceConfiguration, environment: Environment): Unit = { + // Bridge this service's logs to the OTel collector under its own service.name. + org.apache.texera.observability.OtelInit.init("config-service") // Serve backend at /api environment.jersey.setUrlPattern("/api/*") diff --git a/file-service/LICENSE-binary b/file-service/LICENSE-binary index b84e5912d67..681bf5444a9 100644 --- a/file-service/LICENSE-binary +++ b/file-service/LICENSE-binary @@ -334,6 +334,18 @@ Scala/Java jars: - io.netty.netty-transport-native-epoll-4.2.15.Final.jar - io.netty.netty-transport-native-unix-common-4.2.15.Final.jar - org.jspecify.jspecify-1.0.0.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.perfmark.perfmark-api-0.27.0.jar - io.r2dbc.r2dbc-spi-1.0.0.RELEASE.jar - jakarta.inject.jakarta.inject-api-2.0.1.jar diff --git a/file-service/src/main/resources/file-service-web-config.yaml b/file-service/src/main/resources/file-service-web-config.yaml index 41f8d1b1748..db5a7ec6645 100644 --- a/file-service/src/main/resources/file-service-web-config.yaml +++ b/file-service/src/main/resources/file-service-web-config.yaml @@ -28,6 +28,18 @@ logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console - type: file diff --git a/file-service/src/main/scala/org/apache/texera/service/FileService.scala b/file-service/src/main/scala/org/apache/texera/service/FileService.scala index 63cee25a7af..086a45480db 100644 --- a/file-service/src/main/scala/org/apache/texera/service/FileService.scala +++ b/file-service/src/main/scala/org/apache/texera/service/FileService.scala @@ -65,6 +65,8 @@ class FileService extends Application[FileServiceConfiguration] with LazyLogging } override def run(configuration: FileServiceConfiguration, environment: Environment): Unit = { + // Bridge this service's logs to the OTel collector under its own service.name. + org.apache.texera.observability.OtelInit.init("file-service") // Serve backend at /api environment.jersey.setUrlPattern("/api/*") SqlServer.initConnection( diff --git a/notebook-migration-service/LICENSE-binary b/notebook-migration-service/LICENSE-binary index 7b9b8f84b45..d8ae4313541 100644 --- a/notebook-migration-service/LICENSE-binary +++ b/notebook-migration-service/LICENSE-binary @@ -243,8 +243,9 @@ Scala/Java jars: - com.google.j2objc.j2objc-annotations-2.8.jar - com.helger.profiler-1.1.1.jar - com.squareup.okhttp3.logging-interceptor-3.12.12.jar - - com.squareup.okhttp3.okhttp-3.12.12.jar - - com.squareup.okio.okio-1.15.0.jar + - com.squareup.okhttp3.okhttp-4.12.0.jar + - com.squareup.okio.okio-3.6.0.jar + - com.squareup.okio.okio-jvm-3.6.0.jar - com.thesamet.scalapb.lenses_2.13-0.11.20.jar - com.thesamet.scalapb.scalapb-json4s_2.13-0.12.0.jar - com.thesamet.scalapb.scalapb-runtime_2.13-0.11.20.jar @@ -303,6 +304,18 @@ Scala/Java jars: - io.fabric8.kubernetes-model-scheduling-6.12.1.jar - io.fabric8.kubernetes-model-storageclass-6.12.1.jar - io.fabric8.zjsonpatch-0.3.0.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.r2dbc.r2dbc-spi-1.0.0.RELEASE.jar - jakarta.inject.jakarta.inject-api-2.0.1.jar - jakarta.validation.jakarta.validation-api-3.0.2.jar @@ -321,6 +334,11 @@ Scala/Java jars: - org.hibernate.validator.hibernate-validator-7.0.5.Final.jar - org.javassist.javassist-3.30.2-GA.jar - org.jboss.logging.jboss-logging-3.5.3.Final.jar + - org.jetbrains.annotations-13.0.jar + - org.jetbrains.kotlin.kotlin-stdlib-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-common-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-jdk7-1.9.10.jar + - org.jetbrains.kotlin.kotlin-stdlib-jdk8-1.9.10.jar - org.jooq.jooq-3.19.36.jar - org.json4s.json4s-ast_2.13-4.0.1.jar - org.json4s.json4s-jackson-core_2.13-4.0.1.jar diff --git a/notebook-migration-service/src/main/scala/org/apache/texera/service/NotebookMigrationService.scala b/notebook-migration-service/src/main/scala/org/apache/texera/service/NotebookMigrationService.scala index 567cf2b4c69..d9d883b6cca 100644 --- a/notebook-migration-service/src/main/scala/org/apache/texera/service/NotebookMigrationService.scala +++ b/notebook-migration-service/src/main/scala/org/apache/texera/service/NotebookMigrationService.scala @@ -62,6 +62,7 @@ class NotebookMigrationService configuration: NotebookMigrationServiceConfiguration, environment: Environment ): Unit = { + org.apache.texera.observability.OtelInit.init("notebook-migration-service") // Refuse to boot a misconfigured per-user Jupyter rather than failing per request. JupyterTokenDeriver.validateConfiguration() diff --git a/workflow-compiling-service/LICENSE-binary b/workflow-compiling-service/LICENSE-binary index 96c723694ae..90befd23b9a 100644 --- a/workflow-compiling-service/LICENSE-binary +++ b/workflow-compiling-service/LICENSE-binary @@ -336,6 +336,18 @@ Scala/Java jars: - io.netty.netty-transport-native-epoll-4.2.15.Final.jar - io.netty.netty-transport-native-unix-common-4.2.15.Final.jar - org.jspecify.jspecify-1.0.0.jar + - io.opentelemetry.opentelemetry-api-1.50.0.jar + - io.opentelemetry.opentelemetry-context-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar + - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar + - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar - io.perfmark.perfmark-api-0.27.0.jar - io.r2dbc.r2dbc-spi-1.0.0.RELEASE.jar - jakarta.inject.jakarta.inject-api-2.0.1.jar diff --git a/workflow-compiling-service/src/main/resources/workflow-compiling-service-config.yaml b/workflow-compiling-service/src/main/resources/workflow-compiling-service-config.yaml index 5b9016af1b6..37e413c15b6 100644 --- a/workflow-compiling-service/src/main/resources/workflow-compiling-service-config.yaml +++ b/workflow-compiling-service/src/main/resources/workflow-compiling-service-config.yaml @@ -28,6 +28,18 @@ logging: level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO} loggers: "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO} + # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code + # (org.apache.texera) without the framework firehose. + "org.apache.pekko": WARN + "org.apache.iceberg": WARN + "org.apache.hadoop": WARN + "org.apache.kafka": WARN + "org.eclipse.jetty": WARN + "org.glassfish.jersey": WARN + "io.grpc": WARN + "io.netty": WARN + "com.zaxxer.hikari": WARN + "software.amazon.awssdk": WARN appenders: - type: console - type: file diff --git a/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala b/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala index 71aff7800db..368bbecaa71 100644 --- a/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala +++ b/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala @@ -51,6 +51,8 @@ class WorkflowCompilingService extends Application[WorkflowCompilingServiceConfi configuration: WorkflowCompilingServiceConfiguration, environment: Environment ): Unit = { + // Bridge this service's logs to the OTel collector under its own service.name. + org.apache.texera.observability.OtelInit.init("workflow-compiling-service") ObjectMapperUtils.warmupObjectMapperForOperatorsSerde() // serve backend at /api From b372b82feab1f3ccada7aea5328a6adf7f4a5230 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Wed, 7 Oct 2026 04:04:26 -0700 Subject: [PATCH 2/2] feat(observability): backend metrics + tracing primitives Build on the logging foundation (PR 1) with the backend emit path: workflow lifecycle metrics and a run-level setup trace span, plus the tracing/metrics primitives they use. This is PR 2 of the observability stack. Primitives (common/observability): - TexeraTracer: lazy accessor for the process tracer off GlobalOpenTelemetry. - SpanAttrs: typed AttributeKey constants so span attributes use standard keys rather than ad hoc strings. - WorkflowMetrics: the OTel instruments (start/completion/failure/cancellation counters and run-duration histogram) keyed by workflow kind. - TraceparentValidator: validates W3C traceparent headers before use. Emit path (amber): - WorkflowMetricsRecorder: single owner of the metric instruments. init() wires them once; onStart stamps a run's start; onStateChange records terminal counters and duration exactly once on the first transition into a terminal state (idempotent, safe to call on every transition). - WorkflowService.initExecutionService runs inside a run-level setup span so setup-path logs carry its trace id. The span covers only the synchronous setup; synchronous setup failures are recorded on it in the catch block, and the span is ended in finally. The async errorHandler deliberately does not touch the span: it is invoked after setup returns and the span has ended, so the failure is surfaced through the metadata store instead. - ExecutionStateStore.updateWorkflowState is the single chokepoint that feeds every state transition to WorkflowMetricsRecorder.onStateChange. - ComputingUnitMaster initializes the recorder at startup. All observability sources live under common/observability (the module from PR 1); the tracing/metrics classes are not duplicated into common/config. Tests: WorkflowMetricsSpec, SpanAttrsSpec, and TraceparentValidatorSpec. Review follow-ups addressed: span attributes use SpanAttrs keys instead of plain strings; the error handler no longer records onto a span that may have already ended (documented and handled via the metadata store). Stacked on PR 1 (obs/pr1/foundations). --- .../texera/web/ComputingUnitMaster.scala | 1 + .../WorkflowMetricsRecorder.scala | 107 ++++++++ .../texera/web/service/WorkflowService.scala | 238 +++++++++-------- .../web/storage/ExecutionStateStore.scala | 4 + .../texera/observability/SpanAttrs.scala | 62 +++++ .../texera/observability/TexeraTracer.scala | 61 +++++ .../observability/TraceparentValidator.scala | 98 +++++++ .../observability/WorkflowMetrics.scala | 243 ++++++++++++++++++ .../texera/observability/SpanAttrsSpec.scala | Bin 0 -> 4084 bytes .../TraceparentValidatorSpec.scala | Bin 0 -> 5084 bytes .../observability/WorkflowMetricsSpec.scala | 195 ++++++++++++++ 11 files changed, 904 insertions(+), 105 deletions(-) create mode 100644 amber/src/main/scala/org/apache/texera/web/observability/WorkflowMetricsRecorder.scala create mode 100644 common/observability/src/main/scala/org/apache/texera/observability/SpanAttrs.scala create mode 100644 common/observability/src/main/scala/org/apache/texera/observability/TexeraTracer.scala create mode 100644 common/observability/src/main/scala/org/apache/texera/observability/TraceparentValidator.scala create mode 100644 common/observability/src/main/scala/org/apache/texera/observability/WorkflowMetrics.scala create mode 100644 common/observability/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala create mode 100644 common/observability/src/test/scala/org/apache/texera/observability/TraceparentValidatorSpec.scala create mode 100644 common/observability/src/test/scala/org/apache/texera/observability/WorkflowMetricsSpec.scala diff --git a/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala b/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala index b5bdcbf5573..d85f127b740 100644 --- a/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala +++ b/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala @@ -139,6 +139,7 @@ class ComputingUnitMaster extends io.dropwizard.Application[Configuration] with override def run(configuration: Configuration, environment: Environment): Unit = { org.apache.texera.observability.OtelInit.init("computing-unit-master") + org.apache.texera.web.observability.WorkflowMetricsRecorder.init() ObjectMapperUtils.warmupObjectMapperForOperatorsSerde() SqlServer.initConnection( diff --git a/amber/src/main/scala/org/apache/texera/web/observability/WorkflowMetricsRecorder.scala b/amber/src/main/scala/org/apache/texera/web/observability/WorkflowMetricsRecorder.scala new file mode 100644 index 00000000000..0f9ab2401aa --- /dev/null +++ b/amber/src/main/scala/org/apache/texera/web/observability/WorkflowMetricsRecorder.scala @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.web.observability + +import com.typesafe.scalalogging.LazyLogging +import org.apache.texera.amber.core.virtualidentity.ExecutionIdentity +import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAggregatedState +import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAggregatedState.{ + COMPLETED, + FAILED, + KILLED, + PAUSED, + PAUSING, + RESUMING, + RUNNING +} +import org.apache.texera.observability.WorkflowMetrics +import org.apache.texera.observability.WorkflowMetrics.WorkflowKind +import org.apache.texera.web.service.WorkflowService + +import java.util.concurrent.ConcurrentHashMap + +/** + * Drives [[WorkflowMetrics]] from amber's execution lifecycle. The metric + * instruments live in common/config; this object is the single place + * that records them, so the lifecycle code only needs one-line calls. + * + * - start / terminal counters and the duration histogram are recorded + * from [[onStart]] and [[onStateChange]]; + * - the always-polled `texera.workflow.active` gauge is sourced from the + * live WorkflowService registry via the supplier registered in [[init]]. + */ +object WorkflowMetricsRecorder extends LazyLogging { + + // Start time + kind per in-flight run, so a terminal transition can emit + // a duration and attribute the outcome. Keyed by the execution identity. + private val inFlight = new ConcurrentHashMap[ExecutionIdentity, (Long, WorkflowKind)]() + + private val ActiveStates: Set[WorkflowAggregatedState] = Set(RUNNING, PAUSING, PAUSED, RESUMING) + private val TerminalStates: Set[WorkflowAggregatedState] = Set(COMPLETED, FAILED, KILLED) + + /** Register the active-executions gauge supplier and bind the instruments. + * Call once at startup, after OtelInit.init. The supplier is polled on + * every metric collection, so it must never throw. + */ + def init(): Unit = { + WorkflowMetrics.setActiveExecutionsSupplier(() => + try { + WorkflowService.getAllWorkflowServices.iterator + .flatMap(s => Option(s.executionService.getValue)) + .map(_.executionStateStore.metadataStore.getState.state) + .count(ActiveStates.contains) + .toLong + } catch { + case _: Throwable => 0L + } + ) + WorkflowMetrics.ensureBound() + } + + /** Record that a run started. */ + def onStart( + executionId: ExecutionIdentity, + kind: WorkflowKind = WorkflowKind.Interactive + ): Unit = { + inFlight.put(executionId, (System.currentTimeMillis(), kind)) + WorkflowMetrics.recordStart(kind) + } + + /** Record terminal counters + duration exactly once, on the first + * transition from a non-terminal into a terminal state. Safe to call on + * every state change; non-terminal and repeat-terminal calls are no-ops. + */ + def onStateChange( + executionId: ExecutionIdentity, + oldState: WorkflowAggregatedState, + newState: WorkflowAggregatedState + ): Unit = { + if (!TerminalStates.contains(newState) || TerminalStates.contains(oldState)) return + val entry = Option(inFlight.remove(executionId)) + val kind = entry.map(_._2).getOrElse(WorkflowKind.Interactive) + val durationSec = entry.map(e => (System.currentTimeMillis() - e._1) / 1000.0).getOrElse(0.0) + newState match { + case COMPLETED => WorkflowMetrics.recordCompletion(kind, durationSec) + case FAILED => WorkflowMetrics.recordFailure(kind, durationSec) + case KILLED => WorkflowMetrics.recordCancellation(kind) + case _ => () + } + } +} diff --git a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala index 8ec650ba715..1af3e69c258 100644 --- a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala +++ b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala @@ -21,9 +21,11 @@ package org.apache.texera.web.service import com.google.protobuf.timestamp.Timestamp import com.typesafe.scalalogging.LazyLogging +import io.opentelemetry.api.trace.StatusCode import io.reactivex.rxjava3.disposables.{CompositeDisposable, Disposable} import io.reactivex.rxjava3.subjects.BehaviorSubject import org.apache.texera.common.config.{ApplicationConfig, StorageConfig} +import org.apache.texera.observability.{SpanAttrs, TexeraTracer} import org.apache.texera.dao.SqlServer import org.apache.texera.dao.jooq.generated.Tables.USER_WAREHOUSE import org.apache.texera.amber.core.WorkflowRuntimeException @@ -229,126 +231,152 @@ class WorkflowService( userOpt: Option[User], sessionUri: URI ): Unit = { + val span = TexeraTracer.tracer + .spanBuilder("WorkflowService.initExecutionService") + .setAttribute(SpanAttrs.WorkflowId, Long.box(workflowId.id)) + .startSpan() + val scope = span.makeCurrent() + try { - val (uidOpt, userEmailOpt) = userOpt.map(user => (user.getUid, user.getEmail)).unzip + val (uidOpt, userEmailOpt) = userOpt.map(user => (user.getUid, user.getEmail)).unzip - // Validate before touching the execution already in flight: a request that is - // going to be refused must not take the running one's subscriptions with it. - // uid is NOT NULL in the DB; fail early here rather than letting the insert fail downstream. - val uid = uidOpt.getOrElse( - throw new IllegalArgumentException( - "Cannot start execution: a user id (uid) is required but none was provided." + // Validate before touching the execution already in flight: a request that is + // going to be refused must not take the running one's subscriptions with it. + // uid is NOT NULL in the DB; fail early here rather than letting the insert fail downstream. + val uid = uidOpt.getOrElse( + throw new IllegalArgumentException( + "Cannot start execution: a user id (uid) is required but none was provided." + ) ) - ) - val warehouseName = WorkflowService.resolveLakekeeperWarehouseName(req.warehouseId, uid) + val warehouseName = WorkflowService.resolveLakekeeperWarehouseName(req.warehouseId, uid) - if (executionService.hasValue) { - executionService.getValue.unsubscribeAll() - } - - val workflowContext: WorkflowContext = createWorkflowContext() - workflowContext.warehouse = warehouseName - var coordinatorConf = CoordinatorConfig.default - - // clean up results from previous run - val previousExecutionId = - WorkflowExecutionService.getLatestExecutionId(workflowId, req.computingUnitId) - previousExecutionId.foreach(eid => { - clearExecutionResources(eid) - }) // TODO: change this behavior after enabling cache. - - workflowContext.executionId = ExecutionsMetadataPersistService.insertNewExecution( - workflowContext.workflowId, - uid, - req.executionName, - convertToJson(req.engineVersion), - req.computingUnitId, - req.warehouseId - ) + if (executionService.hasValue) { + executionService.getValue.unsubscribeAll() + } - if (ApplicationConfig.faultToleranceLogRootFolder.isDefined) { - val writeLocation = ApplicationConfig.faultToleranceLogRootFolder.get.resolve( - s"${workflowContext.workflowId}/${workflowContext.executionId}/" + val workflowContext: WorkflowContext = createWorkflowContext() + workflowContext.warehouse = warehouseName + var coordinatorConf = CoordinatorConfig.default + + // clean up results from previous run + val previousExecutionId = + WorkflowExecutionService.getLatestExecutionId(workflowId, req.computingUnitId) + previousExecutionId.foreach(eid => { + clearExecutionResources(eid) + }) // TODO: change this behavior after enabling cache. + + workflowContext.executionId = ExecutionsMetadataPersistService.insertNewExecution( + workflowContext.workflowId, + uid, + req.executionName, + convertToJson(req.engineVersion), + req.computingUnitId, + req.warehouseId ) - ExecutionsMetadataPersistService.tryUpdateExistingExecution(workflowContext.executionId) { - execution => execution.setLogLocation(writeLocation.toString) + span.setAttribute(SpanAttrs.ExecutionId, Long.box(workflowContext.executionId.id)) + // A run has started: record the start counter and stamp its start time. + org.apache.texera.web.observability.WorkflowMetricsRecorder + .onStart(workflowContext.executionId) + + if (ApplicationConfig.faultToleranceLogRootFolder.isDefined) { + val writeLocation = ApplicationConfig.faultToleranceLogRootFolder.get.resolve( + s"${workflowContext.workflowId}/${workflowContext.executionId}/" + ) + ExecutionsMetadataPersistService.tryUpdateExistingExecution(workflowContext.executionId) { + execution => execution.setLogLocation(writeLocation.toString) + } + coordinatorConf = coordinatorConf.copy(faultToleranceConfOpt = + Some(FaultToleranceConfig(writeTo = writeLocation)) + ) } - coordinatorConf = coordinatorConf.copy(faultToleranceConfOpt = - Some(FaultToleranceConfig(writeTo = writeLocation)) - ) - } - if (req.replayFromExecution.isDefined) { - val replayInfo = req.replayFromExecution.get - ExecutionsMetadataPersistService - .tryGetExistingExecution(ExecutionIdentity(replayInfo.eid)) - .foreach { execution => - val readLocation = new URI(execution.getLogLocation) - coordinatorConf = coordinatorConf.copy(stateRestoreConfOpt = - Some( - StateRestoreConfig( - readFrom = readLocation, - replayDestination = EmbeddedControlMessageIdentity(replayInfo.interaction) + if (req.replayFromExecution.isDefined) { + val replayInfo = req.replayFromExecution.get + ExecutionsMetadataPersistService + .tryGetExistingExecution(ExecutionIdentity(replayInfo.eid)) + .foreach { execution => + val readLocation = new URI(execution.getLogLocation) + coordinatorConf = coordinatorConf.copy(stateRestoreConfOpt = + Some( + StateRestoreConfig( + readFrom = readLocation, + replayDestination = EmbeddedControlMessageIdentity(replayInfo.interaction) + ) ) ) - ) - } - } + } + } - val executionStateStore = new ExecutionStateStore() - // assign execution id to find the execution from DB in case the constructor fails. - executionStateStore.metadataStore.updateState(state => - state.withExecutionId(workflowContext.executionId) - ) - val errorHandler: Throwable => Unit = { t => - { - val fromActorOpt = t match { - case ex: WorkflowRuntimeException => - ex.relatedWorkerId - case other => - None - } - val (operatorId, workerId) = getOperatorFromActorIdOpt(fromActorOpt) - logger.error("error during execution", t) - executionStateStore.statsStore.updateState(stats => - stats.withEndTimeStamp(System.currentTimeMillis()) - ) - executionStateStore.metadataStore.updateState { metadataStore => - updateWorkflowState(FAILED, metadataStore).addFatalErrors( - WorkflowFatalError( - EXECUTION_FAILURE, - Timestamp(Instant.now), - t.toString, - getStackTraceWithAllCauses(t), - operatorId, - workerId - ) + val executionStateStore = new ExecutionStateStore() + // assign execution id to find the execution from DB in case the constructor fails. + executionStateStore.metadataStore.updateState(state => + state.withExecutionId(workflowContext.executionId) + ) + val errorHandler: Throwable => Unit = { t => + { + val fromActorOpt = t match { + case ex: WorkflowRuntimeException => + ex.relatedWorkerId + case other => + None + } + val (operatorId, workerId) = getOperatorFromActorIdOpt(fromActorOpt) + logger.error("error during execution", t) + // Do NOT touch `span` here: this handler is passed into + // WorkflowExecutionService and invoked asynchronously (runtime, + // websocket, startWorkflow callbacks) after initExecutionService has + // returned and the setup span has already ended, so recording onto it + // would be a silent no-op. The failure is surfaced via the metadata + // store below; setup-span errors are recorded in the catch block. + executionStateStore.statsStore.updateState(stats => + stats.withEndTimeStamp(System.currentTimeMillis()) ) + executionStateStore.metadataStore.updateState { metadataStore => + updateWorkflowState(FAILED, metadataStore).addFatalErrors( + WorkflowFatalError( + EXECUTION_FAILURE, + Timestamp(Instant.now), + t.toString, + getStackTraceWithAllCauses(t), + operatorId, + workerId + ) + ) + } } } - } - // WorkflowExecutionService construction does no external work and cannot - // throw; it registers its error/state diff handler up front. Once published - // via `executionService.onNext`, any failure in `executeWorkflow()` is - // recorded by `errorHandler` into the metadata store, whose handler emits a - // WorkflowErrorEvent that `connectToExecution` forwards. - try { - val execution = new WorkflowExecutionService( - coordinatorConf, - workflowContext, - resultService, - req, - executionStateStore, - errorHandler, - userEmailOpt, - sessionUri - ) - lifeCycleManager.registerCleanUpOnStateChange(executionStateStore) - executionService.onNext(execution) - execution.executeWorkflow() + // WorkflowExecutionService construction does no external work and cannot + // throw; it registers its error/state diff handler up front. Once published + // via `executionService.onNext`, any failure in `executeWorkflow()` is + // recorded by `errorHandler` into the metadata store, whose handler emits a + // WorkflowErrorEvent that `connectToExecution` forwards. + try { + val execution = new WorkflowExecutionService( + coordinatorConf, + workflowContext, + resultService, + req, + executionStateStore, + errorHandler, + userEmailOpt, + sessionUri + ) + lifeCycleManager.registerCleanUpOnStateChange(executionStateStore) + executionService.onNext(execution) + execution.executeWorkflow() + } catch { + case e: Throwable => errorHandler(e) + } + } catch { - case e: Throwable => errorHandler(e) + case t: Throwable => + // Synchronous setup failure (before the run's own errorHandler is wired). + span.recordException(t) + span.setStatus(StatusCode.ERROR) + throw t + } finally { + scope.close() + span.end() } - } def convertToJson(frontendVersion: String): String = { diff --git a/amber/src/main/scala/org/apache/texera/web/storage/ExecutionStateStore.scala b/amber/src/main/scala/org/apache/texera/web/storage/ExecutionStateStore.scala index 654acbbefd1..bc97b39d185 100644 --- a/amber/src/main/scala/org/apache/texera/web/storage/ExecutionStateStore.scala +++ b/amber/src/main/scala/org/apache/texera/web/storage/ExecutionStateStore.scala @@ -27,6 +27,7 @@ import org.apache.texera.amber.engine.common.executionruntimestate.{ ExecutionMetadataStore, ExecutionStatsStore } +import org.apache.texera.web.observability.WorkflowMetricsRecorder import org.apache.texera.web.service.ExecutionsMetadataPersistService import java.sql.Timestamp @@ -44,6 +45,9 @@ object ExecutionStateStore { execution.setStatus(maptoStatusCode(state)) execution.setLastUpdateTime(new Timestamp(System.currentTimeMillis())) } + // Single chokepoint for every state transition: emit lifecycle metrics + // once on the first transition into a terminal state. + WorkflowMetricsRecorder.onStateChange(metadataStore.executionId, metadataStore.state, state) metadataStore.withState(state) } } diff --git a/common/observability/src/main/scala/org/apache/texera/observability/SpanAttrs.scala b/common/observability/src/main/scala/org/apache/texera/observability/SpanAttrs.scala new file mode 100644 index 00000000000..c818f69bc16 --- /dev/null +++ b/common/observability/src/main/scala/org/apache/texera/observability/SpanAttrs.scala @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import io.opentelemetry.api.common.AttributeKey + +/** + * Standard Texera span-attribute keys plus a free-text sanitizer. + * + * These are the shared label keys so every callsite tags spans with the + * same names. Set them with the standard OTel API at the callsite, e.g. + * ``spanBuilder.setAttribute(SpanAttrs.WorkflowId, id)`` or + * ``span.setAttribute(SpanAttrs.WorkflowId, id)``. Run any free-text value + * through [[sanitizeFreeText]] first to strip CRLF and cap its length. + */ +object SpanAttrs { + + /** Maximum length for free-text span attribute values. */ + val FreeTextMaxLen: Int = 256 + + // ---- Standard Texera correlation labels ------------------------------ + + val WorkflowId: AttributeKey[java.lang.Long] = AttributeKey.longKey("texera.workflow.id") + val ExecutionId: AttributeKey[java.lang.Long] = AttributeKey.longKey("texera.execution.id") + val ProjectId: AttributeKey[java.lang.Long] = AttributeKey.longKey("texera.project.id") + val UserId: AttributeKey[java.lang.Long] = AttributeKey.longKey("texera.user.id") + val OperatorId: AttributeKey[String] = AttributeKey.stringKey("texera.operator.id") + val OperatorName: AttributeKey[String] = AttributeKey.stringKey("texera.operator.name") + val Outcome: AttributeKey[String] = AttributeKey.stringKey("texera.outcome") + + // ---- Pure helpers (exposed for testing) ------------------------------ + + /** + * Strip CR/LF and other C0 control characters from a free-text + * value, then cap at [[FreeTextMaxLen]]. Returns null for + * null/empty input (caller skips the setAttribute call). + */ + def sanitizeFreeText(value: String): String = { + if (value == null || value.isEmpty) return null + val stripped = value.filter(c => c >= 0x20 && c != 0x7f) + if (stripped.isEmpty) null + else if (stripped.length <= FreeTextMaxLen) stripped + else stripped.substring(0, FreeTextMaxLen) + } +} diff --git a/common/observability/src/main/scala/org/apache/texera/observability/TexeraTracer.scala b/common/observability/src/main/scala/org/apache/texera/observability/TexeraTracer.scala new file mode 100644 index 00000000000..38e1c36b620 --- /dev/null +++ b/common/observability/src/main/scala/org/apache/texera/observability/TexeraTracer.scala @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import io.opentelemetry.api.GlobalOpenTelemetry +import io.opentelemetry.api.trace.Tracer +import io.opentelemetry.context.Context + +/** + * Accessor for the Texera OTel tracer. + * + * The only thing this adds over calling ``GlobalOpenTelemetry.getTracer`` + * directly is a single instrumentation scope name (``org.apache.texera``), + * so every Texera-produced span shows up under one logical scope in the + * backend, separable from anything emitted by transitive libraries. + * + * Start and end spans with the standard OTel API at the callsite (see the + * OpenTelemetry Java demo for the recommended pattern): + * + * {{{ + * val span = TexeraTracer.tracer.spanBuilder("MyClass.myMethod").startSpan() + * val scope = span.makeCurrent() + * try { ... } catch { + * case t: Throwable => span.recordException(t); span.setStatus(ERROR); throw t + * } finally { scope.close(); span.end() } + * }}} + * + * When the SDK is disabled, ``GlobalOpenTelemetry.getTracer`` returns a + * no-op tracer, so calling this is safe at any time. + */ +object TexeraTracer { + + private val InstrumentationScope = "org.apache.texera" + + def tracer: Tracer = GlobalOpenTelemetry.getTracer(InstrumentationScope) + + /** + * Snapshot the current OTel ``Context`` so async callbacks can + * re-attach it via ``Context.makeCurrent`` later. Useful at the + * Scala to Python boundary where the calling thread is not the + * receiving thread. + */ + def currentContext: Context = Context.current() +} diff --git a/common/observability/src/main/scala/org/apache/texera/observability/TraceparentValidator.scala b/common/observability/src/main/scala/org/apache/texera/observability/TraceparentValidator.scala new file mode 100644 index 00000000000..91868449027 --- /dev/null +++ b/common/observability/src/main/scala/org/apache/texera/observability/TraceparentValidator.scala @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +/** + * Pure validators for W3C Trace Context headers crossing the + * Scala↔Python boundary. The single rule: if the inbound bytes do + * not match the strict regex, we discard the value and start a fresh + * trace. We never echo a rejected value back into a span, log, or + * error message. + * + * Spec reference: https://www.w3.org/TR/trace-context/ + * + * The regexes here intentionally do NOT use any context-sensitive + * grouping or backreferences — keeps the validators safe against + * pathological inputs (ReDoS) and trivially fast. + */ +object TraceparentValidator { + + /** W3C traceparent format: `---`. + * We accept only version `00` (the only published version) with the + * canonical 32-hex / 16-hex / 2-hex layout. All hex lowercase per + * spec — uppercase is invalid. + */ + private val TraceparentPattern = + "^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$".r.pattern + + /** Bounded tracestate length. Spec recommends ≤512 chars. We are + * stricter to remove a small DoS surface — an attacker can't send + * a 1 MiB tracestate to balloon downstream context allocations. + */ + val MaxTracestateLength: Int = 512 + + /** Validate a traceparent header. Returns the input unchanged on + * success, None on any failure (rejected — caller starts a fresh + * trace). Null and empty are silent failures. + */ + def validateTraceparent(header: String): Option[String] = { + if (header == null || header.isEmpty) return None + // Trace-id and parent-id must not be all-zero per spec — an + // all-zero ID is a sentinel for "no value" and MUST be rejected. + if (!TraceparentPattern.matcher(header).matches()) return None + val parts = header.split('-') + val traceId = parts(1) + val spanId = parts(2) + if (isAllZero(traceId) || isAllZero(spanId)) return None + Some(header) + } + + /** Validate a tracestate header. Spec: comma-separated list of + * key=value pairs, ASCII-printable only, total length capped. + * Returns the input unchanged on success, None on rejection. + */ + def validateTracestate(header: String): Option[String] = { + if (header == null || header.isEmpty) return None + if (header.length > MaxTracestateLength) return None + if (!isAsciiPrintable(header)) return None + Some(header) + } + + private def isAllZero(s: String): Boolean = { + var i = 0 + while (i < s.length) { + if (s.charAt(i) != '0') return false + i += 1 + } + true + } + + private def isAsciiPrintable(s: String): Boolean = { + var i = 0 + while (i < s.length) { + val c = s.charAt(i) + // Allow printable ASCII range (0x20-0x7E). Tab and CR/LF are + // rejected — a tracestate must not span lines. + if (c < 0x20 || c > 0x7e) return false + i += 1 + } + true + } +} diff --git a/common/observability/src/main/scala/org/apache/texera/observability/WorkflowMetrics.scala b/common/observability/src/main/scala/org/apache/texera/observability/WorkflowMetrics.scala new file mode 100644 index 00000000000..1035e8a52ec --- /dev/null +++ b/common/observability/src/main/scala/org/apache/texera/observability/WorkflowMetrics.scala @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import com.typesafe.scalalogging.LazyLogging +import io.opentelemetry.api.GlobalOpenTelemetry +import io.opentelemetry.api.common.{AttributeKey, Attributes} +import io.opentelemetry.api.metrics.Meter + +/** + * Strongly-typed facade for the workflow-execution metrics cluster. + * + * This facade pattern is deliberate here because these instruments are a + * small, fixed, correlated set with strict cardinality rules, and it is + * worth centralizing that control in one place. It is NOT the default + * pattern for metrics in general: most call sites should just call the OTel + * meter API directly next to the business logic, e.g. + * + * {{{ + * private val meter = GlobalOpenTelemetry.getMeter("org.apache.texera") + * private val requests = meter.counterBuilder("myfeature.requests").build() + * // in the handler: + * requests.add(1, Attributes.of(AttributeKey.stringKey("route"), route)) + * }}} + * + * Only reach for a facade like this one when the metrics are complex or + * need centralized, standardized control. Do not copy it by default. + * + * Cardinality safety here is enforced by the API surface, not by + * documentation: there is no public method that accepts an arbitrary + * string as a label key or value. The only labels that ever land on + * an instrument are the two enums [[Outcome]] and [[WorkflowKind]], + * each restricted to a fixed set. ``workflow.id`` / ``execution.id`` + * are deliberately NOT metric labels: per-execution detail belongs + * in traces and logs, joined on ``trace_id`` at query time. + * + * Histogram bucket bounds are hard-coded constants so they can't be + * coerced by request input. The OTel SDK applies its own default + * attribute-value-length cap to anything that does slip through. + */ +object WorkflowMetrics extends LazyLogging { + + /** Outcome enum, the only mutable label on lifecycle counters. */ + sealed abstract class Outcome(val name: String) + object Outcome { + case object Success extends Outcome("success") + case object Failure extends Outcome("failure") + case object Cancelled extends Outcome("cancelled") + } + + /** Workflow kind enum. Distinguishes interactive vs. scheduled + * workflows for the dashboard's basic split — extend deliberately. + */ + sealed abstract class WorkflowKind(val name: String) + object WorkflowKind { + case object Interactive extends WorkflowKind("interactive") + case object Scheduled extends WorkflowKind("scheduled") + case object Unknown extends WorkflowKind("unknown") + } + + private val OutcomeKey: AttributeKey[String] = AttributeKey.stringKey("texera.outcome") + private val WorkflowKindKey: AttributeKey[String] = AttributeKey.stringKey("texera.workflow.kind") + + /** Histogram bucket bounds in seconds. Hard-coded — constants so + * request input can't reshape the histogram. Range covers + * fast (<1s) to long (>1h) workflows. + */ + private val DurationBuckets: java.util.List[java.lang.Double] = { + val builder = new java.util.ArrayList[java.lang.Double]() + Seq(0.1, 0.5, 1.0, 5.0, 10.0, 30.0, 60.0, 300.0, 600.0, 1800.0, 3600.0) + .foreach(b => builder.add(java.lang.Double.valueOf(b))) + builder + } + + private val InstrumentationScope = "org.apache.texera" + + // Instruments are lazy + memoised. The first call after SDK init + // builds them against the active GlobalOpenTelemetry meter; once + // built they hold the meter instance, so a later GlobalOpenTelemetry + // reset (in tests) wouldn't be visible here — see [[resetForTest]]. + @volatile private var _starts: io.opentelemetry.api.metrics.LongCounter = _ + @volatile private var _completions: io.opentelemetry.api.metrics.LongCounter = _ + @volatile private var _cancellations: io.opentelemetry.api.metrics.LongCounter = _ + // `texera.workflow.active` is an OBSERVABLE gauge, not a manual up/down + // counter. A manual +1/-1 counter leaks whenever a run starts but its + // terminal event never fires (the controller is killed, the process + // restarts mid-run, or the +1 and -1 are split across the engine and web + // tiers and one side is missed) — the gauge then drifts upward forever and + // the dashboard shows phantom "active" executions. An observable gauge + // instead reports the TRUE in-progress count from the live execution + // registry on every collection, so it cannot leak. The count is supplied + // by the host process via [[setActiveExecutionsSupplier]]. + @volatile private var _active: io.opentelemetry.api.metrics.ObservableLongGauge = _ + @volatile private var _duration: io.opentelemetry.api.metrics.DoubleHistogram = _ + + // Supplier of the current in-progress execution count. Defaults to 0 until + // the host process registers the real source (so a process that never + // registers reports a flat 0 rather than a wrong number). Read by the gauge + // callback on every metric collection. + @volatile private var activeExecutionsSupplier: () => Long = () => 0L + + /** Register the authoritative source of "currently active executions". + * The supplier is polled on every metric collection, so the gauge always + * reflects ground truth and can never leak. Called once at process + * startup (e.g. by ComputingUnitMaster). + */ + def setActiveExecutionsSupplier(supplier: () => Long): Unit = + synchronized { + activeExecutionsSupplier = supplier + ensureBound() + } + + /** Bind instruments to the current global meter. Idempotent — the + * first call wins; later calls are no-ops. Tests can call + * [[bindForTest]] with an explicit Meter, then [[resetForTest]] to + * rebind. + */ + def ensureBound(): Unit = + synchronized { + if (_starts == null) bind(GlobalOpenTelemetry.getMeter(InstrumentationScope)) + } + + private[observability] def bindForTest(meter: Meter): Unit = + synchronized { + bind(meter) + } + + private[observability] def resetForTest(): Unit = + synchronized { + _starts = null + _completions = null + _cancellations = null + // The observable gauge registered a collection callback — close it so the + // previous test's meter provider stops being polled after it's discarded. + if (_active != null) _active.close() + _active = null + _duration = null + } + + private def bind(meter: Meter): Unit = { + _starts = meter + .counterBuilder("texera.workflow.starts") + .setDescription("Number of workflow executions started.") + .build() + // Completions carry texera.outcome={success|failure}. Both success + // and self-terminating failures land here so the success/failure-rate + // queries (non-success ÷ all completions) have a denominator that + // means "runs that finished on their own". User-initiated kills are + // NOT completions — see _cancellations. + _completions = meter + .counterBuilder("texera.workflow.completions") + .setDescription("Number of workflow executions that ran to completion (success or failure).") + .build() + // Cancellations (user kills) are tracked separately so they decrement + // the active gauge without polluting the success/failure-rate + // denominator. Deliberately label-free apart from workflow.kind. + _cancellations = meter + .counterBuilder("texera.workflow.cancellations") + .setDescription("Number of workflow executions cancelled/killed before finishing.") + .build() + // Observable gauge: the callback runs on each collection and reports the + // live in-progress count, so the value cannot leak. No per-execution + // labels (cardinality-safe); the dashboard queries sum(texera_workflow_active). + _active = meter + .gaugeBuilder("texera.workflow.active") + .ofLongs() + .setDescription( + "Number of workflow executions currently in progress (observed from the live registry)." + ) + .buildWithCallback(obs => obs.record(activeExecutionsSupplier())) + _duration = meter + .histogramBuilder("texera.workflow.duration") + .setDescription("End-to-end duration of a workflow execution.") + .setUnit("s") + .setExplicitBucketBoundariesAdvice(DurationBuckets) + .build() + logger.info(s"Texera metric instruments bound to meter scope '$InstrumentationScope'") + } + + // ---- Public emitters — typed, no untyped escape hatch -------------- + + def recordStart(kind: WorkflowKind): Unit = { + ensureBound() + logger.debug(s"metric: workflow started (kind=${kind.name}) — starts +1") + _starts.add(1L, Attributes.of(WorkflowKindKey, kind.name)) + // `active` is no longer mutated here — it is an observable gauge sourced + // from the live execution registry (see setActiveExecutionsSupplier). + } + + def recordCompletion(kind: WorkflowKind, durationSec: Double): Unit = { + ensureBound() + logger.debug( + f"metric: workflow completed successfully (kind=${kind.name}, ${durationSec}%.1fs) — completions +1" + ) + val attrs = Attributes.of(OutcomeKey, Outcome.Success.name, WorkflowKindKey, kind.name) + _completions.add(1L, attrs) + _duration.record(durationSec, attrs) + } + + def recordFailure(kind: WorkflowKind, durationSec: Double): Unit = { + ensureBound() + logger.debug( + f"metric: workflow failed (kind=${kind.name}, ${durationSec}%.1fs) — completions +1 (outcome=failure)" + ) + // A failure is a completion with outcome=failure — it shares the + // completions counter (and duration histogram) with successes so the + // failure-rate query has both numerator and denominator. + val attrs = Attributes.of(OutcomeKey, Outcome.Failure.name, WorkflowKindKey, kind.name) + _completions.add(1L, attrs) + _duration.record(durationSec, attrs) + } + + /** A user-initiated kill/cancel. Bumps a dedicated counter. Deliberately + * NOT recorded as a completion: a cancelled run never finished on its own, + * so it must not drag down the success rate. No duration is recorded for + * the same reason — a killed run's wall-clock time is not a real runtime + * and would skew the duration percentiles. (The active gauge is observed + * from the live registry and needs no decrement here.) + */ + def recordCancellation(kind: WorkflowKind): Unit = { + ensureBound() + logger.debug(s"metric: workflow cancelled (kind=${kind.name}) — cancellations +1") + val attrs = Attributes.of(WorkflowKindKey, kind.name) + _cancellations.add(1L, attrs) + } +} diff --git a/common/observability/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala b/common/observability/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala new file mode 100644 index 0000000000000000000000000000000000000000..5553a36e997809029167db2558a2c36b0c8f6243 GIT binary patch literal 4084 zcmcgvZExE)5MICfPwXHJm_0ajR}9$Dt^?dO#*C)UP`N7x6xe8qwzVU0W5)6TkUT0)h~bIqn@LE7M86CGj0POYTNwOPS*xp#WC4rF&mIltB=?9Dlp zbkaKGYsIZu#Jaat6G|uQ28<#73Pl|9Bk_>>xWFcp=4mo`5~VkCPsRqSJckCiSv#ql zOqBud!m&+RXlb%2-9)Eaz~SR>aDkQBMUc2JV<=>o5vax($pG7Io@gCH`FJ$i?RG-O zED7sYqXuq2x&*MZ^x_OYHzKc1p}YrHe^uH6;l-Y037;~#fTxArasFbOV*263&H-<; z8nQv(mOwX2b@myu)~(lkxz**Kl1LwYlv|9tg&PH)DqUX5q-=|xIc zuju@0_Ix^@Ud`}pLgU$QEcs$Odp;zEazLS}o6>O*@KiIND(_TN>jT$HlOeIlZ1*HKfzwg12duAI5zuad59&raDeS^4 zx0MTB=&ML}a$;&;Dm2N88`pAHAo;(@Z$GgFDoRf zfxQ}s;5^$xjRa+KPaXR;o3#4rU9U$p8qpd5k(UN#{PV=Li4vzmXrt%$a0lH_-+u)3FtgUrPC%vIzvk`?Pf(4$AU-Ks?QB#vp=&XZNH_y zEXuVU+;ZaN;XVFjvIJ98TTn0M&80FO1h<%Ag#`sYrQ{|7*YuBxzb%Ak-wh&MEMuzx zk>lNyB4MTVbzBvNGAj(xx)M<{N5R`h?unpVwY#-46zim9+QjInfMpy@v3_}1h~?ag z@M!A9$Art{iw9j$xwcy;j%^u0%9!e)?u9DoTJ1fvcurd_qw%ufIe7&drSUJ*yZsv5 zk|&ct)pP7luL9<*bKsU@53SBtD_%QPt#?uBg7+Kf?*zxG=(l7%q!2lEfXo=Rs|mJ$ z%$Z-{L48GdwD+@GWk2DyW?9%BXkBGhK&i((`-Om0%x1hmsZM3QX#}SEkp7fgDR`!d zOQc-Dqe@*Z``^4ANmjNYft(kT!5T6uV?X2X-ycVx_`F8|~LBXNhT08!x0!dGd(0k~!Zv0EZcl zzo>-I@5Ql0k3jB&0v*IU65oGVpo>jMB4H$sD8SxbBI8)GKcZxLB}2)R0lXf|7a_{R zYfM=5GjHNpxtq?|T5NDtA>F=HqsIn3mJQ?Z0y+Mw9|pHO`Odro(__YmC=SW5 zJ+Eh zrv81K?(lS_mZcVvV@1usOvd#_#aV9|1c&$O6B!@`(;hiHl(>4jEpulOiDoD?_i=#t zQf{}yUfjQDS8k28zBClrs_PiX>jA9aeL*-{ZP!8(njhfDIf;R5kTA+`H N_o1fW_r3SMe*wJvRcHVJ literal 0 HcmV?d00001 diff --git a/common/observability/src/test/scala/org/apache/texera/observability/TraceparentValidatorSpec.scala b/common/observability/src/test/scala/org/apache/texera/observability/TraceparentValidatorSpec.scala new file mode 100644 index 0000000000000000000000000000000000000000..8c4bd670fc1994160c1bb916f1443f08d9d3f20c GIT binary patch literal 5084 zcmcIo+j84B5bd+RVuw$1V^WgkI+uc|;OxpndalAweJ0$2dDtYq?Sedv#M z7bG3pk`l>vJs#VnfZcO;&n^~j{oH_`;X))_S)PK^;AR|pIZN=@SZ8j@42OYUsFXRO z6*PO}!2#eSH=q@#-P*uh?fiW>Iy(f8azLT+o7@lyL@FqrJgupz!a!Onvt)Cg zh)g6%PfZtW%3-RnxltsQoSV6@lvs-(QxwNsI4>zne%H7~__oz(V5~opA&hnvc*k$J zVUdn4H`go{Qn*#4fdI8Z9pXBhFv%RZF3RvXfr@%+HNd~e`1j_OsBSDhVP^DCZolp~pbGhdh5*nK=18Y- z3PHOawpsS{R?4Ha^h?oK5m}P=b9)y>8BncWCA-u@mk-70Xb3JUwj#=5zX}=-ZEtesi7`cFg_r zT#JO^fAYkET1e@uhtG4j!ZhX!SBuBaNgzA5B#iutf^yi|KoG2Nzn{GlGf6vihG8SH zOorC2u>aXy>x|k0bB?&!hp-5usDvNwjM0x2bD!}3{eLe;Z(hGX zzv}nK=P=_pm>@cE%ZbW@)Jtv>X4g`*$Z^=HPPE>jJkk1R$KA6tI?>Kw;)&MxCtCjz za^&6ZoFqJxLn4#R7z)o}BcRi`65&0up`=S5??3$Uf1k|gY-bj}jfIk{Dc&0_pZcpt z4Lo+TiV|H{JX<=}4NlZ$r}9rFyC2DJ>)3&QDzV?Xk=S8xz0gA?g?+s|V7l?7)}y#b z7bC#y2DUUYK96~t;_ae;buobEj5E3)3Pr;Lo-g-%S7Ib~^x8T1kQ3sg`KaE9101yZ z3s`Zx=LAIF5;i$}-)Rzc=1=wnp9$zxW0-vL(TH2fkW?{rCha{5^cqEA|H)W~wb%zl9K6vznq1 z-Hc&t!>K1>c?~-qqjWE{;xnh+oiY) zGL_R>3d+%U3ngmSy>WjygjVry7A}Pp`(BPE(Ej=XLm}a zG7E!LnvZPqr)~D`N36Vd)V^TIKX)g_VSGA1!8ZKV?GXC257m~4G?f?J) literal 0 HcmV?d00001 diff --git a/common/observability/src/test/scala/org/apache/texera/observability/WorkflowMetricsSpec.scala b/common/observability/src/test/scala/org/apache/texera/observability/WorkflowMetricsSpec.scala new file mode 100644 index 00000000000..17c9c4c74dd --- /dev/null +++ b/common/observability/src/test/scala/org/apache/texera/observability/WorkflowMetricsSpec.scala @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import io.opentelemetry.sdk.metrics.SdkMeterProvider +import io.opentelemetry.sdk.metrics.data.MetricData +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader +import org.scalatest.BeforeAndAfterEach +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import scala.jdk.CollectionConverters._ + +class WorkflowMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { + + private var reader: InMemoryMetricReader = _ + private var provider: SdkMeterProvider = _ + + override def beforeEach(): Unit = { + reader = InMemoryMetricReader.create() + provider = SdkMeterProvider.builder().registerMetricReader(reader).build() + WorkflowMetrics.resetForTest() + WorkflowMetrics.bindForTest(provider.get("org.apache.texera")) + } + + override def afterEach(): Unit = { + WorkflowMetrics.resetForTest() + provider.close() + } + + private def collectAll(): Map[String, MetricData] = { + reader.collectAllMetrics().asScala.map(m => m.getName -> m).toMap + } + + // ----- positive: lifecycle emissions ---------------------------------- + + "WorkflowMetrics" should "increment workflow.starts on recordStart" in { + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + + val metrics = collectAll() + metrics.keySet should contain("texera.workflow.starts") + + val starts = metrics("texera.workflow.starts").getLongSumData.getPoints.asScala.head + starts.getValue shouldBe 1L + // recordStart no longer mutates `active` — that gauge is observed from the + // live registry (see "report the active-execution count…" below). + } + + it should "report the active-execution count from the registered supplier, never a manual counter" in { + // The whole point of the fix: `active` is an observable gauge sourced from + // ground truth, so it cannot leak. Drive it with a stub supplier and + // confirm the gauge reports exactly what the supplier returns — regardless + // of how many starts/completions were recorded. + @volatile var live = 3L + WorkflowMetrics.setActiveExecutionsSupplier(() => live) + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + + val first = collectAll()("texera.workflow.active").getLongGaugeData.getPoints.asScala.head + first.getValue shouldBe 3L + + // It tracks the live source on the next collection — a manual +1/-1 + // counter could never drop like this without an explicit decrement. + live = 0L + val second = collectAll()("texera.workflow.active").getLongGaugeData.getPoints.asScala.head + second.getValue shouldBe 0L + } + + it should "record a completion and duration sample on recordCompletion" in { + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + WorkflowMetrics.recordCompletion(WorkflowMetrics.WorkflowKind.Interactive, durationSec = 2.5) + + val metrics = collectAll() + metrics.keySet should contain allOf ( + "texera.workflow.completions", + "texera.workflow.duration" + ) + + metrics( + "texera.workflow.completions" + ).getLongSumData.getPoints.asScala.head.getValue shouldBe 1L + + val histogram = metrics("texera.workflow.duration").getHistogramData.getPoints.asScala.head + histogram.getCount shouldBe 1L + histogram.getSum shouldBe 2.5 + } + + it should "record a failure as a non-success completion (so failure-rate queries work)" in { + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Scheduled) + WorkflowMetrics.recordFailure(WorkflowMetrics.WorkflowKind.Scheduled, durationSec = 12.0) + + val metrics = collectAll() + // A failure shares the completions counter with successes — the + // failure-rate query divides non-success completions by all + // completions, so failures must live here (not in a separate + // series the query never reads). + val completion = metrics("texera.workflow.completions").getLongSumData.getPoints.asScala.head + completion.getValue shouldBe 1L + completion.getAttributes.asMap.asScala.map { + case (k, v) => k.getKey -> v.toString + } should contain( + "texera.outcome" -> "failure" + ) + metrics("texera.workflow.duration").getHistogramData.getPoints.asScala.head.getSum shouldBe 12.0 + // The orphan counter the old wiring used must be gone. + metrics.keySet should not contain "texera.workflow.failures" + } + + it should "record a cancellation that is not a completion" in { + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + WorkflowMetrics.recordCancellation(WorkflowMetrics.WorkflowKind.Interactive) + + val metrics = collectAll() + metrics( + "texera.workflow.cancellations" + ).getLongSumData.getPoints.asScala.head.getValue shouldBe 1L + // A kill is not a completion and records no duration: it must not + // drag down the success rate nor skew the duration percentiles. + metrics.keySet should not contain "texera.workflow.completions" + metrics.keySet should not contain "texera.workflow.duration" + } + + // ----- security: cardinality safety ----------------------------------- + + it should "only emit the texera.outcome and texera.workflow.kind labels" in { + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + WorkflowMetrics.recordCompletion(WorkflowMetrics.WorkflowKind.Interactive, durationSec = 1.0) + + val attrKeys = collectAll().values.flatMap { md => + val pointSet = md.getType.name() match { + case "HISTOGRAM" => md.getHistogramData.getPoints.asScala + case "LONG_GAUGE" => md.getLongGaugeData.getPoints.asScala + case _ => md.getLongSumData.getPoints.asScala + } + pointSet.flatMap(_.getAttributes.asMap.keySet.asScala.map(_.getKey)) + }.toSet + + attrKeys.foreach { key => + Set("texera.outcome", "texera.workflow.kind") should contain(key) + } + attrKeys should not contain "texera.workflow.id" + attrKeys should not contain "texera.execution.id" + } + + it should "expose no public API to attach an arbitrary string label" in { + // The class has only typed emitters whose attribute set is + // hard-coded. This test is intentionally a compile-time check + // disguised as a runtime one — if a future contributor adds an + // untyped public method like recordStart(attrs: Attributes), + // this assertion still passes but the design intent is broken. + // Make the intent explicit: + val methodNames = classOf[WorkflowMetrics.type].getDeclaredMethods + .map(_.getName) + .toSet + methodNames should contain allOf ("recordStart", "recordCompletion", "recordFailure") + methodNames should not contain "recordWithAttributes" + } + + // ----- histogram buckets are constants -------------------------------- + + it should "use the hard-coded explicit bucket boundaries for duration" in { + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + // Hit a few bucket bounds. + Seq(0.05, 0.6, 7.0, 65.0, 400.0).foreach { d => + WorkflowMetrics.recordCompletion(WorkflowMetrics.WorkflowKind.Interactive, durationSec = d) + } + + val histogram = collectAll()("texera.workflow.duration").getHistogramData + val point = histogram.getPoints.asScala.head + point.getCount shouldBe 5L + // The point exposes the SDK-configured boundaries; we don't + // assert exact values here (would couple the test to the impl) + // but we do assert there ARE explicit boundaries — anything + // empty would mean the .setExplicitBucketBoundariesAdvice call + // was lost during a refactor. + point.getBoundaries.size should be > 0 + } +}