diff --git a/java-spanner/google-cloud-spanner/pom.xml b/java-spanner/google-cloud-spanner/pom.xml index 683f5548e242..6063bd61297f 100644 --- a/java-spanner/google-cloud-spanner/pom.xml +++ b/java-spanner/google-cloud-spanner/pom.xml @@ -518,7 +518,20 @@ io.opentelemetry opentelemetry-sdk-trace - test + + + io.opentelemetry + opentelemetry-exporter-otlp + + + io.opentelemetry + opentelemetry-exporter-sender-okhttp + + + + + io.opentelemetry + opentelemetry-exporter-sender-grpc-managed-channel io.opentelemetry diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInTracingProvider.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInTracingProvider.java new file mode 100644 index 000000000000..8a6233a8a3b4 --- /dev/null +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInTracingProvider.java @@ -0,0 +1,203 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner; + +import com.google.auth.Credentials; +import com.google.auth.oauth2.GoogleCredentials; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; +import io.opentelemetry.sdk.trace.export.SpanExporter; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import java.io.IOException; +import java.net.URI; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.jspecify.annotations.Nullable; + +/** + * Creates and caches the OpenTelemetry instances that are used for built-in tracing. One instance + * is created per project and sampling rate, so that spans are always exported to the project of the + * client that created them. + */ +final class BuiltInTracingProvider { + + static final BuiltInTracingProvider INSTANCE = new BuiltInTracingProvider(); + + private static final Logger logger = Logger.getLogger(BuiltInTracingProvider.class.getName()); + + private static final String TELEMETRY_API_ENDPOINT = "https://telemetry.googleapis.com"; + private static final String CLOUD_PLATFORM_SCOPE = + "https://www.googleapis.com/auth/cloud-platform"; + private static final AttributeKey GCP_PROJECT_ID_KEY = + AttributeKey.stringKey("gcp.project_id"); + private static final AttributeKey SERVICE_NAME_KEY = + AttributeKey.stringKey("service.name"); + private static final String OTEL_SERVICE_NAME = "OTEL_SERVICE_NAME"; + + private final Map, OpenTelemetrySdk> openTelemetries = new HashMap<>(); + private BiFunction spanExporterFactory = + BuiltInTracingProvider::createTelemetryApiExporter; + private Thread shutdownHook; + private boolean openCensusWarningLogged; + private boolean customTracingInfoLogged; + + private BuiltInTracingProvider() {} + + @Nullable + synchronized OpenTelemetry getOrCreateOpenTelemetry( + String projectId, Credentials credentials, double samplingRate) { + List key = ImmutableList.of(projectId, samplingRate); + OpenTelemetrySdk openTelemetry = openTelemetries.get(key); + if (openTelemetry == null) { + try { + openTelemetry = + createOpenTelemetry( + projectId, spanExporterFactory.apply(projectId, credentials), samplingRate); + } catch (Throwable throwable) { + logger.log( + Level.WARNING, + "Unable to create the exporter for built-in tracing, built-in tracing is disabled", + throwable); + return null; + } + openTelemetries.put(key, openTelemetry); + if (shutdownHook == null) { + shutdownHook = new Thread(this::shutdown); + Runtime.getRuntime().addShutdownHook(shutdownHook); + } + } + return openTelemetry; + } + + synchronized void logOpenCensusWarningOnce() { + if (!openCensusWarningLogged) { + openCensusWarningLogged = true; + logger.log( + Level.WARNING, + "Built-in tracing is enabled, but this application uses OpenCensus tracing. Built-in" + + " tracing requires OpenTelemetry and is not used. Call" + + " SpannerOptions.enableOpenTelemetryTraces() before creating any Spanner client to" + + " use built-in tracing."); + } + } + + synchronized void logCustomTracingInfoOnce(boolean setInOptions) { + if (!customTracingInfoLogged) { + customTracingInfoLogged = true; + logger.log( + Level.INFO, + setInOptions + ? "Built-in tracing is enabled, but an OpenTelemetry instance was set with" + + " SpannerOptions.Builder#setOpenTelemetry. Spanner uses that instance instead" + + " of built-in tracing." + : "Built-in tracing is enabled, but a global OpenTelemetry instance with tracing is" + + " registered, for example by the OpenTelemetry Java agent. Spanner uses that" + + " instance instead of built-in tracing."); + } + } + + @VisibleForTesting + synchronized void setSpanExporterFactory( + BiFunction spanExporterFactory) { + this.spanExporterFactory = spanExporterFactory; + } + + @VisibleForTesting + synchronized void reset() { + shutdown(); + openTelemetries.clear(); + spanExporterFactory = BuiltInTracingProvider::createTelemetryApiExporter; + openCensusWarningLogged = false; + customTracingInfoLogged = false; + } + + private synchronized void shutdown() { + for (OpenTelemetrySdk openTelemetry : openTelemetries.values()) { + openTelemetry.close(); + } + } + + private static OpenTelemetrySdk createOpenTelemetry( + String projectId, SpanExporter spanExporter, double samplingRate) { + SdkTracerProvider tracerProvider = + SdkTracerProvider.builder() + .setResource(createResource(projectId, System.getenv(OTEL_SERVICE_NAME))) + .setSampler(Sampler.parentBased(Sampler.traceIdRatioBased(samplingRate))) + .addSpanProcessor(BatchSpanProcessor.builder(spanExporter).build()) + .build(); + return OpenTelemetrySdk.builder() + .setTracerProvider(tracerProvider) + .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance())) + .build(); + } + + @VisibleForTesting + static Resource createResource(String projectId, @Nullable String serviceName) { + AttributesBuilder attributes = Attributes.builder().put(GCP_PROJECT_ID_KEY, projectId); + if (!Strings.isNullOrEmpty(serviceName)) { + attributes.put(SERVICE_NAME_KEY, serviceName); + } + return Resource.getDefault().merge(Resource.create(attributes.build())); + } + + private static SpanExporter createTelemetryApiExporter( + String projectId, Credentials credentials) { + Credentials scopedCredentials = + credentials instanceof GoogleCredentials + && ((GoogleCredentials) credentials).createScopedRequired() + ? ((GoogleCredentials) credentials).createScoped(CLOUD_PLATFORM_SCOPE) + : credentials; + return OtlpGrpcSpanExporter.builder() + .setEndpoint(TELEMETRY_API_ENDPOINT) + .setHeaders(() -> getRequestHeaders(scopedCredentials)) + .build(); + } + + @VisibleForTesting + static Map getRequestHeaders(Credentials credentials) { + try { + Map headers = new HashMap<>(); + for (Map.Entry> header : + credentials.getRequestMetadata(URI.create(TELEMETRY_API_ENDPOINT)).entrySet()) { + if (!header.getValue().isEmpty()) { + headers.put(header.getKey(), header.getValue().get(0)); + } + } + return headers; + } catch (IOException exception) { + logger.log(Level.WARNING, "Unable to get credentials for built-in tracing", exception); + return Collections.emptyMap(); + } + } +} diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerImpl.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerImpl.java index 755b1acc8290..582cac58fe2e 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerImpl.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerImpl.java @@ -61,7 +61,7 @@ class SpannerImpl extends BaseService implements Spanner { new TraceWrapper( Tracing.getTracer(), getOptions() - .getOpenTelemetry() + .getTracingOpenTelemetry() .getTracer( MetricRegistryConstants.INSTRUMENTATION_SCOPE, GaxProperties.getLibraryVersion(this.getOptions().getClass())), @@ -174,6 +174,11 @@ private void logSpannerOptions(SpannerOptions options) { + options.isEnableExtendedTracing() + "\nEnd to end tracing enabled: " + options.isEndToEndTracingEnabled() + + "\nBuilt-in tracing enabled: " + + options.isEnableBuiltInTracing() + + (options.isEnableBuiltInTracing() + ? "\nBuilt-in tracing sampling rate: " + options.getBuiltInTracingSamplingRate() + : "") + "\nBuilt-in metrics enabled: " + options.isEnableBuiltInMetrics()); if (options.getSessionPoolOptions() != null) { diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java index b33065b65c84..36f281de6a45 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java @@ -92,6 +92,7 @@ import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.TracerProvider; import io.opentelemetry.sdk.OpenTelemetrySdk; import java.io.File; import java.io.IOException; @@ -355,6 +356,8 @@ static GcpChannelPoolOptions mergeWithDefaultChannelPoolOptions( private final boolean enableLocationApi; private final boolean enableExtendedTracing; private final boolean enableEndToEndTracing; + private final boolean enableBuiltInTracing; + private final double builtInTracingSamplingRate; private final String monitoringHost; private final TransactionOptions defaultTransactionOptions; private final RequestOptions.ClientContext clientContext; @@ -385,6 +388,8 @@ public enum InstanceType { private static volatile TracingFramework activeTracingFramework; + static final double DEFAULT_BUILT_IN_TRACING_SAMPLING_RATE = 0.1; + /** Interface that can be used to provide {@link CallCredentials} to {@link SpannerOptions}. */ public interface CallCredentialsProvider { /** Return the {@link CallCredentials} to use for a gRPC call. */ @@ -1088,6 +1093,15 @@ protected SpannerOptions(Builder builder) { enableLocationApi = builder.enableLocationApi; } enableEndToEndTracing = builder.enableEndToEndTracing; + enableBuiltInTracing = builder.enableBuiltInTracing; + builtInTracingSamplingRate = builder.builtInTracingSamplingRate; + if (enableBuiltInTracing) { + synchronized (lock) { + if (activeTracingFramework == null) { + activeTracingFramework = TracingFramework.OPEN_TELEMETRY; + } + } + } monitoringHost = builder.monitoringHost; defaultTransactionOptions = builder.defaultTransactionOptions; clientContext = builder.clientContext; @@ -1173,6 +1187,14 @@ default boolean isEnableEndToEndTracing() { return false; } + default boolean isEnableBuiltInTracing() { + return false; + } + + default double getBuiltInTracingSamplingRate() { + return DEFAULT_BUILT_IN_TRACING_SAMPLING_RATE; + } + default boolean isEnableLocationApi() { return false; } @@ -1219,6 +1241,8 @@ private static class SpannerEnvironmentImpl implements SpannerEnvironment { "GOOGLE_SPANNER_EXPERIMENTAL_GCP_FALLBACK_RECOVERY"; private static final String SPANNER_ENABLE_END_TO_END_TRACING = "SPANNER_ENABLE_END_TO_END_TRACING"; + private static final String SPANNER_ENABLE_TRACING = "SPANNER_ENABLE_TRACING"; + private static final String OTEL_TRACES_SAMPLER_ARG = "OTEL_TRACES_SAMPLER_ARG"; private static final String SPANNER_DISABLE_BUILTIN_METRICS = "SPANNER_DISABLE_BUILTIN_METRICS"; private static final String SPANNER_DISABLE_DIRECT_ACCESS_GRPC_BUILTIN_METRICS = "SPANNER_DISABLE_DIRECT_ACCESS_GRPC_BUILTIN_METRICS"; @@ -1299,6 +1323,27 @@ public boolean isEnableEndToEndTracing() { return Boolean.parseBoolean(System.getenv(SPANNER_ENABLE_END_TO_END_TRACING)); } + @Override + public boolean isEnableBuiltInTracing() { + return Boolean.parseBoolean(System.getenv(SPANNER_ENABLE_TRACING)); + } + + @Override + public double getBuiltInTracingSamplingRate() { + String samplingRate = System.getenv(OTEL_TRACES_SAMPLER_ARG); + if (samplingRate != null) { + try { + double value = Double.parseDouble(samplingRate); + if (value >= 0.0 && value <= 1.0) { + return value; + } + } catch (NumberFormatException exception) { + return DEFAULT_BUILT_IN_TRACING_SAMPLING_RATE; + } + } + return DEFAULT_BUILT_IN_TRACING_SAMPLING_RATE; + } + @Override public boolean isEnableLocationApi() { return Boolean.parseBoolean(System.getenv(EXPERIMENTAL_LOCATION_API_ENV_VAR)); @@ -1461,6 +1506,9 @@ private static Builder prepareBuilder(Builder builder) { private boolean enableApiTracing = SpannerOptions.environment.isEnableApiTracing(); private boolean enableExtendedTracing = SpannerOptions.environment.isEnableExtendedTracing(); private boolean enableEndToEndTracing = SpannerOptions.environment.isEnableEndToEndTracing(); + private boolean enableBuiltInTracing = SpannerOptions.environment.isEnableBuiltInTracing(); + private double builtInTracingSamplingRate = + SpannerOptions.environment.getBuiltInTracingSamplingRate(); private boolean enableBuiltInMetrics = SpannerOptions.environment.isEnableBuiltInMetrics(); private MetricsProvider metricsProvider = DefaultMetricsProvider.INSTANCE; private boolean enableLocationApi = SpannerOptions.environment.isEnableLocationApi(); @@ -1582,6 +1630,8 @@ protected Builder() { this.metricsProvider = options.metricsProvider; this.enableLocationApi = options.enableLocationApi; this.enableEndToEndTracing = options.enableEndToEndTracing; + this.enableBuiltInTracing = options.enableBuiltInTracing; + this.builtInTracingSamplingRate = options.builtInTracingSamplingRate; this.monitoringHost = options.monitoringHost; this.defaultTransactionOptions = options.defaultTransactionOptions; this.clientContext = options.clientContext; @@ -2500,6 +2550,32 @@ public Builder setEnableEndToEndTracing(boolean enableEndToEndTracing) { return this; } + /** + * Sets whether to enable built-in tracing. When enabled, the client exports traces to Cloud + * Trace in the project of the client and enables end to end tracing, without the need to set up + * OpenTelemetry. If OpenTelemetry tracing has already been configured for the client or the + * application, then that configuration is used instead, and only end to end tracing is enabled. + * Built-in tracing is disabled by default, and can also be enabled by setting the environment + * variable SPANNER_ENABLE_TRACING to true. Cloud Trace charges for the spans that are exported. + */ + public Builder setEnableBuiltInTracing(boolean enableBuiltInTracing) { + this.enableBuiltInTracing = enableBuiltInTracing; + return this; + } + + /** + * Sets the fraction of requests that are traced by built-in tracing. The value must be between + * 0.0 and 1.0. The default is the value of the environment variable OTEL_TRACES_SAMPLER_ARG if + * that is set, and otherwise 0.1. + */ + public Builder setBuiltInTracingSamplingRate(double builtInTracingSamplingRate) { + Preconditions.checkArgument( + builtInTracingSamplingRate >= 0.0 && builtInTracingSamplingRate <= 1.0, + "The sampling rate must be between 0.0 and 1.0"); + this.builtInTracingSamplingRate = builtInTracingSamplingRate; + return this; + } + /** * Provides the default read-write transaction options for all databases. These defaults are * overridden by any explicit {@link com.google.cloud.spanner.Options.TransactionOption} @@ -2615,7 +2691,8 @@ public SpannerOptions build() { synchronized (lock) { if (activeTracingFramework == null) { - activeTracingFramework = TracingFramework.OPEN_CENSUS; + activeTracingFramework = + enableBuiltInTracing ? TracingFramework.OPEN_TELEMETRY : TracingFramework.OPEN_CENSUS; } } return new SpannerOptions(this); @@ -2964,6 +3041,50 @@ public OpenTelemetry getOpenTelemetry() { } } + /** + * Returns the OpenTelemetry instance that is used for tracing. This is the built-in tracing + * instance if built-in tracing is enabled and no OpenTelemetry tracing has been configured for + * the client or the application, and otherwise the same as {@link #getOpenTelemetry()}. + */ + @InternalApi + public OpenTelemetry getTracingOpenTelemetry() { + if (!enableBuiltInTracing) { + return getOpenTelemetry(); + } + if (activeTracingFramework != TracingFramework.OPEN_TELEMETRY) { + BuiltInTracingProvider.INSTANCE.logOpenCensusWarningOnce(); + return getOpenTelemetry(); + } + OpenTelemetry openTelemetry = getOpenTelemetry(); + if (isTracingEnabled(openTelemetry)) { + BuiltInTracingProvider.INSTANCE.logCustomTracingInfoOnce(this.openTelemetry != null); + return openTelemetry; + } + if (usesNoCredentials() + || emulatorHost != null + || instanceType == InstanceType.OMNI + || !GOOGLE_DEFAULT_UNIVERSE.equals(getResolvedUniverseDomain())) { + return openTelemetry; + } + OpenTelemetry builtInOpenTelemetry = + BuiltInTracingProvider.INSTANCE.getOrCreateOpenTelemetry( + getProjectId(), getCredentials(), builtInTracingSamplingRate); + return builtInOpenTelemetry == null ? openTelemetry : builtInOpenTelemetry; + } + + private boolean isTracingEnabled(OpenTelemetry openTelemetry) { + try { + return openTelemetry + .getTracer( + MetricRegistryConstants.INSTRUMENTATION_SCOPE, + GaxProperties.getLibraryVersion(getClass())) + .isEnabled(); + } catch (Throwable throwable) { + return this.openTelemetry != null + || openTelemetry.getTracerProvider() != TracerProvider.noop(); + } + } + @Override public ApiTracerFactory getApiTracerFactory() { return createApiTracerFactory(false, false); @@ -3073,7 +3194,7 @@ private ApiTracerFactory getDefaultApiTracerFactory() { if (isEnableApiTracing()) { if (activeTracingFramework == TracingFramework.OPEN_TELEMETRY) { return new OpenTelemetryApiTracerFactory( - getOpenTelemetry() + getTracingOpenTelemetry() .getTracer( MetricRegistryConstants.INSTRUMENTATION_SCOPE, GaxProperties.getLibraryVersion(getClass())), @@ -3218,10 +3339,22 @@ public boolean isEnableExtendedTracing() { /** * Returns whether end to end tracing is enabled. If this option is enabled then trace spans will - * be created at the Spanner layer. + * be created at the Spanner layer. End to end tracing is also enabled when built-in tracing is + * enabled and OpenTelemetry tracing is used. */ public boolean isEndToEndTracingEnabled() { - return enableEndToEndTracing; + return enableEndToEndTracing + || (enableBuiltInTracing && activeTracingFramework == TracingFramework.OPEN_TELEMETRY); + } + + /** Returns whether built-in tracing is enabled. */ + public boolean isEnableBuiltInTracing() { + return enableBuiltInTracing; + } + + /** Returns the fraction of requests that are traced by built-in tracing. */ + public double getBuiltInTracingSamplingRate() { + return builtInTracingSamplingRate; } /** Returns the default query options to use for the specific database. */ diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionImpl.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionImpl.java index 2f9a5de2ea2a..536788822eb9 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionImpl.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionImpl.java @@ -323,7 +323,7 @@ static UnitOfWorkType of(TransactionMode transactionMode) { this.tracer = spanner .getOptions() - .getOpenTelemetry() + .getTracingOpenTelemetry() .getTracer( INSTRUMENTATION_SCOPE, GaxProperties.getLibraryVersion(spanner.getOptions().getClass())); diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java index a6d5e3a1caf1..8288c19cf3c2 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java @@ -978,7 +978,7 @@ private InstantiatingGrpcChannelProvider.Builder createBaseChannelProviderBuilde options.getInterceptorProvider(), SpannerInterceptorProvider.createDefault(options.getOpenTelemetry()))) // This sets the trace context headers. - .withTraceContext(endToEndTracingEnabled, options.getOpenTelemetry()) + .withTraceContext(endToEndTracingEnabled, options.getTracingOpenTelemetry()) // This sets the response compressor (Server -> Client). .withEncoding(compressorName)) .setHeaderProvider(headerProviderWithUserAgent) diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/BatchClientImplTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/BatchClientImplTest.java index d8d123925073..46bb4fd6c3d4 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/BatchClientImplTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/BatchClientImplTest.java @@ -90,6 +90,7 @@ public void setUp() { when(spannerOptions.getSpannerRpcV1()).thenReturn(gapicRpc); when(spannerOptions.getSessionLabels()).thenReturn(Collections.emptyMap()); when(spannerOptions.getOpenTelemetry()).thenReturn(OpenTelemetry.noop()); + when(spannerOptions.getTracingOpenTelemetry()).thenReturn(OpenTelemetry.noop()); GrpcTransportOptions transportOptions = mock(GrpcTransportOptions.class); when(transportOptions.getExecutorFactory()).thenReturn(mock(ExecutorFactory.class)); when(spannerOptions.getTransportOptions()).thenReturn(transportOptions); diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/BuiltInTracingTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/BuiltInTracingTest.java new file mode 100644 index 000000000000..3e68c1eb0ef9 --- /dev/null +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/BuiltInTracingTest.java @@ -0,0 +1,344 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner; + +import static com.google.cloud.spanner.MockSpannerTestUtil.SELECT1; +import static com.google.cloud.spanner.MockSpannerTestUtil.SELECT1_RESULTSET; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import com.google.auth.oauth2.AccessToken; +import com.google.auth.oauth2.OAuth2Credentials; +import com.google.cloud.NoCredentials; +import com.google.cloud.spanner.MockSpannerServiceImpl.StatementResult; +import com.google.spanner.v1.SpannerGrpc; +import io.grpc.ManagedChannelBuilder; +import io.grpc.Metadata; +import io.grpc.Server; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.TracerProvider; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.net.InetSocketAddress; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class BuiltInTracingTest { + private static final Metadata.Key TRACE_PARENT_KEY = + Metadata.Key.of("traceparent", Metadata.ASCII_STRING_MARSHALLER); + private static final Metadata.Key END_TO_END_TRACING_KEY = + Metadata.Key.of("x-goog-spanner-end-to-end-tracing", Metadata.ASCII_STRING_MARSHALLER); + private static final OAuth2Credentials CREDENTIALS = + OAuth2Credentials.create(new AccessToken("token", null)); + + private static final AtomicReference lastExecuteSqlHeaders = new AtomicReference<>(); + private static MockSpannerServiceImpl mockSpanner; + private static Server server; + + private InMemorySpanExporter spanExporter; + + @BeforeClass + public static void startMockServer() throws Exception { + mockSpanner = new MockSpannerServiceImpl(); + mockSpanner.setAbortProbability(0.0D); + mockSpanner.putStatementResult(StatementResult.query(SELECT1, SELECT1_RESULTSET)); + server = + NettyServerBuilder.forAddress(new InetSocketAddress("localhost", 0)) + .addService(mockSpanner) + .intercept( + new ServerInterceptor() { + @Override + public ServerCall.Listener interceptCall( + ServerCall call, + Metadata headers, + ServerCallHandler next) { + if (call.getMethodDescriptor() + .getFullMethodName() + .equals(SpannerGrpc.getExecuteStreamingSqlMethod().getFullMethodName())) { + lastExecuteSqlHeaders.set(headers); + } + return next.startCall(call, headers); + } + }) + .build() + .start(); + } + + @AfterClass + public static void stopMockServer() throws InterruptedException { + server.shutdown(); + server.awaitTermination(); + } + + @Before + public void setUp() { + SpannerOptions.useEnvironment(new SpannerOptions.SpannerEnvironment() {}); + SpannerOptionsHelper.resetActiveTracingFramework(); + GlobalOpenTelemetry.resetForTest(); + spanExporter = InMemorySpanExporter.create(); + BuiltInTracingProvider.INSTANCE.setSpanExporterFactory( + (projectId, credentials) -> spanExporter); + lastExecuteSqlHeaders.set(null); + } + + @After + public void tearDown() { + BuiltInTracingProvider.INSTANCE.reset(); + GlobalOpenTelemetry.resetForTest(); + SpannerOptionsHelper.resetActiveTracingFramework(); + SpannerOptions.useDefaultEnvironment(); + mockSpanner.reset(); + } + + private static OpenTelemetry createOpenTelemetryWithTracing() { + return OpenTelemetrySdk.builder() + .setTracerProvider( + SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(InMemorySpanExporter.create())) + .build()) + .build(); + } + + private static OpenTelemetry createMetricsOnlyOpenTelemetry() { + return OpenTelemetrySdk.builder().setMeterProvider(SdkMeterProvider.builder().build()).build(); + } + + private static SpannerOptions.Builder createOptionsBuilder() { + return SpannerOptions.newBuilder() + .setProjectId("test-project") + .setHost("http://localhost:" + server.getPort()) + .setChannelConfigurator(ManagedChannelBuilder::usePlaintext) + .setEnableDirectAccess(false) + .setCredentials(CREDENTIALS); + } + + @Test + public void builtInTracingIsDisabledByDefault() { + SpannerOptions options = createOptionsBuilder().build(); + + assertThat(options.isEnableBuiltInTracing()).isFalse(); + assertThat(options.getBuiltInTracingSamplingRate()).isEqualTo(0.1); + assertThat(options.isEndToEndTracingEnabled()).isFalse(); + assertThat(options.getTracingOpenTelemetry().getTracerProvider()) + .isSameInstanceAs(TracerProvider.noop()); + } + + @Test + public void builtInTracingIsConfiguredFromTheEnvironment() { + SpannerOptions.useEnvironment( + new SpannerOptions.SpannerEnvironment() { + @Override + public boolean isEnableBuiltInTracing() { + return true; + } + + @Override + public double getBuiltInTracingSamplingRate() { + return 0.5; + } + }); + + SpannerOptions options = createOptionsBuilder().build(); + + assertThat(options.isEnableBuiltInTracing()).isTrue(); + assertThat(options.getBuiltInTracingSamplingRate()).isEqualTo(0.5); + } + + @Test + public void samplingRateMustBeBetweenZeroAndOne() { + assertThrows( + IllegalArgumentException.class, + () -> createOptionsBuilder().setBuiltInTracingSamplingRate(-0.1)); + assertThrows( + IllegalArgumentException.class, + () -> createOptionsBuilder().setBuiltInTracingSamplingRate(1.1)); + } + + @Test + public void builtInTracingSendsTheTraceOfTheRequestToSpanner() { + SpannerOptions options = + createOptionsBuilder() + .setEnableBuiltInTracing(true) + .setBuiltInTracingSamplingRate(1.0) + .build(); + assertThat(options.isEndToEndTracingEnabled()).isTrue(); + + try (Spanner spanner = options.getService()) { + DatabaseClient client = + spanner.getDatabaseClient(DatabaseId.of("test-project", "test-instance", "test-db")); + try (ResultSet resultSet = client.singleUse().executeQuery(SELECT1)) { + while (resultSet.next()) {} + } + } + ((OpenTelemetrySdk) options.getTracingOpenTelemetry()) + .getSdkTracerProvider() + .forceFlush() + .join(10, TimeUnit.SECONDS); + + Metadata headers = lastExecuteSqlHeaders.get(); + assertThat(headers).isNotNull(); + assertThat(headers.get(END_TO_END_TRACING_KEY)).isEqualTo("true"); + String traceParent = headers.get(TRACE_PARENT_KEY); + assertThat(traceParent).isNotNull(); + String[] traceParentParts = traceParent.split("-"); + assertThat(Integer.parseInt(traceParentParts[3], 16) & 1).isEqualTo(1); + List spans = spanExporter.getFinishedSpanItems(); + assertThat(spans).isNotEmpty(); + assertThat(spans.stream().map(SpanData::getTraceId)).contains(traceParentParts[1]); + assertThat(spans.get(0).getResource().getAttribute(AttributeKey.stringKey("gcp.project_id"))) + .isEqualTo("test-project"); + } + + @Test + public void resourceUsesTheServiceNameWhenSet() { + Resource resource = BuiltInTracingProvider.createResource("test-project", "my-service"); + + assertThat(resource.getAttribute(AttributeKey.stringKey("gcp.project_id"))) + .isEqualTo("test-project"); + assertThat(resource.getAttribute(AttributeKey.stringKey("service.name"))) + .isEqualTo("my-service"); + assertThat( + BuiltInTracingProvider.createResource("test-project", null) + .getAttribute(AttributeKey.stringKey("service.name"))) + .isEqualTo(Resource.getDefault().getAttribute(AttributeKey.stringKey("service.name"))); + } + + @Test + public void requestHeadersContainTheCredentials() { + assertThat(BuiltInTracingProvider.getRequestHeaders(CREDENTIALS)) + .containsEntry("Authorization", "Bearer token"); + } + + @Test + public void clientsOfTheSameProjectShareTheBuiltInOpenTelemetry() { + SpannerOptions options = createOptionsBuilder().setEnableBuiltInTracing(true).build(); + SpannerOptions sameProject = createOptionsBuilder().setEnableBuiltInTracing(true).build(); + SpannerOptions otherProject = + createOptionsBuilder().setEnableBuiltInTracing(true).setProjectId("other-project").build(); + + OpenTelemetry openTelemetry = options.getTracingOpenTelemetry(); + + assertThat(openTelemetry).isInstanceOf(OpenTelemetrySdk.class); + assertThat(sameProject.getTracingOpenTelemetry()).isSameInstanceAs(openTelemetry); + assertThat(otherProject.getTracingOpenTelemetry()).isNotSameInstanceAs(openTelemetry); + assertThat(options.getOpenTelemetry()).isNotSameInstanceAs(openTelemetry); + } + + @Test + public void openTelemetryOfTheClientIsUsedInsteadOfBuiltInTracing() { + OpenTelemetry openTelemetry = createOpenTelemetryWithTracing(); + SpannerOptions options = + createOptionsBuilder() + .setEnableBuiltInTracing(true) + .setOpenTelemetry(openTelemetry) + .build(); + + assertThat(options.getTracingOpenTelemetry()).isSameInstanceAs(openTelemetry); + assertThat(options.isEndToEndTracingEnabled()).isTrue(); + } + + @Test + public void globalOpenTelemetryWithTracingIsUsedInsteadOfBuiltInTracing() { + GlobalOpenTelemetry.set(createOpenTelemetryWithTracing()); + SpannerOptions options = createOptionsBuilder().setEnableBuiltInTracing(true).build(); + + assertThat(options.getTracingOpenTelemetry()).isSameInstanceAs(GlobalOpenTelemetry.get()); + assertThat(options.isEndToEndTracingEnabled()).isTrue(); + } + + @Test + public void metricsOnlyOpenTelemetryOfTheClientDoesNotDisableBuiltInTracing() { + OpenTelemetry openTelemetry = createMetricsOnlyOpenTelemetry(); + SpannerOptions options = + createOptionsBuilder() + .setEnableBuiltInTracing(true) + .setOpenTelemetry(openTelemetry) + .build(); + + assertThat(options.getTracingOpenTelemetry()).isInstanceOf(OpenTelemetrySdk.class); + assertThat(options.getTracingOpenTelemetry()).isNotSameInstanceAs(openTelemetry); + assertThat(options.getOpenTelemetry()).isSameInstanceAs(openTelemetry); + } + + @Test + public void metricsOnlyGlobalOpenTelemetryDoesNotDisableBuiltInTracing() { + GlobalOpenTelemetry.set(createMetricsOnlyOpenTelemetry()); + SpannerOptions options = createOptionsBuilder().setEnableBuiltInTracing(true).build(); + + assertThat(options.getTracingOpenTelemetry()).isInstanceOf(OpenTelemetrySdk.class); + assertThat(options.getOpenTelemetry()).isSameInstanceAs(GlobalOpenTelemetry.get()); + } + + @Test + public void globalOpenTelemetryWithoutTracingDoesNotDisableBuiltInTracing() { + GlobalOpenTelemetry.get(); + SpannerOptions options = createOptionsBuilder().setEnableBuiltInTracing(true).build(); + + assertThat(options.getTracingOpenTelemetry()).isInstanceOf(OpenTelemetrySdk.class); + } + + @Test + public void builtInTracingIsNotUsedWithoutCredentials() { + SpannerOptions options = + createOptionsBuilder() + .setEnableBuiltInTracing(true) + .setCredentials(NoCredentials.getInstance()) + .build(); + + assertThat(options.getTracingOpenTelemetry()).isNotInstanceOf(OpenTelemetrySdk.class); + } + + @Test + public void builtInTracingIsNotUsedInOtherUniverses() { + SpannerOptions options = + createOptionsBuilder() + .setEnableBuiltInTracing(true) + .setUniverseDomain("example.com") + .build(); + + assertThat(options.getTracingOpenTelemetry()).isNotInstanceOf(OpenTelemetrySdk.class); + } + + @Test + public void builtInTracingIsNotUsedWithOpenCensus() { + createOptionsBuilder().build(); + SpannerOptions options = createOptionsBuilder().setEnableBuiltInTracing(true).build(); + + assertThat(options.getTracingOpenTelemetry()).isNotInstanceOf(OpenTelemetrySdk.class); + assertThat(options.isEndToEndTracingEnabled()).isFalse(); + } +} diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SessionImplTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SessionImplTest.java index 4e7197356350..bc27aee4ccc2 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SessionImplTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SessionImplTest.java @@ -127,6 +127,7 @@ public void setUp() { when(sessionPoolOptions.getPoolMaintainerClock()).thenReturn(Clock.INSTANCE); when(spannerOptions.getSessionPoolOptions()).thenReturn(sessionPoolOptions); when(spannerOptions.getOpenTelemetry()).thenReturn(OpenTelemetry.noop()); + when(spannerOptions.getTracingOpenTelemetry()).thenReturn(OpenTelemetry.noop()); @SuppressWarnings("resource") SpannerImpl spanner = new SpannerImpl(rpc, spannerOptions); String dbName = "projects/p1/instances/i1/databases/d1"; diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SpannerImplTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SpannerImplTest.java index 1bcfde7d4e27..0169b34c8b3d 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SpannerImplTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SpannerImplTest.java @@ -106,6 +106,7 @@ public void setUp() { when(spannerOptions.getClock()).thenReturn(NanoClock.getDefaultClock()); when(spannerOptions.getSessionLabels()).thenReturn(Collections.emptyMap()); when(spannerOptions.getOpenTelemetry()).thenReturn(OpenTelemetry.noop()); + when(spannerOptions.getTracingOpenTelemetry()).thenReturn(OpenTelemetry.noop()); impl = new SpannerImpl(rpc, spannerOptions); } diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SpannerOptionsTestHelper.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SpannerOptionsTestHelper.java index 8f8c0a30a8ca..bc47577953bd 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SpannerOptionsTestHelper.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SpannerOptionsTestHelper.java @@ -15,9 +15,20 @@ */ package com.google.cloud.spanner; +import io.opentelemetry.sdk.trace.export.SpanExporter; + public class SpannerOptionsTestHelper { public static void resetActiveTracingFramework() { SpannerOptions.resetActiveTracingFramework(); } + + public static void setBuiltInTracingSpanExporter(SpanExporter spanExporter) { + BuiltInTracingProvider.INSTANCE.setSpanExporterFactory( + (projectId, credentials) -> spanExporter); + } + + public static void resetBuiltInTracing() { + BuiltInTracingProvider.INSTANCE.reset(); + } } diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/TransactionManagerImplTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/TransactionManagerImplTest.java index 2aefc2ca6ff7..fafe8e77b0c2 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/TransactionManagerImplTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/TransactionManagerImplTest.java @@ -281,6 +281,7 @@ public void usesPreparedTransaction() { when(options.getSessionLabels()).thenReturn(Collections.emptyMap()); when(options.getDatabaseRole()).thenReturn("role"); when(options.getOpenTelemetry()).thenReturn(OpenTelemetry.noop()); + when(options.getTracingOpenTelemetry()).thenReturn(OpenTelemetry.noop()); SpannerRpc rpc = mock(SpannerRpc.class); when(rpc.asyncDeleteSession(Mockito.anyString(), Mockito.anyMap())) .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); @@ -363,6 +364,7 @@ public void inlineBegin() { when(options.getDefaultQueryOptions(Mockito.any(DatabaseId.class))) .thenReturn(QueryOptions.getDefaultInstance()); when(options.getOpenTelemetry()).thenReturn(OpenTelemetry.noop()); + when(options.getTracingOpenTelemetry()).thenReturn(OpenTelemetry.noop()); SpannerRpc rpc = mock(SpannerRpc.class); when(rpc.asyncDeleteSession(Mockito.anyString(), Mockito.anyMap())) .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/TransactionRunnerImplTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/TransactionRunnerImplTest.java index 3ee62127a2a3..0972892bac6b 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/TransactionRunnerImplTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/TransactionRunnerImplTest.java @@ -217,6 +217,7 @@ public void usesPreparedTransaction() { when(options.getSessionLabels()).thenReturn(Collections.emptyMap()); when(options.getDatabaseRole()).thenReturn("role"); when(options.getOpenTelemetry()).thenReturn(OpenTelemetry.noop()); + when(options.getTracingOpenTelemetry()).thenReturn(OpenTelemetry.noop()); SpannerRpc rpc = mock(SpannerRpc.class); when(rpc.asyncDeleteSession(Mockito.anyString(), Mockito.anyMap())) .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/BuiltInTracingConnectionTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/BuiltInTracingConnectionTest.java new file mode 100644 index 000000000000..5990e8bca368 --- /dev/null +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/BuiltInTracingConnectionTest.java @@ -0,0 +1,88 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner.connection; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.auth.oauth2.AccessToken; +import com.google.auth.oauth2.OAuth2Credentials; +import com.google.cloud.spanner.ResultSet; +import com.google.cloud.spanner.SpannerOptionsTestHelper; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.data.SpanData; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class BuiltInTracingConnectionTest extends AbstractMockServerTest { + private InMemorySpanExporter spanExporter; + + @Before + public void setUpBuiltInTracing() { + SpannerOptionsTestHelper.resetActiveTracingFramework(); + GlobalOpenTelemetry.resetForTest(); + spanExporter = InMemorySpanExporter.create(); + SpannerOptionsTestHelper.setBuiltInTracingSpanExporter(spanExporter); + } + + @After + public void resetBuiltInTracing() { + SpannerPool.closeSpannerPool(); + SpannerOptionsTestHelper.resetBuiltInTracing(); + GlobalOpenTelemetry.resetForTest(); + SpannerOptionsTestHelper.resetActiveTracingFramework(); + } + + @Test + public void connectionSpansAreRecordedByBuiltInTracing() { + try (Connection connection = + ConnectionOptions.newBuilder() + .setUri(getBaseUrl()) + .setConfigurator( + builder -> + builder + .setCredentials(OAuth2Credentials.create(new AccessToken("token", null))) + .setEnableBuiltInTracing(true) + .setBuiltInTracingSamplingRate(1.0)) + .build() + .getConnection()) { + connection.setAutocommit(true); + try (ResultSet resultSet = connection.executeQuery(SELECT1_STATEMENT)) { + while (resultSet.next()) {} + } + ((OpenTelemetrySdk) connection.getSpanner().getOptions().getTracingOpenTelemetry()) + .getSdkTracerProvider() + .forceFlush() + .join(10, TimeUnit.SECONDS); + } + + List spanNames = + spanExporter.getFinishedSpanItems().stream() + .map(SpanData::getName) + .collect(Collectors.toList()); + assertThat(spanNames).contains("CloudSpanner.SingleUseTransaction"); + assertThat(spanNames).contains("CloudSpannerOperation.ExecuteStreamingQuery"); + } +}