diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageOptions.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageOptions.java index 5f5dd064b7f6..514fc77fb8f1 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageOptions.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageOptions.java @@ -334,7 +334,18 @@ private Tuple> resolveSettingsAndOpts() throw Collections.singletonList(InstantiatingGrpcChannelProvider.HardBoundTokenTypes.ALTS)); } - if (!NoopGrpcInterceptorProvider.INSTANCE.equals(grpcInterceptorProvider)) { + StorageClientMetrics clientMetrics = getStorageClientMetrics(); + if (clientMetrics != null) { + StorageGrpcMetricsInterceptor metricsInterceptor = + new StorageGrpcMetricsInterceptor(clientMetrics, getClock()); + GrpcInterceptorProvider userProvider = grpcInterceptorProvider; + channelProviderBuilder.setInterceptorProvider( + () -> + ImmutableList.builder() + .addAll(userProvider.getInterceptors()) + .add(metricsInterceptor) + .build()); + } else if (!NoopGrpcInterceptorProvider.INSTANCE.equals(grpcInterceptorProvider)) { channelProviderBuilder.setInterceptorProvider(grpcInterceptorProvider); } diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageClientMetrics.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageClientMetrics.java index 36175a3ee4f4..efcb506b4be1 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageClientMetrics.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageClientMetrics.java @@ -26,6 +26,7 @@ import io.opentelemetry.api.metrics.LongUpDownCounter; import io.opentelemetry.api.metrics.Meter; import io.opentelemetry.api.metrics.MeterProvider; +import io.opentelemetry.context.Context; import io.opentelemetry.sdk.metrics.SdkMeterProvider; import java.util.logging.Logger; import org.checkerframework.checker.nullness.qual.Nullable; @@ -448,6 +449,26 @@ StorageMetricsContext createMultiStepContext(String operationName) { return StorageMetricsContext.createForMultiStep(this, operationName, baseAttributes); } + void recordRpcCallDuration( + Context otelContext, + double durationSeconds, + String rpcMethod, + String statusCode, + String networkPath, + String errorType) { + if (rpcClientCallDuration == null) { + return; + } + Attributes attributes = + baseAttributes.toBuilder() + .put(KEY_RPC_METHOD, rpcMethod) + .put(KEY_RPC_RESPONSE_STATUS_CODE, statusCode) + .put(KEY_NETWORK_PATH, networkPath) + .put(KEY_ERROR_TYPE, errorType) + .build(); + rpcClientCallDuration.record(durationSeconds, attributes, otelContext); + } + @Nullable static StorageMetricsContext createContext( @Nullable StorageClientMetrics metrics, String operationName) { diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageGrpcMetricsInterceptor.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageGrpcMetricsInterceptor.java new file mode 100644 index 000000000000..b33f1f1a4300 --- /dev/null +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageGrpcMetricsInterceptor.java @@ -0,0 +1,131 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import static java.util.Objects.requireNonNull; + +import com.google.api.core.ApiClock; +import io.grpc.Attributes; +import io.grpc.CallOptions; +import io.grpc.Channel; +import io.grpc.ClientCall; +import io.grpc.ClientInterceptor; +import io.grpc.ForwardingClientCall.SimpleForwardingClientCall; +import io.grpc.ForwardingClientCallListener.SimpleForwardingClientCallListener; +import io.grpc.Grpc; +import io.grpc.Metadata; +import io.grpc.MethodDescriptor; +import io.grpc.Status; +import io.opentelemetry.context.Context; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; + +/** + * {@link ClientInterceptor} that instruments outgoing gRPC calls to record OpenTelemetry client + * metrics. + */ +@NullMarked +final class StorageGrpcMetricsInterceptor implements ClientInterceptor { + + private final StorageClientMetrics metrics; + private final ApiClock clock; + + StorageGrpcMetricsInterceptor(StorageClientMetrics metrics, ApiClock clock) { + this.metrics = requireNonNull(metrics, "metrics cannot be null"); + this.clock = requireNonNull(clock, "clock cannot be null"); + } + + @Override + public ClientCall interceptCall( + MethodDescriptor method, CallOptions callOptions, Channel next) { + String rpcMethod = method.getFullMethodName(); + StorageMetricsContext metricsCtx = StorageMetricsContext.from(callOptions); + Context otelContext = metricsCtx != null ? metricsCtx.getOtelContext() : Context.current(); + + return new SimpleForwardingClientCall(next.newCall(method, callOptions)) { + @Override + public void start(Listener responseListener, Metadata headers) { + long startTimeNanos = clock.nanoTime(); + + super.start( + new SimpleForwardingClientCallListener(responseListener) { + @Override + public void onHeaders(Metadata headers) { + if (metricsCtx != null) { + metricsCtx.setNetworkPath(detectNetworkPath(getAttributes())); + } + super.onHeaders(headers); + } + + @Override + public void onClose(Status status, Metadata trailers) { + try { + double durationSeconds = (clock.nanoTime() - startTimeNanos) / 1_000_000_000.0; + String networkPath = detectNetworkPath(getAttributes()); + if (metricsCtx != null) { + metricsCtx.setNetworkPath(networkPath); + } + String statusCode = status.getCode().name(); + String errorType = StorageErrorTypeMapper.mapGrpcStatus(status); + + metrics.recordRpcCallDuration( + otelContext, durationSeconds, rpcMethod, statusCode, networkPath, errorType); + } finally { + super.onClose(status, trailers); + } + } + }, + headers); + } + }; + } + + static String detectNetworkPath(@Nullable Attributes callAttributes) { + SocketAddress remoteAddr = + callAttributes != null ? callAttributes.get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR) : null; + if (remoteAddr instanceof InetSocketAddress) { + InetAddress inetAddr = ((InetSocketAddress) remoteAddr).getAddress(); + if (inetAddr != null) { + return isDirectPathIp(inetAddr) + ? StorageClientMetrics.NETWORK_PATH_DIRECTPATH + : StorageClientMetrics.NETWORK_PATH_CLOUDPATH; + } + } + return StorageClientMetrics.NETWORK_PATH_UNKNOWN; + } + + static boolean isDirectPathIp(InetAddress addr) { + byte[] b = addr.getAddress(); + if (b.length == 4) { + // 34.126.0.0/18 + return b[0] == 34 && b[1] == 126 && (b[2] & 0xC0) == 0; + } + if (b.length == 16) { + // 2001:4860:8040::/42 + return b[0] == 0x20 + && b[1] == 0x01 + && b[2] == 0x48 + && b[3] == 0x60 + && b[4] == (byte) 0x80 + && (b[5] & 0xC0) == 0x40; + } + return false; + } +} diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageMetricsContext.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageMetricsContext.java index 24d23d2229f3..37d97629ab94 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageMetricsContext.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageMetricsContext.java @@ -57,6 +57,7 @@ final class StorageMetricsContext { private final AtomicInteger attemptCount = new AtomicInteger(0); private final AtomicLong requestBytes = new AtomicLong(0L); private final AtomicLong responseBytes = new AtomicLong(0L); + @Nullable private volatile String networkPath; private StorageMetricsContext( @Nullable StorageClientMetrics metrics, @@ -268,6 +269,10 @@ void recordAttemptFailure(@Nullable StorageClientMetrics metrics, Throwable e) { } } + void setNetworkPath(String networkPath) { + this.networkPath = networkPath; + } + /** * Populates both {@code rpc.response.status_code} and {@code http.response.status_code} on * attempt metrics so every {@code gcp.storage.client.attempts} data point emits a consistent set @@ -324,6 +329,10 @@ private Attributes buildAttemptBaseAttributes(StorageClientMetrics metrics) { } private String resolveNetworkPath() { + String detected = this.networkPath; + if (detected != null && !detected.isEmpty()) { + return detected; + } String path = baseAttributes.get(StorageClientMetrics.KEY_NETWORK_PATH); if (path != null && !path.isEmpty()) { return path; diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageGrpcMetricsInterceptorTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageGrpcMetricsInterceptorTest.java new file mode 100644 index 000000000000..fe0217406cb6 --- /dev/null +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageGrpcMetricsInterceptorTest.java @@ -0,0 +1,584 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.core.ApiClock; +import com.google.api.gax.grpc.GrpcInterceptorProvider; +import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; +import com.google.cloud.NoCredentials; +import com.google.common.collect.ImmutableList; +import io.grpc.Attributes; +import io.grpc.CallOptions; +import io.grpc.Channel; +import io.grpc.ClientCall; +import io.grpc.ClientInterceptor; +import io.grpc.Grpc; +import io.grpc.Metadata; +import io.grpc.MethodDescriptor; +import io.grpc.Status; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.DoubleHistogramBuilder; +import io.opentelemetry.api.metrics.LongHistogramBuilder; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.metrics.MeterBuilder; +import io.opentelemetry.api.metrics.MeterProvider; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.ContextKey; +import io.opentelemetry.context.Scope; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.data.HistogramPointData; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.lang.reflect.Field; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.util.Collections; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import org.jspecify.annotations.Nullable; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +public final class StorageGrpcMetricsInterceptorTest { + + private static final MethodDescriptor READ_OBJECT_METHOD = + MethodDescriptor.newBuilder() + .setType(MethodDescriptor.MethodType.SERVER_STREAMING) + .setFullMethodName("google.storage.v2.Storage/ReadObject") + .setRequestMarshaller(new NoopMarshaller()) + .setResponseMarshaller(new NoopMarshaller()) + .build(); + + private InMemoryMetricReader reader; + private SdkMeterProvider meterProvider; + private FakeClock clock; + private StorageClientMetrics metrics; + private StorageGrpcMetricsInterceptor interceptor; + + @Before + public void setUp() { + reader = InMemoryMetricReader.create(); + meterProvider = SdkMeterProvider.builder().registerMetricReader(reader).build(); + clock = new FakeClock(); + metrics = + StorageOptions.grpc() + .setProjectId("test-project") + .setCredentials(NoCredentials.getInstance()) + .setEnableOtelMetrics(true) + .setMeterProvider(meterProvider) + .build() + .getStorageClientMetrics(); + interceptor = new StorageGrpcMetricsInterceptor(metrics, clock); + } + + @Test + public void isDirectPathIp_ipv4AndIpv6() throws Exception { + assertThat(StorageGrpcMetricsInterceptor.isDirectPathIp(InetAddress.getByName("34.126.0.1"))) + .isTrue(); + assertThat(StorageGrpcMetricsInterceptor.isDirectPathIp(InetAddress.getByName("34.126.63.254"))) + .isTrue(); + assertThat(StorageGrpcMetricsInterceptor.isDirectPathIp(InetAddress.getByName("34.126.64.1"))) + .isFalse(); + assertThat(StorageGrpcMetricsInterceptor.isDirectPathIp(InetAddress.getByName("35.190.1.1"))) + .isFalse(); + assertThat(StorageGrpcMetricsInterceptor.isDirectPathIp(InetAddress.getByName("142.250.1.1"))) + .isFalse(); + + assertThat( + StorageGrpcMetricsInterceptor.isDirectPathIp( + InetAddress.getByName("2001:4860:8040:0000:0000:0000:0000:0001"))) + .isTrue(); + assertThat( + StorageGrpcMetricsInterceptor.isDirectPathIp( + InetAddress.getByName("2001:4860:8040:ffff:ffff:ffff:ffff:ffff"))) + .isTrue(); + assertThat( + StorageGrpcMetricsInterceptor.isDirectPathIp( + InetAddress.getByName("2001:4860:8041::1"))) + .isTrue(); + assertThat( + StorageGrpcMetricsInterceptor.isDirectPathIp( + InetAddress.getByName("2001:4860:8044::1"))) + .isTrue(); + assertThat( + StorageGrpcMetricsInterceptor.isDirectPathIp( + InetAddress.getByName("2001:4860:807f::1"))) + .isTrue(); + assertThat( + StorageGrpcMetricsInterceptor.isDirectPathIp( + InetAddress.getByName("2001:4860:8080:0000:0000:0000:0000:0001"))) + .isFalse(); + assertThat( + StorageGrpcMetricsInterceptor.isDirectPathIp( + InetAddress.getByName("2607:f8b0:4005:805::200e"))) + .isFalse(); + } + + @Test + public void detectNetworkPath_cases() throws Exception { + Attributes directPathAttrs = + Attributes.newBuilder() + .set( + Grpc.TRANSPORT_ATTR_REMOTE_ADDR, + new InetSocketAddress(InetAddress.getByName("34.126.1.2"), 443)) + .build(); + assertThat(StorageGrpcMetricsInterceptor.detectNetworkPath(directPathAttrs)) + .isEqualTo(StorageClientMetrics.NETWORK_PATH_DIRECTPATH); + + Attributes cloudPathAttrs = + Attributes.newBuilder() + .set( + Grpc.TRANSPORT_ATTR_REMOTE_ADDR, + new InetSocketAddress(InetAddress.getByName("142.250.1.1"), 443)) + .build(); + assertThat(StorageGrpcMetricsInterceptor.detectNetworkPath(cloudPathAttrs)) + .isEqualTo(StorageClientMetrics.NETWORK_PATH_CLOUDPATH); + + assertThat(StorageGrpcMetricsInterceptor.detectNetworkPath(Attributes.EMPTY)) + .isEqualTo(StorageClientMetrics.NETWORK_PATH_UNKNOWN); + assertThat(StorageGrpcMetricsInterceptor.detectNetworkPath(null)) + .isEqualTo(StorageClientMetrics.NETWORK_PATH_UNKNOWN); + } + + @Test + public void interceptCall_unaryCallSuccess_recordsMetricWithErrorTypeOk() throws Exception { + Attributes callAttrs = + Attributes.newBuilder() + .set( + Grpc.TRANSPORT_ATTR_REMOTE_ADDR, + new InetSocketAddress(InetAddress.getByName("142.250.1.1"), 443)) + .build(); + TestClientCall testCall = new TestClientCall<>(callAttrs); + TestChannel testChannel = new TestChannel(testCall); + + ClientCall call = + interceptor.interceptCall(READ_OBJECT_METHOD, CallOptions.DEFAULT, testChannel); + TestListener listener = new TestListener<>(); + call.start(listener, new Metadata()); + + clock.advanceNanos(500_000_000L); // 0.5s + testCall.close(Status.OK, new Metadata()); + + MetricData rpcDurationData = findMetric(StorageClientMetrics.METRIC_RPC_CLIENT_CALL_DURATION); + + assertThat(rpcDurationData).isNotNull(); + assertThat(rpcDurationData.getHistogramData().getPoints()).hasSize(1); + HistogramPointData point = rpcDurationData.getHistogramData().getPoints().iterator().next(); + assertThat(point.getCount()).isEqualTo(1L); + assertThat(point.getSum()).isWithin(1e-6).of(0.5); + + io.opentelemetry.api.common.Attributes attrs = point.getAttributes(); + assertThat(attrs.get(StorageClientMetrics.KEY_RPC_SYSTEM_NAME)).isEqualTo("grpc"); + assertThat(attrs.get(StorageClientMetrics.KEY_RPC_METHOD)) + .isEqualTo("google.storage.v2.Storage/ReadObject"); + assertThat(attrs.get(StorageClientMetrics.KEY_RPC_RESPONSE_STATUS_CODE)).isEqualTo("OK"); + assertThat(attrs.get(StorageClientMetrics.KEY_NETWORK_PATH)) + .isEqualTo(StorageClientMetrics.NETWORK_PATH_CLOUDPATH); + assertThat(attrs.get(StorageClientMetrics.KEY_SERVER_ADDRESS)) + .isEqualTo("storage.googleapis.com"); + assertThat(attrs.get(StorageClientMetrics.KEY_ERROR_TYPE)).isEqualTo("OK"); + } + + @Test + public void interceptCall_callFails_recordsErrorType() { + TestClientCall testCall = new TestClientCall<>(Attributes.EMPTY); + TestChannel testChannel = new TestChannel(testCall); + + ClientCall call = + interceptor.interceptCall(READ_OBJECT_METHOD, CallOptions.DEFAULT, testChannel); + call.start(new TestListener<>(), new Metadata()); + + clock.advanceNanos(100_000_000L); // 0.1s + testCall.close(Status.NOT_FOUND, new Metadata()); + + MetricData rpcDurationData = findMetric(StorageClientMetrics.METRIC_RPC_CLIENT_CALL_DURATION); + + assertThat(rpcDurationData).isNotNull(); + HistogramPointData point = rpcDurationData.getHistogramData().getPoints().iterator().next(); + assertThat(point.getAttributes().get(StorageClientMetrics.KEY_RPC_RESPONSE_STATUS_CODE)) + .isEqualTo("NOT_FOUND"); + assertThat(point.getAttributes().get(StorageClientMetrics.KEY_ERROR_TYPE)) + .isEqualTo("NOT_FOUND"); + assertThat(point.getAttributes().get(StorageClientMetrics.KEY_SERVER_ADDRESS)) + .isEqualTo("storage.googleapis.com"); + } + + @Test + public void interceptCall_withStorageMetricsContext_propagatesContextAndFeedsNetworkPath() + throws Exception { + StorageMetricsContext metricsContext = metrics.createContext("ReadObject"); + CallOptions callOptions = metricsContext.attach(CallOptions.DEFAULT); + + Attributes directPathAttrs = + Attributes.newBuilder() + .set( + Grpc.TRANSPORT_ATTR_REMOTE_ADDR, + new InetSocketAddress(InetAddress.getByName("34.126.1.2"), 443)) + .build(); + TestClientCall testCall = new TestClientCall<>(directPathAttrs); + TestChannel testChannel = new TestChannel(testCall); + + ClientCall call = + interceptor.interceptCall(READ_OBJECT_METHOD, callOptions, testChannel); + call.start(new TestListener<>(), new Metadata()); + testCall.close(Status.OK, new Metadata()); + + metricsContext.recordAttemptSuccess(metrics); + + MetricData rpcDurationData = findMetric(StorageClientMetrics.METRIC_RPC_CLIENT_CALL_DURATION); + assertThat(rpcDurationData).isNotNull(); + assertThat(rpcDurationData.getHistogramData().getPoints()).hasSize(1); + HistogramPointData durationPoint = + rpcDurationData.getHistogramData().getPoints().iterator().next(); + assertThat(durationPoint.getAttributes().get(StorageClientMetrics.KEY_NETWORK_PATH)) + .isEqualTo(StorageClientMetrics.NETWORK_PATH_DIRECTPATH); + + MetricData attemptsData = findMetric(StorageClientMetrics.METRIC_GCP_STORAGE_CLIENT_ATTEMPTS); + assertThat(attemptsData).isNotNull(); + assertThat( + attemptsData + .getLongSumData() + .getPoints() + .iterator() + .next() + .getAttributes() + .get(StorageClientMetrics.KEY_NETWORK_PATH)) + .isEqualTo(StorageClientMetrics.NETWORK_PATH_DIRECTPATH); + } + + @Test + public void interceptCall_streamingReadOnHeaders_setsNetworkPathBeforeOnClose() throws Exception { + StorageMetricsContext metricsContext = metrics.createContext("ReadObject"); + CallOptions callOptions = metricsContext.attach(CallOptions.DEFAULT); + + Attributes directPathAttrs = + Attributes.newBuilder() + .set( + Grpc.TRANSPORT_ATTR_REMOTE_ADDR, + new InetSocketAddress(InetAddress.getByName("34.126.0.1"), 443)) + .build(); + TestClientCall testCall = new TestClientCall<>(directPathAttrs); + TestChannel testChannel = new TestChannel(testCall); + + ClientCall call = + interceptor.interceptCall(READ_OBJECT_METHOD, callOptions, testChannel); + TestListener listener = new TestListener<>(); + call.start(listener, new Metadata()); + + // Simulate streaming ReadObject where headers arrive and the attempt completes before onClose + testCall.receiveHeaders(new Metadata()); + assertThat(listener.headersReceived).isTrue(); + metricsContext.recordAttemptSuccess(metrics); + + MetricData attemptsData = findMetric(StorageClientMetrics.METRIC_GCP_STORAGE_CLIENT_ATTEMPTS); + assertThat(attemptsData).isNotNull(); + assertThat( + attemptsData + .getLongSumData() + .getPoints() + .iterator() + .next() + .getAttributes() + .get(StorageClientMetrics.KEY_NETWORK_PATH)) + .isEqualTo(StorageClientMetrics.NETWORK_PATH_DIRECTPATH); + } + + @Test + public void interceptCall_withoutStorageMetricsContext_asyncOnCloseUsesCallerContext() + throws Exception { + ContextKey testKey = ContextKey.named("caller-trace-key"); + Context callerContext = Context.root().with(testKey, "caller-trace-value"); + Context callbackThreadContext = Context.root().with(testKey, "callback-thread-value"); + + AtomicReference recordedContext = new AtomicReference<>(); + DoubleHistogram mockHistogram = Mockito.mock(DoubleHistogram.class); + Mockito.doAnswer( + invocation -> { + recordedContext.set(invocation.getArgument(2)); + return null; + }) + .when(mockHistogram) + .record( + Mockito.anyDouble(), + Mockito.any(io.opentelemetry.api.common.Attributes.class), + Mockito.any(Context.class)); + + DoubleHistogramBuilder mockHistBuilder = + Mockito.mock(DoubleHistogramBuilder.class, Mockito.RETURNS_SELF); + Mockito.when(mockHistBuilder.build()).thenReturn(mockHistogram); + Mockito.when(mockHistBuilder.ofLongs()) + .thenReturn(Mockito.mock(LongHistogramBuilder.class, Mockito.RETURNS_DEEP_STUBS)); + + Meter mockMeter = Mockito.mock(Meter.class, Mockito.RETURNS_DEEP_STUBS); + Mockito.when(mockMeter.histogramBuilder(Mockito.anyString())).thenReturn(mockHistBuilder); + + MeterBuilder mockMeterBuilder = Mockito.mock(MeterBuilder.class, Mockito.RETURNS_SELF); + Mockito.when(mockMeterBuilder.build()).thenReturn(mockMeter); + + MeterProvider mockMeterProvider = Mockito.mock(MeterProvider.class); + Mockito.when(mockMeterProvider.meterBuilder(Mockito.anyString())).thenReturn(mockMeterBuilder); + + StorageClientMetrics mockClientMetrics = + StorageOptions.grpc() + .setProjectId("test-project") + .setCredentials(NoCredentials.getInstance()) + .setEnableOtelMetrics(true) + .setMeterProvider(mockMeterProvider) + .build() + .getStorageClientMetrics(); + StorageGrpcMetricsInterceptor mockInterceptor = + new StorageGrpcMetricsInterceptor(mockClientMetrics, clock); + + TestClientCall testCall = new TestClientCall<>(Attributes.EMPTY); + TestChannel testChannel = new TestChannel(testCall); + + final ClientCall call; + try (Scope ignored = callerContext.makeCurrent()) { + call = mockInterceptor.interceptCall(READ_OBJECT_METHOD, CallOptions.DEFAULT, testChannel); + } + call.start(new TestListener<>(), new Metadata()); + + CountDownLatch latch = new CountDownLatch(1); + Thread asyncCallbackThread = + new Thread( + () -> { + try (Scope ignored = callbackThreadContext.makeCurrent()) { + testCall.close(Status.OK, new Metadata()); + } finally { + latch.countDown(); + } + }); + asyncCallbackThread.start(); + assertThat(latch.await(5, TimeUnit.SECONDS)).isTrue(); + + assertThat(recordedContext.get()).isSameInstanceAs(callerContext); + assertThat(recordedContext.get().get(testKey)).isEqualTo("caller-trace-value"); + } + + @Test + public void grpcStorageOptions_interceptorProviderWiring() throws Exception { + GrpcStorageOptions defaultOptions = + StorageOptions.grpc() + .setProjectId("test-project") + .setCredentials(NoCredentials.getInstance()) + .setEnableOtelMetrics(true) + .setMeterProvider(meterProvider) + .build(); + InstantiatingGrpcChannelProvider defaultChannelProvider = + (InstantiatingGrpcChannelProvider) + defaultOptions.getStorageSettings().getTransportChannelProvider(); + GrpcInterceptorProvider defaultInterceptorProvider = + extractInterceptorProvider(defaultChannelProvider); + assertThat(defaultInterceptorProvider).isNotNull(); + assertThat(defaultInterceptorProvider.getInterceptors()).hasSize(1); + assertThat(defaultInterceptorProvider.getInterceptors().get(0)) + .isInstanceOf(StorageGrpcMetricsInterceptor.class); + + ClientInterceptor customInterceptor = StorageGrpcMetricsInterceptorTest::passthroughCall; + + GrpcStorageOptions customOptions = + StorageOptions.grpc() + .setProjectId("test-project") + .setCredentials(NoCredentials.getInstance()) + .setEnableOtelMetrics(true) + .setMeterProvider(meterProvider) + .setGrpcInterceptorProvider(() -> ImmutableList.of(customInterceptor)) + .build(); + InstantiatingGrpcChannelProvider customChannelProvider = + (InstantiatingGrpcChannelProvider) + customOptions.getStorageSettings().getTransportChannelProvider(); + GrpcInterceptorProvider combinedProvider = extractInterceptorProvider(customChannelProvider); + assertThat(combinedProvider).isNotNull(); + assertThat(combinedProvider.getInterceptors()).hasSize(2); + assertThat(combinedProvider.getInterceptors().get(0)).isSameInstanceAs(customInterceptor); + assertThat(combinedProvider.getInterceptors().get(1)) + .isInstanceOf(StorageGrpcMetricsInterceptor.class); + + GrpcStorageOptions emptyCustomOptions = + StorageOptions.grpc() + .setProjectId("test-project") + .setCredentials(NoCredentials.getInstance()) + .setEnableOtelMetrics(true) + .setMeterProvider(meterProvider) + .setGrpcInterceptorProvider(Collections::emptyList) + .build(); + InstantiatingGrpcChannelProvider emptyCustomChannelProvider = + (InstantiatingGrpcChannelProvider) + emptyCustomOptions.getStorageSettings().getTransportChannelProvider(); + GrpcInterceptorProvider emptyCombinedProvider = + extractInterceptorProvider(emptyCustomChannelProvider); + assertThat(emptyCombinedProvider).isNotNull(); + assertThat(emptyCombinedProvider.getInterceptors()).hasSize(1); + assertThat(emptyCombinedProvider.getInterceptors().get(0)) + .isInstanceOf(StorageGrpcMetricsInterceptor.class); + + GrpcInterceptorProvider userProvider = () -> ImmutableList.of(customInterceptor); + GrpcStorageOptions disabledCustomOptions = + StorageOptions.grpc() + .setProjectId("test-project") + .setCredentials(NoCredentials.getInstance()) + .setEnableOtelMetrics(false) + .setGrpcInterceptorProvider(userProvider) + .build(); + InstantiatingGrpcChannelProvider disabledCustomChannelProvider = + (InstantiatingGrpcChannelProvider) + disabledCustomOptions.getStorageSettings().getTransportChannelProvider(); + GrpcInterceptorProvider disabledCustomExtracted = + extractInterceptorProvider(disabledCustomChannelProvider); + assertThat(disabledCustomExtracted).isSameInstanceAs(userProvider); + assertThat(disabledCustomExtracted.getInterceptors()).containsExactly(customInterceptor); + + GrpcStorageOptions disabledDefaultOptions = + StorageOptions.grpc() + .setProjectId("test-project") + .setCredentials(NoCredentials.getInstance()) + .setEnableOtelMetrics(false) + .build(); + InstantiatingGrpcChannelProvider disabledDefaultChannelProvider = + (InstantiatingGrpcChannelProvider) + disabledDefaultOptions.getStorageSettings().getTransportChannelProvider(); + assertThat(extractInterceptorProvider(disabledDefaultChannelProvider)).isNull(); + } + + private @Nullable MetricData findMetric(String name) { + return reader.collectAllMetrics().stream() + .filter(m -> m.getName().equals(name)) + .findFirst() + .orElse(null); + } + + private static ClientCall passthroughCall( + MethodDescriptor method, CallOptions callOptions, Channel next) { + return next.newCall(method, callOptions); + } + + private static GrpcInterceptorProvider extractInterceptorProvider( + InstantiatingGrpcChannelProvider channelProvider) throws Exception { + Field field = InstantiatingGrpcChannelProvider.class.getDeclaredField("interceptorProvider"); + field.setAccessible(true); + return (GrpcInterceptorProvider) field.get(channelProvider); + } + + private static final class TestClientCall extends ClientCall { + private Listener listener; + private final Attributes attributes; + + TestClientCall(Attributes attributes) { + this.attributes = attributes; + } + + @Override + public void start(Listener responseListener, Metadata headers) { + this.listener = responseListener; + } + + @Override + public void request(int numMessages) {} + + @Override + public void cancel(String message, Throwable cause) {} + + @Override + public void halfClose() {} + + @Override + public void sendMessage(ReqT message) {} + + @Override + public Attributes getAttributes() { + return attributes; + } + + void receiveHeaders(Metadata headers) { + if (listener != null) { + listener.onHeaders(headers); + } + } + + void close(Status status, Metadata trailers) { + if (listener != null) { + listener.onClose(status, trailers); + } + } + } + + private static final class TestChannel extends Channel { + private final TestClientCall call; + + TestChannel(TestClientCall call) { + this.call = call; + } + + @SuppressWarnings("unchecked") + @Override + public ClientCall newCall( + MethodDescriptor methodDescriptor, CallOptions callOptions) { + return (ClientCall) call; + } + + @Override + public String authority() { + return "storage.googleapis.com"; + } + } + + private static final class TestListener extends ClientCall.Listener { + boolean headersReceived = false; + + @Override + public void onHeaders(Metadata headers) { + headersReceived = true; + } + } + + private static final class NoopMarshaller implements MethodDescriptor.Marshaller { + @Override + public InputStream stream(Void value) { + return new ByteArrayInputStream(new byte[0]); + } + + @Override + public Void parse(InputStream stream) { + return null; + } + } + + private static final class FakeClock implements ApiClock { + private final AtomicLong nanos = new AtomicLong(1_000_000_000L); + + @Override + public long nanoTime() { + return nanos.get(); + } + + @Override + public long millisTime() { + return nanos.get() / 1_000_000L; + } + + void advanceNanos(long amount) { + nanos.addAndGet(amount); + } + } +}