diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageClientMetrics.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageClientMetrics.java index efcb506b4be1..2f8c326b909d 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageClientMetrics.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageClientMetrics.java @@ -469,6 +469,26 @@ void recordRpcCallDuration( rpcClientCallDuration.record(durationSeconds, attributes, otelContext); } + void recordHttpRequestDuration( + double durationSeconds, + String httpMethod, + long statusCode, + String urlTemplate, + String errorType) { + if (httpClientRequestDuration == null) { + return; + } + Attributes attributes = + baseAttributes.toBuilder() + .put(KEY_HTTP_REQUEST_METHOD, httpMethod) + .put(KEY_HTTP_RESPONSE_STATUS_CODE, statusCode) + .put(KEY_NETWORK_PATH, NETWORK_PATH_CLOUDPATH) + .put(KEY_URL_TEMPLATE, urlTemplate) + .put(KEY_ERROR_TYPE, errorType) + .build(); + httpClientRequestDuration.record(durationSeconds, attributes, Context.current()); + } + @Nullable static StorageMetricsContext createContext( @Nullable StorageClientMetrics metrics, String operationName) { diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageHttpMetricsInterceptor.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageHttpMetricsInterceptor.java new file mode 100644 index 000000000000..9aed94d4bb48 --- /dev/null +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageHttpMetricsInterceptor.java @@ -0,0 +1,242 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import static java.util.Objects.requireNonNull; + +import com.google.api.client.http.GenericUrl; +import com.google.api.client.http.HttpExecuteInterceptor; +import com.google.api.client.http.HttpIOExceptionHandler; +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpResponseInterceptor; +import com.google.api.client.http.HttpUnsuccessfulResponseHandler; +import com.google.api.core.ApiClock; +import com.google.common.collect.ImmutableSet; +import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; + +/** + * {@link HttpRequestInitializer} that instruments outgoing HTTP requests to record OpenTelemetry + * client metrics. + * + *

Note: {@code google-http-client} invokes {@link HttpResponseInterceptor#interceptResponse} + * inside {@link HttpRequest#execute()} when the HTTP response headers arrive, so download durations + * recorded here end at response headers and exclude streaming body transfer (a follow-up is tracked + * for recording when the response body is closed). End-to-end download duration including body + * transfer is captured by {@code gcp.client.request.duration}. + */ +@NullMarked +final class StorageHttpMetricsInterceptor implements HttpRequestInitializer { + + private static final ImmutableSet OBJECT_SUB_RESOURCES = + ImmutableSet.of("acl", "compose", "restore"); + + @Nullable private final HttpRequestInitializer delegate; + private final StorageClientMetrics metrics; + private final ApiClock clock; + + StorageHttpMetricsInterceptor( + @Nullable HttpRequestInitializer delegate, StorageClientMetrics metrics, ApiClock clock) { + this.delegate = delegate; + this.metrics = requireNonNull(metrics, "metrics cannot be null"); + this.clock = requireNonNull(clock, "clock cannot be null"); + } + + @Override + public void initialize(HttpRequest request) throws IOException { + if (delegate != null) { + delegate.initialize(request); + } + + HttpExecuteInterceptor existingExec = request.getInterceptor(); + HttpResponseInterceptor existingResp = request.getResponseInterceptor(); + HttpUnsuccessfulResponseHandler existingUnsuccessfulHandler = + request.getUnsuccessfulResponseHandler(); + HttpIOExceptionHandler existingIoHandler = request.getIOExceptionHandler(); + + AtomicLong startTimeNanos = new AtomicLong(); + AtomicBoolean recorded = new AtomicBoolean(true); + + request.setInterceptor( + req -> { + if (existingExec != null) { + existingExec.intercept(req); + } + startTimeNanos.set(clock.nanoTime()); + recorded.set(false); + }); + + if (existingUnsuccessfulHandler != null) { + request.setUnsuccessfulResponseHandler( + (req, response, supportsRetry) -> { + long endNanos = clock.nanoTime(); + boolean handled = + existingUnsuccessfulHandler.handleResponse(req, response, supportsRetry); + if (handled && response.getRequest() == req && recorded.compareAndSet(false, true)) { + recordResponse(req, startTimeNanos.get(), endNanos, response.getStatusCode()); + } + return handled; + }); + } + + request.setResponseInterceptor( + response -> { + long endNanos = clock.nanoTime(); + if (existingResp != null) { + existingResp.interceptResponse(response); + } + if (recorded.compareAndSet(false, true)) { + recordResponse(request, startTimeNanos.get(), endNanos, response.getStatusCode()); + } + }); + + // HttpIOExceptionHandler does not receive the thrown IOException instance from + // google-http-client, so transport-level IOExceptions default to STATUS_CONNECTION_ERROR here + // (specific subtypes like TIMEOUT, DNS_FAILURE, and TLS_FAILURE are mapped in Retrying). + request.setIOExceptionHandler( + (req, supportsRetry) -> { + if (recorded.compareAndSet(false, true)) { + recordDuration( + req, + startTimeNanos.get(), + clock.nanoTime(), + 0L, + StorageErrorTypeMapper.STATUS_CONNECTION_ERROR); + } + return existingIoHandler != null + && existingIoHandler.handleIOException(req, supportsRetry); + }); + } + + private void recordResponse(HttpRequest request, long startNanos, long endNanos, int statusCode) { + String errorType = + statusCode >= 400 + ? StorageErrorTypeMapper.mapHttpResponseCode(statusCode) + : StorageErrorTypeMapper.STATUS_OK; + recordDuration(request, startNanos, endNanos, statusCode, errorType); + } + + private void recordDuration( + HttpRequest request, long startNanos, long endNanos, long statusCode, String errorType) { + double durationSeconds = (endNanos - startNanos) / 1_000_000_000.0; + metrics.recordHttpRequestDuration( + durationSeconds, + request.getRequestMethod(), + statusCode, + extractUrlTemplate(request.getUrl()), + errorType); + } + + static String extractUrlTemplate(GenericUrl url) { + String path = url.getRawPath(); + if (path == null || path.isEmpty()) { + return "/"; + } + String host = url.getHost(); + if (host != null + && host.endsWith(".storage.googleapis.com") + && !host.equals("storage.googleapis.com")) { + return "/".equals(path) ? "/" : "/{object}"; + } + if (path.length() > 1 && path.endsWith("/")) { + path = path.substring(0, path.length() - 1); + } + if (path.startsWith("/upload/storage/v1/b/")) { + return "/upload/storage/v1/b/{bucket}/o"; + } + if (path.startsWith("/download/storage/v1/b/")) { + return "/download/storage/v1/b/{bucket}/o/{object}"; + } + if (path.startsWith("/batch/storage/v1")) { + return "/batch/storage/v1"; + } + if (path.startsWith("/storage/v1/")) { + return extractV1UrlTemplate(path.substring("/storage/v1/".length())); + } + String trimmed = path.startsWith("/") ? path.substring(1) : path; + if (trimmed.isEmpty()) { + return "/"; + } + String[] parts = trimmed.split("/", 2); + return parts.length == 1 ? "/{bucket}" : "/{bucket}/{object}"; + } + + private static String extractV1UrlTemplate(String rest) { + if ("b".equals(rest)) { + return "/storage/v1/b"; + } + String[] segments = rest.split("/"); + if (segments.length >= 2 && "b".equals(segments[0])) { + if (segments.length == 2) { + return "/storage/v1/b/{bucket}"; + } + if ("o".equals(segments[2])) { + return extractObjectUrlTemplate(segments); + } + if ("iam".equals(segments[2]) && segments.length > 3) { + return "/storage/v1/b/{bucket}/iam/" + segments[3]; + } + if (segments.length >= 4) { + if ("acl".equals(segments[2]) || "defaultObjectAcl".equals(segments[2])) { + return "/storage/v1/b/{bucket}/" + segments[2] + "/{entity}"; + } + if ("notificationConfigs".equals(segments[2])) { + return "/storage/v1/b/{bucket}/notificationConfigs/{notification}"; + } + } + return "/storage/v1/b/{bucket}/" + segments[2]; + } + if (segments.length >= 2 && "projects".equals(segments[0])) { + segments[1] = "{project}"; + if (segments.length >= 4 && "hmacKeys".equals(segments[2])) { + segments[3] = "{access_id}"; + } + return "/storage/v1/" + String.join("/", segments); + } + return "/storage/v1"; + } + + private static String extractObjectUrlTemplate(String[] segments) { + if (segments.length == 3) { + return "/storage/v1/b/{bucket}/o"; + } + if (segments.length == 4) { + return "/storage/v1/b/{bucket}/o/{object}"; + } + String action = segments[4]; + if ("rewriteTo".equals(action) + && segments.length >= 8 + && "b".equals(segments[5]) + && "o".equals(segments[7])) { + return "/storage/v1/b/{bucket}/o/{object}/rewriteTo/b/{bucket}/o/{object}"; + } + if ("moveTo".equals(action) && segments.length >= 7 && "o".equals(segments[5])) { + return "/storage/v1/b/{bucket}/o/{object}/moveTo/o/{object}"; + } + if ("acl".equals(action) && segments.length >= 6) { + return "/storage/v1/b/{bucket}/o/{object}/acl/{entity}"; + } + if (OBJECT_SUB_RESOURCES.contains(action)) { + return "/storage/v1/b/{bucket}/o/{object}/" + action; + } + return "/storage/v1/b/{bucket}/o/{object}"; + } +} diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageOptions.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageOptions.java index c9403850c698..ffbe17f71bc3 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageOptions.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageOptions.java @@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument; +import com.google.api.client.http.HttpRequestInitializer; import com.google.api.core.BetaApi; import com.google.api.core.InternalApi; import com.google.cloud.NoCredentials; @@ -402,6 +403,16 @@ synchronized void closeStorageClientMetrics() { } } + @InternalApi + @Nullable + public HttpRequestInitializer decorateWithMetrics(@Nullable HttpRequestInitializer initializer) { + StorageClientMetrics metrics = getStorageClientMetrics(); + if (metrics == null) { + return initializer; + } + return new StorageHttpMetricsInterceptor(initializer, metrics, getClock()); + } + /** * The metric export interval configured for periodic metric reading. * diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index 3e9c08a6a739..f1dd5b2594cf 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -159,7 +159,9 @@ public HttpStorageRpc(StorageOptions options, JsonFactory jsonFactory) { CensusHttpModule censusHttpModule = new CensusHttpModule(tracer, IS_RECORD_EVENTS); initializer = censusHttpModule.getHttpRequestInitializer(initializer); initializer = new InvocationIdInitializer(initializer, applicationName, tm); - batchRequestInitializer = censusHttpModule.getHttpRequestInitializer(null); + initializer = options.decorateWithMetrics(initializer); + batchRequestInitializer = + options.decorateWithMetrics(censusHttpModule.getHttpRequestInitializer(null)); String host = options.getHost(); Storage.Builder storageBuilder = new Storage.Builder(transport, jsonFactory, initializer) @@ -283,7 +285,7 @@ public void addDelete( StorageObject storageObject, RpcBatch.Callback callback, Map options) { try { if (currentBatchSize == MAX_BATCH_SIZE) { - batches.add(storage.batch()); + batches.add(storage.batch(batchRequestInitializer)); currentBatchSize = 0; } Delete call = deleteCall(storageObject, options); @@ -302,7 +304,7 @@ public void addPatch( Map options) { try { if (currentBatchSize == MAX_BATCH_SIZE) { - batches.add(storage.batch()); + batches.add(storage.batch(batchRequestInitializer)); currentBatchSize = 0; } Patch call = patchCall(storageObject, options); @@ -321,7 +323,7 @@ public void addGet( Map options) { try { if (currentBatchSize == MAX_BATCH_SIZE) { - batches.add(storage.batch()); + batches.add(storage.batch(batchRequestInitializer)); currentBatchSize = 0; } Get call = getCall(storageObject, options); diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageHttpMetricsInterceptorTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageHttpMetricsInterceptorTest.java new file mode 100644 index 000000000000..9499e9874dfa --- /dev/null +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageHttpMetricsInterceptorTest.java @@ -0,0 +1,542 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; + +import com.google.api.client.googleapis.batch.BatchCallback; +import com.google.api.client.googleapis.batch.BatchRequest; +import com.google.api.client.http.GenericUrl; +import com.google.api.client.http.HttpHeaders; +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.LowLevelHttpRequest; +import com.google.api.client.http.LowLevelHttpResponse; +import com.google.api.client.testing.http.MockHttpTransport; +import com.google.api.client.testing.http.MockLowLevelHttpRequest; +import com.google.api.client.testing.http.MockLowLevelHttpResponse; +import com.google.api.core.ApiClock; +import com.google.cloud.NoCredentials; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.data.HistogramPointData; +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import org.junit.Before; +import org.junit.Test; + +public final class StorageHttpMetricsInterceptorTest { + + private InMemoryMetricReader reader; + private FakeClock clock; + private HttpStorageOptions httpOptions; + private StorageClientMetrics metrics; + + @Before + public void setUp() { + reader = InMemoryMetricReader.create(); + SdkMeterProvider meterProvider = + SdkMeterProvider.builder().registerMetricReader(reader).build(); + clock = new FakeClock(); + httpOptions = + StorageOptions.http() + .setProjectId("test-project") + .setCredentials(NoCredentials.getInstance()) + .setClock(clock) + .setEnableOtelMetrics(true) + .setMeterProvider(meterProvider) + .build(); + metrics = httpOptions.getStorageClientMetrics(); + } + + @Test + public void extractUrlTemplate_cases() { + String[][] cases = { + {"https://storage.googleapis.com", "/"}, + {"https://storage.googleapis.com/", "/"}, + { + "https://storage.googleapis.com/upload/storage/v1/b/my-b/o", + "/upload/storage/v1/b/{bucket}/o" + }, + { + "https://storage.googleapis.com/download/storage/v1/b/my-b/o/my-obj", + "/download/storage/v1/b/{bucket}/o/{object}" + }, + {"https://storage.googleapis.com/batch/storage/v1", "/batch/storage/v1"}, + {"https://storage.googleapis.com/storage/v1/b", "/storage/v1/b"}, + {"https://storage.googleapis.com/storage/v1/b/my-b", "/storage/v1/b/{bucket}"}, + {"https://storage.googleapis.com/storage/v1/b/my-b/o", "/storage/v1/b/{bucket}/o"}, + { + "https://storage.googleapis.com/storage/v1/b/my-b/o/my-obj", + "/storage/v1/b/{bucket}/o/{object}" + }, + { + "https://storage.googleapis.com/storage/v1/b/my-b/o/dir/sub/file.txt", + "/storage/v1/b/{bucket}/o/{object}" + }, + { + "https://storage.googleapis.com/storage/v1/b/my-b/o/my-obj/compose", + "/storage/v1/b/{bucket}/o/{object}/compose" + }, + { + "https://storage.googleapis.com/storage/v1/b/my-b/o/my-obj/restore", + "/storage/v1/b/{bucket}/o/{object}/restore" + }, + { + "https://storage.googleapis.com/storage/v1/b/my-b/o/my-obj/acl", + "/storage/v1/b/{bucket}/o/{object}/acl" + }, + { + "https://storage.googleapis.com/storage/v1/b/my-b/o/my-obj/acl/allUsers", + "/storage/v1/b/{bucket}/o/{object}/acl/{entity}" + }, + { + "https://storage.googleapis.com/storage/v1/b/b1/o/o1/rewriteTo/b/b2/o/o2", + "/storage/v1/b/{bucket}/o/{object}/rewriteTo/b/{bucket}/o/{object}" + }, + { + "https://storage.googleapis.com/storage/v1/b/b1/o/o1/moveTo/o/o2", + "/storage/v1/b/{bucket}/o/{object}/moveTo/o/{object}" + }, + {"https://storage.googleapis.com/storage/v1/b/b1/acl", "/storage/v1/b/{bucket}/acl"}, + { + "https://storage.googleapis.com/storage/v1/b/b1/acl/allUsers", + "/storage/v1/b/{bucket}/acl/{entity}" + }, + { + "https://storage.googleapis.com/storage/v1/b/b1/defaultObjectAcl", + "/storage/v1/b/{bucket}/defaultObjectAcl" + }, + { + "https://storage.googleapis.com/storage/v1/b/b1/defaultObjectAcl/allUsers", + "/storage/v1/b/{bucket}/defaultObjectAcl/{entity}" + }, + { + "https://storage.googleapis.com/storage/v1/b/b1/notificationConfigs", + "/storage/v1/b/{bucket}/notificationConfigs" + }, + { + "https://storage.googleapis.com/storage/v1/b/b1/notificationConfigs/notif1", + "/storage/v1/b/{bucket}/notificationConfigs/{notification}" + }, + {"https://storage.googleapis.com/storage/v1/b/b1/iam", "/storage/v1/b/{bucket}/iam"}, + { + "https://storage.googleapis.com/storage/v1/b/b1/iam/testPermissions", + "/storage/v1/b/{bucket}/iam/testPermissions" + }, + { + "https://storage.googleapis.com/storage/v1/projects/my-p/hmacKeys/key1", + "/storage/v1/projects/{project}/hmacKeys/{access_id}" + }, + { + "https://storage.googleapis.com/storage/v1/projects/my-p/serviceAccount", + "/storage/v1/projects/{project}/serviceAccount" + }, + {"https://storage.googleapis.com/my-b", "/{bucket}"}, + {"https://storage.googleapis.com/my-b/my-obj", "/{bucket}/{object}"}, + {"https://my-bucket.storage.googleapis.com/", "/"}, + {"https://my-bucket.storage.googleapis.com/my-obj", "/{object}"}, + {"https://my-bucket.storage.googleapis.com/dir/my-obj", "/{object}"}, + }; + + for (String[] testCase : cases) { + String url = testCase[0]; + String expectedTemplate = testCase[1]; + assertWithMessage(url) + .that(StorageHttpMetricsInterceptor.extractUrlTemplate(new GenericUrl(url))) + .isEqualTo(expectedTemplate); + } + } + + @Test + public void initialize_statusCodes_recordsDurationAndErrorType() throws IOException { + Map cases = + ImmutableMap.of( + 200, "OK", + 308, "OK", + 503, "UNAVAILABLE"); + + for (int statusCode : cases.keySet()) { + AtomicBoolean delegateCalled = new AtomicBoolean(false); + AtomicBoolean execCalled = new AtomicBoolean(false); + AtomicBoolean respCalled = new AtomicBoolean(false); + HttpRequestInitializer delegate = + request -> { + delegateCalled.set(true); + request.setInterceptor(req -> execCalled.set(true)); + request.setResponseInterceptor( + resp -> { + clock.advanceNanos(50_000_000L); + respCalled.set(true); + }); + }; + HttpRequestInitializer interceptor = httpOptions.decorateWithMetrics(delegate); + + HttpTransport transport = createAdvancingTransport(statusCode, 250_000_000L); + HttpRequest request = + transport + .createRequestFactory(interceptor) + .buildGetRequest( + new GenericUrl("https://storage.googleapis.com/storage/v1/b/my-b/o/my-obj")); + if (statusCode != 200) { + request.setThrowExceptionOnExecuteError(false); + } + request.execute(); + + assertWithMessage("status=%s", statusCode).that(delegateCalled.get()).isTrue(); + assertWithMessage("status=%s", statusCode).that(execCalled.get()).isTrue(); + assertWithMessage("status=%s", statusCode).that(respCalled.get()).isTrue(); + } + + List points = durationPoints(); + assertThat(points).hasSize(cases.size()); + for (Map.Entry entry : cases.entrySet()) { + long statusCode = entry.getKey(); + HistogramPointData point = findPointByStatus(points, statusCode); + assertWithMessage("status=%s", statusCode).that(point.getCount()).isEqualTo(1L); + assertWithMessage("status=%s", statusCode).that(point.getSum()).isWithin(1e-6).of(0.25); + + Attributes attrs = point.getAttributes(); + assertThat(attrs.get(StorageClientMetrics.KEY_GCP_CLIENT_SERVICE)) + .isEqualTo(StorageClientMetrics.SERVICE_STORAGE); + assertThat(attrs.get(StorageClientMetrics.KEY_GCP_CLIENT_VERSION)) + .isEqualTo(StorageOptions.version()); + assertThat(attrs.get(StorageClientMetrics.KEY_GCP_CLIENT_ARTIFACT)) + .isEqualTo(StorageClientMetrics.ARTIFACT_STORAGE); + assertThat(attrs.get(StorageClientMetrics.KEY_RPC_SYSTEM_NAME)).isEqualTo("http"); + assertThat(attrs.get(StorageClientMetrics.KEY_HTTP_REQUEST_METHOD)).isEqualTo("GET"); + assertThat(attrs.get(StorageClientMetrics.KEY_HTTP_RESPONSE_STATUS_CODE)) + .isEqualTo(statusCode); + assertThat(attrs.get(StorageClientMetrics.KEY_NETWORK_PATH)) + .isEqualTo(StorageClientMetrics.NETWORK_PATH_CLOUDPATH); + assertThat(attrs.get(StorageClientMetrics.KEY_URL_TEMPLATE)) + .isEqualTo("/storage/v1/b/{bucket}/o/{object}"); + assertThat(attrs.get(StorageClientMetrics.KEY_SERVER_ADDRESS)) + .isEqualTo("storage.googleapis.com"); + assertWithMessage("status=%s", statusCode) + .that(attrs.get(StorageClientMetrics.KEY_ERROR_TYPE)) + .isEqualTo(entry.getValue()); + } + } + + @Test + public void initialize_unsuccessfulResponseHandlerReturnsFalseOn404_recordsOnce() + throws IOException { + AtomicBoolean handlerCalled = new AtomicBoolean(false); + HttpRequestInitializer delegate = + request -> + request.setUnsuccessfulResponseHandler( + (req, response, supportsRetry) -> { + handlerCalled.set(true); + return false; + }); + StorageHttpMetricsInterceptor interceptor = + new StorageHttpMetricsInterceptor(delegate, metrics, clock); + + HttpTransport transport = createAdvancingTransport(404, 80_000_000L); + HttpRequest request = + transport + .createRequestFactory(interceptor) + .buildGetRequest( + new GenericUrl("https://storage.googleapis.com/storage/v1/b/missing-bucket")); + request.setThrowExceptionOnExecuteError(false); + request.execute(); + + assertThat(handlerCalled.get()).isTrue(); + + List points = durationPoints(); + assertThat(points).hasSize(1); + HistogramPointData point = points.get(0); + assertThat(point.getCount()).isEqualTo(1L); + assertThat(point.getSum()).isWithin(1e-6).of(0.08); + Attributes attrs = point.getAttributes(); + assertThat(attrs.get(StorageClientMetrics.KEY_RPC_SYSTEM_NAME)).isEqualTo("http"); + assertThat(attrs.get(StorageClientMetrics.KEY_HTTP_RESPONSE_STATUS_CODE)).isEqualTo(404L); + assertThat(attrs.get(StorageClientMetrics.KEY_ERROR_TYPE)).isEqualTo("NOT_FOUND"); + assertThat(attrs.get(StorageClientMetrics.KEY_SERVER_ADDRESS)) + .isEqualTo("storage.googleapis.com"); + } + + @Test + public void initialize_ioExceptionThenRetrySucceeds_recordsTwoPoints() throws IOException { + AtomicBoolean firstAttempt = new AtomicBoolean(true); + HttpRequestInitializer retryDelegate = + request -> request.setIOExceptionHandler((req, supportsRetry) -> true); + StorageHttpMetricsInterceptor interceptor = + new StorageHttpMetricsInterceptor(retryDelegate, metrics, clock); + + HttpTransport transport = + new MockHttpTransport() { + @Override + public LowLevelHttpRequest buildRequest(String method, String url) { + return new MockLowLevelHttpRequest() { + @Override + public LowLevelHttpResponse execute() throws IOException { + if (firstAttempt.compareAndSet(true, false)) { + clock.advanceNanos(100_000_000L); + throw new IOException("Connection reset by peer"); + } + clock.advanceNanos(200_000_000L); + MockLowLevelHttpResponse response = new MockLowLevelHttpResponse(); + response.setStatusCode(200); + return response; + } + }; + } + }; + + HttpRequest req = + transport + .createRequestFactory(interceptor) + .buildGetRequest(new GenericUrl("https://storage.googleapis.com/storage/v1/b/my-b")); + req.execute(); + + List points = durationPoints(); + assertThat(points).hasSize(2); + + HistogramPointData errorPoint = findPointByStatus(points, 0L); + assertThat(errorPoint.getSum()).isWithin(1e-6).of(0.10); + assertThat(errorPoint.getAttributes().get(StorageClientMetrics.KEY_RPC_SYSTEM_NAME)) + .isEqualTo("http"); + assertThat(errorPoint.getAttributes().get(StorageClientMetrics.KEY_SERVER_ADDRESS)) + .isEqualTo("storage.googleapis.com"); + assertThat(errorPoint.getAttributes().get(StorageClientMetrics.KEY_ERROR_TYPE)) + .isEqualTo(StorageErrorTypeMapper.STATUS_CONNECTION_ERROR); + + HistogramPointData okPoint = findPointByStatus(points, 200L); + assertThat(okPoint.getSum()).isWithin(1e-6).of(0.20); + assertThat(okPoint.getAttributes().get(StorageClientMetrics.KEY_ERROR_TYPE)) + .isEqualTo(StorageErrorTypeMapper.STATUS_OK); + } + + @Test + public void initialize_401RefreshRetryThen200_recordsBothAttempts() throws IOException { + AtomicBoolean firstAttempt = new AtomicBoolean(true); + HttpRequestInitializer refreshDelegate = + request -> + request.setUnsuccessfulResponseHandler( + (req, response, supportsRetry) -> { + if (response.getStatusCode() == 401) { + // Simulate token refresh latency inside the UnsuccessfulResponseHandler; + // the 401 request duration must be captured before this handler runs. + clock.advanceNanos(200_000_000L); + return true; + } + return false; + }); + StorageHttpMetricsInterceptor interceptor = + new StorageHttpMetricsInterceptor(refreshDelegate, metrics, clock); + + HttpTransport transport = + new MockHttpTransport() { + @Override + public LowLevelHttpRequest buildRequest(String method, String url) { + return new MockLowLevelHttpRequest() { + @Override + public LowLevelHttpResponse execute() { + MockLowLevelHttpResponse response = new MockLowLevelHttpResponse(); + if (firstAttempt.compareAndSet(true, false)) { + clock.advanceNanos(90_000_000L); + response.setStatusCode(401); + } else { + clock.advanceNanos(180_000_000L); + response.setStatusCode(200); + } + return response; + } + }; + } + }; + + HttpRequest req = + transport + .createRequestFactory(interceptor) + .buildGetRequest(new GenericUrl("https://storage.googleapis.com/storage/v1/b/my-b")); + req.execute(); + + List points = durationPoints(); + assertThat(points).hasSize(2); + + HistogramPointData unauthorizedPoint = findPointByStatus(points, 401L); + assertThat(unauthorizedPoint.getSum()).isWithin(1e-6).of(0.09); + assertThat(unauthorizedPoint.getAttributes().get(StorageClientMetrics.KEY_ERROR_TYPE)) + .isEqualTo("UNAUTHENTICATED"); + + HistogramPointData okPoint = findPointByStatus(points, 200L); + assertThat(okPoint.getSum()).isWithin(1e-6).of(0.18); + assertThat(okPoint.getAttributes().get(StorageClientMetrics.KEY_ERROR_TYPE)) + .isEqualTo(StorageErrorTypeMapper.STATUS_OK); + } + + @Test + public void initialize_batchPart401_doesNotRecordPartDuration() throws IOException { + AtomicBoolean refreshHandlerCalled = new AtomicBoolean(false); + AtomicBoolean partSucceeded = new AtomicBoolean(false); + AtomicBoolean firstBatch = new AtomicBoolean(true); + + HttpRequestInitializer refreshDelegate = + request -> + request.setUnsuccessfulResponseHandler( + (req, response, supportsRetry) -> { + if (response.getStatusCode() == 401) { + refreshHandlerCalled.set(true); + return true; + } + return false; + }); + StorageHttpMetricsInterceptor partInterceptor = + new StorageHttpMetricsInterceptor(refreshDelegate, metrics, clock); + StorageHttpMetricsInterceptor batchInterceptor = + new StorageHttpMetricsInterceptor(null, metrics, clock); + + HttpTransport transport = + new MockHttpTransport() { + @Override + public LowLevelHttpRequest buildRequest(String method, String url) { + return new MockLowLevelHttpRequest() { + @Override + public LowLevelHttpResponse execute() { + clock.advanceNanos(100_000_000L); + String partStatusLine = + firstBatch.compareAndSet(true, false) + ? "HTTP/1.1 401 Unauthorized" + : "HTTP/1.1 200 OK"; + String body = + "--batch_boundary\r\n" + + "Content-Type: application/http\r\n" + + "\r\n" + + partStatusLine + + "\r\n" + + "\r\n" + + "--batch_boundary--\r\n"; + MockLowLevelHttpResponse response = new MockLowLevelHttpResponse(); + response.setStatusCode(200); + response.setContentType("multipart/mixed; boundary=batch_boundary"); + response.setContent(body); + return response; + } + }; + } + }; + + HttpRequest partRequest = + transport + .createRequestFactory(partInterceptor) + .buildGetRequest( + new GenericUrl("https://storage.googleapis.com/storage/v1/b/my-b/o/my-obj")); + + BatchRequest batchRequest = new BatchRequest(transport, batchInterceptor); + batchRequest.setBatchUrl(new GenericUrl("https://storage.googleapis.com/batch/storage/v1")); + batchRequest.setSleeper(millis -> {}); + batchRequest.queue( + partRequest, + Void.class, + Void.class, + new BatchCallback() { + @Override + public void onSuccess(Void t, HttpHeaders responseHeaders) { + partSucceeded.set(true); + } + + @Override + public void onFailure(Void e, HttpHeaders responseHeaders) { + throw new AssertionError("Expected retried batch part to succeed"); + } + }); + + batchRequest.execute(); + + assertThat(refreshHandlerCalled.get()).isTrue(); + assertThat(partSucceeded.get()).isTrue(); + + List points = durationPoints(); + assertThat(points).hasSize(1); + HistogramPointData point = points.get(0); + assertThat(point.getCount()).isEqualTo(2L); + assertThat(point.getSum()).isWithin(1e-6).of(0.20); + Attributes attrs = point.getAttributes(); + assertThat(attrs.get(StorageClientMetrics.KEY_URL_TEMPLATE)).isEqualTo("/batch/storage/v1"); + assertThat(attrs.get(StorageClientMetrics.KEY_HTTP_REQUEST_METHOD)).isEqualTo("POST"); + assertThat(attrs.get(StorageClientMetrics.KEY_HTTP_RESPONSE_STATUS_CODE)).isEqualTo(200L); + assertThat(attrs.get(StorageClientMetrics.KEY_ERROR_TYPE)) + .isEqualTo(StorageErrorTypeMapper.STATUS_OK); + } + + private List durationPoints() { + return reader.collectAllMetrics().stream() + .filter(m -> m.getName().equals(StorageClientMetrics.METRIC_HTTP_CLIENT_REQUEST_DURATION)) + .findFirst() + .map(m -> ImmutableList.copyOf(m.getHistogramData().getPoints())) + .orElse(ImmutableList.of()); + } + + private static HistogramPointData findPointByStatus( + List points, long statusCode) { + return points.stream() + .filter( + p -> + Objects.equals( + p.getAttributes().get(StorageClientMetrics.KEY_HTTP_RESPONSE_STATUS_CODE), + statusCode)) + .findFirst() + .orElseThrow(() -> new AssertionError("No point for status " + statusCode)); + } + + private HttpTransport createAdvancingTransport(int statusCode, long advanceNanos) { + return new MockHttpTransport() { + @Override + public LowLevelHttpRequest buildRequest(String method, String url) { + return new MockLowLevelHttpRequest() { + @Override + public LowLevelHttpResponse execute() { + clock.advanceNanos(advanceNanos); + MockLowLevelHttpResponse response = new MockLowLevelHttpResponse(); + response.setStatusCode(statusCode); + return response; + } + }; + } + }; + } + + private static final class FakeClock implements ApiClock { + private final AtomicLong nanos = new AtomicLong(1_000_000_000L); + + @Override + public long nanoTime() { + return nanos.get(); + } + + @Override + public long millisTime() { + return nanos.get() / 1_000_000L; + } + + void advanceNanos(long amount) { + nanos.addAndGet(amount); + } + } +}