diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageClientMetrics.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageClientMetrics.java index 2f8c326b909d..b5b7a5ab45e9 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageClientMetrics.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageClientMetrics.java @@ -489,6 +489,19 @@ void recordHttpRequestDuration( httpClientRequestDuration.record(durationSeconds, attributes, Context.current()); } + void recordTtfb( + Context otelContext, String operationName, double ttfbSeconds, String networkPath) { + if (ttfb == null) { + return; + } + Attributes attributes = + baseAttributes.toBuilder() + .put(KEY_GCP_STORAGE_OPERATION, operationName) + .put(KEY_NETWORK_PATH, networkPath) + .build(); + ttfb.record(ttfbSeconds, attributes, otelContext); + } + @Nullable static StorageMetricsContext createContext( @Nullable StorageClientMetrics metrics, String operationName) { diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageGrpcMetricsInterceptor.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageGrpcMetricsInterceptor.java index b33f1f1a4300..f95b2ac7441a 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageGrpcMetricsInterceptor.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageGrpcMetricsInterceptor.java @@ -19,6 +19,8 @@ import static java.util.Objects.requireNonNull; import com.google.api.core.ApiClock; +import com.google.storage.v2.BidiWriteObjectRequest; +import com.google.storage.v2.WriteObjectRequest; import io.grpc.Attributes; import io.grpc.CallOptions; import io.grpc.Channel; @@ -34,6 +36,8 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.SocketAddress; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; import org.jspecify.annotations.NullMarked; import org.jspecify.annotations.Nullable; @@ -58,11 +62,19 @@ public ClientCall interceptCall( String rpcMethod = method.getFullMethodName(); StorageMetricsContext metricsCtx = StorageMetricsContext.from(callOptions); Context otelContext = metricsCtx != null ? metricsCtx.getOtelContext() : Context.current(); + String operationName = + metricsCtx != null + ? metricsCtx.getOperationName() + : defaultOperationName(method.getBareMethodName()); return new SimpleForwardingClientCall(next.newCall(method, callOptions)) { + private final AtomicBoolean ttfbRecorded = new AtomicBoolean(false); + private final AtomicLong ttfbStartNanos = new AtomicLong(); + @Override public void start(Listener responseListener, Metadata headers) { long startTimeNanos = clock.nanoTime(); + ttfbStartNanos.set(startTimeNanos); super.start( new SimpleForwardingClientCallListener(responseListener) { @@ -74,6 +86,18 @@ public void onHeaders(Metadata headers) { super.onHeaders(headers); } + @Override + public void onMessage(@Nullable RespT message) { + // The first response message from the server (including metadata-only + // responses for 0-byte objects) marks Time To First Byte (TTFB). + if (ttfbRecorded.compareAndSet(false, true)) { + double ttfbSeconds = (clock.nanoTime() - ttfbStartNanos.get()) / 1_000_000_000.0; + String networkPath = detectNetworkPath(getAttributes()); + metrics.recordTtfb(otelContext, operationName, ttfbSeconds, networkPath); + } + super.onMessage(message); + } + @Override public void onClose(Status status, Metadata trailers) { try { @@ -94,9 +118,44 @@ public void onClose(Status status, Metadata trailers) { }, headers); } + + @Override + public void sendMessage(ReqT message) { + if (!ttfbRecorded.get() && isWriteFlush(message)) { + ttfbStartNanos.set(clock.nanoTime()); + } + super.sendMessage(message); + } }; } + static String defaultOperationName(@Nullable String bareMethod) { + if (bareMethod == null || bareMethod.isEmpty()) { + return "UNKNOWN"; + } + switch (bareMethod) { + case "BidiReadObject": + return "ReadObject"; + case "BidiWriteObject": + case "StartResumableWrite": + case "QueryWriteStatus": + return "WriteObject"; + default: + return bareMethod; + } + } + + static boolean isWriteFlush(@Nullable Object message) { + if (message instanceof BidiWriteObjectRequest) { + BidiWriteObjectRequest req = (BidiWriteObjectRequest) message; + return req.getFlush() || req.getFinishWrite(); + } + if (message instanceof WriteObjectRequest) { + return ((WriteObjectRequest) message).getFinishWrite(); + } + return false; + } + static String detectNetworkPath(@Nullable Attributes callAttributes) { SocketAddress remoteAddr = callAttributes != null ? callAttributes.get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR) : null; diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageHttpMetricsInterceptor.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageHttpMetricsInterceptor.java index 9aed94d4bb48..39d682c86dca 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageHttpMetricsInterceptor.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageHttpMetricsInterceptor.java @@ -27,6 +27,7 @@ import com.google.api.client.http.HttpUnsuccessfulResponseHandler; import com.google.api.core.ApiClock; import com.google.common.collect.ImmutableSet; +import io.opentelemetry.context.Context; import java.io.IOException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -42,6 +43,10 @@ * recorded here end at response headers and exclude streaming body transfer (a follow-up is tracked * for recording when the response body is closed). End-to-end download duration including body * transfer is captured by {@code gcp.client.request.duration}. + * + *

{@code gcp.storage.client.ttfb} is recorded here for media download requests across all HTTP + * response status codes when response headers arrive (HTTP unary and upload TTFB are tracked for a + * follow-up). */ @NullMarked final class StorageHttpMetricsInterceptor implements HttpRequestInitializer { @@ -132,6 +137,16 @@ private void recordResponse(HttpRequest request, long startNanos, long endNanos, ? StorageErrorTypeMapper.mapHttpResponseCode(statusCode) : StorageErrorTypeMapper.STATUS_OK; recordDuration(request, startNanos, endNanos, statusCode, errorType); + // Evaluate isMediaDownloadRequest(request) at response time rather than in + // initialize(HttpRequest), because callers such as MediaHttpDownloader may append + // query parameters (e.g. alt=media) to request.getUrl() after buildGetRequest(). + if (isMediaDownloadRequest(request)) { + metrics.recordTtfb( + Context.current(), + "ReadObject", + (endNanos - startNanos) / 1_000_000_000.0, + StorageClientMetrics.NETWORK_PATH_CLOUDPATH); + } } private void recordDuration( @@ -145,6 +160,16 @@ private void recordDuration( errorType); } + static boolean isMediaDownloadRequest(HttpRequest request) { + if (!"GET".equalsIgnoreCase(request.getRequestMethod())) { + return false; + } + GenericUrl url = request.getUrl(); + String path = url.getRawPath(); + return (path != null && path.startsWith("/download/storage/v1/")) + || "media".equals(url.getFirst("alt")); + } + static String extractUrlTemplate(GenericUrl url) { String path = url.getRawPath(); if (path == null || path.isEmpty()) { diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageGrpcMetricsInterceptorTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageGrpcMetricsInterceptorTest.java index fe0217406cb6..89894fb177bb 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageGrpcMetricsInterceptorTest.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageGrpcMetricsInterceptorTest.java @@ -23,6 +23,11 @@ import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; import com.google.cloud.NoCredentials; import com.google.common.collect.ImmutableList; +import com.google.storage.v2.BidiWriteObjectRequest; +import com.google.storage.v2.BidiWriteObjectResponse; +import com.google.storage.v2.ReadObjectRequest; +import com.google.storage.v2.StorageGrpc; +import com.google.storage.v2.WriteObjectRequest; import io.grpc.Attributes; import io.grpc.CallOptions; import io.grpc.Channel; @@ -50,11 +55,12 @@ import java.lang.reflect.Field; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; import org.jspecify.annotations.Nullable; import org.junit.Before; import org.junit.Test; @@ -161,6 +167,52 @@ public void detectNetworkPath_cases() throws Exception { .isEqualTo(StorageClientMetrics.NETWORK_PATH_UNKNOWN); } + @Test + public void defaultOperationName_cases() { + assertThat(StorageGrpcMetricsInterceptor.defaultOperationName("BidiReadObject")) + .isEqualTo("ReadObject"); + assertThat(StorageGrpcMetricsInterceptor.defaultOperationName("BidiWriteObject")) + .isEqualTo("WriteObject"); + assertThat(StorageGrpcMetricsInterceptor.defaultOperationName("StartResumableWrite")) + .isEqualTo("WriteObject"); + assertThat(StorageGrpcMetricsInterceptor.defaultOperationName("QueryWriteStatus")) + .isEqualTo("WriteObject"); + assertThat(StorageGrpcMetricsInterceptor.defaultOperationName("ReadObject")) + .isEqualTo("ReadObject"); + assertThat(StorageGrpcMetricsInterceptor.defaultOperationName("GetBucket")) + .isEqualTo("GetBucket"); + assertThat(StorageGrpcMetricsInterceptor.defaultOperationName("")).isEqualTo("UNKNOWN"); + assertThat(StorageGrpcMetricsInterceptor.defaultOperationName(null)).isEqualTo("UNKNOWN"); + } + + @Test + public void isWriteFlush_cases() { + assertThat( + StorageGrpcMetricsInterceptor.isWriteFlush( + BidiWriteObjectRequest.newBuilder().setFlush(true).build())) + .isTrue(); + assertThat( + StorageGrpcMetricsInterceptor.isWriteFlush( + BidiWriteObjectRequest.newBuilder().setFinishWrite(true).build())) + .isTrue(); + assertThat( + StorageGrpcMetricsInterceptor.isWriteFlush( + BidiWriteObjectRequest.newBuilder().setStateLookup(true).build())) + .isFalse(); + assertThat( + StorageGrpcMetricsInterceptor.isWriteFlush(BidiWriteObjectRequest.getDefaultInstance())) + .isFalse(); + assertThat( + StorageGrpcMetricsInterceptor.isWriteFlush( + WriteObjectRequest.newBuilder().setFinishWrite(true).build())) + .isTrue(); + assertThat(StorageGrpcMetricsInterceptor.isWriteFlush(WriteObjectRequest.getDefaultInstance())) + .isFalse(); + assertThat(StorageGrpcMetricsInterceptor.isWriteFlush(ReadObjectRequest.getDefaultInstance())) + .isFalse(); + assertThat(StorageGrpcMetricsInterceptor.isWriteFlush(null)).isFalse(); + } + @Test public void interceptCall_unaryCallSuccess_recordsMetricWithErrorTypeOk() throws Exception { Attributes callAttrs = @@ -177,7 +229,11 @@ public void interceptCall_unaryCallSuccess_recordsMetricWithErrorTypeOk() throws TestListener listener = new TestListener<>(); call.start(listener, new Metadata()); - clock.advanceNanos(500_000_000L); // 0.5s + clock.advanceNanos(200_000_000L); // 0.2s: first response message triggers TTFB + testCall.deliverMessage(null); + clock.advanceNanos(150_000_000L); // 0.35s: second message must not record TTFB again + testCall.deliverMessage(null); + clock.advanceNanos(150_000_000L); // 0.5s total duration testCall.close(Status.OK, new Metadata()); MetricData rpcDurationData = findMetric(StorageClientMetrics.METRIC_RPC_CLIENT_CALL_DURATION); @@ -198,6 +254,65 @@ public void interceptCall_unaryCallSuccess_recordsMetricWithErrorTypeOk() throws assertThat(attrs.get(StorageClientMetrics.KEY_SERVER_ADDRESS)) .isEqualTo("storage.googleapis.com"); assertThat(attrs.get(StorageClientMetrics.KEY_ERROR_TYPE)).isEqualTo("OK"); + + MetricData ttfbData = findMetric(StorageClientMetrics.METRIC_GCP_STORAGE_CLIENT_TTFB); + assertThat(ttfbData).isNotNull(); + assertThat(ttfbData.getHistogramData().getPoints()).hasSize(1); + HistogramPointData ttfbPoint = ttfbData.getHistogramData().getPoints().iterator().next(); + assertThat(ttfbPoint.getCount()).isEqualTo(1L); + assertThat(ttfbPoint.getSum()).isWithin(1e-6).of(0.2); + assertThat(ttfbPoint.getAttributes()) + .isEqualTo( + io.opentelemetry.api.common.Attributes.builder() + .put( + StorageClientMetrics.KEY_GCP_CLIENT_SERVICE, + StorageClientMetrics.SERVICE_STORAGE) + .put(StorageClientMetrics.KEY_GCP_CLIENT_VERSION, StorageOptions.version()) + .put( + StorageClientMetrics.KEY_GCP_CLIENT_ARTIFACT, + StorageClientMetrics.ARTIFACT_STORAGE) + .put(StorageClientMetrics.KEY_RPC_SYSTEM_NAME, "grpc") + .put(StorageClientMetrics.KEY_GCP_STORAGE_OPERATION, "ReadObject") + .put( + StorageClientMetrics.KEY_NETWORK_PATH, + StorageClientMetrics.NETWORK_PATH_CLOUDPATH) + .put(StorageClientMetrics.KEY_SERVER_ADDRESS, "storage.googleapis.com") + .build()); + } + + @Test + public void interceptCall_bidiWriteObject_recordsTtfbFromFlush() { + TestClientCall writeCall = + new TestClientCall<>(Attributes.EMPTY); + TestChannel writeChannel = new TestChannel(writeCall); + + ClientCall call = + interceptor.interceptCall( + StorageGrpc.getBidiWriteObjectMethod(), CallOptions.DEFAULT, writeChannel); + call.start(new TestListener<>(), new Metadata()); + + clock.advanceNanos(100_000_000L); // t = 0.10s: flush re-anchors ttfbStartNanos + call.sendMessage(BidiWriteObjectRequest.newBuilder().setFlush(true).build()); + + clock.advanceNanos(100_000_000L); // t = 0.20s: trailing non-flush chunk must not re-anchor + call.sendMessage(BidiWriteObjectRequest.getDefaultInstance()); + + clock.advanceNanos(150_000_000L); // t = 0.35s: first server ack -> TTFB = 0.35 - 0.10 = 0.25s + writeCall.deliverMessage(BidiWriteObjectResponse.getDefaultInstance()); + writeCall.close(Status.OK, new Metadata()); + + MetricData ttfbData = findMetric(StorageClientMetrics.METRIC_GCP_STORAGE_CLIENT_TTFB); + assertThat(ttfbData).isNotNull(); + assertThat(ttfbData.getHistogramData().getPoints()).hasSize(1); + HistogramPointData ttfbPoint = ttfbData.getHistogramData().getPoints().iterator().next(); + assertThat(ttfbPoint.getCount()).isEqualTo(1L); + assertThat(ttfbPoint.getSum()).isWithin(1e-6).of(0.25); + assertThat(ttfbPoint.getAttributes().get(StorageClientMetrics.KEY_GCP_STORAGE_OPERATION)) + .isEqualTo("WriteObject"); + assertThat(ttfbPoint.getAttributes().get(StorageClientMetrics.KEY_RPC_SYSTEM_NAME)) + .isEqualTo("grpc"); + assertThat(ttfbPoint.getAttributes().get(StorageClientMetrics.KEY_SERVER_ADDRESS)) + .isEqualTo("storage.googleapis.com"); } @Test @@ -222,12 +337,14 @@ public void interceptCall_callFails_recordsErrorType() { .isEqualTo("NOT_FOUND"); assertThat(point.getAttributes().get(StorageClientMetrics.KEY_SERVER_ADDRESS)) .isEqualTo("storage.googleapis.com"); + + assertThat(findMetric(StorageClientMetrics.METRIC_GCP_STORAGE_CLIENT_TTFB)).isNull(); } @Test public void interceptCall_withStorageMetricsContext_propagatesContextAndFeedsNetworkPath() throws Exception { - StorageMetricsContext metricsContext = metrics.createContext("ReadObject"); + StorageMetricsContext metricsContext = metrics.createContext("CustomOperation"); CallOptions callOptions = metricsContext.attach(CallOptions.DEFAULT); Attributes directPathAttrs = @@ -242,6 +359,8 @@ public void interceptCall_withStorageMetricsContext_propagatesContextAndFeedsNet ClientCall call = interceptor.interceptCall(READ_OBJECT_METHOD, callOptions, testChannel); call.start(new TestListener<>(), new Metadata()); + clock.advanceNanos(150_000_000L); + testCall.deliverMessage(null); testCall.close(Status.OK, new Metadata()); metricsContext.recordAttemptSuccess(metrics); @@ -254,6 +373,16 @@ public void interceptCall_withStorageMetricsContext_propagatesContextAndFeedsNet assertThat(durationPoint.getAttributes().get(StorageClientMetrics.KEY_NETWORK_PATH)) .isEqualTo(StorageClientMetrics.NETWORK_PATH_DIRECTPATH); + MetricData ttfbData = findMetric(StorageClientMetrics.METRIC_GCP_STORAGE_CLIENT_TTFB); + assertThat(ttfbData).isNotNull(); + assertThat(ttfbData.getHistogramData().getPoints()).hasSize(1); + HistogramPointData ttfbPoint = ttfbData.getHistogramData().getPoints().iterator().next(); + assertThat(ttfbPoint.getSum()).isWithin(1e-6).of(0.15); + assertThat(ttfbPoint.getAttributes().get(StorageClientMetrics.KEY_GCP_STORAGE_OPERATION)) + .isEqualTo("CustomOperation"); + assertThat(ttfbPoint.getAttributes().get(StorageClientMetrics.KEY_NETWORK_PATH)) + .isEqualTo(StorageClientMetrics.NETWORK_PATH_DIRECTPATH); + MetricData attemptsData = findMetric(StorageClientMetrics.METRIC_GCP_STORAGE_CLIENT_ATTEMPTS); assertThat(attemptsData).isNotNull(); assertThat( @@ -311,11 +440,11 @@ public void interceptCall_withoutStorageMetricsContext_asyncOnCloseUsesCallerCon Context callerContext = Context.root().with(testKey, "caller-trace-value"); Context callbackThreadContext = Context.root().with(testKey, "callback-thread-value"); - AtomicReference recordedContext = new AtomicReference<>(); + List recordedContexts = Collections.synchronizedList(new ArrayList<>()); DoubleHistogram mockHistogram = Mockito.mock(DoubleHistogram.class); Mockito.doAnswer( invocation -> { - recordedContext.set(invocation.getArgument(2)); + recordedContexts.add(invocation.getArgument(2)); return null; }) .when(mockHistogram) @@ -364,6 +493,7 @@ public void interceptCall_withoutStorageMetricsContext_asyncOnCloseUsesCallerCon new Thread( () -> { try (Scope ignored = callbackThreadContext.makeCurrent()) { + testCall.deliverMessage(null); testCall.close(Status.OK, new Metadata()); } finally { latch.countDown(); @@ -372,8 +502,11 @@ public void interceptCall_withoutStorageMetricsContext_asyncOnCloseUsesCallerCon asyncCallbackThread.start(); assertThat(latch.await(5, TimeUnit.SECONDS)).isTrue(); - assertThat(recordedContext.get()).isSameInstanceAs(callerContext); - assertThat(recordedContext.get().get(testKey)).isEqualTo("caller-trace-value"); + assertThat(recordedContexts).hasSize(2); + for (Context ctx : recordedContexts) { + assertThat(ctx).isSameInstanceAs(callerContext); + assertThat(ctx.get(testKey)).isEqualTo("caller-trace-value"); + } } @Test @@ -516,6 +649,12 @@ void receiveHeaders(Metadata headers) { } } + void deliverMessage(@Nullable RespT message) { + if (listener != null) { + listener.onMessage(message); + } + } + void close(Status status, Metadata trailers) { if (listener != null) { listener.onClose(status, trailers); diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageHttpMetricsInterceptorTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageHttpMetricsInterceptorTest.java index 9499e9874dfa..5570101e51e4 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageHttpMetricsInterceptorTest.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageHttpMetricsInterceptorTest.java @@ -24,6 +24,7 @@ import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpRequestFactory; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.http.LowLevelHttpRequest; @@ -171,13 +172,36 @@ public void extractUrlTemplate_cases() { } } + @Test + public void isMediaDownloadRequest_cases() throws IOException { + HttpRequestFactory factory = new MockHttpTransport().createRequestFactory(); + Object[][] cases = { + {"GET", "https://storage.googleapis.com/download/storage/v1/b/my-b/o/my-obj", true}, + {"GET", "https://storage.googleapis.com/storage/v1/b/my-b/o/my-obj?alt=media", true}, + {"GET", "https://storage.googleapis.com/storage/v1/b/my-b/o/my-obj?alt=json", false}, + {"GET", "https://storage.googleapis.com/storage/v1/b/my-b/o/my-obj", false}, + {"GET", "https://storage.googleapis.com", false}, + {"POST", "https://storage.googleapis.com/download/storage/v1/b/my-b/o/my-obj", false}, + }; + + for (Object[] testCase : cases) { + String method = (String) testCase[0]; + String url = (String) testCase[1]; + boolean expected = (boolean) testCase[2]; + HttpRequest request = factory.buildRequest(method, new GenericUrl(url), null); + assertWithMessage("%s %s", method, url) + .that(StorageHttpMetricsInterceptor.isMediaDownloadRequest(request)) + .isEqualTo(expected); + } + } + @Test public void initialize_statusCodes_recordsDurationAndErrorType() throws IOException { Map cases = ImmutableMap.of( - 200, "OK", 308, "OK", - 503, "UNAVAILABLE"); + 503, "UNAVAILABLE", + 200, "OK"); for (int statusCode : cases.keySet()) { AtomicBoolean delegateCalled = new AtomicBoolean(false); @@ -196,11 +220,12 @@ public void initialize_statusCodes_recordsDurationAndErrorType() throws IOExcept HttpRequestInitializer interceptor = httpOptions.decorateWithMetrics(delegate); HttpTransport transport = createAdvancingTransport(statusCode, 250_000_000L); + String url = + statusCode == 200 + ? "https://storage.googleapis.com/storage/v1/b/my-b/o/my-obj?alt=media" + : "https://storage.googleapis.com/storage/v1/b/my-b/o/my-obj"; HttpRequest request = - transport - .createRequestFactory(interceptor) - .buildGetRequest( - new GenericUrl("https://storage.googleapis.com/storage/v1/b/my-b/o/my-obj")); + transport.createRequestFactory(interceptor).buildGetRequest(new GenericUrl(url)); if (statusCode != 200) { request.setThrowExceptionOnExecuteError(false); } @@ -209,6 +234,9 @@ public void initialize_statusCodes_recordsDurationAndErrorType() throws IOExcept assertWithMessage("status=%s", statusCode).that(delegateCalled.get()).isTrue(); assertWithMessage("status=%s", statusCode).that(execCalled.get()).isTrue(); assertWithMessage("status=%s", statusCode).that(respCalled.get()).isTrue(); + if (statusCode != 200) { + assertWithMessage("status=%s", statusCode).that(ttfbPoints()).isEmpty(); + } } List points = durationPoints(); @@ -240,6 +268,53 @@ public void initialize_statusCodes_recordsDurationAndErrorType() throws IOExcept .that(attrs.get(StorageClientMetrics.KEY_ERROR_TYPE)) .isEqualTo(entry.getValue()); } + + List ttfb = ttfbPoints(); + assertThat(ttfb).hasSize(1); + HistogramPointData ttfbPoint = ttfb.get(0); + assertThat(ttfbPoint.getCount()).isEqualTo(1L); + assertThat(ttfbPoint.getSum()).isWithin(1e-6).of(0.25); + assertThat(ttfbPoint.getAttributes()) + .isEqualTo( + Attributes.builder() + .put( + StorageClientMetrics.KEY_GCP_CLIENT_SERVICE, + StorageClientMetrics.SERVICE_STORAGE) + .put(StorageClientMetrics.KEY_GCP_CLIENT_VERSION, StorageOptions.version()) + .put( + StorageClientMetrics.KEY_GCP_CLIENT_ARTIFACT, + StorageClientMetrics.ARTIFACT_STORAGE) + .put(StorageClientMetrics.KEY_RPC_SYSTEM_NAME, "http") + .put(StorageClientMetrics.KEY_GCP_STORAGE_OPERATION, "ReadObject") + .put( + StorageClientMetrics.KEY_NETWORK_PATH, + StorageClientMetrics.NETWORK_PATH_CLOUDPATH) + .put(StorageClientMetrics.KEY_SERVER_ADDRESS, "storage.googleapis.com") + .build()); + } + + @Test + public void initialize_altMediaAddedAfterBuildGetRequest_recordsTtfb() throws IOException { + HttpRequestInitializer interceptor = httpOptions.decorateWithMetrics(null); + + HttpTransport transport = createAdvancingTransport(200, 120_000_000L); + HttpRequest request = + transport + .createRequestFactory(interceptor) + .buildGetRequest( + new GenericUrl("https://storage.googleapis.com/storage/v1/b/my-b/o/my-obj")); + request.getUrl().put("alt", "media"); + request.execute(); + + List ttfb = ttfbPoints(); + assertThat(ttfb).hasSize(1); + HistogramPointData ttfbPoint = ttfb.get(0); + assertThat(ttfbPoint.getCount()).isEqualTo(1L); + assertThat(ttfbPoint.getSum()).isWithin(1e-6).of(0.12); + assertThat(ttfbPoint.getAttributes().get(StorageClientMetrics.KEY_GCP_STORAGE_OPERATION)) + .isEqualTo("ReadObject"); + assertThat(ttfbPoint.getAttributes().get(StorageClientMetrics.KEY_SERVER_ADDRESS)) + .isEqualTo("storage.googleapis.com"); } @Test @@ -261,7 +336,8 @@ public void initialize_unsuccessfulResponseHandlerReturnsFalseOn404_recordsOnce( transport .createRequestFactory(interceptor) .buildGetRequest( - new GenericUrl("https://storage.googleapis.com/storage/v1/b/missing-bucket")); + new GenericUrl( + "https://storage.googleapis.com/download/storage/v1/b/my-b/o/missing-obj?alt=media")); request.setThrowExceptionOnExecuteError(false); request.execute(); @@ -275,9 +351,21 @@ public void initialize_unsuccessfulResponseHandlerReturnsFalseOn404_recordsOnce( Attributes attrs = point.getAttributes(); assertThat(attrs.get(StorageClientMetrics.KEY_RPC_SYSTEM_NAME)).isEqualTo("http"); assertThat(attrs.get(StorageClientMetrics.KEY_HTTP_RESPONSE_STATUS_CODE)).isEqualTo(404L); + assertThat(attrs.get(StorageClientMetrics.KEY_URL_TEMPLATE)) + .isEqualTo("/download/storage/v1/b/{bucket}/o/{object}"); assertThat(attrs.get(StorageClientMetrics.KEY_ERROR_TYPE)).isEqualTo("NOT_FOUND"); assertThat(attrs.get(StorageClientMetrics.KEY_SERVER_ADDRESS)) .isEqualTo("storage.googleapis.com"); + + List ttfb = ttfbPoints(); + assertThat(ttfb).hasSize(1); + HistogramPointData ttfbPoint = ttfb.get(0); + assertThat(ttfbPoint.getCount()).isEqualTo(1L); + assertThat(ttfbPoint.getSum()).isWithin(1e-6).of(0.08); + assertThat(ttfbPoint.getAttributes().get(StorageClientMetrics.KEY_GCP_STORAGE_OPERATION)) + .isEqualTo("ReadObject"); + assertThat(ttfbPoint.getAttributes().get(StorageClientMetrics.KEY_SERVER_ADDRESS)) + .isEqualTo("storage.googleapis.com"); } @Test @@ -330,6 +418,8 @@ public LowLevelHttpResponse execute() throws IOException { assertThat(okPoint.getSum()).isWithin(1e-6).of(0.20); assertThat(okPoint.getAttributes().get(StorageClientMetrics.KEY_ERROR_TYPE)) .isEqualTo(StorageErrorTypeMapper.STATUS_OK); + + assertThat(ttfbPoints()).isEmpty(); } @Test @@ -341,7 +431,7 @@ public void initialize_401RefreshRetryThen200_recordsBothAttempts() throws IOExc (req, response, supportsRetry) -> { if (response.getStatusCode() == 401) { // Simulate token refresh latency inside the UnsuccessfulResponseHandler; - // the 401 request duration must be captured before this handler runs. + // the 401 request duration and TTFB must be captured before this handler runs. clock.advanceNanos(200_000_000L); return true; } @@ -374,7 +464,9 @@ public LowLevelHttpResponse execute() { HttpRequest req = transport .createRequestFactory(interceptor) - .buildGetRequest(new GenericUrl("https://storage.googleapis.com/storage/v1/b/my-b")); + .buildGetRequest( + new GenericUrl( + "https://storage.googleapis.com/download/storage/v1/b/my-b/o/my-obj?alt=media")); req.execute(); List points = durationPoints(); @@ -382,13 +474,27 @@ public LowLevelHttpResponse execute() { HistogramPointData unauthorizedPoint = findPointByStatus(points, 401L); assertThat(unauthorizedPoint.getSum()).isWithin(1e-6).of(0.09); + assertThat(unauthorizedPoint.getAttributes().get(StorageClientMetrics.KEY_URL_TEMPLATE)) + .isEqualTo("/download/storage/v1/b/{bucket}/o/{object}"); assertThat(unauthorizedPoint.getAttributes().get(StorageClientMetrics.KEY_ERROR_TYPE)) .isEqualTo("UNAUTHENTICATED"); HistogramPointData okPoint = findPointByStatus(points, 200L); assertThat(okPoint.getSum()).isWithin(1e-6).of(0.18); + assertThat(okPoint.getAttributes().get(StorageClientMetrics.KEY_URL_TEMPLATE)) + .isEqualTo("/download/storage/v1/b/{bucket}/o/{object}"); assertThat(okPoint.getAttributes().get(StorageClientMetrics.KEY_ERROR_TYPE)) .isEqualTo(StorageErrorTypeMapper.STATUS_OK); + + List ttfb = ttfbPoints(); + assertThat(ttfb).hasSize(1); + HistogramPointData ttfbPoint = ttfb.get(0); + assertThat(ttfbPoint.getCount()).isEqualTo(2L); + assertThat(ttfbPoint.getSum()).isWithin(1e-6).of(0.27); + assertThat(ttfbPoint.getAttributes().get(StorageClientMetrics.KEY_GCP_STORAGE_OPERATION)) + .isEqualTo("ReadObject"); + assertThat(ttfbPoint.getAttributes().get(StorageClientMetrics.KEY_SERVER_ADDRESS)) + .isEqualTo("storage.googleapis.com"); } @Test @@ -483,11 +589,20 @@ public void onFailure(Void e, HttpHeaders responseHeaders) { assertThat(attrs.get(StorageClientMetrics.KEY_HTTP_RESPONSE_STATUS_CODE)).isEqualTo(200L); assertThat(attrs.get(StorageClientMetrics.KEY_ERROR_TYPE)) .isEqualTo(StorageErrorTypeMapper.STATUS_OK); + assertThat(ttfbPoints()).isEmpty(); } private List durationPoints() { + return histogramPoints(StorageClientMetrics.METRIC_HTTP_CLIENT_REQUEST_DURATION); + } + + private List ttfbPoints() { + return histogramPoints(StorageClientMetrics.METRIC_GCP_STORAGE_CLIENT_TTFB); + } + + private List histogramPoints(String metricName) { return reader.collectAllMetrics().stream() - .filter(m -> m.getName().equals(StorageClientMetrics.METRIC_HTTP_CLIENT_REQUEST_DURATION)) + .filter(m -> m.getName().equals(metricName)) .findFirst() .map(m -> ImmutableList.copyOf(m.getHistogramData().getPoints())) .orElse(ImmutableList.of());