diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/BaseStorageWriteChannel.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/BaseStorageWriteChannel.java index 9efba201b73a..db127faee18d 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/BaseStorageWriteChannel.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/BaseStorageWriteChannel.java @@ -140,6 +140,9 @@ public final int write(ByteBuffer src) throws IOException { return 0; } int write = tmp.write(src); + if (write > 0 && metricsContext != null) { + metricsContext.addRequestBytes(write); + } return write; } catch (StorageException e) { lastOperationError = e; diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java index 86d4d0175af8..d8ae74b4331f 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java @@ -78,6 +78,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Streams; import com.google.common.io.ByteStreams; +import com.google.common.io.CountingInputStream; import com.google.common.util.concurrent.MoreExecutors; import com.google.iam.v1.GetIamPolicyRequest; import com.google.iam.v1.SetIamPolicyRequest; @@ -290,8 +291,13 @@ public Blob create(BlobInfo blobInfo, InputStream content, BlobWriteOption... op .setRequest(req) .build(); + CountingInputStream countingStream = new CountingInputStream(inputStreamParam); try (UnbufferedWritableByteChannel c = session.open()) { - ByteStreams.copy(Channels.newChannel(inputStreamParam), c); + ByteStreams.copy(Channels.newChannel(countingStream), c); + } finally { + if (metricsContext != null) { + metricsContext.setRequestBytes(countingStream.getCount()); + } } ApiFuture responseApiFuture = session.getResult(); return this.getBlob(responseApiFuture); @@ -326,6 +332,10 @@ public Blob internalCreateFrom(Path path, BlobInfo info, Opts o getOptions().getStorageClientMetrics(), "WriteObject"); Throwable error = null; try { + long size = Files.size(path); + if (multiContext != null) { + multiContext.setRequestBytes(size); + } GrpcCallContext grpcCallContext = opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault()); GrpcCallContext callContextWithMetrics = @@ -416,12 +426,17 @@ public Blob createFrom( .build(); // Specifically not in the try-with, so we don't close the provided stream - ReadableByteChannel src = - Channels.newChannel(firstNonNull(in, new ByteArrayInputStream(ZERO_BYTES))); + CountingInputStream countingStream = + new CountingInputStream(firstNonNull(in, new ByteArrayInputStream(ZERO_BYTES))); + ReadableByteChannel src = Channels.newChannel(countingStream); try (BufferedWritableByteChannel dst = session.open()) { ByteStreams.copy(src, dst); } catch (Exception e) { throw StorageException.coalesce(e); + } finally { + if (multiContext != null) { + multiContext.setRequestBytes(countingStream.getCount()); + } } return getBlob(session.getResult()); } catch (Throwable t) { @@ -1006,6 +1021,10 @@ private BlobInfo internalDirectUpload( optsWithDefaults.grpcMetadataMapper().apply(GrpcCallContext.createDefault()); WriteObjectRequest req = getWriteObjectRequest(blobInfo, optsWithDefaults); Hasher hasher = opts.getHasher(); + int totalBytes = buf.remaining(); + if (metricsContext != null) { + metricsContext.setRequestBytes(totalBytes); + } GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext(metricsContext)); RewindableContent content = RewindableContent.of(buf); return retrier.run( diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageClientMetrics.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageClientMetrics.java index b5b7a5ab45e9..dff0b43ea78b 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageClientMetrics.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageClientMetrics.java @@ -58,7 +58,6 @@ final class StorageClientMetrics implements AutoCloseable { AttributeKey.longKey("http.response.status_code"); static final AttributeKey KEY_URL_TEMPLATE = AttributeKey.stringKey("url.template"); static final AttributeKey KEY_SERVER_ADDRESS = AttributeKey.stringKey("server.address"); - static final AttributeKey KEY_SERVER_PORT = AttributeKey.longKey("server.port"); static final AttributeKey KEY_ERROR_TYPE = AttributeKey.stringKey("error.type"); static final AttributeKey KEY_NETWORK_PATH = AttributeKey.stringKey("network.path"); static final AttributeKey KEY_NETWORK_IO_DIRECTION = diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index 4eac3436c73d..34c4d83ac8ff 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -65,6 +65,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.io.BaseEncoding; +import com.google.common.io.CountingInputStream; import com.google.common.io.CountingOutputStream; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -204,12 +205,24 @@ public Blob create(BlobInfo blobInfo, InputStream content, BlobWriteOption... op StorageObject blobPb = codecs.blobInfo().encode(updated); InputStream inputStreamParam = firstNonNull(content, new ByteArrayInputStream(EMPTY_BYTE_ARRAY)); + StorageMetricsContext metricsContext = + StorageClientMetrics.createContext(getOptions().getStorageClientMetrics(), "WriteObject"); // retries are not safe when the input is an InputStream, so we can't retry. - return run( - "WriteObject", + return retrier.run( Retrying.neverRetry(), - () -> storageRpc.create(blobPb, inputStreamParam, optionsMap), - (x) -> Conversions.json().blobInfo().decode(x).asBlob(this)); + () -> { + if (metricsContext == null) { + return storageRpc.create(blobPb, inputStreamParam, optionsMap); + } + CountingInputStream countingStream = new CountingInputStream(inputStreamParam); + try { + return storageRpc.create(blobPb, countingStream, optionsMap); + } finally { + metricsContext.setRequestBytes(countingStream.getCount()); + } + }, + (x) -> Conversions.json().blobInfo().decode(x).asBlob(this), + metricsContext); } @Override @@ -1944,6 +1957,9 @@ public BlobInfo internalCreateFrom(Path path, BlobInfo info, Opts optionsMap = opts.getRpcOptions(); BlobInfo.Builder builder = info.toBuilder().setMd5(null).setCrc32c(null); BlobInfo updated = opts.blobInfoMapper().apply(builder).build(); @@ -2002,6 +2018,10 @@ private BlobInfo internalDirectUpload( Opts opts, ByteBuffer buf, @Nullable StorageMetricsContext metricsContext) { + int totalBytes = buf.remaining(); + if (metricsContext != null) { + metricsContext.setRequestBytes(totalBytes); + } BlobInfo.Builder builder = opts.blobInfoMapper().apply(info.toBuilder().clearMd5().clearCrc32c()); @Nullable Crc32cLengthKnown hash = opts.getHasher().hash(buf.duplicate()); diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageMetricsContext.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageMetricsContext.java index 37d97629ab94..92dc47972967 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageMetricsContext.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageMetricsContext.java @@ -195,9 +195,12 @@ void recordOperation( clientMethod = operationName; } - Attributes operationAttributes = + Attributes bodySizeAttributes = buildAttemptBaseAttributes(metrics).toBuilder() .put(StorageClientMetrics.KEY_GCP_CLIENT_METHOD, clientMethod) + .build(); + Attributes operationAttributes = + bodySizeAttributes.toBuilder() .put(StorageClientMetrics.KEY_ERROR_TYPE, terminalErrorType) .build(); @@ -216,6 +219,18 @@ void recordOperation( .build(); metrics.getGcpClientRequestDuration().record(getElapsedSeconds(), durationAttributes); } + + recordRequestBodySize(metrics, bodySizeAttributes); + } + } + + private void recordRequestBodySize(StorageClientMetrics metrics, Attributes bodySizeAttributes) { + if (metrics.getRequestBodySize() == null) { + return; + } + long reqBytes = requestBytes.get(); + if ("WriteObject".equals(operationName) && reqBytes >= 0) { + metrics.getRequestBodySize().record(reqBytes, bodySizeAttributes, otelContext); } } @@ -380,6 +395,10 @@ long addRequestBytes(long bytes) { return requestBytes.addAndGet(bytes); } + void setRequestBytes(long bytes) { + requestBytes.set(bytes); + } + long getRequestBytes() { return requestBytes.get(); } diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageMetricsContextTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageMetricsContextTest.java index 4ceb2160d459..f510eb23bc47 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageMetricsContextTest.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageMetricsContextTest.java @@ -18,18 +18,48 @@ import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.HttpResponseException; +import com.google.api.client.http.LowLevelHttpRequest; +import com.google.api.client.http.LowLevelHttpResponse; +import com.google.api.client.json.gson.GsonFactory; +import com.google.api.client.testing.http.MockHttpTransport; +import com.google.api.client.testing.http.MockLowLevelHttpRequest; +import com.google.api.client.testing.http.MockLowLevelHttpResponse; import com.google.api.core.ApiClock; +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutures; import com.google.api.gax.grpc.GrpcCallContext; import com.google.api.gax.grpc.GrpcStatusCode; import com.google.api.gax.rpc.ApiException; import com.google.api.gax.rpc.StatusCode; +import com.google.api.services.storage.model.StorageObject; +import com.google.cloud.NoCredentials; +import com.google.cloud.RestorableState; +import com.google.cloud.ServiceOptions; +import com.google.cloud.WriteChannel; +import com.google.cloud.storage.BufferedWritableByteChannelSession.BufferedWritableByteChannel; +import com.google.cloud.storage.Conversions.Decoder; +import com.google.cloud.storage.UnifiedOpts.Opts; +import com.google.cloud.storage.spi.StorageRpcFactory; +import com.google.cloud.storage.spi.v1.StorageRpc; +import com.google.common.io.ByteStreams; +import com.google.storage.v2.StartResumableWriteRequest; +import com.google.storage.v2.StartResumableWriteResponse; +import com.google.storage.v2.StorageGrpc; +import com.google.storage.v2.WriteObjectRequest; +import com.google.storage.v2.WriteObjectResponse; import io.grpc.CallOptions; +import io.grpc.Server; import io.grpc.Status; import io.grpc.StatusException; import io.grpc.StatusRuntimeException; +import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; +import io.grpc.stub.StreamObserver; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.context.Context; @@ -38,15 +68,29 @@ import io.opentelemetry.sdk.metrics.data.LongPointData; import io.opentelemetry.sdk.metrics.data.MetricData; import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; import java.util.Collection; +import java.util.Locale; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import org.jspecify.annotations.Nullable; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; public final class StorageMetricsContextTest { + @Rule public final TemporaryFolder tempFolder = new TemporaryFolder(); + private InMemoryMetricReader reader; private SdkMeterProvider provider; private StorageClientMetrics metrics; @@ -856,6 +900,308 @@ public void recordAttemptFailure_grpcStatusExceptionsWithNullStatus_fallsBackToU .isEqualTo(StorageErrorTypeMapper.STATUS_UNKNOWN); } + @Test + public void recordOperation_recordsRequestBodySize_whenWriteObjectAndBytesGreaterThanZero() { + StorageMetricsContext context = metrics.createContext("WriteObject"); + context.setRequestBytes(2048L); + context.recordOperation(metrics, null); + + Collection collected = reader.collectAllMetrics(); + HistogramPointData point = + getSingleHistogramPoint( + collected, StorageClientMetrics.METRIC_GCP_STORAGE_CLIENT_REQUEST_BODY_SIZE); + assertThat(point.getCount()).isEqualTo(1L); + assertThat(point.getSum()).isEqualTo(2048.0); + Attributes attrs = point.getAttributes(); + assertThat(attrs.asMap().keySet()) + .containsExactly( + StorageClientMetrics.KEY_GCP_CLIENT_SERVICE, + StorageClientMetrics.KEY_GCP_CLIENT_VERSION, + StorageClientMetrics.KEY_GCP_CLIENT_ARTIFACT, + StorageClientMetrics.KEY_GCP_STORAGE_OPERATION, + StorageClientMetrics.KEY_GCP_CLIENT_METHOD); + assertThat(attrs.get(StorageClientMetrics.KEY_GCP_CLIENT_SERVICE)) + .isEqualTo(StorageClientMetrics.SERVICE_STORAGE); + assertThat(attrs.get(StorageClientMetrics.KEY_GCP_CLIENT_VERSION)) + .isEqualTo(StorageOptions.version()); + assertThat(attrs.get(StorageClientMetrics.KEY_GCP_CLIENT_ARTIFACT)) + .isEqualTo(StorageClientMetrics.ARTIFACT_STORAGE); + assertThat(attrs.get(StorageClientMetrics.KEY_GCP_STORAGE_OPERATION)).isEqualTo("WriteObject"); + assertThat(attrs.get(StorageClientMetrics.KEY_GCP_CLIENT_METHOD)).isEqualTo("WriteObject"); + } + + @Test + public void recordOperation_doesNotRecordRequestBodySize_whenNotWriteObject() { + StorageMetricsContext context = metrics.createContext("ReadObject"); + context.setRequestBytes(2048L); + context.recordOperation(metrics, null); + + Collection collected = reader.collectAllMetrics(); + assertThat( + findMetric(collected, StorageClientMetrics.METRIC_GCP_STORAGE_CLIENT_REQUEST_BODY_SIZE)) + .isNull(); + } + + @Test + public void baseStorageWriteChannel_recordsRequestBodySize_whenZeroBytesWritten() + throws IOException { + StorageMetricsContext context = metrics.createMultiStepContext("WriteObject"); + + BaseStorageWriteChannel channel = createTestWriteChannel(context, metrics, false); + channel.close(); + + Collection collected = reader.collectAllMetrics(); + HistogramPointData bodyPoint = + getSingleHistogramPoint( + collected, StorageClientMetrics.METRIC_GCP_STORAGE_CLIENT_REQUEST_BODY_SIZE); + assertThat(bodyPoint.getCount()).isEqualTo(1L); + assertThat(bodyPoint.getSum()).isEqualTo(0.0); + LongPointData opPoint = + getSinglePoint(collected, StorageClientMetrics.METRIC_GCP_STORAGE_CLIENT_OPERATIONS); + assertThat(opPoint.getValue()).isEqualTo(1L); + } + + @Test + public void baseStorageWriteChannel_recordsRequestBodySizeEvenWhenCloseFails() + throws IOException { + StorageMetricsContext context = metrics.createMultiStepContext("WriteObject"); + + BaseStorageWriteChannel channel = createTestWriteChannel(context, metrics, true); + channel.write(ByteBuffer.wrap(new byte[128])); + channel.write(ByteBuffer.wrap(new byte[384])); + assertThrows(StorageException.class, channel::close); + + Collection collected = reader.collectAllMetrics(); + HistogramPointData point = + getSingleHistogramPoint( + collected, StorageClientMetrics.METRIC_GCP_STORAGE_CLIENT_REQUEST_BODY_SIZE); + assertThat(point.getCount()).isEqualTo(1L); + assertThat(point.getSum()).isEqualTo(512.0); + assertThat(point.getAttributes().get(StorageClientMetrics.KEY_GCP_STORAGE_OPERATION)) + .isEqualTo("WriteObject"); + } + + @Test + @SuppressWarnings("deprecation") + public void storageImpl_writeMethods_recordRequestBodySize_andPcuHookSuppresses() + throws Exception { + StorageRpcFactory rpcFactoryMock = mock(StorageRpcFactory.class); + StorageRpc storageRpcMock = mock(StorageRpc.class); + when(rpcFactoryMock.create(any(StorageOptions.class))).thenReturn(storageRpcMock); + when(storageRpcMock.create(any(StorageObject.class), any(InputStream.class), any())) + .thenAnswer( + inv -> { + InputStream in = inv.getArgument(1); + ByteStreams.exhaust(in); + return new StorageObject().setBucket("b").setName("n"); + }); + when(storageRpcMock.open(any(StorageObject.class), any())) + .thenReturn("http://localhost/upload-session-1"); + + MockHttpTransport mockTransport = + new MockHttpTransport() { + @Override + public LowLevelHttpRequest buildRequest(String method, String url) { + return new MockLowLevelHttpRequest(url) { + @Override + public LowLevelHttpResponse execute() { + long uploadedLen = getContentLength(); + String json = + String.format( + Locale.US, + "{\"bucket\":\"b\",\"name\":\"n\",\"size\":\"%d\"}", + uploadedLen); + return new MockLowLevelHttpResponse() + .setStatusCode(200) + .setContentType("application/json; charset=UTF-8") + .addHeader("content-type", "application/json; charset=UTF-8") + .setContent(json); + } + }; + } + }; + com.google.api.services.storage.Storage apiary = + new com.google.api.services.storage.Storage( + mockTransport, GsonFactory.getDefaultInstance(), null); + when(storageRpcMock.getStorage()).thenReturn(apiary); + + HttpStorageOptions options = + HttpStorageOptions.newBuilder() + .setProjectId("test-proj") + .setCredentials(NoCredentials.getInstance()) + .setServiceRpcFactory(rpcFactoryMock) + .setRetrySettings(ServiceOptions.getNoRetrySettings()) + .setEnableOtelMetrics(true) + .setMeterProvider(provider) + .build(); + StorageImpl storage = (StorageImpl) options.getService(); + BlobInfo blobInfo = BlobInfo.newBuilder("b", "n").build(); + + // 1. create(BlobInfo, byte[]) -> 100 bytes + storage.create(blobInfo, new byte[100]); + // 2. create(BlobInfo, InputStream) -> 200 bytes + storage.create(blobInfo, new ByteArrayInputStream(new byte[200])); + // 3. createFrom(BlobInfo, Path) -> 300 bytes + Path file1 = tempFolder.newFile("f1.bin").toPath(); + Files.write(file1, new byte[300]); + storage.createFrom(blobInfo, file1); + // 4. create(BlobInfo) -> 0 bytes + storage.create(blobInfo); + // 5. 3-arg internalDirectUpload (PCU part upload hook) -> must NOT record request.body.size + storage.internalDirectUpload(blobInfo, Opts.empty(), ByteBuffer.wrap(new byte[999])); + + Collection collected = reader.collectAllMetrics(); + HistogramPointData point = + getSingleHistogramPoint( + collected, StorageClientMetrics.METRIC_GCP_STORAGE_CLIENT_REQUEST_BODY_SIZE); + assertThat(point.getCount()).isEqualTo(4L); + assertThat(point.getSum()).isEqualTo(600.0); + Attributes attrs = point.getAttributes(); + assertThat(attrs.asMap().keySet()) + .containsExactly( + StorageClientMetrics.KEY_GCP_CLIENT_SERVICE, + StorageClientMetrics.KEY_GCP_CLIENT_VERSION, + StorageClientMetrics.KEY_GCP_CLIENT_ARTIFACT, + StorageClientMetrics.KEY_RPC_SYSTEM_NAME, + StorageClientMetrics.KEY_SERVER_ADDRESS, + StorageClientMetrics.KEY_GCP_STORAGE_OPERATION, + StorageClientMetrics.KEY_GCP_CLIENT_METHOD); + assertThat(attrs.get(StorageClientMetrics.KEY_RPC_SYSTEM_NAME)).isEqualTo("http"); + assertThat(attrs.get(StorageClientMetrics.KEY_SERVER_ADDRESS)) + .isEqualTo("storage.googleapis.com"); + assertThat(attrs.get(StorageClientMetrics.KEY_GCP_STORAGE_OPERATION)).isEqualTo("WriteObject"); + assertThat(attrs.get(StorageClientMetrics.KEY_GCP_CLIENT_METHOD)).isEqualTo("WriteObject"); + } + + @Test + public void grpcStorageImpl_writeMethods_recordRequestBodySize_andPcuHookSuppresses() + throws Exception { + StorageGrpc.StorageImplBase service = + new StorageGrpc.StorageImplBase() { + @Override + public void startResumableWrite( + StartResumableWriteRequest request, + StreamObserver responseObserver) { + responseObserver.onNext( + StartResumableWriteResponse.newBuilder().setUploadId("upload-1").build()); + responseObserver.onCompleted(); + } + + @Override + public StreamObserver writeObject( + StreamObserver responseObserver) { + return new StreamObserver() { + long totalSize = 0; + + @Override + public void onNext(WriteObjectRequest value) { + totalSize += value.getChecksummedData().getContent().size(); + } + + @Override + public void onError(Throwable t) {} + + @Override + public void onCompleted() { + responseObserver.onNext( + WriteObjectResponse.newBuilder() + .setResource( + com.google.storage.v2.Object.newBuilder() + .setBucket("projects/_/buckets/b") + .setName("n") + .setSize(totalSize) + .build()) + .build()); + responseObserver.onCompleted(); + } + }; + } + }; + + Server server = + NettyServerBuilder.forAddress(new InetSocketAddress("127.0.0.1", 0)) + .addService(service) + .build() + .start(); + try { + String endpoint = String.format(Locale.US, "http://127.0.0.1:%d", server.getPort()); + GrpcStorageOptions options = + StorageOptions.grpc() + .setHost(endpoint) + .setProjectId("test-proj") + .setCredentials(NoCredentials.getInstance()) + .setEnableGrpcClientMetrics(false) + .setAttemptDirectPath(false) + .setRetrySettings(ServiceOptions.getNoRetrySettings()) + .setEnableOtelMetrics(true) + .setMeterProvider(provider) + .build(); + try (GrpcStorageImpl storage = (GrpcStorageImpl) options.getService()) { + BlobInfo blobInfo = BlobInfo.newBuilder("b", "n").build(); + + // 1. create(BlobInfo, byte[]) -> 120 bytes + storage.create(blobInfo, new byte[120]); + // 2. create(BlobInfo, InputStream) -> 230 bytes + storage.create(blobInfo, new ByteArrayInputStream(new byte[230])); + // 3. createFrom(BlobInfo, InputStream) -> 340 bytes + storage.createFrom(blobInfo, new ByteArrayInputStream(new byte[340])); + // 4. createFrom(BlobInfo, Path) / internalCreateFrom -> 410 bytes + Path file = tempFolder.newFile("grpc.bin").toPath(); + Files.write(file, new byte[410]); + storage.createFrom(blobInfo, file); + // 5. writer(BlobInfo) -> 500 bytes + try (WriteChannel writer = storage.writer(blobInfo)) { + writer.write(ByteBuffer.wrap(new byte[500])); + } + // 6. 3-arg internalDirectUpload (PCU hook) -> must NOT record request.body.size + storage.internalDirectUpload(blobInfo, Opts.empty(), ByteBuffer.wrap(new byte[777])); + + // 7. internalCreateFrom with nonexistent Path -> throws NoSuchFileException inside try + // block and still records the failed WriteObject operation metric + Path missingPath = tempFolder.getRoot().toPath().resolve("does-not-exist.bin"); + assertThrows( + NoSuchFileException.class, + () -> storage.internalCreateFrom(missingPath, blobInfo, Opts.empty())); + + Collection collected = reader.collectAllMetrics(); + HistogramPointData bodyPoint = + getSingleHistogramPoint( + collected, StorageClientMetrics.METRIC_GCP_STORAGE_CLIENT_REQUEST_BODY_SIZE); + // 5 successful WriteObject ops (1600 B total) + 1 failed internalCreateFrom(missingPath) (0 + // B) + assertThat(bodyPoint.getCount()).isEqualTo(6L); + assertThat(bodyPoint.getSum()).isEqualTo(1600.0); + Attributes attrs = bodyPoint.getAttributes(); + assertThat(attrs.asMap().keySet()) + .containsExactly( + StorageClientMetrics.KEY_GCP_CLIENT_SERVICE, + StorageClientMetrics.KEY_GCP_CLIENT_VERSION, + StorageClientMetrics.KEY_GCP_CLIENT_ARTIFACT, + StorageClientMetrics.KEY_RPC_SYSTEM_NAME, + StorageClientMetrics.KEY_SERVER_ADDRESS, + StorageClientMetrics.KEY_GCP_STORAGE_OPERATION, + StorageClientMetrics.KEY_GCP_CLIENT_METHOD); + assertThat(attrs.get(StorageClientMetrics.KEY_RPC_SYSTEM_NAME)).isEqualTo("grpc"); + assertThat(attrs.get(StorageClientMetrics.KEY_SERVER_ADDRESS)).isEqualTo("127.0.0.1"); + assertThat(attrs.get(StorageClientMetrics.KEY_GCP_STORAGE_OPERATION)) + .isEqualTo("WriteObject"); + assertThat(attrs.get(StorageClientMetrics.KEY_GCP_CLIENT_METHOD)).isEqualTo("WriteObject"); + + MetricData opsMetric = + findMetric(collected, StorageClientMetrics.METRIC_GCP_STORAGE_CLIENT_OPERATIONS); + assertThat(opsMetric).isNotNull(); + long totalOps = + opsMetric.getLongSumData().getPoints().stream() + .mapToLong(LongPointData::getValue) + .sum(); + // 5 successful WriteObject ops + 1 failed internalCreateFrom(missingPath) op = 6 ops + assertThat(totalOps).isEqualTo(6L); + } + } finally { + server.shutdownNow().awaitTermination(5, TimeUnit.SECONDS); + } + } + private static @Nullable MetricData findMetric( Collection metrics, String metricName) { return metrics.stream().filter(m -> m.getName().equals(metricName)).findFirst().orElse(null); @@ -867,4 +1213,72 @@ private static LongPointData getSinglePoint(Collection metrics, Stri assertThat(metric.getLongSumData().getPoints()).hasSize(1); return metric.getLongSumData().getPoints().iterator().next(); } + + private static HistogramPointData getSingleHistogramPoint( + Collection collected, String metricName) { + MetricData metric = findMetric(collected, metricName); + assertThat(metric).isNotNull(); + assertThat(metric.getHistogramData().getPoints()).hasSize(1); + return metric.getHistogramData().getPoints().iterator().next(); + } + + private static BaseStorageWriteChannel createTestWriteChannel( + StorageMetricsContext context, StorageClientMetrics clientMetrics, boolean failOnClose) { + BlobInfo resultBlob = BlobInfo.newBuilder("b", "n").build(); + BufferedWritableByteChannel bufferedChannel = + new BufferedWritableByteChannel() { + private boolean open = true; + + @Override + public void flush() {} + + @Override + public int write(ByteBuffer src) { + int remaining = src.remaining(); + src.position(src.limit()); + return remaining; + } + + @Override + public boolean isOpen() { + return open; + } + + @Override + public void close() throws IOException { + open = false; + if (failOnClose) { + throw new IOException("simulated close failure"); + } + } + }; + BufferedWritableByteChannelSession session = + new BufferedWritableByteChannelSession() { + @Override + public ApiFuture openAsync() { + return ApiFutures.immediateFuture(bufferedChannel); + } + + @Override + public BufferedWritableByteChannel open() { + return bufferedChannel; + } + + @Override + public ApiFuture getResult() { + return ApiFutures.immediateFuture(resultBlob); + } + }; + return new BaseStorageWriteChannel(Decoder.identity(), context, clientMetrics) { + @Override + public RestorableState capture() { + return null; + } + + @Override + protected LazyWriteChannel newLazyWriteChannel() { + return new LazyWriteChannel<>(() -> session); + } + }; + } }