diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/OpenTelemetryBootstrappingUtils.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/OpenTelemetryBootstrappingUtils.java index 1a7ab2e5879e..8fe1565badd4 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/OpenTelemetryBootstrappingUtils.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/OpenTelemetryBootstrappingUtils.java @@ -33,6 +33,9 @@ import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.common.AttributesBuilder; import io.opentelemetry.api.internal.StringUtils; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.metrics.MeterProvider; import io.opentelemetry.contrib.gcp.resource.GCPResourceProvider; import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.common.CompletableResultCode; @@ -61,12 +64,29 @@ import java.util.Objects; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BiConsumer; +import java.util.function.Supplier; import java.util.logging.Logger; import java.util.stream.Collectors; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; final class OpenTelemetryBootstrappingUtils { + static final String CHANNEL_CREATION_LATENCY_INSTRUMENT = + "internal/client/channel_creation_latency"; + static final String CHANNEL_CREATION_LATENCY_VIEW_NAME = + "storage.googleapis.com/internal/client/channel_creation_latency"; + static final String CHANNEL_CREATION_LATENCY_DESCRIPTION = + "Time from client construction until the first gRPC channel became ready."; + static final String CHANNEL_CREATION_LATENCY_UNIT = "s"; + static final AttributeKey TRANSPORT_TYPE_KEY = AttributeKey.stringKey("transport_type"); + + private static final String C2P_PREFIX = "google-c2p:///"; + private static final String C2P_EXPERIMENTAL_PREFIX = "google-c2p-experimental:///"; + private static final String DIRECT_PATH_INTERCONNECT_HOST = "storage-direct.googleapis.com"; + private static final String DIRECT_PATH_INTERCONNECT_PREPROD_HOST = + "storage-preprod-direct.sandbox.googleapis.com"; + private static final Collection METRICS_TO_ENABLE = ImmutableList.of( "grpc.lb.wrr.rr_fallback", @@ -128,7 +148,30 @@ static ChannelConfigurator enableGrpcMetrics( grpcOpenTelemetry.configureChannelBuilder(b); return b; }; - return otelConfigurator.andThen(channelConfigurator); + return new GrpcMetricsChannelConfigurator( + otelConfigurator.andThen(channelConfigurator), provider); + } + + @SuppressWarnings("rawtypes") // ManagedChannelBuilder + static final class GrpcMetricsChannelConfigurator + implements ChannelConfigurator, Supplier { + private final ChannelConfigurator delegate; + private final MeterProvider meterProvider; + + GrpcMetricsChannelConfigurator(ChannelConfigurator delegate, MeterProvider meterProvider) { + this.delegate = delegate; + this.meterProvider = meterProvider; + } + + @Override + public ManagedChannelBuilder apply(ManagedChannelBuilder input) { + return delegate.apply(input); + } + + @Override + public MeterProvider get() { + return meterProvider; + } } @SuppressWarnings("rawtypes") // ManagedChannelBuilder @@ -168,13 +211,21 @@ public ManagedChannelBuilder apply(ManagedChannelBuilder input) { } @VisibleForTesting - static String getCloudMonitoringEndpoint(String endpoint, String universeDomain) { + static String getCloudMonitoringEndpoint( + @Nullable String endpoint, @Nullable String universeDomain) { + if (endpoint == null || endpoint.isEmpty()) { + return "monitoring.googleapis.com:443"; + } String metricServiceEndpoint = "monitoring.googleapis.com"; + HostAndPort parsed = parseHostAndPort(endpoint); + String host = parsed != null ? parsed.getHost() : extractHostFallback(endpoint); + int port = parsed != null ? parsed.getPortOrDefault(443) : 443; - // use contains instead of equals because endpoint has a port in it - if (universeDomain != null && endpoint.contains("storage." + universeDomain)) { + if (universeDomain != null + && (host.equals("storage." + universeDomain) + || host.equals("storage-direct." + universeDomain))) { metricServiceEndpoint = "monitoring." + universeDomain; - } else if (!endpoint.contains("storage.googleapis.com")) { + } else if (!host.equals("storage.googleapis.com") && !isDirectPathInterconnectHost(host)) { String canonicalEndpoint = "storage.googleapis.com"; String privateEndpoint = "private.googleapis.com"; String restrictedEndpoint = "restricted.googleapis.com"; @@ -183,35 +234,76 @@ static String getCloudMonitoringEndpoint(String endpoint, String universeDomain) privateEndpoint = "private." + universeDomain; restrictedEndpoint = "restricted." + universeDomain; } + final String resolvedHost = host; String match = ImmutableList.of(canonicalEndpoint, privateEndpoint, restrictedEndpoint).stream() - .filter(s -> endpoint.contains(s) || endpoint.contains("google-c2p:///" + s)) + .filter(resolvedHost::contains) .collect(Collectors.joining()); if (!StringUtils.isNullOrEmpty(match)) { metricServiceEndpoint = match; } } - HostAndPort parsed = parseHostAndPort(endpoint); - int port = parsed != null ? parsed.getPortOrDefault(443) : 443; return metricServiceEndpoint + ":" + port; } + private static boolean isDirectPathInterconnectHost(String host) { + return DIRECT_PATH_INTERCONNECT_HOST.equals(host) + || DIRECT_PATH_INTERCONNECT_PREPROD_HOST.equals(host); + } + + private static String extractHostFallback(String endpoint) { + String s = stripEndpointScheme(endpoint.trim()); + int end = indexOfPathQueryOrFragment(s); + String hostPort = end >= 0 ? s.substring(0, end) : s; + int colon = hostPort.lastIndexOf(':'); + if (colon >= 0) { + int closeBracket = hostPort.indexOf(']'); + if (closeBracket >= 0 && colon < closeBracket) { + return hostPort; + } + if (closeBracket < 0 && hostPort.indexOf(':') != colon) { + return hostPort; + } + return hostPort.substring(0, colon); + } + return hostPort; + } + + private static String stripEndpointScheme(String s) { + if (s.startsWith(C2P_PREFIX)) { + return s.substring(C2P_PREFIX.length()); + } + if (s.startsWith(C2P_EXPERIMENTAL_PREFIX)) { + return s.substring(C2P_EXPERIMENTAL_PREFIX.length()); + } + if (s.startsWith("https://")) { + return s.substring("https://".length()); + } + if (s.startsWith("http://")) { + return s.substring("http://".length()); + } + return s; + } + + private static int indexOfPathQueryOrFragment(String s) { + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (c == '/' || c == '?' || c == '#') { + return i; + } + } + return -1; + } + @Nullable private static HostAndPort parseHostAndPort(@Nullable String host) { if (host == null || host.trim().isEmpty()) { return null; } - String s = host.trim(); - if (s.startsWith("google-c2p:///")) { - s = s.substring("google-c2p:///".length()); - } else if (s.startsWith("https://")) { - s = s.substring("https://".length()); - } else if (s.startsWith("http://")) { - s = s.substring("http://".length()); - } - int slashIdx = s.indexOf('/'); - if (slashIdx >= 0) { - s = s.substring(0, slashIdx); + String s = stripEndpointScheme(host.trim()); + int endIdx = indexOfPathQueryOrFragment(s); + if (endIdx >= 0) { + s = s.substring(0, endIdx); } if (s.isEmpty()) { return null; @@ -415,6 +507,7 @@ static SdkMeterProvider createMeterProvider( "By", "A view of grpc/client/attempt/sent_total_compressed_message_size with histogram boundaries" + " more appropriate for Google Cloud Storage RPCs"); + registerGrpcChannelViews(providerBuilder); return buildMeterProvider( cloudMonitoringExporter, @@ -569,6 +662,41 @@ private static SdkMeterProvider buildMeterProvider( return providerBuilder.build(); } + static SdkMeterProviderBuilder registerGrpcChannelViews(SdkMeterProviderBuilder providerBuilder) { + addHistogramView( + providerBuilder, + latencyHistogramBoundaries(), + CHANNEL_CREATION_LATENCY_INSTRUMENT, + CHANNEL_CREATION_LATENCY_VIEW_NAME, + StorageClientMetrics.METER_NAME, + CHANNEL_CREATION_LATENCY_UNIT, + CHANNEL_CREATION_LATENCY_DESCRIPTION); + return providerBuilder; + } + + static BiConsumer createChannelCreationLatencyRecorder( + @Nullable MeterProvider meterProvider) { + if (meterProvider == null) { + return (transportType, latency) -> {}; + } + Meter meter = + meterProvider + .meterBuilder(StorageClientMetrics.METER_NAME) + .setInstrumentationVersion(StorageOptions.version()) + .build(); + DoubleHistogram histogram = + meter + .histogramBuilder(CHANNEL_CREATION_LATENCY_INSTRUMENT) + .setDescription(CHANNEL_CREATION_LATENCY_DESCRIPTION) + .setUnit(CHANNEL_CREATION_LATENCY_UNIT) + .setExplicitBucketBoundariesAdvice(latencyHistogramBoundaries()) + .build(); + return (transportType, latency) -> { + double seconds = latency.toNanos() / 1_000_000_000.0; + histogram.record(seconds, Attributes.of(TRANSPORT_TYPE_KEY, transportType)); + }; + } + @VisibleForTesting static SdkMeterProviderBuilder registerClientViews(SdkMeterProviderBuilder providerBuilder) { for (String metric : CLIENT_COUNTERS) { diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/OpenTelemetryBootstrappingUtilsTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/OpenTelemetryBootstrappingUtilsTest.java index 01cc527922a9..efa2cda17f22 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/OpenTelemetryBootstrappingUtilsTest.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/OpenTelemetryBootstrappingUtilsTest.java @@ -17,18 +17,31 @@ package com.google.cloud.storage; import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; import static org.junit.Assert.assertThrows; import static org.junit.Assume.assumeFalse; import static org.mockito.Mockito.mock; import com.google.cloud.storage.OpenTelemetryBootstrappingUtils.ChannelConfigurator; +import com.google.cloud.storage.OpenTelemetryBootstrappingUtils.GrpcMetricsChannelConfigurator; +import com.google.common.collect.ImmutableList; import io.grpc.ManagedChannelBuilder; import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.MeterProvider; import io.opentelemetry.sdk.metrics.SdkMeterProvider; import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; +import io.opentelemetry.sdk.metrics.data.HistogramPointData; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; import java.time.Duration; +import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; import org.junit.Test; public final class OpenTelemetryBootstrappingUtilsTest { @@ -272,10 +285,135 @@ public void getCloudMonitoringEndpoint_extractsPortFromIpv4AndIpv6Endpoints() { OpenTelemetryBootstrappingUtils.getCloudMonitoringEndpoint( "[::1]:8080", "googleapis.com")) .isEqualTo("monitoring.googleapis.com:8080"); + assertThat( + OpenTelemetryBootstrappingUtils.getCloudMonitoringEndpoint( + "[::1]:8443", "googleapis.com")) + .isEqualTo("monitoring.googleapis.com:8443"); + assertThat(OpenTelemetryBootstrappingUtils.getCloudMonitoringEndpoint("::1", "googleapis.com")) + .isEqualTo("monitoring.googleapis.com:443"); + assertThat( + OpenTelemetryBootstrappingUtils.getCloudMonitoringEndpoint( + "[::1]:invalid", "googleapis.com")) + .isEqualTo("monitoring.googleapis.com:443"); assertThat( OpenTelemetryBootstrappingUtils.getCloudMonitoringEndpoint( "storage.my-universe.com:8443", "my-universe.com")) .isEqualTo("monitoring.my-universe.com:8443"); + assertThat( + OpenTelemetryBootstrappingUtils.getCloudMonitoringEndpoint( + "storage.googleapis.com:443", null)) + .isEqualTo("monitoring.googleapis.com:443"); + } + + @Test + public void getCloudMonitoringEndpoint_directPathInterconnectAndC2pEndpoints() { + ImmutableList cases = + ImmutableList.of( + new String[] { + "storage-direct.googleapis.com:443", null, "monitoring.googleapis.com:443" + }, + new String[] { + "storage-preprod-direct.sandbox.googleapis.com:443", + null, + "monitoring.googleapis.com:443" + }, + new String[] { + "storage-direct.apis-tpclp.goog:443", + "apis-tpclp.goog", + "monitoring.apis-tpclp.goog:443" + }, + new String[] { + "google-c2p:///storage-direct.googleapis.com?force-xds", + null, + "monitoring.googleapis.com:443" + }, + new String[] { + "google-c2p-experimental:///storage-preprod-direct.sandbox.googleapis.com:443?force-xds", + null, + "monitoring.googleapis.com:443" + }, + new String[] {null, null, "monitoring.googleapis.com:443"}, + new String[] {"", null, "monitoring.googleapis.com:443"}); + for (String[] c : cases) { + assertWithMessage("endpoint=%s, universe=%s", c[0], c[1]) + .that(OpenTelemetryBootstrappingUtils.getCloudMonitoringEndpoint(c[0], c[1])) + .isEqualTo(c[2]); + } + } + + @Test + public void registerGrpcChannelViews_exportsChannelCreationLatencyWithSecondsBuckets() { + InMemoryMetricReader metricReader = InMemoryMetricReader.create(); + SdkMeterProviderBuilder providerBuilder = + SdkMeterProvider.builder().registerMetricReader(metricReader); + SdkMeterProviderBuilder returned = + OpenTelemetryBootstrappingUtils.registerGrpcChannelViews(providerBuilder); + assertThat(returned).isSameInstanceAs(providerBuilder); + + try (SdkMeterProvider meterProvider = providerBuilder.build()) { + BiConsumer recorder = + OpenTelemetryBootstrappingUtils.createChannelCreationLatencyRecorder(meterProvider); + recorder.accept("DirectPath", Duration.ofMillis(3)); + recorder.accept("DirectPathInterconnect", Duration.ofMillis(1200)); + recorder.accept("CloudPath", Duration.ofSeconds(45)); + + Collection metrics = metricReader.collectAllMetrics(); + assertThat(metrics).hasSize(1); + MetricData metricData = metrics.iterator().next(); + assertThat(metricData.getName()) + .isEqualTo(OpenTelemetryBootstrappingUtils.CHANNEL_CREATION_LATENCY_VIEW_NAME); + assertThat(metricData.getUnit()) + .isEqualTo(OpenTelemetryBootstrappingUtils.CHANNEL_CREATION_LATENCY_UNIT); + assertThat(metricData.getDescription()) + .isEqualTo(OpenTelemetryBootstrappingUtils.CHANNEL_CREATION_LATENCY_DESCRIPTION); + + Map pointsByTransport = + metricData.getHistogramData().getPoints().stream() + .collect( + Collectors.toMap( + p -> + p.getAttributes().get(OpenTelemetryBootstrappingUtils.TRANSPORT_TYPE_KEY), + Function.identity())); + assertThat(pointsByTransport.keySet()) + .containsExactly("DirectPath", "DirectPathInterconnect", "CloudPath"); + + HistogramPointData dpPoint = pointsByTransport.get("DirectPath"); + HistogramPointData dpInterconnectPoint = pointsByTransport.get("DirectPathInterconnect"); + HistogramPointData cloudPathPoint = pointsByTransport.get("CloudPath"); + + assertThat(dpPoint.getSum()).isWithin(1e-6).of(0.003); + assertThat(dpInterconnectPoint.getSum()).isWithin(1e-6).of(1.2); + assertThat(cloudPathPoint.getSum()).isWithin(1e-6).of(45.0); + + List boundaries = dpPoint.getBoundaries(); + int idx003 = findBucketIndex(boundaries, 0.003); + int idx12 = findBucketIndex(boundaries, 1.2); + int idx45 = findBucketIndex(boundaries, 45.0); + assertThat(idx003).isEqualTo(2); + assertThat(idx12).isGreaterThan(idx003); + assertThat(idx45).isGreaterThan(idx12); + assertThat(dpPoint.getCounts().get(idx003)).isEqualTo(1L); + assertThat(dpInterconnectPoint.getCounts().get(idx12)).isEqualTo(1L); + assertThat(cloudPathPoint.getCounts().get(idx45)).isEqualTo(1L); + } + } + + @SuppressWarnings("rawtypes") // ManagedChannelBuilder + @Test + public void grpcMetricsChannelConfigurator_delegatesAndSuppliesMeterProvider() { + ManagedChannelBuilder b1 = mock(ManagedChannelBuilder.class, "b1"); + ManagedChannelBuilder b2 = mock(ManagedChannelBuilder.class, "b2"); + ChannelConfigurator delegate = + b -> { + assertThat(b).isSameInstanceAs(b1); + return b2; + }; + try (SdkMeterProvider meterProvider = SdkMeterProvider.builder().build()) { + GrpcMetricsChannelConfigurator configurator = + new GrpcMetricsChannelConfigurator(delegate, meterProvider); + assertThat(configurator.apply(b1)).isSameInstanceAs(b2); + assertThat(((Supplier) configurator).get()).isSameInstanceAs(meterProvider); + } } @Test @@ -326,4 +464,47 @@ public void createDefaultClientMeterProvider_returnsNullWhenProjectIdCannotBeDet Attributes.empty()); assertThat(nullProvider).isNull(); } + + @Test + public void + createChannelCreationLatencyRecorder_appliesExplicitBucketBoundariesAdviceWithoutCustomViews() { + InMemoryMetricReader metricReader = InMemoryMetricReader.create(); + try (SdkMeterProvider meterProvider = + SdkMeterProvider.builder().registerMetricReader(metricReader).build()) { + BiConsumer recorder = + OpenTelemetryBootstrappingUtils.createChannelCreationLatencyRecorder(meterProvider); + recorder.accept("DirectPathInterconnect", Duration.ofMillis(125)); + + Collection metrics = metricReader.collectAllMetrics(); + assertThat(metrics).hasSize(1); + MetricData metricData = metrics.iterator().next(); + assertThat(metricData.getName()) + .isEqualTo(OpenTelemetryBootstrappingUtils.CHANNEL_CREATION_LATENCY_INSTRUMENT); + HistogramPointData point = metricData.getHistogramData().getPoints().iterator().next(); + assertThat(point.getBoundaries()) + .isEqualTo(OpenTelemetryBootstrappingUtils.latencyHistogramBoundaries()); + assertThat(point.getSum()).isWithin(1e-6).of(0.125); + } + } + + @Test + public void createChannelCreationLatencyRecorder_nullMeterProvider_isNoop() { + InMemoryMetricReader metricReader = InMemoryMetricReader.create(); + try (SdkMeterProvider unusedProvider = + SdkMeterProvider.builder().registerMetricReader(metricReader).build()) { + BiConsumer noopRecorder = + OpenTelemetryBootstrappingUtils.createChannelCreationLatencyRecorder(null); + noopRecorder.accept("CloudPath", Duration.ofMillis(10)); + assertThat(metricReader.collectAllMetrics()).isEmpty(); + } + } + + private static int findBucketIndex(List boundaries, double value) { + for (int i = 0; i < boundaries.size(); i++) { + if (value <= boundaries.get(i)) { + return i; + } + } + return boundaries.size(); + } }