diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcChannelTelemetry.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcChannelTelemetry.java index b9c949dbbba5..79f418e427bf 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcChannelTelemetry.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcChannelTelemetry.java @@ -16,18 +16,40 @@ package com.google.cloud.storage; +import com.google.api.core.ApiClock; +import com.google.api.core.ApiFunction; +import com.google.api.core.NanoClock; +import com.google.api.gax.grpc.ChannelPrimer; import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; import com.google.auth.Credentials; import com.google.cloud.NoCredentials; +import com.google.cloud.storage.OpenTelemetryBootstrappingUtils.GrpcMetricsChannelConfigurator; import com.google.common.annotations.VisibleForTesting; +import io.grpc.ConnectivityState; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; import io.grpc.internal.GrpcUtil; +import io.opentelemetry.api.metrics.MeterProvider; +import java.lang.ref.WeakReference; +import java.lang.reflect.Method; +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; import org.checkerframework.checker.nullness.qual.Nullable; /** - * Package-private transport classification and startup configuration diagnostics for gRPC channels, - * supporting CloudPath, DirectPath, and DirectPath over Cloud Interconnect. + * Package-private telemetry and diagnostics for gRPC channels, supporting CloudPath, DirectPath, + * and DirectPath over Cloud Interconnect. */ final class GrpcChannelTelemetry { @@ -37,6 +59,9 @@ final class GrpcChannelTelemetry { static final String DIRECT_PATH_INTERCONNECT_HOST = "storage-direct.googleapis.com"; static final String DIRECT_PATH_INTERCONNECT_PREPROD_HOST = "storage-preprod-direct.sandbox.googleapis.com"; + static final String XDS_LB_POLICY_NAME = "xds_cluster_manager_experimental"; + static final String DNS_LB_POLICY_NAME = "pick_first"; + static final Duration DEFAULT_CHANNEL_READY_TIMEOUT = Duration.ofSeconds(30); @VisibleForTesting static final Logger LOG = Logger.getLogger(GrpcChannelTelemetry.class.getName()); @@ -64,6 +89,14 @@ public String toString() { } } + /** + * Callback invoked once the first gRPC channel transitions to {@link ConnectivityState#READY}. + */ + @FunctionalInterface + interface ChannelReadyCallback { + void onChannelReady(TransportType selectedTransport, Duration latency); + } + static boolean hasQueryParameter(@Nullable String uri, String key) { if (uri == null || key == null) { return false; @@ -104,22 +137,7 @@ static TransportType detectTransportType(@Nullable String endpoint) { } static String extractHost(@Nullable String endpointOrAuthority) { - if (endpointOrAuthority == null || endpointOrAuthority.isEmpty()) { - return ""; - } - String normalized = endpointOrAuthority; - if (normalized.startsWith(C2P_EXPERIMENTAL_PREFIX)) { - normalized = C2P_PREFIX + normalized.substring(C2P_EXPERIMENTAL_PREFIX.length()); - } - int queryIdx = normalized.indexOf('?'); - if (queryIdx >= 0) { - normalized = normalized.substring(0, queryIdx); - } - int fragmentIdx = normalized.indexOf('#'); - if (fragmentIdx >= 0) { - normalized = normalized.substring(0, fragmentIdx); - } - String extracted = OpenTelemetryBootstrappingUtils.extractServerAddress(normalized); + String extracted = OpenTelemetryBootstrappingUtils.extractServerAddress(endpointOrAuthority); return extracted != null ? extracted : ""; } @@ -129,6 +147,19 @@ static boolean isDirectPathInterconnectHost(@Nullable String host) { || DIRECT_PATH_INTERCONNECT_PREPROD_HOST.equals(h); } + static TransportType selectedTransportType( + TransportType requestedTransport, + @Nullable String lbPolicyName, + boolean grpcSupportsForceXds) { + if (requestedTransport == TransportType.CLOUD_PATH || DNS_LB_POLICY_NAME.equals(lbPolicyName)) { + return TransportType.CLOUD_PATH; + } + if (requestedTransport == TransportType.DIRECT_PATH_INTERCONNECT && !grpcSupportsForceXds) { + return TransportType.DIRECT_PATH; + } + return requestedTransport; + } + static boolean grpcForceXdsIsSupported() { return GRPC_FORCE_XDS_SUPPORTED; } @@ -178,6 +209,37 @@ static String resolveEffectiveTarget( return endpoint; } + @SuppressWarnings("rawtypes") // ManagedChannelBuilder + static ChannelPrimer configureChannelTelemetry( + InstantiatingGrpcChannelProvider.Builder channelProviderBuilder, + @Nullable Credentials credentials, + boolean attemptDirectPathXdsOverInterconnect, + @Nullable ApiClock clock) { + InstantiatingGrpcChannelProvider baseProvider = channelProviderBuilder.build(); + String effectiveTarget = + logChannelConfiguration(baseProvider, credentials, attemptDirectPathXdsOverInterconnect); + ApiFunction configurator = + channelProviderBuilder.getChannelConfigurator(); + MeterProvider grpcMeterProvider = + configurator instanceof GrpcMetricsChannelConfigurator + ? ((GrpcMetricsChannelConfigurator) configurator).get() + : null; + BiConsumer latencyRecorder = + OpenTelemetryBootstrappingUtils.createChannelCreationLatencyRecorder(grpcMeterProvider); + ChannelReadyCallback onReady = + (selectedTransport, latency) -> + latencyRecorder.accept(selectedTransport.toString(), latency); + ChannelPrimer primer = + createChannelPrimer( + effectiveTarget, + grpcForceXdsIsSupported(), + onReady, + clock != null ? clock : NanoClock.getDefaultClock(), + GrpcChannelTelemetry::newTelemetryScheduler); + channelProviderBuilder.setChannelPrimer(primer); + return primer; + } + static String logChannelConfiguration( InstantiatingGrpcChannelProvider baseProvider, @Nullable Credentials credentials, @@ -255,8 +317,168 @@ static void logChannelConfiguration( + grpcVersion); } + static String channelReadyNote( + TransportType requestedTransport, + TransportType selectedTransport, + @Nullable String lbPolicyName, + boolean ready) { + if (!ready) { + if (requestedTransport == TransportType.DIRECT_PATH_INTERCONNECT) { + return " Verify that TD bootstrap and xDS reachability are configured, or enable FINE" + + " java.util.logging for io.grpc.xds and io.grpc.googleapis."; + } + if (requestedTransport == TransportType.DIRECT_PATH) { + return " Enable FINE java.util.logging for io.grpc.xds and io.grpc.googleapis."; + } + return ""; + } + if (requestedTransport != TransportType.CLOUD_PATH + && selectedTransport == TransportType.CLOUD_PATH) { + return " Note: google-c2p resolver fell back to DNS (" + + DNS_LB_POLICY_NAME + + "); traffic is using CloudPath instead of " + + requestedTransport + + "."; + } + if (requestedTransport != TransportType.CLOUD_PATH + && lbPolicyName != null + && !lbPolicyName.isEmpty() + && !XDS_LB_POLICY_NAME.equals(lbPolicyName) + && !DNS_LB_POLICY_NAME.equals(lbPolicyName)) { + return " Note: unexpected LB policy '" + + lbPolicyName + + "' for " + + requestedTransport + + " (expected '" + + XDS_LB_POLICY_NAME + + "')."; + } + return ""; + } + + static void logChannelReady( + TransportType requestedTransport, + TransportType selectedTransport, + @Nullable String lbPolicyName, + Duration elapsed, + boolean ready, + @Nullable String statusDescription) { + String lb = (lbPolicyName == null || lbPolicyName.isEmpty()) ? "unknown" : lbPolicyName; + String note = channelReadyNote(requestedTransport, selectedTransport, lbPolicyName, ready); + long elapsedMs = elapsed.toMillis(); + if (ready) { + LOG.log( + Level.INFO, + "gRPC channel [0] is ready. transport_type=" + + selectedTransport + + ", requested_transport_type=" + + requestedTransport + + ", lb_policy=" + + lb + + ", elapsed_ms=" + + elapsedMs + + note); + } else { + LOG.log( + Level.INFO, + "gRPC channel [0] did not become ready before deadline. transport_type=" + + selectedTransport + + ", requested_transport_type=" + + requestedTransport + + ", lb_policy=" + + lb + + ", elapsed_ms=" + + elapsedMs + + (statusDescription != null && !statusDescription.isEmpty() + ? ", status=" + statusDescription + : "") + + note); + } + } + + static CompletableFuture startChannelTelemetry( + ManagedChannel channel, + String effectiveEndpoint, + boolean grpcSupportsForceXds, + Duration timeout, + @Nullable ChannelReadyCallback onReady, + ApiClock clock, + ScheduledExecutorService scheduler) { + return startChannelTelemetry( + new WeakReference<>(channel), + effectiveEndpoint, + grpcSupportsForceXds, + timeout, + onReady, + scheduler, + clock, + GrpcChannelTelemetry::defaultExtractLbPolicyName); + } + + @VisibleForTesting + static CompletableFuture startChannelTelemetry( + WeakReference channelRef, + String effectiveEndpoint, + boolean grpcSupportsForceXds, + Duration timeout, + @Nullable ChannelReadyCallback onReady, + ScheduledExecutorService scheduler, + ApiClock clock, + Function lbPolicyExtractor) { + TransportType requestedTransport = detectTransportType(effectiveEndpoint); + ChannelReadyWatcher watcher = + new ChannelReadyWatcher( + channelRef, + requestedTransport, + grpcSupportsForceXds, + timeout, + onReady, + scheduler, + clock, + lbPolicyExtractor); + return watcher.start(); + } + + static ChannelPrimer createChannelPrimer( + String effectiveEndpoint, + boolean grpcSupportsForceXds, + @Nullable ChannelReadyCallback onReady, + ApiClock clock, + Supplier ownedSchedulerSupplier) { + AtomicBoolean firstChannelObserved = new AtomicBoolean(false); + return managedChannel -> { + if (firstChannelObserved.compareAndSet(false, true)) { + ScheduledExecutorService scheduler = ownedSchedulerSupplier.get(); + startChannelTelemetry( + managedChannel, + effectiveEndpoint, + grpcSupportsForceXds, + DEFAULT_CHANNEL_READY_TIMEOUT, + onReady, + clock, + scheduler) + .whenComplete((unused, error) -> scheduler.shutdown()); + } + }; + } + private GrpcChannelTelemetry() {} + private static ScheduledExecutorService newTelemetryScheduler() { + ScheduledThreadPoolExecutor executor = + new ScheduledThreadPoolExecutor( + 1, + runnable -> { + Thread t = new Thread(runnable, "gcs-grpc-channel-telemetry"); + t.setDaemon(true); + return t; + }); + executor.setKeepAliveTime(5L, TimeUnit.SECONDS); + executor.allowCoreThreadTimeOut(true); + executor.setRemoveOnCancelPolicy(true); + return executor; + } + private static boolean detectGrpcForceXdsSupport() { try { Class clazz = Class.forName("io.grpc.googleapis.GoogleCloudToProdNameResolver"); @@ -278,4 +500,150 @@ private static String rewriteDirectPathInterconnectHost(String serviceAddress) { } return serviceAddress; } + + @Nullable + static String defaultExtractLbPolicyName(@Nullable ManagedChannel channel) { + if (channel == null) { + return null; + } + try { + Method m = channel.getClass().getMethod("getLoadBalancingPolicyName"); + m.setAccessible(true); + Object result = m.invoke(channel); + if (result instanceof String) { + return (String) result; + } + } catch (Exception ignored) { + // Standard io.grpc.ManagedChannel does not expose getLoadBalancingPolicyName publicly. + } + return null; + } + + private static final class ChannelReadyWatcher { + private final WeakReference channelRef; + private final TransportType requestedTransport; + private final boolean grpcSupportsForceXds; + private final Duration timeout; + @Nullable private final ChannelReadyCallback onReady; + private final ScheduledExecutorService scheduler; + private final ApiClock clock; + private final Function lbPolicyExtractor; + private final AtomicBoolean completed = new AtomicBoolean(false); + private final CompletableFuture completionFuture = new CompletableFuture<>(); + private volatile long startNanos; + @Nullable private volatile ScheduledFuture timeoutFuture; + + ChannelReadyWatcher( + WeakReference channelRef, + TransportType requestedTransport, + boolean grpcSupportsForceXds, + Duration timeout, + @Nullable ChannelReadyCallback onReady, + ScheduledExecutorService scheduler, + ApiClock clock, + Function lbPolicyExtractor) { + this.channelRef = channelRef; + this.requestedTransport = requestedTransport; + this.grpcSupportsForceXds = grpcSupportsForceXds; + this.timeout = timeout; + this.onReady = onReady; + this.scheduler = scheduler; + this.clock = clock; + this.lbPolicyExtractor = lbPolicyExtractor; + } + + CompletableFuture start() { + this.startNanos = clock.nanoTime(); + try { + this.timeoutFuture = + scheduler.schedule( + this::onTimeout, Math.max(0L, timeout.toMillis()), TimeUnit.MILLISECONDS); + } catch (RejectedExecutionException e) { + completeFailure(channelRef.get(), "telemetry scheduler shut down"); + return completionFuture; + } + step(); + return completionFuture; + } + + private void step() { + if (completed.get()) { + return; + } + ManagedChannel ch = channelRef.get(); + if (ch == null) { + completeFailure(null, "channel destroyed before becoming ready"); + return; + } + ConnectivityState state; + try { + state = ch.getState(true); + } catch (UnsupportedOperationException e) { + completeFailure(ch, "channel does not support connectivity state inspection"); + return; + } + if (state == ConnectivityState.READY) { + if (completed.compareAndSet(false, true)) { + try { + cancelTimeout(); + Duration elapsed = elapsedDuration(); + String lbPolicyName = lbPolicyExtractor.apply(ch); + TransportType selected = + selectedTransportType(requestedTransport, lbPolicyName, grpcSupportsForceXds); + logChannelReady(requestedTransport, selected, lbPolicyName, elapsed, true, null); + if (onReady != null) { + onReady.onChannelReady(selected, elapsed); + } + } catch (RuntimeException e) { + LOG.log(Level.FINE, "Channel telemetry callback failed", e); + } finally { + completionFuture.complete(null); + } + } + return; + } + if (state == ConnectivityState.SHUTDOWN) { + completeFailure(ch, "channel shut down before becoming ready"); + return; + } + try { + ch.notifyWhenStateChanged(state, this::step); + } catch (UnsupportedOperationException e) { + completeFailure(ch, "channel does not support state change notifications"); + } + } + + private void onTimeout() { + completeFailure(channelRef.get(), "DEADLINE_EXCEEDED"); + } + + private void completeFailure(@Nullable ManagedChannel ch, String statusDescription) { + if (completed.compareAndSet(false, true)) { + try { + cancelTimeout(); + Duration elapsed = elapsedDuration(); + String lbPolicyName = ch != null ? lbPolicyExtractor.apply(ch) : null; + TransportType selected = + selectedTransportType(requestedTransport, lbPolicyName, grpcSupportsForceXds); + logChannelReady( + requestedTransport, selected, lbPolicyName, elapsed, false, statusDescription); + } catch (RuntimeException e) { + LOG.log(Level.FINE, "Channel telemetry failure callback failed", e); + } finally { + completionFuture.complete(null); + } + } + } + + private Duration elapsedDuration() { + return Duration.ofNanos(clock.nanoTime() - startNanos); + } + + private void cancelTimeout() { + ScheduledFuture f = this.timeoutFuture; + if (f != null) { + f.cancel(false); + } + } + } } diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageOptions.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageOptions.java index 9eb93c6ef9c1..9858783cb446 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageOptions.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageOptions.java @@ -357,10 +357,9 @@ private Tuple> resolveSettingsAndOpts() throw channelProviderBuilder.setChannelConfigurator(channelConfigurator); } - InstantiatingGrpcChannelProvider baseProvider = channelProviderBuilder.build(); - GrpcChannelTelemetry.logChannelConfiguration( - baseProvider, credentials, attemptDirectPathXdsOverInterconnect); - builder.setTransportChannelProvider(baseProvider); + GrpcChannelTelemetry.configureChannelTelemetry( + channelProviderBuilder, credentials, attemptDirectPathXdsOverInterconnect, getClock()); + builder.setTransportChannelProvider(channelProviderBuilder.build()); RetrySettings baseRetrySettings = getRetrySettings(); RetrySettings readRetrySettings = baseRetrySettings.toBuilder() diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/OpenTelemetryBootstrappingUtils.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/OpenTelemetryBootstrappingUtils.java index 1a7ab2e5879e..8fe1565badd4 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/OpenTelemetryBootstrappingUtils.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/OpenTelemetryBootstrappingUtils.java @@ -33,6 +33,9 @@ import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.common.AttributesBuilder; import io.opentelemetry.api.internal.StringUtils; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.metrics.MeterProvider; import io.opentelemetry.contrib.gcp.resource.GCPResourceProvider; import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.common.CompletableResultCode; @@ -61,12 +64,29 @@ import java.util.Objects; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BiConsumer; +import java.util.function.Supplier; import java.util.logging.Logger; import java.util.stream.Collectors; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; final class OpenTelemetryBootstrappingUtils { + static final String CHANNEL_CREATION_LATENCY_INSTRUMENT = + "internal/client/channel_creation_latency"; + static final String CHANNEL_CREATION_LATENCY_VIEW_NAME = + "storage.googleapis.com/internal/client/channel_creation_latency"; + static final String CHANNEL_CREATION_LATENCY_DESCRIPTION = + "Time from client construction until the first gRPC channel became ready."; + static final String CHANNEL_CREATION_LATENCY_UNIT = "s"; + static final AttributeKey TRANSPORT_TYPE_KEY = AttributeKey.stringKey("transport_type"); + + private static final String C2P_PREFIX = "google-c2p:///"; + private static final String C2P_EXPERIMENTAL_PREFIX = "google-c2p-experimental:///"; + private static final String DIRECT_PATH_INTERCONNECT_HOST = "storage-direct.googleapis.com"; + private static final String DIRECT_PATH_INTERCONNECT_PREPROD_HOST = + "storage-preprod-direct.sandbox.googleapis.com"; + private static final Collection METRICS_TO_ENABLE = ImmutableList.of( "grpc.lb.wrr.rr_fallback", @@ -128,7 +148,30 @@ static ChannelConfigurator enableGrpcMetrics( grpcOpenTelemetry.configureChannelBuilder(b); return b; }; - return otelConfigurator.andThen(channelConfigurator); + return new GrpcMetricsChannelConfigurator( + otelConfigurator.andThen(channelConfigurator), provider); + } + + @SuppressWarnings("rawtypes") // ManagedChannelBuilder + static final class GrpcMetricsChannelConfigurator + implements ChannelConfigurator, Supplier { + private final ChannelConfigurator delegate; + private final MeterProvider meterProvider; + + GrpcMetricsChannelConfigurator(ChannelConfigurator delegate, MeterProvider meterProvider) { + this.delegate = delegate; + this.meterProvider = meterProvider; + } + + @Override + public ManagedChannelBuilder apply(ManagedChannelBuilder input) { + return delegate.apply(input); + } + + @Override + public MeterProvider get() { + return meterProvider; + } } @SuppressWarnings("rawtypes") // ManagedChannelBuilder @@ -168,13 +211,21 @@ public ManagedChannelBuilder apply(ManagedChannelBuilder input) { } @VisibleForTesting - static String getCloudMonitoringEndpoint(String endpoint, String universeDomain) { + static String getCloudMonitoringEndpoint( + @Nullable String endpoint, @Nullable String universeDomain) { + if (endpoint == null || endpoint.isEmpty()) { + return "monitoring.googleapis.com:443"; + } String metricServiceEndpoint = "monitoring.googleapis.com"; + HostAndPort parsed = parseHostAndPort(endpoint); + String host = parsed != null ? parsed.getHost() : extractHostFallback(endpoint); + int port = parsed != null ? parsed.getPortOrDefault(443) : 443; - // use contains instead of equals because endpoint has a port in it - if (universeDomain != null && endpoint.contains("storage." + universeDomain)) { + if (universeDomain != null + && (host.equals("storage." + universeDomain) + || host.equals("storage-direct." + universeDomain))) { metricServiceEndpoint = "monitoring." + universeDomain; - } else if (!endpoint.contains("storage.googleapis.com")) { + } else if (!host.equals("storage.googleapis.com") && !isDirectPathInterconnectHost(host)) { String canonicalEndpoint = "storage.googleapis.com"; String privateEndpoint = "private.googleapis.com"; String restrictedEndpoint = "restricted.googleapis.com"; @@ -183,35 +234,76 @@ static String getCloudMonitoringEndpoint(String endpoint, String universeDomain) privateEndpoint = "private." + universeDomain; restrictedEndpoint = "restricted." + universeDomain; } + final String resolvedHost = host; String match = ImmutableList.of(canonicalEndpoint, privateEndpoint, restrictedEndpoint).stream() - .filter(s -> endpoint.contains(s) || endpoint.contains("google-c2p:///" + s)) + .filter(resolvedHost::contains) .collect(Collectors.joining()); if (!StringUtils.isNullOrEmpty(match)) { metricServiceEndpoint = match; } } - HostAndPort parsed = parseHostAndPort(endpoint); - int port = parsed != null ? parsed.getPortOrDefault(443) : 443; return metricServiceEndpoint + ":" + port; } + private static boolean isDirectPathInterconnectHost(String host) { + return DIRECT_PATH_INTERCONNECT_HOST.equals(host) + || DIRECT_PATH_INTERCONNECT_PREPROD_HOST.equals(host); + } + + private static String extractHostFallback(String endpoint) { + String s = stripEndpointScheme(endpoint.trim()); + int end = indexOfPathQueryOrFragment(s); + String hostPort = end >= 0 ? s.substring(0, end) : s; + int colon = hostPort.lastIndexOf(':'); + if (colon >= 0) { + int closeBracket = hostPort.indexOf(']'); + if (closeBracket >= 0 && colon < closeBracket) { + return hostPort; + } + if (closeBracket < 0 && hostPort.indexOf(':') != colon) { + return hostPort; + } + return hostPort.substring(0, colon); + } + return hostPort; + } + + private static String stripEndpointScheme(String s) { + if (s.startsWith(C2P_PREFIX)) { + return s.substring(C2P_PREFIX.length()); + } + if (s.startsWith(C2P_EXPERIMENTAL_PREFIX)) { + return s.substring(C2P_EXPERIMENTAL_PREFIX.length()); + } + if (s.startsWith("https://")) { + return s.substring("https://".length()); + } + if (s.startsWith("http://")) { + return s.substring("http://".length()); + } + return s; + } + + private static int indexOfPathQueryOrFragment(String s) { + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (c == '/' || c == '?' || c == '#') { + return i; + } + } + return -1; + } + @Nullable private static HostAndPort parseHostAndPort(@Nullable String host) { if (host == null || host.trim().isEmpty()) { return null; } - String s = host.trim(); - if (s.startsWith("google-c2p:///")) { - s = s.substring("google-c2p:///".length()); - } else if (s.startsWith("https://")) { - s = s.substring("https://".length()); - } else if (s.startsWith("http://")) { - s = s.substring("http://".length()); - } - int slashIdx = s.indexOf('/'); - if (slashIdx >= 0) { - s = s.substring(0, slashIdx); + String s = stripEndpointScheme(host.trim()); + int endIdx = indexOfPathQueryOrFragment(s); + if (endIdx >= 0) { + s = s.substring(0, endIdx); } if (s.isEmpty()) { return null; @@ -415,6 +507,7 @@ static SdkMeterProvider createMeterProvider( "By", "A view of grpc/client/attempt/sent_total_compressed_message_size with histogram boundaries" + " more appropriate for Google Cloud Storage RPCs"); + registerGrpcChannelViews(providerBuilder); return buildMeterProvider( cloudMonitoringExporter, @@ -569,6 +662,41 @@ private static SdkMeterProvider buildMeterProvider( return providerBuilder.build(); } + static SdkMeterProviderBuilder registerGrpcChannelViews(SdkMeterProviderBuilder providerBuilder) { + addHistogramView( + providerBuilder, + latencyHistogramBoundaries(), + CHANNEL_CREATION_LATENCY_INSTRUMENT, + CHANNEL_CREATION_LATENCY_VIEW_NAME, + StorageClientMetrics.METER_NAME, + CHANNEL_CREATION_LATENCY_UNIT, + CHANNEL_CREATION_LATENCY_DESCRIPTION); + return providerBuilder; + } + + static BiConsumer createChannelCreationLatencyRecorder( + @Nullable MeterProvider meterProvider) { + if (meterProvider == null) { + return (transportType, latency) -> {}; + } + Meter meter = + meterProvider + .meterBuilder(StorageClientMetrics.METER_NAME) + .setInstrumentationVersion(StorageOptions.version()) + .build(); + DoubleHistogram histogram = + meter + .histogramBuilder(CHANNEL_CREATION_LATENCY_INSTRUMENT) + .setDescription(CHANNEL_CREATION_LATENCY_DESCRIPTION) + .setUnit(CHANNEL_CREATION_LATENCY_UNIT) + .setExplicitBucketBoundariesAdvice(latencyHistogramBoundaries()) + .build(); + return (transportType, latency) -> { + double seconds = latency.toNanos() / 1_000_000_000.0; + histogram.record(seconds, Attributes.of(TRANSPORT_TYPE_KEY, transportType)); + }; + } + @VisibleForTesting static SdkMeterProviderBuilder registerClientViews(SdkMeterProviderBuilder providerBuilder) { for (String metric : CLIENT_COUNTERS) { diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/GrpcChannelTelemetryTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/GrpcChannelTelemetryTest.java index a6fc9e998904..89ba98d00e7c 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/GrpcChannelTelemetryTest.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/GrpcChannelTelemetryTest.java @@ -19,15 +19,42 @@ import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertWithMessage; import static org.junit.Assume.assumeFalse; +import static org.mockito.Mockito.mock; +import com.google.api.core.ApiClock; +import com.google.api.core.NanoClock; +import com.google.api.gax.grpc.ChannelPrimer; import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; import com.google.auth.oauth2.AccessToken; import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.NoCredentials; +import com.google.cloud.storage.GrpcChannelTelemetry.ChannelReadyCallback; import com.google.cloud.storage.GrpcChannelTelemetry.TransportType; +import com.google.cloud.storage.OpenTelemetryBootstrappingUtils.ChannelConfigurator; +import com.google.cloud.storage.OpenTelemetryBootstrappingUtils.GrpcMetricsChannelConfigurator; import com.google.storage.v2.StorageSettings; +import io.grpc.CallOptions; +import io.grpc.ClientCall; +import io.grpc.ConnectivityState; +import io.grpc.ManagedChannel; +import io.grpc.MethodDescriptor; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; +import io.opentelemetry.sdk.metrics.data.HistogramPointData; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; +import java.lang.ref.WeakReference; +import java.time.Duration; import java.util.ArrayList; +import java.util.Collection; import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiConsumer; import java.util.logging.Handler; import java.util.logging.Level; import java.util.logging.LogRecord; @@ -37,9 +64,13 @@ public final class GrpcChannelTelemetryTest { + private static final String DP_INTERCONNECT_TARGET = + "google-c2p:///storage-direct.googleapis.com?force-xds"; + private CapturingLogHandler logHandler; private Level previousLevel; private boolean previousUseParentHandlers; + private ScheduledExecutorService scheduler; @Before public void setUp() { @@ -49,6 +80,7 @@ public void setUp() { GrpcChannelTelemetry.LOG.setLevel(Level.ALL); GrpcChannelTelemetry.LOG.setUseParentHandlers(false); GrpcChannelTelemetry.LOG.addHandler(logHandler); + scheduler = Executors.newSingleThreadScheduledExecutor(); } @After @@ -56,6 +88,7 @@ public void tearDown() { GrpcChannelTelemetry.LOG.removeHandler(logHandler); GrpcChannelTelemetry.LOG.setLevel(previousLevel); GrpcChannelTelemetry.LOG.setUseParentHandlers(previousUseParentHandlers); + scheduler.shutdownNow(); } @Test @@ -67,10 +100,7 @@ public void detectTransportType_allCases() { {"https://storage.googleapis.com", TransportType.CLOUD_PATH}, {"google-c2p:///storage.googleapis.com", TransportType.DIRECT_PATH}, {"google-c2p-experimental:///storage.googleapis.com", TransportType.DIRECT_PATH}, - { - "google-c2p:///storage-direct.googleapis.com?force-xds", - TransportType.DIRECT_PATH_INTERCONNECT - }, + {DP_INTERCONNECT_TARGET, TransportType.DIRECT_PATH_INTERCONNECT}, { "google-c2p:///storage-direct.googleapis.com?foo=1&force-xds=true", TransportType.DIRECT_PATH_INTERCONNECT @@ -99,7 +129,7 @@ public void detectTransportType_allCases() { @Test public void extractHost_extractsNormalizedServerAddress() { String[][] cases = { - {"google-c2p:///storage-direct.googleapis.com?force-xds", "storage-direct.googleapis.com"}, + {DP_INTERCONNECT_TARGET, "storage-direct.googleapis.com"}, { "google-c2p-experimental:///storage-preprod-direct.sandbox.googleapis.com:443?force-xds", "storage-preprod-direct.sandbox.googleapis.com" @@ -139,16 +169,66 @@ public void isDirectPathInterconnectHost_matchesDirectHostsOnly() { } @Test - public void logChannelConfiguration_validConfigurations_logInfo() { + public void selectedTransportType_allCases() { Object[][] cases = { - {"storage.googleapis.com:443", false, "CloudPath", "storage.googleapis.com"}, - {"google-c2p:///storage.googleapis.com", false, "DirectPath", "storage.googleapis.com"}, + {TransportType.CLOUD_PATH, null, true, TransportType.CLOUD_PATH}, { - "google-c2p:///storage-direct.googleapis.com?force-xds", + TransportType.CLOUD_PATH, + GrpcChannelTelemetry.XDS_LB_POLICY_NAME, true, - "DirectPathInterconnect", - "storage-direct.googleapis.com" + TransportType.CLOUD_PATH + }, + {TransportType.DIRECT_PATH, null, true, TransportType.DIRECT_PATH}, + { + TransportType.DIRECT_PATH, + GrpcChannelTelemetry.XDS_LB_POLICY_NAME, + true, + TransportType.DIRECT_PATH }, + { + TransportType.DIRECT_PATH, + GrpcChannelTelemetry.DNS_LB_POLICY_NAME, + true, + TransportType.CLOUD_PATH + }, + {TransportType.DIRECT_PATH_INTERCONNECT, null, true, TransportType.DIRECT_PATH_INTERCONNECT}, + { + TransportType.DIRECT_PATH_INTERCONNECT, + GrpcChannelTelemetry.XDS_LB_POLICY_NAME, + true, + TransportType.DIRECT_PATH_INTERCONNECT + }, + { + TransportType.DIRECT_PATH_INTERCONNECT, + GrpcChannelTelemetry.DNS_LB_POLICY_NAME, + true, + TransportType.CLOUD_PATH + }, + { + TransportType.DIRECT_PATH_INTERCONNECT, + GrpcChannelTelemetry.XDS_LB_POLICY_NAME, + false, + TransportType.DIRECT_PATH + } + }; + for (Object[] c : cases) { + TransportType requested = (TransportType) c[0]; + String lbPolicy = (String) c[1]; + boolean forceXdsSupported = (Boolean) c[2]; + TransportType expected = (TransportType) c[3]; + assertWithMessage( + "requested=%s, lbPolicy=%s, forceXds=%s", requested, lbPolicy, forceXdsSupported) + .that(GrpcChannelTelemetry.selectedTransportType(requested, lbPolicy, forceXdsSupported)) + .isEqualTo(expected); + } + } + + @Test + public void logChannelConfiguration_validConfigurations_logInfo() { + Object[][] cases = { + {"storage.googleapis.com:443", false, "CloudPath", "storage.googleapis.com"}, + {"google-c2p:///storage.googleapis.com", false, "DirectPath", "storage.googleapis.com"}, + {DP_INTERCONNECT_TARGET, true, "DirectPathInterconnect", "storage-direct.googleapis.com"}, { "google-c2p:///storage-preprod-direct.sandbox.googleapis.com?force-xds", true, @@ -175,8 +255,7 @@ public void logChannelConfiguration_validConfigurations_logInfo() { @Test public void logChannelConfiguration_unsupportedGrpcForceXds_logsWarning() { - GrpcChannelTelemetry.logChannelConfiguration( - "google-c2p:///storage-direct.googleapis.com?force-xds", true, false); + GrpcChannelTelemetry.logChannelConfiguration(DP_INTERCONNECT_TARGET, true, false); assertThat(logHandler.records).hasSize(1); assertThat(logHandler.records.get(0).getLevel()).isEqualTo(Level.WARNING); assertThat(logHandler.records.get(0).getMessage()) @@ -218,6 +297,348 @@ public void logChannelConfiguration_nonDirectAuthorityWithForceXds_logsWarning() + " over Interconnect host"); } + @Test + public void logChannelReady_directPathInterconnectReady_logsInfo() { + GrpcChannelTelemetry.logChannelReady( + TransportType.DIRECT_PATH_INTERCONNECT, + TransportType.DIRECT_PATH_INTERCONNECT, + GrpcChannelTelemetry.XDS_LB_POLICY_NAME, + Duration.ofMillis(42), + true, + null); + assertThat(logHandler.records).hasSize(1); + assertThat(logHandler.records.get(0).getLevel()).isEqualTo(Level.INFO); + assertThat(logHandler.records.get(0).getMessage()) + .isEqualTo( + "gRPC channel [0] is ready. transport_type=DirectPathInterconnect," + + " requested_transport_type=DirectPathInterconnect," + + " lb_policy=xds_cluster_manager_experimental, elapsed_ms=42"); + } + + @Test + public void logChannelReady_dnsFallback_logsCloudPathNote() { + GrpcChannelTelemetry.logChannelReady( + TransportType.DIRECT_PATH_INTERCONNECT, + TransportType.CLOUD_PATH, + GrpcChannelTelemetry.DNS_LB_POLICY_NAME, + Duration.ofMillis(15), + true, + null); + assertThat(logHandler.records).hasSize(1); + assertThat(logHandler.records.get(0).getMessage()) + .contains( + "Note: google-c2p resolver fell back to DNS (pick_first); traffic is using CloudPath" + + " instead of DirectPathInterconnect."); + } + + @Test + public void logChannelReady_unexpectedLbPolicy_logsNote() { + GrpcChannelTelemetry.logChannelReady( + TransportType.DIRECT_PATH, + TransportType.DIRECT_PATH, + "round_robin", + Duration.ofMillis(20), + true, + null); + assertThat(logHandler.records).hasSize(1); + assertThat(logHandler.records.get(0).getMessage()) + .contains( + "Note: unexpected LB policy 'round_robin' for DirectPath (expected" + + " 'xds_cluster_manager_experimental')."); + } + + @Test + public void logChannelReady_deadlineExceeded_logsJavaLoggingGuidance() { + GrpcChannelTelemetry.logChannelReady( + TransportType.DIRECT_PATH_INTERCONNECT, + TransportType.DIRECT_PATH_INTERCONNECT, + null, + Duration.ofMillis(30000), + false, + "DEADLINE_EXCEEDED"); + assertThat(logHandler.records).hasSize(1); + assertThat(logHandler.records.get(0).getMessage()) + .contains("gRPC channel [0] did not become ready before deadline."); + assertThat(logHandler.records.get(0).getMessage()).contains("status=DEADLINE_EXCEEDED"); + assertThat(logHandler.records.get(0).getMessage()) + .contains("enable FINE java.util.logging for io.grpc.xds and io.grpc.googleapis"); + } + + @Test + public void startChannelTelemetry_immediateReady_invokesCallbackOnce() throws Exception { + FakeManagedChannel channel = + new FakeManagedChannel(ConnectivityState.READY, GrpcChannelTelemetry.XDS_LB_POLICY_NAME); + TestApiClock clock = TestApiClock.tickBy(1_000_000_000L, Duration.ofMillis(25)); + AtomicInteger callbackCount = new AtomicInteger(0); + AtomicReference reportedTransport = new AtomicReference<>(); + AtomicReference reportedLatency = new AtomicReference<>(); + + watchInterconnect( + channel, + Duration.ofSeconds(30), + clock, + (transport, latency) -> { + callbackCount.incrementAndGet(); + reportedTransport.set(transport); + reportedLatency.set(latency); + }) + .get(5, TimeUnit.SECONDS); + + assertThat(callbackCount.get()).isEqualTo(1); + assertThat(reportedTransport.get()).isEqualTo(TransportType.DIRECT_PATH_INTERCONNECT); + assertThat(reportedLatency.get()).isEqualTo(Duration.ofMillis(25)); + assertThat(logHandler.records).hasSize(1); + assertThat(logHandler.records.get(0).getMessage()).contains("gRPC channel [0] is ready."); + } + + @Test + public void startChannelTelemetry_asyncTransitionToReady_invokesCallbackOnce() throws Exception { + FakeManagedChannel channel = + new FakeManagedChannel(ConnectivityState.IDLE, GrpcChannelTelemetry.XDS_LB_POLICY_NAME); + TestApiClock clock = TestApiClock.of(0L, prev -> prev); + AtomicInteger callbackCount = new AtomicInteger(0); + AtomicReference reportedTransport = new AtomicReference<>(); + AtomicReference reportedLatency = new AtomicReference<>(); + + CompletableFuture future = + watchInterconnect( + channel, + Duration.ofSeconds(30), + clock, + (transport, latency) -> { + callbackCount.incrementAndGet(); + reportedTransport.set(transport); + reportedLatency.set(latency); + }); + + assertThat(callbackCount.get()).isEqualTo(0); + channel.transitionTo(ConnectivityState.CONNECTING); + assertThat(callbackCount.get()).isEqualTo(0); + + clock.advance(Duration.ofMillis(55)); + channel.transitionTo(ConnectivityState.READY); + future.get(5, TimeUnit.SECONDS); + + assertThat(callbackCount.get()).isEqualTo(1); + assertThat(reportedTransport.get()).isEqualTo(TransportType.DIRECT_PATH_INTERCONNECT); + assertThat(reportedLatency.get()).isEqualTo(Duration.ofMillis(55)); + } + + @Test + public void startChannelTelemetry_shutdownBeforeReady_doesNotInvokeCallback() throws Exception { + AtomicInteger callbackCount = new AtomicInteger(0); + FakeManagedChannel channel = new FakeManagedChannel(ConnectivityState.CONNECTING, null); + CompletableFuture future = + watchInterconnect( + channel, + Duration.ofSeconds(30), + NanoClock.getDefaultClock(), + (t, d) -> callbackCount.incrementAndGet()); + channel.transitionTo(ConnectivityState.SHUTDOWN); + future.get(5, TimeUnit.SECONDS); + assertThat(callbackCount.get()).isEqualTo(0); + assertThat(logHandler.records).hasSize(1); + assertThat(logHandler.records.get(0).getMessage()) + .contains("status=channel shut down before becoming ready"); + } + + @Test + public void startChannelTelemetry_timeoutBeforeReady_doesNotInvokeCallback() throws Exception { + AtomicInteger callbackCount = new AtomicInteger(0); + FakeManagedChannel channel = new FakeManagedChannel(ConnectivityState.CONNECTING, null); + watchInterconnect( + channel, + Duration.ofMillis(20), + NanoClock.getDefaultClock(), + (t, d) -> callbackCount.incrementAndGet()) + .get(5, TimeUnit.SECONDS); + assertThat(callbackCount.get()).isEqualTo(0); + assertThat(logHandler.records).hasSize(1); + assertThat(logHandler.records.get(0).getMessage()).contains("status=DEADLINE_EXCEEDED"); + } + + @Test + public void startChannelTelemetry_clearedWeakReference_doesNotInvokeCallback() throws Exception { + AtomicInteger callbackCount = new AtomicInteger(0); + watchInterconnect( + null, + Duration.ofSeconds(30), + NanoClock.getDefaultClock(), + (t, d) -> callbackCount.incrementAndGet()) + .get(5, TimeUnit.SECONDS); + assertThat(callbackCount.get()).isEqualTo(0); + assertThat(logHandler.records).hasSize(1); + assertThat(logHandler.records.get(0).getMessage()) + .contains("status=channel destroyed before becoming ready"); + } + + @Test + public void startChannelTelemetry_rejectedScheduler_completesCleanly() throws Exception { + ScheduledExecutorService closedScheduler = Executors.newSingleThreadScheduledExecutor(); + closedScheduler.shutdownNow(); + FakeManagedChannel connectingChannel = + new FakeManagedChannel(ConnectivityState.CONNECTING, null); + GrpcChannelTelemetry.startChannelTelemetry( + new WeakReference<>(connectingChannel), + DP_INTERCONNECT_TARGET, + true, + Duration.ofSeconds(30), + null, + closedScheduler, + NanoClock.getDefaultClock(), + FakeManagedChannel::getLbPolicy) + .get(5, TimeUnit.SECONDS); + assertThat(logHandler.records).hasSize(1); + assertThat(logHandler.records.get(0).getMessage()) + .contains("status=telemetry scheduler shut down"); + } + + @Test + public void startChannelTelemetry_throwingCallback_completesCleanly() throws Exception { + FakeManagedChannel readyChannel = + new FakeManagedChannel(ConnectivityState.READY, GrpcChannelTelemetry.XDS_LB_POLICY_NAME); + watchInterconnect( + readyChannel, + Duration.ofSeconds(30), + NanoClock.getDefaultClock(), + (transport, latency) -> { + throw new IllegalStateException("simulated callback failure"); + }) + .get(5, TimeUnit.SECONDS); + assertThat(logHandler.records).hasSize(2); + assertThat(logHandler.records.get(0).getLevel()).isEqualTo(Level.INFO); + assertThat(logHandler.records.get(1).getLevel()).isEqualTo(Level.FINE); + } + + @Test + public void createChannelPrimer_onlyObservesFirstChannelAndShutsDownOwnedScheduler() { + AtomicInteger callbackCount = new AtomicInteger(0); + AtomicInteger ownedSupplierCalls = new AtomicInteger(0); + AtomicReference createdScheduler = new AtomicReference<>(); + ChannelPrimer ownedPrimer = + GrpcChannelTelemetry.createChannelPrimer( + DP_INTERCONNECT_TARGET, + true, + (transport, latency) -> callbackCount.incrementAndGet(), + NanoClock.getDefaultClock(), + () -> { + ownedSupplierCalls.incrementAndGet(); + ScheduledExecutorService s = Executors.newSingleThreadScheduledExecutor(); + createdScheduler.set(s); + return s; + }); + + FakeManagedChannel ch0 = + new FakeManagedChannel(ConnectivityState.READY, GrpcChannelTelemetry.XDS_LB_POLICY_NAME); + FakeManagedChannel ch1 = + new FakeManagedChannel(ConnectivityState.READY, GrpcChannelTelemetry.XDS_LB_POLICY_NAME); + + assertThat(ownedSupplierCalls.get()).isEqualTo(0); + ownedPrimer.primeChannel(ch0); + ownedPrimer.primeChannel(ch1); + assertThat(callbackCount.get()).isEqualTo(1); + assertThat(logHandler.records).hasSize(1); + assertThat(ownedSupplierCalls.get()).isEqualTo(1); + assertThat(createdScheduler.get()).isNotNull(); + assertThat(createdScheduler.get().isShutdown()).isTrue(); + } + + @Test + public void defaultExtractLbPolicyName_extractsFromPackagePrivateAndHandlesMissingMethod() { + FakeManagedChannel channel = + new FakeManagedChannel(ConnectivityState.READY, GrpcChannelTelemetry.XDS_LB_POLICY_NAME); + assertThat(GrpcChannelTelemetry.defaultExtractLbPolicyName(channel)) + .isEqualTo(GrpcChannelTelemetry.XDS_LB_POLICY_NAME); + assertThat(GrpcChannelTelemetry.defaultExtractLbPolicyName(mock(ManagedChannel.class))) + .isNull(); + assertThat(GrpcChannelTelemetry.defaultExtractLbPolicyName(null)).isNull(); + } + + @Test + public void channelReadyCallback_withLatencyRecorder_recordsExpectedHistogramMetric() + throws Exception { + InMemoryMetricReader metricReader = InMemoryMetricReader.create(); + SdkMeterProviderBuilder providerBuilder = + OpenTelemetryBootstrappingUtils.registerGrpcChannelViews( + SdkMeterProvider.builder().registerMetricReader(metricReader)); + + try (SdkMeterProvider meterProvider = providerBuilder.build()) { + BiConsumer recorder = + OpenTelemetryBootstrappingUtils.createChannelCreationLatencyRecorder(meterProvider); + ChannelReadyCallback callback = + (selectedTransport, latency) -> recorder.accept(selectedTransport.toString(), latency); + + TestApiClock clock = TestApiClock.tickBy(0L, Duration.ofMillis(125)); + FakeManagedChannel channel = + new FakeManagedChannel(ConnectivityState.READY, GrpcChannelTelemetry.XDS_LB_POLICY_NAME); + watchInterconnect(channel, Duration.ofSeconds(30), clock, callback).get(5, TimeUnit.SECONDS); + + Collection metrics = metricReader.collectAllMetrics(); + assertThat(metrics).hasSize(1); + MetricData metricData = metrics.iterator().next(); + assertThat(metricData.getName()) + .isEqualTo(OpenTelemetryBootstrappingUtils.CHANNEL_CREATION_LATENCY_VIEW_NAME); + HistogramPointData point = metricData.getHistogramData().getPoints().iterator().next(); + assertThat(point.getAttributes().get(OpenTelemetryBootstrappingUtils.TRANSPORT_TYPE_KEY)) + .isEqualTo("DirectPathInterconnect"); + assertThat(point.getCount()).isEqualTo(1L); + assertThat(point.getSum()).isWithin(1e-6).of(0.125); + } + } + + @Test + public void configureChannelTelemetry_extractsMeterProviderFromGrpcMetricsConfigurator() { + InMemoryMetricReader metricReader = InMemoryMetricReader.create(); + SdkMeterProviderBuilder providerBuilder = + OpenTelemetryBootstrappingUtils.registerGrpcChannelViews( + SdkMeterProvider.builder().registerMetricReader(metricReader)); + + try (SdkMeterProvider meterProvider = providerBuilder.build()) { + InstantiatingGrpcChannelProvider.Builder channelProviderBuilder = + InstantiatingGrpcChannelProvider.newBuilder() + .setEndpoint("storage.googleapis.com:443") + .setChannelConfigurator( + new GrpcMetricsChannelConfigurator( + ChannelConfigurator.identity(), meterProvider)); + TestApiClock clock = TestApiClock.tickBy(0L, Duration.ofMillis(80)); + + ChannelPrimer primer = + GrpcChannelTelemetry.configureChannelTelemetry( + channelProviderBuilder, NoCredentials.getInstance(), false, clock); + primer.primeChannel( + new FakeManagedChannel(ConnectivityState.READY, GrpcChannelTelemetry.DNS_LB_POLICY_NAME)); + + assertThat(logHandler.records).hasSize(2); + Collection metrics = metricReader.collectAllMetrics(); + assertThat(metrics).hasSize(1); + HistogramPointData point = + metrics.iterator().next().getHistogramData().getPoints().iterator().next(); + assertThat(point.getAttributes().get(OpenTelemetryBootstrappingUtils.TRANSPORT_TYPE_KEY)) + .isEqualTo("CloudPath"); + assertThat(point.getCount()).isEqualTo(1L); + assertThat(point.getSum()).isWithin(1e-6).of(0.080); + } + } + + @Test + public void startChannelTelemetry_dnsFallback_reportsCloudPath() throws Exception { + FakeManagedChannel channel = + new FakeManagedChannel(ConnectivityState.READY, GrpcChannelTelemetry.DNS_LB_POLICY_NAME); + AtomicReference reportedTransport = new AtomicReference<>(); + + watchInterconnect( + channel, + Duration.ofSeconds(30), + NanoClock.getDefaultClock(), + (transport, latency) -> reportedTransport.set(transport)) + .get(5, TimeUnit.SECONDS); + + assertThat(reportedTransport.get()).isEqualTo(TransportType.CLOUD_PATH); + assertThat(logHandler.records).hasSize(1); + assertThat(logHandler.records.get(0).getMessage()) + .contains("transport_type=CloudPath, requested_transport_type=DirectPathInterconnect"); + } + @Test public void isDirectPathInterconnectEnabled_envPrecedence() { Object[][] cases = { @@ -277,7 +698,7 @@ public void grpcStorageOptions_directPathOverInterconnect_resolvesProdDirectTarg .getTransportChannelProvider() .withCredentials(options.getCredentials()); assertThat(GrpcChannelTelemetry.resolveEffectiveTarget(provider, true)) - .isEqualTo("google-c2p:///storage-direct.googleapis.com?force-xds"); + .isEqualTo(DP_INTERCONNECT_TARGET); } @Test @@ -319,6 +740,99 @@ private static void assumeDirectPathNotDisabledByEnv() { assumeFalse(Boolean.parseBoolean(System.getenv("GOOGLE_CLOUD_DISABLE_DIRECT_PATH"))); } + private CompletableFuture watchInterconnect( + ManagedChannel channel, Duration timeout, ApiClock clock, ChannelReadyCallback callback) { + return GrpcChannelTelemetry.startChannelTelemetry( + new WeakReference<>(channel), + DP_INTERCONNECT_TARGET, + true, + timeout, + callback, + scheduler, + clock, + FakeManagedChannel::getLbPolicy); + } + + private static final class FakeManagedChannel extends ManagedChannel { + private volatile ConnectivityState state; + private final String lbPolicy; + private volatile Runnable stateChangeCallback; + + FakeManagedChannel(ConnectivityState initialState, String lbPolicy) { + this.state = initialState; + this.lbPolicy = lbPolicy; + } + + public String getLoadBalancingPolicyName() { + return lbPolicy; + } + + static String getLbPolicy(ManagedChannel channel) { + return channel instanceof FakeManagedChannel ? ((FakeManagedChannel) channel).lbPolicy : null; + } + + @Override + public ConnectivityState getState(boolean requestConnection) { + return state; + } + + @Override + public void notifyWhenStateChanged(ConnectivityState source, Runnable callback) { + if (state != source) { + callback.run(); + } else { + this.stateChangeCallback = callback; + } + } + + void transitionTo(ConnectivityState newState) { + this.state = newState; + Runnable cb = this.stateChangeCallback; + this.stateChangeCallback = null; + if (cb != null) { + cb.run(); + } + } + + @Override + public ManagedChannel shutdown() { + transitionTo(ConnectivityState.SHUTDOWN); + return this; + } + + @Override + public boolean isShutdown() { + return state == ConnectivityState.SHUTDOWN; + } + + @Override + public boolean isTerminated() { + return state == ConnectivityState.SHUTDOWN; + } + + @Override + public ManagedChannel shutdownNow() { + transitionTo(ConnectivityState.SHUTDOWN); + return this; + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) { + return true; + } + + @Override + public ClientCall newCall( + MethodDescriptor methodDescriptor, CallOptions callOptions) { + throw new UnsupportedOperationException(); + } + + @Override + public String authority() { + return "storage-direct.googleapis.com"; + } + } + private static final class CapturingLogHandler extends Handler { private final List records = new ArrayList<>(); diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/OpenTelemetryBootstrappingUtilsTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/OpenTelemetryBootstrappingUtilsTest.java index 01cc527922a9..efa2cda17f22 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/OpenTelemetryBootstrappingUtilsTest.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/OpenTelemetryBootstrappingUtilsTest.java @@ -17,18 +17,31 @@ package com.google.cloud.storage; import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; import static org.junit.Assert.assertThrows; import static org.junit.Assume.assumeFalse; import static org.mockito.Mockito.mock; import com.google.cloud.storage.OpenTelemetryBootstrappingUtils.ChannelConfigurator; +import com.google.cloud.storage.OpenTelemetryBootstrappingUtils.GrpcMetricsChannelConfigurator; +import com.google.common.collect.ImmutableList; import io.grpc.ManagedChannelBuilder; import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.MeterProvider; import io.opentelemetry.sdk.metrics.SdkMeterProvider; import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; +import io.opentelemetry.sdk.metrics.data.HistogramPointData; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; import java.time.Duration; +import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; import org.junit.Test; public final class OpenTelemetryBootstrappingUtilsTest { @@ -272,10 +285,135 @@ public void getCloudMonitoringEndpoint_extractsPortFromIpv4AndIpv6Endpoints() { OpenTelemetryBootstrappingUtils.getCloudMonitoringEndpoint( "[::1]:8080", "googleapis.com")) .isEqualTo("monitoring.googleapis.com:8080"); + assertThat( + OpenTelemetryBootstrappingUtils.getCloudMonitoringEndpoint( + "[::1]:8443", "googleapis.com")) + .isEqualTo("monitoring.googleapis.com:8443"); + assertThat(OpenTelemetryBootstrappingUtils.getCloudMonitoringEndpoint("::1", "googleapis.com")) + .isEqualTo("monitoring.googleapis.com:443"); + assertThat( + OpenTelemetryBootstrappingUtils.getCloudMonitoringEndpoint( + "[::1]:invalid", "googleapis.com")) + .isEqualTo("monitoring.googleapis.com:443"); assertThat( OpenTelemetryBootstrappingUtils.getCloudMonitoringEndpoint( "storage.my-universe.com:8443", "my-universe.com")) .isEqualTo("monitoring.my-universe.com:8443"); + assertThat( + OpenTelemetryBootstrappingUtils.getCloudMonitoringEndpoint( + "storage.googleapis.com:443", null)) + .isEqualTo("monitoring.googleapis.com:443"); + } + + @Test + public void getCloudMonitoringEndpoint_directPathInterconnectAndC2pEndpoints() { + ImmutableList cases = + ImmutableList.of( + new String[] { + "storage-direct.googleapis.com:443", null, "monitoring.googleapis.com:443" + }, + new String[] { + "storage-preprod-direct.sandbox.googleapis.com:443", + null, + "monitoring.googleapis.com:443" + }, + new String[] { + "storage-direct.apis-tpclp.goog:443", + "apis-tpclp.goog", + "monitoring.apis-tpclp.goog:443" + }, + new String[] { + "google-c2p:///storage-direct.googleapis.com?force-xds", + null, + "monitoring.googleapis.com:443" + }, + new String[] { + "google-c2p-experimental:///storage-preprod-direct.sandbox.googleapis.com:443?force-xds", + null, + "monitoring.googleapis.com:443" + }, + new String[] {null, null, "monitoring.googleapis.com:443"}, + new String[] {"", null, "monitoring.googleapis.com:443"}); + for (String[] c : cases) { + assertWithMessage("endpoint=%s, universe=%s", c[0], c[1]) + .that(OpenTelemetryBootstrappingUtils.getCloudMonitoringEndpoint(c[0], c[1])) + .isEqualTo(c[2]); + } + } + + @Test + public void registerGrpcChannelViews_exportsChannelCreationLatencyWithSecondsBuckets() { + InMemoryMetricReader metricReader = InMemoryMetricReader.create(); + SdkMeterProviderBuilder providerBuilder = + SdkMeterProvider.builder().registerMetricReader(metricReader); + SdkMeterProviderBuilder returned = + OpenTelemetryBootstrappingUtils.registerGrpcChannelViews(providerBuilder); + assertThat(returned).isSameInstanceAs(providerBuilder); + + try (SdkMeterProvider meterProvider = providerBuilder.build()) { + BiConsumer recorder = + OpenTelemetryBootstrappingUtils.createChannelCreationLatencyRecorder(meterProvider); + recorder.accept("DirectPath", Duration.ofMillis(3)); + recorder.accept("DirectPathInterconnect", Duration.ofMillis(1200)); + recorder.accept("CloudPath", Duration.ofSeconds(45)); + + Collection metrics = metricReader.collectAllMetrics(); + assertThat(metrics).hasSize(1); + MetricData metricData = metrics.iterator().next(); + assertThat(metricData.getName()) + .isEqualTo(OpenTelemetryBootstrappingUtils.CHANNEL_CREATION_LATENCY_VIEW_NAME); + assertThat(metricData.getUnit()) + .isEqualTo(OpenTelemetryBootstrappingUtils.CHANNEL_CREATION_LATENCY_UNIT); + assertThat(metricData.getDescription()) + .isEqualTo(OpenTelemetryBootstrappingUtils.CHANNEL_CREATION_LATENCY_DESCRIPTION); + + Map pointsByTransport = + metricData.getHistogramData().getPoints().stream() + .collect( + Collectors.toMap( + p -> + p.getAttributes().get(OpenTelemetryBootstrappingUtils.TRANSPORT_TYPE_KEY), + Function.identity())); + assertThat(pointsByTransport.keySet()) + .containsExactly("DirectPath", "DirectPathInterconnect", "CloudPath"); + + HistogramPointData dpPoint = pointsByTransport.get("DirectPath"); + HistogramPointData dpInterconnectPoint = pointsByTransport.get("DirectPathInterconnect"); + HistogramPointData cloudPathPoint = pointsByTransport.get("CloudPath"); + + assertThat(dpPoint.getSum()).isWithin(1e-6).of(0.003); + assertThat(dpInterconnectPoint.getSum()).isWithin(1e-6).of(1.2); + assertThat(cloudPathPoint.getSum()).isWithin(1e-6).of(45.0); + + List boundaries = dpPoint.getBoundaries(); + int idx003 = findBucketIndex(boundaries, 0.003); + int idx12 = findBucketIndex(boundaries, 1.2); + int idx45 = findBucketIndex(boundaries, 45.0); + assertThat(idx003).isEqualTo(2); + assertThat(idx12).isGreaterThan(idx003); + assertThat(idx45).isGreaterThan(idx12); + assertThat(dpPoint.getCounts().get(idx003)).isEqualTo(1L); + assertThat(dpInterconnectPoint.getCounts().get(idx12)).isEqualTo(1L); + assertThat(cloudPathPoint.getCounts().get(idx45)).isEqualTo(1L); + } + } + + @SuppressWarnings("rawtypes") // ManagedChannelBuilder + @Test + public void grpcMetricsChannelConfigurator_delegatesAndSuppliesMeterProvider() { + ManagedChannelBuilder b1 = mock(ManagedChannelBuilder.class, "b1"); + ManagedChannelBuilder b2 = mock(ManagedChannelBuilder.class, "b2"); + ChannelConfigurator delegate = + b -> { + assertThat(b).isSameInstanceAs(b1); + return b2; + }; + try (SdkMeterProvider meterProvider = SdkMeterProvider.builder().build()) { + GrpcMetricsChannelConfigurator configurator = + new GrpcMetricsChannelConfigurator(delegate, meterProvider); + assertThat(configurator.apply(b1)).isSameInstanceAs(b2); + assertThat(((Supplier) configurator).get()).isSameInstanceAs(meterProvider); + } } @Test @@ -326,4 +464,47 @@ public void createDefaultClientMeterProvider_returnsNullWhenProjectIdCannotBeDet Attributes.empty()); assertThat(nullProvider).isNull(); } + + @Test + public void + createChannelCreationLatencyRecorder_appliesExplicitBucketBoundariesAdviceWithoutCustomViews() { + InMemoryMetricReader metricReader = InMemoryMetricReader.create(); + try (SdkMeterProvider meterProvider = + SdkMeterProvider.builder().registerMetricReader(metricReader).build()) { + BiConsumer recorder = + OpenTelemetryBootstrappingUtils.createChannelCreationLatencyRecorder(meterProvider); + recorder.accept("DirectPathInterconnect", Duration.ofMillis(125)); + + Collection metrics = metricReader.collectAllMetrics(); + assertThat(metrics).hasSize(1); + MetricData metricData = metrics.iterator().next(); + assertThat(metricData.getName()) + .isEqualTo(OpenTelemetryBootstrappingUtils.CHANNEL_CREATION_LATENCY_INSTRUMENT); + HistogramPointData point = metricData.getHistogramData().getPoints().iterator().next(); + assertThat(point.getBoundaries()) + .isEqualTo(OpenTelemetryBootstrappingUtils.latencyHistogramBoundaries()); + assertThat(point.getSum()).isWithin(1e-6).of(0.125); + } + } + + @Test + public void createChannelCreationLatencyRecorder_nullMeterProvider_isNoop() { + InMemoryMetricReader metricReader = InMemoryMetricReader.create(); + try (SdkMeterProvider unusedProvider = + SdkMeterProvider.builder().registerMetricReader(metricReader).build()) { + BiConsumer noopRecorder = + OpenTelemetryBootstrappingUtils.createChannelCreationLatencyRecorder(null); + noopRecorder.accept("CloudPath", Duration.ofMillis(10)); + assertThat(metricReader.collectAllMetrics()).isEmpty(); + } + } + + private static int findBucketIndex(List boundaries, double value) { + for (int i = 0; i < boundaries.size(); i++) { + if (value <= boundaries.get(i)) { + return i; + } + } + return boundaries.size(); + } }