diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java index 1ef72a91937a..f1db13451e75 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java @@ -308,6 +308,21 @@ public synchronized SessionStream newStream( return new SessionStreamImpl(innerCall) { // null until onBeforeSessionStart fires @Nullable AfeId afeId = null; + volatile boolean sessionEstablished = false; + + @Override + public void afterSessionStart() { + // Only mark the session established and reset consecutiveFailures after + // OpenSessionResponse is processed. onBeforeSessionStart is not enough because the AFE + // sends initial PeerInfo response headers before completing OpenSession, so streams + // rejected during OpenSession (e.g. RESOURCE_EXHAUSTED) still fire + // onBeforeSessionStart before closing with an error. + sessionEstablished = true; + synchronized (ChannelPoolDpImpl.this) { + fw.consecutiveFailures = 0; + recycleBackoff = INITIAL_RECYCLE_BACKOFF; + } + } @Override public void start(Listener responseListener, Metadata headers) { @@ -317,8 +332,6 @@ public void start(Listener responseListener, Metadata headers) { public void onBeforeSessionStart(PeerInfo peerInfo) { afeId = AfeId.extract(peerInfo); synchronized (ChannelPoolDpImpl.this) { - fw.consecutiveFailures = 0; - recycleBackoff = INITIAL_RECYCLE_BACKOFF; sessionsPerAfeId.add(afeId); if (!fw.channel.isShutdown()) { routeObservations.put(new RouteKey(fw.id, tenant), afeId); @@ -333,8 +346,12 @@ public void onClose(Status status, Metadata trailers) { synchronized (ChannelPoolDpImpl.this) { if (afeId != null) { sessionsPerAfeId.remove(afeId); - } else if (!status.isOk() && status.getCode() != Code.CANCELLED) { + } + if (!sessionEstablished && !status.isOk() && status.getCode() != Code.CANCELLED) { fw.consecutiveFailures++; + if (afeId != null) { + routeObservations.remove(new RouteKey(fw.id, tenant), afeId); + } } releaseChannel(fw, status); } @@ -572,8 +589,11 @@ enum State { final ManagedChannel channel; final Instant createdAt; volatile State state = State.ACTIVE; + // Guarded by the enclosing ChannelPoolDpImpl instance. int numOutstanding = 0; + // Guarded by the enclosing ChannelPoolDpImpl instance. int consecutiveFailures = 0; + // Guarded by the enclosing ChannelPoolDpImpl instance. Instant lastUsedAt; ChannelWrapper(long id, ManagedChannel channel, Clock clock) { diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/FailingSessionStream.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/FailingSessionStream.java index df2ae9ad35c1..ae3f23630642 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/FailingSessionStream.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/FailingSessionStream.java @@ -39,6 +39,11 @@ public void start(Listener responseListener, Metadata headers) { responseListener.onClose(status, new Metadata()); } + @Override + public void afterSessionStart() { + // noop + } + @Override public void sendMessage(SessionRequest request) { // noop diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ForwardingSessionStream.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ForwardingSessionStream.java index 54652c044ee5..6fb8cae917f3 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ForwardingSessionStream.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ForwardingSessionStream.java @@ -37,6 +37,11 @@ public void start(Listener responseListener, Metadata headers) { delegate.start(responseListener, headers); } + @Override + public void afterSessionStart() { + delegate.afterSessionStart(); + } + @Override public void sendMessage(SessionRequest request) { delegate.sendMessage(request); diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/SessionStream.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/SessionStream.java index eb6dbfa339eb..5f99778e1c8c 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/SessionStream.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/SessionStream.java @@ -31,6 +31,8 @@ public interface SessionStream { public void start(Listener responseListener, Metadata headers); + public void afterSessionStart(); + public void sendMessage(SessionRequest request); public void halfClose(); diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/SessionStreamImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/SessionStreamImpl.java index fa46914df03b..4d271b5cbc10 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/SessionStreamImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/SessionStreamImpl.java @@ -103,6 +103,9 @@ public void onClose(Status status, Metadata trailers) { call.request(Integer.MAX_VALUE); } + @Override + public void afterSessionStart() {} + @Override public void sendMessage(SessionRequest request) { call.sendMessage(request); diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionImpl.java index d07bbc348d39..d80f301f63ac 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionImpl.java @@ -682,6 +682,7 @@ private void handleOpenSessionResponse(OpenSessionResponse openSession) { } PeerInfo localPeerInfo = stream.getPeerInfo(); updateState(SessionState.READY); + stream.afterSessionStart(); tracer.onOpen(localPeerInfo); sessionListener.onReady(openSession); } diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImplTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImplTest.java index 4efca0494718..bfdedf35f734 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImplTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImplTest.java @@ -517,9 +517,10 @@ void testResetConsecutiveFailuresOnSuccess() { } verify(channel, times(0)).shutdown(); - // A success: onHeaders (which calls onBeforeSessionStart) - pool.newStream(FakeSessionGrpc.getOpenSessionMethod(), CallOptions.DEFAULT) - .start(mock(Listener.class), new Metadata()); + // A success: onHeaders (which calls onBeforeSessionStart) followed by afterSessionStart() + SessionStream stream = + pool.newStream(FakeSessionGrpc.getOpenSessionMethod(), CallOptions.DEFAULT); + stream.start(mock(Listener.class), new Metadata()); PeerInfo peerInfo = PeerInfo.newBuilder().setApplicationFrontendId(555).build(); Metadata headers = new Metadata(); @@ -527,6 +528,7 @@ void testResetConsecutiveFailuresOnSuccess() { SessionStreamImpl.PEER_INFO_KEY, Base64.getEncoder().encodeToString(peerInfo.toByteArray())); listener.getValue().onHeaders(headers); + stream.afterSessionStart(); listener.getValue().onClose(Status.OK, new Metadata()); // Another 4 failures - should still not recycle because counter was reset @@ -632,4 +634,35 @@ void testRecycleChannelBackoff() { pool.close(); } + + @Test + void testPreOpenSessionFailureWithPeerInfoIncrementsFailures() { + when(channelSupplier.get()).thenReturn(channel); + when(channel.newCall(any(), any())).thenReturn(clientCall); + doNothing().when(clientCall).start(listener.capture(), any()); + doReturn(Attributes.EMPTY).when(clientCall).getAttributes(); + + ChannelPoolDpImpl pool = + new ChannelPoolDpImpl(channelSupplier, defaultConfig, debugTagTracer, bgExecutor); + + PeerInfo peerInfo = PeerInfo.newBuilder().setApplicationFrontendId(555).build(); + Metadata headers = new Metadata(); + headers.put( + SessionStreamImpl.PEER_INFO_KEY, + Base64.getEncoder().encodeToString(peerInfo.toByteArray())); + + // 5 pre-OpenSessionResponse rejections (onHeaders with PeerInfo followed by onClose(!OK) + // without OpenSessionResponse) must count as consecutive failures and recycle the channel. + for (int i = 0; i < 5; i++) { + pool.newStream(FakeSessionGrpc.getOpenSessionMethod(), CallOptions.DEFAULT) + .start(mock(Listener.class), new Metadata()); + listener.getValue().onHeaders(headers); + listener.getValue().onClose(Status.RESOURCE_EXHAUSTED, new Metadata()); + } + + verify(channel, times(1)).shutdown(); + verify(channelSupplier, times(2)).get(); + + pool.close(); + } }