diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java index c1abfb90232c..5951a458b047 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java @@ -144,6 +144,10 @@ private enum PoolState { @GuardedBy("poolLock") private int consecutiveFailures = 0; + @GuardedBy("poolLock") + @Nullable + private VRpcResult lastFailureResult = null; + /** * When the client fallback to a non-session AFE session creation will return unimplemented * errors. In which case the requests should fallback to classic client instead of waiting for an @@ -281,6 +285,7 @@ public SessionPoolImpl( poolSizer.updateConfig(newConfig); picker.updateConfig(newConfig); if (poolSizer.isDisabled()) { + resetConsecutiveFailures(); if (retryCreateSessionTimeout != null) { retryCreateSessionTimeout.cancel(); retryCreateSessionTimeout = null; @@ -597,8 +602,7 @@ private void onSessionReady(SessionHandle handle, OpenSessionResponse ignored) { "Session {0} in state {1}", new Object[] {handle.getSession().getLogName(), handle.getSession().getState()}); - consecutiveFailures = 0; - consecutiveUnimplementedFailures = 0; + resetConsecutiveFailures(); if (poolState != PoolState.STARTED) { logger.fine( @@ -688,6 +692,7 @@ private void onSessionClose( SessionHandle handle, SessionState prevState, Status status, Metadata trailers) { List> toBeClosed = new ArrayList<>(); + VRpcResult failureResult = null; poolLock.lock(); try { @@ -729,13 +734,15 @@ private void onSessionClose( // rather than session establishment failures and should not fail pending vRPCs. if (prevState == SessionState.STARTING) { consecutiveFailures++; + lastFailureResult = createConsecutiveFailureResult(status, trailers); if (status.getCode() == Status.Code.UNIMPLEMENTED) { consecutiveUnimplementedFailures++; } else { consecutiveUnimplementedFailures = 0; } // TODO: decide if max consecutive failures should be capped per client - if (consecutiveFailures >= getMaxConsecutiveFailures(configManager)) { + if (isFailingWithNoEstablishedSessions()) { + failureResult = lastFailureResult; toBeClosed = popClosableRpcs(); } } @@ -757,17 +764,9 @@ private void onSessionClose( if (!toBeClosed.isEmpty()) { // vRPCs failing with consecutive failures should fail - VRpcResult result = - VRpcResult.createRejectedError( - Status.fromCode(status.getCode()) - .withDescription( - String.format( - "Session failed with consecutive failures. Most recent server status: %s," - + " metadata: %s.", - status, trailers))); for (PendingVRpc vrpc : toBeClosed) { try { - vrpc.cancelWithResult(result); + vrpc.cancelWithResult(failureResult); } catch (Throwable t) { logger.log(Level.WARNING, "Exception dispatching close to op executor", t); } @@ -775,6 +774,36 @@ private void onSessionClose( } } + @GuardedBy("poolLock") + private void resetConsecutiveFailures() { + consecutiveFailures = 0; + lastFailureResult = null; + consecutiveUnimplementedFailures = 0; + } + + @GuardedBy("poolLock") + private boolean hasEstablishedSession() { + return sessions.getStats().getReadyCount() + sessions.getStats().getInUseCount() > 0; + } + + @GuardedBy("poolLock") + private boolean isFailingWithNoEstablishedSessions() { + return consecutiveFailures >= getMaxConsecutiveFailures(configManager) + && lastFailureResult != null + && !hasEstablishedSession(); + } + + private static VRpcResult createConsecutiveFailureResult( + Status status, @Nullable Metadata trailers) { + Status errorStatus = status.isOk() ? Status.UNAVAILABLE : Status.fromCode(status.getCode()); + return VRpcResult.createRejectedError( + errorStatus.withDescription( + String.format( + "Session failed with consecutive failures. Most recent server status: %s," + + " metadata: %s.", + status, trailers))); + } + @GuardedBy("poolLock") private void tryDrainPendingRpcs() { while (!pendingRpcs.isEmpty()) { @@ -895,7 +924,7 @@ public boolean hasSession() { return false; } try { - return sessions.getStats().getReadyCount() + sessions.getStats().getInUseCount() > 0; + return hasEstablishedSession(); } finally { poolLock.unlock(); } @@ -970,6 +999,10 @@ public void start(ReqT req, VRpcCallContext ctx, VRpcListener listener) { }); return; } + if (isFailingWithNoEstablishedSessions()) { + cancelWithResult(lastFailureResult); + return; + } // Only arm the deadline monitor after we've committed to queueing; otherwise the // fast-fail early return above would leak a timer that fires later and emits a phantom // tracer.onAttemptFinish on the Active state's stale listener. diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java index 55afab0b79dd..d6a240a778da 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java @@ -52,6 +52,7 @@ import com.google.cloud.bigtable.data.v2.internal.middleware.VRpc; import com.google.cloud.bigtable.data.v2.internal.middleware.VRpc.VRpcCallContext; import com.google.cloud.bigtable.data.v2.internal.middleware.VRpc.VRpcResult; +import com.google.cloud.bigtable.data.v2.internal.session.Session.SessionState; import com.google.cloud.bigtable.data.v2.internal.session.SessionList.SessionHandle; import com.google.cloud.bigtable.data.v2.internal.session.fake.FakeClock; import com.google.cloud.bigtable.data.v2.internal.session.fake.FakeServiceBuilder; @@ -83,6 +84,7 @@ import io.grpc.Status; import java.io.IOException; import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; @@ -1448,6 +1450,171 @@ void configUpdateToEmptyClosesSessionsAndScalesBackUpWhenEnabled() throws Except assertThat(sessionPool.hasSession()).isTrue(); } + @Test + @SuppressWarnings("GuardedBy") + void consecutiveFailuresFastFailsDuringBudgetLockout() throws Exception { + com.google.rpc.Status errorStatus = + com.google.rpc.Status.newBuilder() + .setCode(Code.INTERNAL_VALUE) + .setMessage("fake internal error") + .build(); + OpenFakeSessionRequest request = + OpenFakeSessionRequest.newBuilder() + .setStreamError( + StreamError.newBuilder() + .setStatus(errorStatus) + .setDelay(Durations.fromMillis(0)) + .build()) + .build(); + sessionPool.start(request, new Metadata()); + + // Wait until all initial SessionCreationBudget slots have been exhausted and closed on the + // client + ReentrantLock poolLock = extractPoolLock(sessionPool); + Field consecutiveFailuresField = SessionPoolImpl.class.getDeclaredField("consecutiveFailures"); + consecutiveFailuresField.setAccessible(true); + long deadline = System.currentTimeMillis() + 5000; + while (System.currentTimeMillis() < deadline) { + poolLock.lock(); + try { + if (sessionPool.sessions.getAllSessions().isEmpty() + && consecutiveFailuresField.getInt(sessionPool) >= 10) { + break; + } + } finally { + poolLock.unlock(); + } + Thread.sleep(10); + } + poolLock.lock(); + try { + assertThat(sessionPool.sessions.getAllSessions()).isEmpty(); + assertThat(consecutiveFailuresField.getInt(sessionPool)).isAtLeast(10); + } finally { + poolLock.unlock(); + } + + // A new call arriving during the 60s budget lockout must fail immediately instead of queueing + UnaryResponseFuture f = new UnaryResponseFuture<>(); + VRpc rpc = + sessionPool.newCall(FakeDescriptor.SCRIPTED); + rpc.start( + SessionFakeScriptedRequest.getDefaultInstance(), + VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer), + f); + + ExecutionException ex = + assertThrows(ExecutionException.class, () -> f.get(1, TimeUnit.SECONDS)); + assertThat(ex.getCause()) + .hasMessageThat() + .contains("INTERNAL: Session failed with consecutive failures."); + } + + @Test + @SuppressWarnings("GuardedBy") + void consecutiveFailuresPreservesPendingRpcsWhilePoolHasInUseSessions() throws Exception { + sessionPool.start(OpenFakeSessionRequest.getDefaultInstance(), new Metadata()); + ReentrantLock poolLock = extractPoolLock(sessionPool); + long readyDeadline = System.currentTimeMillis() + 5000; + while (System.currentTimeMillis() < readyDeadline) { + poolLock.lock(); + try { + if (sessionPool.sessions.getStats().getReadyCount() == 5) { + break; + } + } finally { + poolLock.unlock(); + } + Thread.sleep(10); + } + + Field consecutiveFailuresField = SessionPoolImpl.class.getDeclaredField("consecutiveFailures"); + consecutiveFailuresField.setAccessible(true); + Field budgetField = SessionPoolImpl.class.getDeclaredField("budget"); + budgetField.setAccessible(true); + Method onSessionCloseMethod = + SessionPoolImpl.class.getDeclaredMethod( + "onSessionClose", + SessionHandle.class, + SessionState.class, + Status.class, + Metadata.class); + onSessionCloseMethod.setAccessible(true); + + UnaryResponseFuture checkedOutFuture = new UnaryResponseFuture<>(); + UnaryResponseFuture queuedBeforeCloseFuture = + new UnaryResponseFuture<>(); + UnaryResponseFuture queuedAfterCloseFuture = + new UnaryResponseFuture<>(); + VRpc checkedOutRpc; + poolLock.lock(); + try { + assertThat(sessionPool.sessions.getStats().getReadyCount()).isEqualTo(5); + // Exhaust the session creation budget so scale-up triggers in start()/onSessionClose() do not + // open new background sessions that could race with the single in-use session under test. + SessionCreationBudget budget = (SessionCreationBudget) budgetField.get(sessionPool); + while (budget.tryReserveSession()) { + budget.onSessionCreationFailure(); + } + + // Mark 4 of the 5 ready sessions closing, then check out the remaining 1 ready session via + // newCall() so readyCount == 0 and inUseCount == 1. + List handles = new ArrayList<>(sessionPool.sessions.getAllSessions()); + for (int i = 1; i < handles.size(); i++) { + handles.get(i).onSessionClosing(); + } + checkedOutRpc = sessionPool.newCall(FakeDescriptor.SCRIPTED); + assertThat(sessionPool.sessions.getStats().getReadyCount()).isEqualTo(0); + assertThat(sessionPool.sessions.getStats().getInUseCount()).isEqualTo(1); + + // Queue a first PendingVRpc before the 10th STARTING failure arrives in onSessionClose. + consecutiveFailuresField.setInt(sessionPool, 9); + VRpc queuedBeforeCloseRpc = + sessionPool.newCall(FakeDescriptor.SCRIPTED); + queuedBeforeCloseRpc.start( + SessionFakeScriptedRequest.newBuilder().setTag(0).build(), + VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer), + queuedBeforeCloseFuture); + assertThat(queuedBeforeCloseFuture.isDone()).isFalse(); + + Session mockStartingSession = mock(Session.class); + when(mockStartingSession.getLogName()).thenReturn("mock-starting"); + SessionHandle failingStartingHandle = sessionPool.sessions.newHandle(mockStartingSession); + + // Trigger the 10th STARTING failure via onSessionClose while inUseCount == 1; + // queuedBeforeCloseFuture must remain queued rather than being aborted. + onSessionCloseMethod.invoke( + sessionPool, + failingStartingHandle, + SessionState.STARTING, + Status.INTERNAL.withDescription("fake internal error"), + new Metadata()); + assertThat(consecutiveFailuresField.getInt(sessionPool)).isEqualTo(10); + assertThat(queuedBeforeCloseFuture.isDone()).isFalse(); + + // A second PendingVRpc started after consecutiveFailures == 10 while inUseCount == 1 must + // also queue instead of failing fast. + VRpc queuedAfterCloseRpc = + sessionPool.newCall(FakeDescriptor.SCRIPTED); + queuedAfterCloseRpc.start( + SessionFakeScriptedRequest.newBuilder().setTag(0).build(), + VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer), + queuedAfterCloseFuture); + assertThat(queuedAfterCloseFuture.isDone()).isFalse(); + } finally { + poolLock.unlock(); + } + + // Complete the checked-out RPC so its session returns to the pool and drains both pending RPCs. + checkedOutRpc.start( + SessionFakeScriptedRequest.newBuilder().setTag(0).build(), + VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer), + checkedOutFuture); + assertThat(checkedOutFuture.get(5, TimeUnit.SECONDS)).isNotNull(); + assertThat(queuedBeforeCloseFuture.get(5, TimeUnit.SECONDS)).isNotNull(); + assertThat(queuedAfterCloseFuture.get(5, TimeUnit.SECONDS)).isNotNull(); + } + private void waitForOpenRequestCount(int minCount, Duration timeout) throws InterruptedException { long deadline = System.currentTimeMillis() + timeout.toMillis(); while (System.currentTimeMillis() < deadline) {