diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java index 3e9d166d9ae0..598e061f65ff 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java @@ -29,7 +29,7 @@ class PoolSizer { // 50% increase in latency (since that also increases requests-in-flight). private volatile float idlesSessionHeadRoom; private volatile int minIdleSessions; - private volatile int maxIdleSessions; + private volatile int maxSessions; private final PoolStats stats; private final Sized pendingRpcs; @@ -60,12 +60,12 @@ interface Sized { void updateConfig(SessionClientConfiguration.SessionPoolConfiguration poolConfiguration) { this.idlesSessionHeadRoom = poolConfiguration.getHeadroom(); this.minIdleSessions = poolConfiguration.getMinSessionCount(); - this.maxIdleSessions = poolConfiguration.getMaxSessionCount(); + this.maxSessions = poolConfiguration.getMaxSessionCount(); this.pendingVRpcsPerSession = poolConfiguration.getNewSessionQueueLength(); this.disabled = poolConfiguration.equals( SessionClientConfiguration.SessionPoolConfiguration.getDefaultInstance()) - || this.maxIdleSessions <= 0; + || this.maxSessions <= 0; } boolean isDisabled() { @@ -82,9 +82,9 @@ public int getScaleDelta() { int sessionsInUse = effectivePending + stats.getInUseCount(); int unboundedDesiredIdleSessions = (int) Math.ceil(sessionsInUse * idlesSessionHeadRoom); int desiredIdleSessions = - Math.max(Math.min(unboundedDesiredIdleSessions, maxIdleSessions), minIdleSessions); + Math.max(Math.min(unboundedDesiredIdleSessions, maxSessions), minIdleSessions); - int desiredCapacity = sessionsInUse + desiredIdleSessions; + int desiredCapacity = Math.min(sessionsInUse + desiredIdleSessions, maxSessions); int eventualCapacity = stats.getExpectedCapacity(); int immediateCapacity = eventualCapacity - stats.getStartingCount(); @@ -100,6 +100,14 @@ public int getScaleDelta() { } } + /** Returns the maximum number of pending vRPCs allowed in the pool queue. */ + int getMaxPendingRpcs() { + if (isDisabled()) { + return 0; + } + return maxSessions * Math.max(1, pendingVRpcsPerSession); + } + /** Returns true if the session should be replaced */ @CheckReturnValue public boolean handleGoAway(GoAwayResponse msg) { diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionList.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionList.java index 79fc1109fe23..a75d1b7b2ecc 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionList.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionList.java @@ -63,7 +63,6 @@ class SessionList { // - starting sessions // - closing sessions private final Set allSessions = new HashSet<>(); - private final Set inUseSessions = new HashSet<>(); // pool level statistics across all the afes private final PoolStats poolStats = new PoolStats(); @@ -119,7 +118,6 @@ public Optional checkoutSession(AfeHandle afeHandle) { handle -> { poolStats.readyCount--; poolStats.inUseCount++; - inUseSessions.add(handle); if (handle.afe.get().sessions.isEmpty()) { afesWithReadySessions.remove(afeHandle); } @@ -221,7 +219,6 @@ private void releaseToPool() { AfeHandle afeHandle = this.afe.get(); poolStats.inUseCount--; - inUseSessions.remove(this); if (!closing && session.getState() == SessionState.READY) { poolStats.readyCount++; diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java index c1abfb90232c..209f1f1e5a90 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java @@ -144,6 +144,12 @@ private enum PoolState { @GuardedBy("poolLock") private int consecutiveFailures = 0; + @GuardedBy("poolLock") + private Status lastCloseStatus = Status.UNAVAILABLE; + + @GuardedBy("poolLock") + private Metadata lastCloseTrailers = new Metadata(); + /** * When the client fallback to a non-session AFE session creation will return unimplemented * errors. In which case the requests should fallback to classic client instead of waiting for an @@ -729,6 +735,8 @@ private void onSessionClose( // rather than session establishment failures and should not fail pending vRPCs. if (prevState == SessionState.STARTING) { consecutiveFailures++; + lastCloseStatus = status; + lastCloseTrailers = trailers; if (status.getCode() == Status.Code.UNIMPLEMENTED) { consecutiveUnimplementedFailures++; } else { @@ -911,7 +919,7 @@ class PendingVRpc implements VRpc realCall; - private BigtableTimer.Timeout deadlineTimeout; + private volatile BigtableTimer.Timeout deadlineTimeout; public PendingVRpc(VRpcDescriptor desc) { this.desc = desc; @@ -922,7 +930,7 @@ public PendingVRpc(VRpcDescriptor desc) { // region. Bounded acquisition on the request hot path; see poolLock. @SuppressWarnings("GuardedBy") public void start(ReqT req, VRpcCallContext ctx, VRpcListener listener) { - Preconditions.checkState(this.req == null, "request is already started"); + Preconditions.checkState(this.ctx == null, "request is already started"); Preconditions.checkNotNull(req, "request can't be null"); Preconditions.checkNotNull(ctx, "ctx can't be null"); Preconditions.checkNotNull(listener, "listener can't be null"); @@ -940,13 +948,7 @@ public void start(ReqT req, VRpcCallContext ctx, VRpcListener listener) { VRpcResult.createUncommitedError( Status.UNAVAILABLE.withCause( new IllegalStateException("SessionPool lock unavailable (pool wedged)"))); - ctx.getExecutor() - .execute( - () -> { - if (isCancelled) return; - isCancelled = true; - listener.onClose(result); - }); + cancelWithResult(result); return; } try { @@ -954,20 +956,44 @@ public void start(ReqT req, VRpcCallContext ctx, VRpcListener listener) { VRpcResult result = VRpcResult.createUncommitedError( Status.UNAVAILABLE.withCause(new IllegalStateException("SessionPool is closed"))); - ctx.getExecutor().execute(() -> listener.onClose(result)); + cancelWithResult(result); return; } if (poolSizer.isDisabled()) { VRpcResult result = VRpcResult.createRejectedError( Status.UNAVAILABLE.withDescription("SessionPool configuration disabled")); - ctx.getExecutor() - .execute( - () -> { - if (isCancelled) return; - isCancelled = true; - listener.onClose(result); - }); + cancelWithResult(result); + return; + } + if (consecutiveFailures >= getMaxConsecutiveFailures(configManager) + && sessions.getStats().getReadyCount() == 0) { + if (poolSizer.handleNewCall()) { + createSession(openParams); + } + VRpcResult result = + VRpcResult.createRejectedError( + Status.fromCode(lastCloseStatus.getCode()) + .withDescription( + String.format( + "Session failed with consecutive failures. Most recent server status:" + + " %s, metadata: %s.", + lastCloseStatus, lastCloseTrailers))); + cancelWithResult(result); + return; + } + int maxPendingRpcs = poolSizer.getMaxPendingRpcs(); + if (maxPendingRpcs > 0 && pendingRpcs.size() >= maxPendingRpcs) { + if (poolSizer.handleNewCall()) { + createSession(openParams); + } + VRpcResult result = + VRpcResult.createRejectedError( + Status.RESOURCE_EXHAUSTED.withDescription( + String.format( + "SessionPool pending RPC queue is full (%d/%d)", + pendingRpcs.size(), maxPendingRpcs))); + cancelWithResult(result); return; } // Only arm the deadline monitor after we've committed to queueing; otherwise the @@ -1013,8 +1039,10 @@ public void cancel(@Nullable String message, @Nullable Throwable cause) { // region below. @SuppressWarnings("GuardedBy") private void cancel(Status status, boolean onlyCancelPendingCall) { - if (deadlineTimeout != null) { - deadlineTimeout.cancel(); + BigtableTimer.Timeout timeout = deadlineTimeout; + deadlineTimeout = null; + if (timeout != null) { + timeout.cancel(); } // The eager removal is only an optimization to skip a wasted session pick in // tryDrainPendingRpcs. @@ -1037,26 +1065,36 @@ private void cancel(Status status, boolean onlyCancelPendingCall) { () -> { if (isCancelled) return; isCancelled = true; + VRpcListener pendingListener = listener; + req = null; + listener = null; if (realCall != null) { if (!onlyCancelPendingCall) { realCall.cancel(status.getDescription(), status.getCause()); } - } else { - listener.onClose(VRpcResult.createRejectedError(status)); + } else if (pendingListener != null) { + pendingListener.onClose(VRpcResult.createRejectedError(status)); } }); } void cancelWithResult(VRpcResult result) { - if (deadlineTimeout != null) { - deadlineTimeout.cancel(); + BigtableTimer.Timeout timeout = deadlineTimeout; + deadlineTimeout = null; + if (timeout != null) { + timeout.cancel(); } ctx.getExecutor() .execute( () -> { if (isCancelled) return; isCancelled = true; - listener.onClose(result); + VRpcListener pendingListener = listener; + req = null; + listener = null; + if (pendingListener != null) { + pendingListener.onClose(result); + } }); } @@ -1077,8 +1115,10 @@ public void requestNext() { } private void drainTo(SessionHandle handle) { - if (deadlineTimeout != null) { - deadlineTimeout.cancel(); + BigtableTimer.Timeout timeout = deadlineTimeout; + deadlineTimeout = null; + if (timeout != null) { + timeout.cancel(); } ctx.getExecutor() .execute( @@ -1087,15 +1127,15 @@ private void drainTo(SessionHandle handle) { SessionPoolImpl.this.onPendingVRpcCancelled(handle); return; } + ReqT pendingReq = req; + VRpcListener pendingListener = listener; + req = null; + listener = null; realCall = newRealCall(desc, handle); - realCall.start(req, ctx, listener); + realCall.start(pendingReq, ctx, pendingListener); }); } - private VRpcListener getListener() { - return listener; - } - private BigtableTimer.Timeout monitorDeadline(Deadline deadline) { // Body runs on backgroundExecutor (off the tick thread). onlyCancelPendingCall=true avoids // racing with a user cancel that already attached a real call. diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java index a343c7973ce8..8f2822cbb3f1 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java @@ -31,14 +31,14 @@ class PoolSizerTest { private static final float IDLE_SESSION_HEADROOM_RATIO = 0.5f; private static final int MIN_IDLE_SESSIONS = 5; - private static final int MAX_IDLE_SESSIONS = 400; + private static final int MAX_SESSIONS = 400; private static final int PENDING_RPCS_PER_SESSION = 10; private static final SessionClientConfiguration.SessionPoolConfiguration DEFAULT_CONFIG = SessionClientConfiguration.SessionPoolConfiguration.newBuilder() .setHeadroom(IDLE_SESSION_HEADROOM_RATIO) .setMinSessionCount(MIN_IDLE_SESSIONS) - .setMaxSessionCount(MAX_IDLE_SESSIONS) + .setMaxSessionCount(MAX_SESSIONS) .setNewSessionQueueLength(PENDING_RPCS_PER_SESSION) .build(); @@ -193,7 +193,7 @@ void testEquilibriumWhenStartingWithLoad() { int sessionsNeeded = forPending + stats.getInUseCount(); int desiredIdleSessions = (int) Math.ceil(sessionsNeeded * IDLE_SESSION_HEADROOM_RATIO); desiredIdleSessions = - Math.max(Math.min(desiredIdleSessions, MAX_IDLE_SESSIONS), MIN_IDLE_SESSIONS); + Math.max(Math.min(desiredIdleSessions, MAX_SESSIONS), MIN_IDLE_SESSIONS); int desiredCapacity = sessionsNeeded + desiredIdleSessions; assertThat(sizer.getScaleDelta()) @@ -249,4 +249,27 @@ void testEmptyConfig() { assertThat(sizer.handleGoAway(GoAwayResponse.getDefaultInstance())).isFalse(); assertThat(sizer.handleSessionClose(Status.getDefaultInstance())).isFalse(); } + + @Test + void testDesiredCapacityCappedAtMaxSessions() { + PoolStats stats = new PoolStats(); + AtomicInteger pendingCount = new AtomicInteger(0); + PoolSizer sizer = new PoolSizer(stats, pendingCount::get, DEFAULT_CONFIG); + + // MAX_SESSIONS in use + 2 ready -> immediateCapacity = MAX_SESSIONS + 2. + // Total desiredCapacity is capped at MAX_SESSIONS, so scaleDelta is -2. + stats.getTestHelper().setInUseCount(MAX_SESSIONS); + stats.getTestHelper().setReadyCount(2); + stats.getTestHelper().setExpectedCapacity(MAX_SESSIONS + 2); + assertThat(sizer.getScaleDelta()).isEqualTo(-2); + + // When MAX_SESSIONS are in use and 50 RPCs are queued, scaleDelta must be 0 + // rather than scaling above MAX_SESSIONS. + stats.getTestHelper().setReadyCount(0); + stats.getTestHelper().setExpectedCapacity(MAX_SESSIONS); + pendingCount.set(50); + assertThat(sizer.getScaleDelta()).isEqualTo(0); + assertThat(sizer.handleNewCall()).isFalse(); + assertThat(sizer.getMaxPendingRpcs()).isEqualTo(MAX_SESSIONS * PENDING_RPCS_PER_SESSION); + } } diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java index 55afab0b79dd..c376220405ad 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java @@ -1448,6 +1448,159 @@ void configUpdateToEmptyClosesSessionsAndScalesBackUpWhenEnabled() throws Except assertThat(sessionPool.hasSession()).isTrue(); } + @Test + void pendingVRpcClearsReferencesAfterDrainAndCancel() throws Exception { + sessionPool.start(OpenFakeSessionRequest.getDefaultInstance(), new Metadata()); + + // Create both PendingVRpcs before any session becomes READY so both return PendingVRpc + UnaryResponseFuture drainedFuture = new UnaryResponseFuture<>(); + VRpc drainedRpc = + sessionPool.newCall(FakeDescriptor.SCRIPTED); + UnaryResponseFuture cancelledFuture = new UnaryResponseFuture<>(); + VRpc cancelledRpc = + sessionPool.newCall(FakeDescriptor.SCRIPTED); + + cancelledRpc.start( + SessionFakeScriptedRequest.getDefaultInstance(), + VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer), + cancelledFuture); + cancelledRpc.cancel("user cancelled", null); + + drainedRpc.start( + SessionFakeScriptedRequest.getDefaultInstance(), + VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer), + drainedFuture); + drainedFuture.get(5, TimeUnit.SECONDS); + + Field reqField = drainedRpc.getClass().getDeclaredField("req"); + Field listenerField = drainedRpc.getClass().getDeclaredField("listener"); + Field timeoutField = drainedRpc.getClass().getDeclaredField("deadlineTimeout"); + reqField.setAccessible(true); + listenerField.setAccessible(true); + timeoutField.setAccessible(true); + + assertThat(reqField.get(drainedRpc)).isNull(); + assertThat(listenerField.get(drainedRpc)).isNull(); + assertThat(timeoutField.get(drainedRpc)).isNull(); + + assertThat(reqField.get(cancelledRpc)).isNull(); + assertThat(listenerField.get(cancelledRpc)).isNull(); + assertThat(timeoutField.get(cancelledRpc)).isNull(); + } + + @Test + void consecutiveFailuresFastFailsDuringBudgetLockout() throws Exception { + com.google.rpc.Status errorStatus = + com.google.rpc.Status.newBuilder() + .setCode(Code.INTERNAL_VALUE) + .setMessage("fake internal error") + .build(); + OpenFakeSessionRequest request = + OpenFakeSessionRequest.newBuilder() + .setStreamError( + StreamError.newBuilder() + .setStatus(errorStatus) + .setDelay(Durations.fromMillis(0)) + .build()) + .build(); + sessionPool.start(request, new Metadata()); + + // Wait until all 20 initial SessionCreationBudget slots have been exhausted and closed + waitForOpenRequestCount(20, Duration.ofSeconds(5)); + + // A new call arriving during the 60s budget lockout must fail immediately instead of queueing + UnaryResponseFuture f = new UnaryResponseFuture<>(); + VRpc rpc = + sessionPool.newCall(FakeDescriptor.SCRIPTED); + rpc.start( + SessionFakeScriptedRequest.getDefaultInstance(), + VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer), + f); + + ExecutionException ex = + assertThrows(ExecutionException.class, () -> f.get(1, TimeUnit.SECONDS)); + assertThat(ex.getCause()) + .hasMessageThat() + .contains("INTERNAL: Session failed with consecutive failures."); + } + + @Test + void pendingRpcsQueueBoundRejectsExcessCalls() throws Exception { + try (ChannelPool delayedPool = + new SingleChannelPool( + Suppliers.ofInstance( + Grpc.newChannelBuilderForAddress( + "localhost", server.getPort(), InsecureChannelCredentials.create()) + .intercept(new DelayedClientInterceptor(Duration.ofMillis(200))) + .build()))) { + delayedPool.start(); + + ClientConfigurationManager boundedConfigManager = mock(ClientConfigurationManager.class); + ClientConfigurationManager.ListenerHandle handle = + mock(ClientConfigurationManager.ListenerHandle.class); + ArgumentCaptor captor = + ArgumentCaptor.forClass(ClientConfigurationManager.ConfigListener.class); + when(boundedConfigManager.getClientConfiguration()) + .thenReturn(ClientConfigurationManager.loadDefault()); + when(boundedConfigManager.addListener(any(), captor.capture())).thenReturn(handle); + + SessionPoolImpl boundedPool = + new SessionPoolImpl<>( + metrics, + FeatureFlags.getDefaultInstance(), + CLIENT_INFO, + boundedConfigManager, + delayedPool, + CallOptions.DEFAULT, + FakeDescriptor.FAKE_SESSION, + "bounded-pool", + testTimer, + MoreExecutors.directExecutor()); + try { + // Set maxSessionCount=2, newSessionQueueLength=2 -> maxPendingRpcs = 4 + SessionPoolConfiguration smallConfig = + ClientConfigurationManager.loadDefault() + .getSessionConfiguration() + .getSessionPoolConfiguration() + .toBuilder() + .setMinSessionCount(1) + .setMaxSessionCount(2) + .setNewSessionQueueLength(2) + .build(); + captor.getValue().onChange(smallConfig); + + boundedPool.start(OpenFakeSessionRequest.getDefaultInstance(), new Metadata()); + + for (int i = 0; i < 4; i++) { + UnaryResponseFuture f = new UnaryResponseFuture<>(); + VRpc rpc = + boundedPool.newCall(FakeDescriptor.SCRIPTED); + rpc.start( + SessionFakeScriptedRequest.getDefaultInstance(), + VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer), + f); + } + + // 5th pending call exceeds maxPendingRpcs (4) and must be rejected immediately + UnaryResponseFuture fifthFuture = new UnaryResponseFuture<>(); + VRpc fifthRpc = + boundedPool.newCall(FakeDescriptor.SCRIPTED); + fifthRpc.start( + SessionFakeScriptedRequest.getDefaultInstance(), + VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer), + fifthFuture); + + ExecutionException ex = + assertThrows(ExecutionException.class, () -> fifthFuture.get(1, TimeUnit.SECONDS)); + assertThat(ex.getCause()) + .hasMessageThat() + .contains("RESOURCE_EXHAUSTED: SessionPool pending RPC queue is full (4/4)"); + } finally { + boundedPool.close(CloseSessionRequest.getDefaultInstance()); + } + } + } + private void waitForOpenRequestCount(int minCount, Duration timeout) throws InterruptedException { long deadline = System.currentTimeMillis() + timeout.toMillis(); while (System.currentTimeMillis() < deadline) {