From 24d65742d97b7cd016b0634981b536ec1955e7b3 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Mon, 16 Nov 2026 15:11:56 +0000 Subject: [PATCH 1/6] fix(bigtable): cap PoolSizer desiredCapacity at maxIdleSessions --- .../data/v2/internal/session/PoolSizer.java | 2 +- .../v2/internal/session/PoolSizerTest.java | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java index 3e9d166d9ae0..3aea1e6b7e70 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java @@ -84,7 +84,7 @@ public int getScaleDelta() { int desiredIdleSessions = Math.max(Math.min(unboundedDesiredIdleSessions, maxIdleSessions), minIdleSessions); - int desiredCapacity = sessionsInUse + desiredIdleSessions; + int desiredCapacity = Math.min(sessionsInUse + desiredIdleSessions, maxIdleSessions); int eventualCapacity = stats.getExpectedCapacity(); int immediateCapacity = eventualCapacity - stats.getStartingCount(); diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java index a343c7973ce8..897079f2244c 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java @@ -249,4 +249,26 @@ void testEmptyConfig() { assertThat(sizer.handleGoAway(GoAwayResponse.getDefaultInstance())).isFalse(); assertThat(sizer.handleSessionClose(Status.getDefaultInstance())).isFalse(); } + + @Test + void testDesiredCapacityCappedAtMaxIdleSessions() { + PoolStats stats = new PoolStats(); + AtomicInteger pendingCount = new AtomicInteger(0); + PoolSizer sizer = new PoolSizer(stats, pendingCount::get, DEFAULT_CONFIG); + + // MAX_IDLE_SESSIONS in use + 2 ready -> immediateCapacity = MAX_IDLE_SESSIONS + 2. + // Total desiredCapacity is capped at MAX_IDLE_SESSIONS, so scaleDelta is -2. + stats.getTestHelper().setInUseCount(MAX_IDLE_SESSIONS); + stats.getTestHelper().setReadyCount(2); + stats.getTestHelper().setExpectedCapacity(MAX_IDLE_SESSIONS + 2); + assertThat(sizer.getScaleDelta()).isEqualTo(-2); + + // When MAX_IDLE_SESSIONS are in use and 50 RPCs are queued, scaleDelta must be 0 + // rather than scaling above MAX_IDLE_SESSIONS. + stats.getTestHelper().setReadyCount(0); + stats.getTestHelper().setExpectedCapacity(MAX_IDLE_SESSIONS); + pendingCount.set(50); + assertThat(sizer.getScaleDelta()).isEqualTo(0); + assertThat(sizer.handleNewCall()).isFalse(); + } } From 86f6d017a1df7842843971ba3fb6db631f9aa916 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Mon, 16 Nov 2026 15:17:51 +0000 Subject: [PATCH 2/6] fix(bigtable): remove dead SessionList.inUseSessions and clear PendingVRpc references --- .../data/v2/internal/session/SessionList.java | 3 - .../v2/internal/session/SessionPoolImpl.java | 64 ++++++++++--------- .../internal/session/SessionPoolImplTest.java | 40 ++++++++++++ 3 files changed, 73 insertions(+), 34 deletions(-) diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionList.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionList.java index 79fc1109fe23..a75d1b7b2ecc 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionList.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionList.java @@ -63,7 +63,6 @@ class SessionList { // - starting sessions // - closing sessions private final Set allSessions = new HashSet<>(); - private final Set inUseSessions = new HashSet<>(); // pool level statistics across all the afes private final PoolStats poolStats = new PoolStats(); @@ -119,7 +118,6 @@ public Optional checkoutSession(AfeHandle afeHandle) { handle -> { poolStats.readyCount--; poolStats.inUseCount++; - inUseSessions.add(handle); if (handle.afe.get().sessions.isEmpty()) { afesWithReadySessions.remove(afeHandle); } @@ -221,7 +219,6 @@ private void releaseToPool() { AfeHandle afeHandle = this.afe.get(); poolStats.inUseCount--; - inUseSessions.remove(this); if (!closing && session.getState() == SessionState.READY) { poolStats.readyCount++; diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java index c1abfb90232c..8b21a48dd7ba 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java @@ -911,7 +911,7 @@ class PendingVRpc implements VRpc realCall; - private BigtableTimer.Timeout deadlineTimeout; + private volatile BigtableTimer.Timeout deadlineTimeout; public PendingVRpc(VRpcDescriptor desc) { this.desc = desc; @@ -922,7 +922,7 @@ public PendingVRpc(VRpcDescriptor desc) { // region. Bounded acquisition on the request hot path; see poolLock. @SuppressWarnings("GuardedBy") public void start(ReqT req, VRpcCallContext ctx, VRpcListener listener) { - Preconditions.checkState(this.req == null, "request is already started"); + Preconditions.checkState(this.ctx == null, "request is already started"); Preconditions.checkNotNull(req, "request can't be null"); Preconditions.checkNotNull(ctx, "ctx can't be null"); Preconditions.checkNotNull(listener, "listener can't be null"); @@ -940,13 +940,7 @@ public void start(ReqT req, VRpcCallContext ctx, VRpcListener listener) { VRpcResult.createUncommitedError( Status.UNAVAILABLE.withCause( new IllegalStateException("SessionPool lock unavailable (pool wedged)"))); - ctx.getExecutor() - .execute( - () -> { - if (isCancelled) return; - isCancelled = true; - listener.onClose(result); - }); + cancelWithResult(result); return; } try { @@ -954,20 +948,14 @@ public void start(ReqT req, VRpcCallContext ctx, VRpcListener listener) { VRpcResult result = VRpcResult.createUncommitedError( Status.UNAVAILABLE.withCause(new IllegalStateException("SessionPool is closed"))); - ctx.getExecutor().execute(() -> listener.onClose(result)); + cancelWithResult(result); return; } if (poolSizer.isDisabled()) { VRpcResult result = VRpcResult.createRejectedError( Status.UNAVAILABLE.withDescription("SessionPool configuration disabled")); - ctx.getExecutor() - .execute( - () -> { - if (isCancelled) return; - isCancelled = true; - listener.onClose(result); - }); + cancelWithResult(result); return; } // Only arm the deadline monitor after we've committed to queueing; otherwise the @@ -1013,8 +1001,10 @@ public void cancel(@Nullable String message, @Nullable Throwable cause) { // region below. @SuppressWarnings("GuardedBy") private void cancel(Status status, boolean onlyCancelPendingCall) { - if (deadlineTimeout != null) { - deadlineTimeout.cancel(); + BigtableTimer.Timeout timeout = deadlineTimeout; + deadlineTimeout = null; + if (timeout != null) { + timeout.cancel(); } // The eager removal is only an optimization to skip a wasted session pick in // tryDrainPendingRpcs. @@ -1037,26 +1027,36 @@ private void cancel(Status status, boolean onlyCancelPendingCall) { () -> { if (isCancelled) return; isCancelled = true; + VRpcListener pendingListener = listener; + req = null; + listener = null; if (realCall != null) { if (!onlyCancelPendingCall) { realCall.cancel(status.getDescription(), status.getCause()); } - } else { - listener.onClose(VRpcResult.createRejectedError(status)); + } else if (pendingListener != null) { + pendingListener.onClose(VRpcResult.createRejectedError(status)); } }); } void cancelWithResult(VRpcResult result) { - if (deadlineTimeout != null) { - deadlineTimeout.cancel(); + BigtableTimer.Timeout timeout = deadlineTimeout; + deadlineTimeout = null; + if (timeout != null) { + timeout.cancel(); } ctx.getExecutor() .execute( () -> { if (isCancelled) return; isCancelled = true; - listener.onClose(result); + VRpcListener pendingListener = listener; + req = null; + listener = null; + if (pendingListener != null) { + pendingListener.onClose(result); + } }); } @@ -1077,8 +1077,10 @@ public void requestNext() { } private void drainTo(SessionHandle handle) { - if (deadlineTimeout != null) { - deadlineTimeout.cancel(); + BigtableTimer.Timeout timeout = deadlineTimeout; + deadlineTimeout = null; + if (timeout != null) { + timeout.cancel(); } ctx.getExecutor() .execute( @@ -1087,15 +1089,15 @@ private void drainTo(SessionHandle handle) { SessionPoolImpl.this.onPendingVRpcCancelled(handle); return; } + ReqT pendingReq = req; + VRpcListener pendingListener = listener; + req = null; + listener = null; realCall = newRealCall(desc, handle); - realCall.start(req, ctx, listener); + realCall.start(pendingReq, ctx, pendingListener); }); } - private VRpcListener getListener() { - return listener; - } - private BigtableTimer.Timeout monitorDeadline(Deadline deadline) { // Body runs on backgroundExecutor (off the tick thread). onlyCancelPendingCall=true avoids // racing with a user cancel that already attached a real call. diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java index 55afab0b79dd..911955b67565 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java @@ -1448,6 +1448,46 @@ void configUpdateToEmptyClosesSessionsAndScalesBackUpWhenEnabled() throws Except assertThat(sessionPool.hasSession()).isTrue(); } + @Test + void pendingVRpcClearsReferencesAfterDrainAndCancel() throws Exception { + sessionPool.start(OpenFakeSessionRequest.getDefaultInstance(), new Metadata()); + + // Create both PendingVRpcs before any session becomes READY so both return PendingVRpc + UnaryResponseFuture drainedFuture = new UnaryResponseFuture<>(); + VRpc drainedRpc = + sessionPool.newCall(FakeDescriptor.SCRIPTED); + UnaryResponseFuture cancelledFuture = new UnaryResponseFuture<>(); + VRpc cancelledRpc = + sessionPool.newCall(FakeDescriptor.SCRIPTED); + + cancelledRpc.start( + SessionFakeScriptedRequest.getDefaultInstance(), + VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer), + cancelledFuture); + cancelledRpc.cancel("user cancelled", null); + + drainedRpc.start( + SessionFakeScriptedRequest.getDefaultInstance(), + VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer), + drainedFuture); + drainedFuture.get(5, TimeUnit.SECONDS); + + Field reqField = drainedRpc.getClass().getDeclaredField("req"); + Field listenerField = drainedRpc.getClass().getDeclaredField("listener"); + Field timeoutField = drainedRpc.getClass().getDeclaredField("deadlineTimeout"); + reqField.setAccessible(true); + listenerField.setAccessible(true); + timeoutField.setAccessible(true); + + assertThat(reqField.get(drainedRpc)).isNull(); + assertThat(listenerField.get(drainedRpc)).isNull(); + assertThat(timeoutField.get(drainedRpc)).isNull(); + + assertThat(reqField.get(cancelledRpc)).isNull(); + assertThat(listenerField.get(cancelledRpc)).isNull(); + assertThat(timeoutField.get(cancelledRpc)).isNull(); + } + private void waitForOpenRequestCount(int minCount, Duration timeout) throws InterruptedException { long deadline = System.currentTimeMillis() + timeout.toMillis(); while (System.currentTimeMillis() < deadline) { From 8fdc29ce87944694268eedb93223752ea4e5fd57 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Mon, 16 Nov 2026 15:22:14 +0000 Subject: [PATCH 3/6] fix(bigtable): bound SessionPool pendingRpcs and fast-fail during consecutive failures --- .../data/v2/internal/session/PoolSizer.java | 8 ++ .../v2/internal/session/SessionPoolImpl.java | 38 ++++++ .../v2/internal/session/PoolSizerTest.java | 1 + .../internal/session/SessionPoolImplTest.java | 113 ++++++++++++++++++ 4 files changed, 160 insertions(+) diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java index 3aea1e6b7e70..a130f05e8c34 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java @@ -100,6 +100,14 @@ public int getScaleDelta() { } } + /** Returns the maximum number of pending vRPCs allowed in the pool queue. */ + int getMaxPendingRpcs() { + if (isDisabled()) { + return 0; + } + return maxIdleSessions * Math.max(1, pendingVRpcsPerSession); + } + /** Returns true if the session should be replaced */ @CheckReturnValue public boolean handleGoAway(GoAwayResponse msg) { diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java index 8b21a48dd7ba..209f1f1e5a90 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java @@ -144,6 +144,12 @@ private enum PoolState { @GuardedBy("poolLock") private int consecutiveFailures = 0; + @GuardedBy("poolLock") + private Status lastCloseStatus = Status.UNAVAILABLE; + + @GuardedBy("poolLock") + private Metadata lastCloseTrailers = new Metadata(); + /** * When the client fallback to a non-session AFE session creation will return unimplemented * errors. In which case the requests should fallback to classic client instead of waiting for an @@ -729,6 +735,8 @@ private void onSessionClose( // rather than session establishment failures and should not fail pending vRPCs. if (prevState == SessionState.STARTING) { consecutiveFailures++; + lastCloseStatus = status; + lastCloseTrailers = trailers; if (status.getCode() == Status.Code.UNIMPLEMENTED) { consecutiveUnimplementedFailures++; } else { @@ -958,6 +966,36 @@ public void start(ReqT req, VRpcCallContext ctx, VRpcListener listener) { cancelWithResult(result); return; } + if (consecutiveFailures >= getMaxConsecutiveFailures(configManager) + && sessions.getStats().getReadyCount() == 0) { + if (poolSizer.handleNewCall()) { + createSession(openParams); + } + VRpcResult result = + VRpcResult.createRejectedError( + Status.fromCode(lastCloseStatus.getCode()) + .withDescription( + String.format( + "Session failed with consecutive failures. Most recent server status:" + + " %s, metadata: %s.", + lastCloseStatus, lastCloseTrailers))); + cancelWithResult(result); + return; + } + int maxPendingRpcs = poolSizer.getMaxPendingRpcs(); + if (maxPendingRpcs > 0 && pendingRpcs.size() >= maxPendingRpcs) { + if (poolSizer.handleNewCall()) { + createSession(openParams); + } + VRpcResult result = + VRpcResult.createRejectedError( + Status.RESOURCE_EXHAUSTED.withDescription( + String.format( + "SessionPool pending RPC queue is full (%d/%d)", + pendingRpcs.size(), maxPendingRpcs))); + cancelWithResult(result); + return; + } // Only arm the deadline monitor after we've committed to queueing; otherwise the // fast-fail early return above would leak a timer that fires later and emits a phantom // tracer.onAttemptFinish on the Active state's stale listener. diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java index 897079f2244c..af6fcb65c1ba 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java @@ -270,5 +270,6 @@ void testDesiredCapacityCappedAtMaxIdleSessions() { pendingCount.set(50); assertThat(sizer.getScaleDelta()).isEqualTo(0); assertThat(sizer.handleNewCall()).isFalse(); + assertThat(sizer.getMaxPendingRpcs()).isEqualTo(MAX_IDLE_SESSIONS * PENDING_RPCS_PER_SESSION); } } diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java index 911955b67565..c376220405ad 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java @@ -1488,6 +1488,119 @@ void pendingVRpcClearsReferencesAfterDrainAndCancel() throws Exception { assertThat(timeoutField.get(cancelledRpc)).isNull(); } + @Test + void consecutiveFailuresFastFailsDuringBudgetLockout() throws Exception { + com.google.rpc.Status errorStatus = + com.google.rpc.Status.newBuilder() + .setCode(Code.INTERNAL_VALUE) + .setMessage("fake internal error") + .build(); + OpenFakeSessionRequest request = + OpenFakeSessionRequest.newBuilder() + .setStreamError( + StreamError.newBuilder() + .setStatus(errorStatus) + .setDelay(Durations.fromMillis(0)) + .build()) + .build(); + sessionPool.start(request, new Metadata()); + + // Wait until all 20 initial SessionCreationBudget slots have been exhausted and closed + waitForOpenRequestCount(20, Duration.ofSeconds(5)); + + // A new call arriving during the 60s budget lockout must fail immediately instead of queueing + UnaryResponseFuture f = new UnaryResponseFuture<>(); + VRpc rpc = + sessionPool.newCall(FakeDescriptor.SCRIPTED); + rpc.start( + SessionFakeScriptedRequest.getDefaultInstance(), + VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer), + f); + + ExecutionException ex = + assertThrows(ExecutionException.class, () -> f.get(1, TimeUnit.SECONDS)); + assertThat(ex.getCause()) + .hasMessageThat() + .contains("INTERNAL: Session failed with consecutive failures."); + } + + @Test + void pendingRpcsQueueBoundRejectsExcessCalls() throws Exception { + try (ChannelPool delayedPool = + new SingleChannelPool( + Suppliers.ofInstance( + Grpc.newChannelBuilderForAddress( + "localhost", server.getPort(), InsecureChannelCredentials.create()) + .intercept(new DelayedClientInterceptor(Duration.ofMillis(200))) + .build()))) { + delayedPool.start(); + + ClientConfigurationManager boundedConfigManager = mock(ClientConfigurationManager.class); + ClientConfigurationManager.ListenerHandle handle = + mock(ClientConfigurationManager.ListenerHandle.class); + ArgumentCaptor captor = + ArgumentCaptor.forClass(ClientConfigurationManager.ConfigListener.class); + when(boundedConfigManager.getClientConfiguration()) + .thenReturn(ClientConfigurationManager.loadDefault()); + when(boundedConfigManager.addListener(any(), captor.capture())).thenReturn(handle); + + SessionPoolImpl boundedPool = + new SessionPoolImpl<>( + metrics, + FeatureFlags.getDefaultInstance(), + CLIENT_INFO, + boundedConfigManager, + delayedPool, + CallOptions.DEFAULT, + FakeDescriptor.FAKE_SESSION, + "bounded-pool", + testTimer, + MoreExecutors.directExecutor()); + try { + // Set maxSessionCount=2, newSessionQueueLength=2 -> maxPendingRpcs = 4 + SessionPoolConfiguration smallConfig = + ClientConfigurationManager.loadDefault() + .getSessionConfiguration() + .getSessionPoolConfiguration() + .toBuilder() + .setMinSessionCount(1) + .setMaxSessionCount(2) + .setNewSessionQueueLength(2) + .build(); + captor.getValue().onChange(smallConfig); + + boundedPool.start(OpenFakeSessionRequest.getDefaultInstance(), new Metadata()); + + for (int i = 0; i < 4; i++) { + UnaryResponseFuture f = new UnaryResponseFuture<>(); + VRpc rpc = + boundedPool.newCall(FakeDescriptor.SCRIPTED); + rpc.start( + SessionFakeScriptedRequest.getDefaultInstance(), + VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer), + f); + } + + // 5th pending call exceeds maxPendingRpcs (4) and must be rejected immediately + UnaryResponseFuture fifthFuture = new UnaryResponseFuture<>(); + VRpc fifthRpc = + boundedPool.newCall(FakeDescriptor.SCRIPTED); + fifthRpc.start( + SessionFakeScriptedRequest.getDefaultInstance(), + VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer), + fifthFuture); + + ExecutionException ex = + assertThrows(ExecutionException.class, () -> fifthFuture.get(1, TimeUnit.SECONDS)); + assertThat(ex.getCause()) + .hasMessageThat() + .contains("RESOURCE_EXHAUSTED: SessionPool pending RPC queue is full (4/4)"); + } finally { + boundedPool.close(CloseSessionRequest.getDefaultInstance()); + } + } + } + private void waitForOpenRequestCount(int minCount, Duration timeout) throws InterruptedException { long deadline = System.currentTimeMillis() + timeout.toMillis(); while (System.currentTimeMillis() < deadline) { From 526574ccdccd3e697d090e80e154f555ce2ca48f Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Mon, 16 Nov 2026 15:25:44 +0000 Subject: [PATCH 4/6] fix(bigtable): count pre-OpenSessionResponse stream failures in ChannelPoolDpImpl --- .../internal/channels/ChannelPoolDpImpl.java | 24 ++++++++++++-- .../channels/ChannelPoolDpImplTest.java | 31 +++++++++++++++++++ 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java index 1ef72a91937a..c5157de5ab5b 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java @@ -308,6 +308,7 @@ public synchronized SessionStream newStream( return new SessionStreamImpl(innerCall) { // null until onBeforeSessionStart fires @Nullable AfeId afeId = null; + boolean sessionEstablished = false; @Override public void start(Listener responseListener, Metadata headers) { @@ -317,8 +318,6 @@ public void start(Listener responseListener, Metadata headers) { public void onBeforeSessionStart(PeerInfo peerInfo) { afeId = AfeId.extract(peerInfo); synchronized (ChannelPoolDpImpl.this) { - fw.consecutiveFailures = 0; - recycleBackoff = INITIAL_RECYCLE_BACKOFF; sessionsPerAfeId.add(afeId); if (!fw.channel.isShutdown()) { routeObservations.put(new RouteKey(fw.id, tenant), afeId); @@ -328,13 +327,32 @@ public void onBeforeSessionStart(PeerInfo peerInfo) { super.onBeforeSessionStart(peerInfo); } + @Override + public void onMessage(SessionResponse msg) { + if (!sessionEstablished && msg.hasOpenSession()) { + synchronized (ChannelPoolDpImpl.this) { + sessionEstablished = true; + fw.consecutiveFailures = 0; + recycleBackoff = INITIAL_RECYCLE_BACKOFF; + } + } + super.onMessage(msg); + } + @Override public void onClose(Status status, Metadata trailers) { synchronized (ChannelPoolDpImpl.this) { if (afeId != null) { sessionsPerAfeId.remove(afeId); - } else if (!status.isOk() && status.getCode() != Code.CANCELLED) { + } + if (status.isOk()) { + fw.consecutiveFailures = 0; + recycleBackoff = INITIAL_RECYCLE_BACKOFF; + } else if (!sessionEstablished && status.getCode() != Code.CANCELLED) { fw.consecutiveFailures++; + if (afeId != null) { + routeObservations.remove(new RouteKey(fw.id, tenant), afeId); + } } releaseChannel(fw, status); } diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImplTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImplTest.java index 4efca0494718..d1e8afb38d2b 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImplTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImplTest.java @@ -632,4 +632,35 @@ void testRecycleChannelBackoff() { pool.close(); } + + @Test + void testPreOpenSessionFailureWithPeerInfoIncrementsFailures() { + when(channelSupplier.get()).thenReturn(channel); + when(channel.newCall(any(), any())).thenReturn(clientCall); + doNothing().when(clientCall).start(listener.capture(), any()); + doReturn(Attributes.EMPTY).when(clientCall).getAttributes(); + + ChannelPoolDpImpl pool = + new ChannelPoolDpImpl(channelSupplier, defaultConfig, debugTagTracer, bgExecutor); + + PeerInfo peerInfo = PeerInfo.newBuilder().setApplicationFrontendId(555).build(); + Metadata headers = new Metadata(); + headers.put( + SessionStreamImpl.PEER_INFO_KEY, + Base64.getEncoder().encodeToString(peerInfo.toByteArray())); + + // 5 pre-OpenSessionResponse rejections (onHeaders with PeerInfo followed by onClose(!OK) + // without OpenSessionResponse) must count as consecutive failures and recycle the channel. + for (int i = 0; i < 5; i++) { + pool.newStream(FakeSessionGrpc.getOpenSessionMethod(), CallOptions.DEFAULT) + .start(mock(Listener.class), new Metadata()); + listener.getValue().onHeaders(headers); + listener.getValue().onClose(Status.RESOURCE_EXHAUSTED, new Metadata()); + } + + verify(channel, times(1)).shutdown(); + verify(channelSupplier, times(2)).get(); + + pool.close(); + } } From c5f3794bbdabc084040fa84c5209438ac6a91c45 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Mon, 16 Nov 2026 16:29:33 +0000 Subject: [PATCH 5/6] refactor(bigtable): rename PoolSizer.maxIdleSessions to maxSessions --- .../data/v2/internal/session/PoolSizer.java | 10 ++++----- .../v2/internal/session/PoolSizerTest.java | 22 +++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java index 3aea1e6b7e70..9c6740e9a339 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java @@ -29,7 +29,7 @@ class PoolSizer { // 50% increase in latency (since that also increases requests-in-flight). private volatile float idlesSessionHeadRoom; private volatile int minIdleSessions; - private volatile int maxIdleSessions; + private volatile int maxSessions; private final PoolStats stats; private final Sized pendingRpcs; @@ -60,12 +60,12 @@ interface Sized { void updateConfig(SessionClientConfiguration.SessionPoolConfiguration poolConfiguration) { this.idlesSessionHeadRoom = poolConfiguration.getHeadroom(); this.minIdleSessions = poolConfiguration.getMinSessionCount(); - this.maxIdleSessions = poolConfiguration.getMaxSessionCount(); + this.maxSessions = poolConfiguration.getMaxSessionCount(); this.pendingVRpcsPerSession = poolConfiguration.getNewSessionQueueLength(); this.disabled = poolConfiguration.equals( SessionClientConfiguration.SessionPoolConfiguration.getDefaultInstance()) - || this.maxIdleSessions <= 0; + || this.maxSessions <= 0; } boolean isDisabled() { @@ -82,9 +82,9 @@ public int getScaleDelta() { int sessionsInUse = effectivePending + stats.getInUseCount(); int unboundedDesiredIdleSessions = (int) Math.ceil(sessionsInUse * idlesSessionHeadRoom); int desiredIdleSessions = - Math.max(Math.min(unboundedDesiredIdleSessions, maxIdleSessions), minIdleSessions); + Math.max(Math.min(unboundedDesiredIdleSessions, maxSessions), minIdleSessions); - int desiredCapacity = Math.min(sessionsInUse + desiredIdleSessions, maxIdleSessions); + int desiredCapacity = Math.min(sessionsInUse + desiredIdleSessions, maxSessions); int eventualCapacity = stats.getExpectedCapacity(); int immediateCapacity = eventualCapacity - stats.getStartingCount(); diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java index 897079f2244c..f7a6df9a4440 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java @@ -31,14 +31,14 @@ class PoolSizerTest { private static final float IDLE_SESSION_HEADROOM_RATIO = 0.5f; private static final int MIN_IDLE_SESSIONS = 5; - private static final int MAX_IDLE_SESSIONS = 400; + private static final int MAX_SESSIONS = 400; private static final int PENDING_RPCS_PER_SESSION = 10; private static final SessionClientConfiguration.SessionPoolConfiguration DEFAULT_CONFIG = SessionClientConfiguration.SessionPoolConfiguration.newBuilder() .setHeadroom(IDLE_SESSION_HEADROOM_RATIO) .setMinSessionCount(MIN_IDLE_SESSIONS) - .setMaxSessionCount(MAX_IDLE_SESSIONS) + .setMaxSessionCount(MAX_SESSIONS) .setNewSessionQueueLength(PENDING_RPCS_PER_SESSION) .build(); @@ -193,7 +193,7 @@ void testEquilibriumWhenStartingWithLoad() { int sessionsNeeded = forPending + stats.getInUseCount(); int desiredIdleSessions = (int) Math.ceil(sessionsNeeded * IDLE_SESSION_HEADROOM_RATIO); desiredIdleSessions = - Math.max(Math.min(desiredIdleSessions, MAX_IDLE_SESSIONS), MIN_IDLE_SESSIONS); + Math.max(Math.min(desiredIdleSessions, MAX_SESSIONS), MIN_IDLE_SESSIONS); int desiredCapacity = sessionsNeeded + desiredIdleSessions; assertThat(sizer.getScaleDelta()) @@ -251,22 +251,22 @@ void testEmptyConfig() { } @Test - void testDesiredCapacityCappedAtMaxIdleSessions() { + void testDesiredCapacityCappedAtMaxSessions() { PoolStats stats = new PoolStats(); AtomicInteger pendingCount = new AtomicInteger(0); PoolSizer sizer = new PoolSizer(stats, pendingCount::get, DEFAULT_CONFIG); - // MAX_IDLE_SESSIONS in use + 2 ready -> immediateCapacity = MAX_IDLE_SESSIONS + 2. - // Total desiredCapacity is capped at MAX_IDLE_SESSIONS, so scaleDelta is -2. - stats.getTestHelper().setInUseCount(MAX_IDLE_SESSIONS); + // MAX_SESSIONS in use + 2 ready -> immediateCapacity = MAX_SESSIONS + 2. + // Total desiredCapacity is capped at MAX_SESSIONS, so scaleDelta is -2. + stats.getTestHelper().setInUseCount(MAX_SESSIONS); stats.getTestHelper().setReadyCount(2); - stats.getTestHelper().setExpectedCapacity(MAX_IDLE_SESSIONS + 2); + stats.getTestHelper().setExpectedCapacity(MAX_SESSIONS + 2); assertThat(sizer.getScaleDelta()).isEqualTo(-2); - // When MAX_IDLE_SESSIONS are in use and 50 RPCs are queued, scaleDelta must be 0 - // rather than scaling above MAX_IDLE_SESSIONS. + // When MAX_SESSIONS are in use and 50 RPCs are queued, scaleDelta must be 0 + // rather than scaling above MAX_SESSIONS. stats.getTestHelper().setReadyCount(0); - stats.getTestHelper().setExpectedCapacity(MAX_IDLE_SESSIONS); + stats.getTestHelper().setExpectedCapacity(MAX_SESSIONS); pendingCount.set(50); assertThat(sizer.getScaleDelta()).isEqualTo(0); assertThat(sizer.handleNewCall()).isFalse(); From 65c90b246686412b2655ed1920f4fa49173288d9 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Mon, 16 Nov 2026 16:30:05 +0000 Subject: [PATCH 6/6] refactor(bigtable): use maxSessions in PoolSizer.getMaxPendingRpcs --- .../cloud/bigtable/data/v2/internal/session/PoolSizer.java | 2 +- .../cloud/bigtable/data/v2/internal/session/PoolSizerTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java index 4b369ac05239..598e061f65ff 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java @@ -105,7 +105,7 @@ int getMaxPendingRpcs() { if (isDisabled()) { return 0; } - return maxIdleSessions * Math.max(1, pendingVRpcsPerSession); + return maxSessions * Math.max(1, pendingVRpcsPerSession); } /** Returns true if the session should be replaced */ diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java index 75fd8967b960..8f2822cbb3f1 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java @@ -270,6 +270,6 @@ void testDesiredCapacityCappedAtMaxSessions() { pendingCount.set(50); assertThat(sizer.getScaleDelta()).isEqualTo(0); assertThat(sizer.handleNewCall()).isFalse(); - assertThat(sizer.getMaxPendingRpcs()).isEqualTo(MAX_IDLE_SESSIONS * PENDING_RPCS_PER_SESSION); + assertThat(sizer.getMaxPendingRpcs()).isEqualTo(MAX_SESSIONS * PENDING_RPCS_PER_SESSION); } }