From 24d65742d97b7cd016b0634981b536ec1955e7b3 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Mon, 16 Nov 2026 15:11:56 +0000 Subject: [PATCH 1/3] fix(bigtable): cap PoolSizer desiredCapacity at maxIdleSessions --- .../data/v2/internal/session/PoolSizer.java | 2 +- .../v2/internal/session/PoolSizerTest.java | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java index 3e9d166d9ae0..3aea1e6b7e70 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java @@ -84,7 +84,7 @@ public int getScaleDelta() { int desiredIdleSessions = Math.max(Math.min(unboundedDesiredIdleSessions, maxIdleSessions), minIdleSessions); - int desiredCapacity = sessionsInUse + desiredIdleSessions; + int desiredCapacity = Math.min(sessionsInUse + desiredIdleSessions, maxIdleSessions); int eventualCapacity = stats.getExpectedCapacity(); int immediateCapacity = eventualCapacity - stats.getStartingCount(); diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java index a343c7973ce8..897079f2244c 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java @@ -249,4 +249,26 @@ void testEmptyConfig() { assertThat(sizer.handleGoAway(GoAwayResponse.getDefaultInstance())).isFalse(); assertThat(sizer.handleSessionClose(Status.getDefaultInstance())).isFalse(); } + + @Test + void testDesiredCapacityCappedAtMaxIdleSessions() { + PoolStats stats = new PoolStats(); + AtomicInteger pendingCount = new AtomicInteger(0); + PoolSizer sizer = new PoolSizer(stats, pendingCount::get, DEFAULT_CONFIG); + + // MAX_IDLE_SESSIONS in use + 2 ready -> immediateCapacity = MAX_IDLE_SESSIONS + 2. + // Total desiredCapacity is capped at MAX_IDLE_SESSIONS, so scaleDelta is -2. + stats.getTestHelper().setInUseCount(MAX_IDLE_SESSIONS); + stats.getTestHelper().setReadyCount(2); + stats.getTestHelper().setExpectedCapacity(MAX_IDLE_SESSIONS + 2); + assertThat(sizer.getScaleDelta()).isEqualTo(-2); + + // When MAX_IDLE_SESSIONS are in use and 50 RPCs are queued, scaleDelta must be 0 + // rather than scaling above MAX_IDLE_SESSIONS. + stats.getTestHelper().setReadyCount(0); + stats.getTestHelper().setExpectedCapacity(MAX_IDLE_SESSIONS); + pendingCount.set(50); + assertThat(sizer.getScaleDelta()).isEqualTo(0); + assertThat(sizer.handleNewCall()).isFalse(); + } } From 86f6d017a1df7842843971ba3fb6db631f9aa916 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Mon, 16 Nov 2026 15:17:51 +0000 Subject: [PATCH 2/3] fix(bigtable): remove dead SessionList.inUseSessions and clear PendingVRpc references --- .../data/v2/internal/session/SessionList.java | 3 - .../v2/internal/session/SessionPoolImpl.java | 64 ++++++++++--------- .../internal/session/SessionPoolImplTest.java | 40 ++++++++++++ 3 files changed, 73 insertions(+), 34 deletions(-) diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionList.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionList.java index 79fc1109fe23..a75d1b7b2ecc 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionList.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionList.java @@ -63,7 +63,6 @@ class SessionList { // - starting sessions // - closing sessions private final Set allSessions = new HashSet<>(); - private final Set inUseSessions = new HashSet<>(); // pool level statistics across all the afes private final PoolStats poolStats = new PoolStats(); @@ -119,7 +118,6 @@ public Optional checkoutSession(AfeHandle afeHandle) { handle -> { poolStats.readyCount--; poolStats.inUseCount++; - inUseSessions.add(handle); if (handle.afe.get().sessions.isEmpty()) { afesWithReadySessions.remove(afeHandle); } @@ -221,7 +219,6 @@ private void releaseToPool() { AfeHandle afeHandle = this.afe.get(); poolStats.inUseCount--; - inUseSessions.remove(this); if (!closing && session.getState() == SessionState.READY) { poolStats.readyCount++; diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java index c1abfb90232c..8b21a48dd7ba 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java @@ -911,7 +911,7 @@ class PendingVRpc implements VRpc realCall; - private BigtableTimer.Timeout deadlineTimeout; + private volatile BigtableTimer.Timeout deadlineTimeout; public PendingVRpc(VRpcDescriptor desc) { this.desc = desc; @@ -922,7 +922,7 @@ public PendingVRpc(VRpcDescriptor desc) { // region. Bounded acquisition on the request hot path; see poolLock. @SuppressWarnings("GuardedBy") public void start(ReqT req, VRpcCallContext ctx, VRpcListener listener) { - Preconditions.checkState(this.req == null, "request is already started"); + Preconditions.checkState(this.ctx == null, "request is already started"); Preconditions.checkNotNull(req, "request can't be null"); Preconditions.checkNotNull(ctx, "ctx can't be null"); Preconditions.checkNotNull(listener, "listener can't be null"); @@ -940,13 +940,7 @@ public void start(ReqT req, VRpcCallContext ctx, VRpcListener listener) { VRpcResult.createUncommitedError( Status.UNAVAILABLE.withCause( new IllegalStateException("SessionPool lock unavailable (pool wedged)"))); - ctx.getExecutor() - .execute( - () -> { - if (isCancelled) return; - isCancelled = true; - listener.onClose(result); - }); + cancelWithResult(result); return; } try { @@ -954,20 +948,14 @@ public void start(ReqT req, VRpcCallContext ctx, VRpcListener listener) { VRpcResult result = VRpcResult.createUncommitedError( Status.UNAVAILABLE.withCause(new IllegalStateException("SessionPool is closed"))); - ctx.getExecutor().execute(() -> listener.onClose(result)); + cancelWithResult(result); return; } if (poolSizer.isDisabled()) { VRpcResult result = VRpcResult.createRejectedError( Status.UNAVAILABLE.withDescription("SessionPool configuration disabled")); - ctx.getExecutor() - .execute( - () -> { - if (isCancelled) return; - isCancelled = true; - listener.onClose(result); - }); + cancelWithResult(result); return; } // Only arm the deadline monitor after we've committed to queueing; otherwise the @@ -1013,8 +1001,10 @@ public void cancel(@Nullable String message, @Nullable Throwable cause) { // region below. @SuppressWarnings("GuardedBy") private void cancel(Status status, boolean onlyCancelPendingCall) { - if (deadlineTimeout != null) { - deadlineTimeout.cancel(); + BigtableTimer.Timeout timeout = deadlineTimeout; + deadlineTimeout = null; + if (timeout != null) { + timeout.cancel(); } // The eager removal is only an optimization to skip a wasted session pick in // tryDrainPendingRpcs. @@ -1037,26 +1027,36 @@ private void cancel(Status status, boolean onlyCancelPendingCall) { () -> { if (isCancelled) return; isCancelled = true; + VRpcListener pendingListener = listener; + req = null; + listener = null; if (realCall != null) { if (!onlyCancelPendingCall) { realCall.cancel(status.getDescription(), status.getCause()); } - } else { - listener.onClose(VRpcResult.createRejectedError(status)); + } else if (pendingListener != null) { + pendingListener.onClose(VRpcResult.createRejectedError(status)); } }); } void cancelWithResult(VRpcResult result) { - if (deadlineTimeout != null) { - deadlineTimeout.cancel(); + BigtableTimer.Timeout timeout = deadlineTimeout; + deadlineTimeout = null; + if (timeout != null) { + timeout.cancel(); } ctx.getExecutor() .execute( () -> { if (isCancelled) return; isCancelled = true; - listener.onClose(result); + VRpcListener pendingListener = listener; + req = null; + listener = null; + if (pendingListener != null) { + pendingListener.onClose(result); + } }); } @@ -1077,8 +1077,10 @@ public void requestNext() { } private void drainTo(SessionHandle handle) { - if (deadlineTimeout != null) { - deadlineTimeout.cancel(); + BigtableTimer.Timeout timeout = deadlineTimeout; + deadlineTimeout = null; + if (timeout != null) { + timeout.cancel(); } ctx.getExecutor() .execute( @@ -1087,15 +1089,15 @@ private void drainTo(SessionHandle handle) { SessionPoolImpl.this.onPendingVRpcCancelled(handle); return; } + ReqT pendingReq = req; + VRpcListener pendingListener = listener; + req = null; + listener = null; realCall = newRealCall(desc, handle); - realCall.start(req, ctx, listener); + realCall.start(pendingReq, ctx, pendingListener); }); } - private VRpcListener getListener() { - return listener; - } - private BigtableTimer.Timeout monitorDeadline(Deadline deadline) { // Body runs on backgroundExecutor (off the tick thread). onlyCancelPendingCall=true avoids // racing with a user cancel that already attached a real call. diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java index 55afab0b79dd..911955b67565 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java @@ -1448,6 +1448,46 @@ void configUpdateToEmptyClosesSessionsAndScalesBackUpWhenEnabled() throws Except assertThat(sessionPool.hasSession()).isTrue(); } + @Test + void pendingVRpcClearsReferencesAfterDrainAndCancel() throws Exception { + sessionPool.start(OpenFakeSessionRequest.getDefaultInstance(), new Metadata()); + + // Create both PendingVRpcs before any session becomes READY so both return PendingVRpc + UnaryResponseFuture drainedFuture = new UnaryResponseFuture<>(); + VRpc drainedRpc = + sessionPool.newCall(FakeDescriptor.SCRIPTED); + UnaryResponseFuture cancelledFuture = new UnaryResponseFuture<>(); + VRpc cancelledRpc = + sessionPool.newCall(FakeDescriptor.SCRIPTED); + + cancelledRpc.start( + SessionFakeScriptedRequest.getDefaultInstance(), + VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer), + cancelledFuture); + cancelledRpc.cancel("user cancelled", null); + + drainedRpc.start( + SessionFakeScriptedRequest.getDefaultInstance(), + VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer), + drainedFuture); + drainedFuture.get(5, TimeUnit.SECONDS); + + Field reqField = drainedRpc.getClass().getDeclaredField("req"); + Field listenerField = drainedRpc.getClass().getDeclaredField("listener"); + Field timeoutField = drainedRpc.getClass().getDeclaredField("deadlineTimeout"); + reqField.setAccessible(true); + listenerField.setAccessible(true); + timeoutField.setAccessible(true); + + assertThat(reqField.get(drainedRpc)).isNull(); + assertThat(listenerField.get(drainedRpc)).isNull(); + assertThat(timeoutField.get(drainedRpc)).isNull(); + + assertThat(reqField.get(cancelledRpc)).isNull(); + assertThat(listenerField.get(cancelledRpc)).isNull(); + assertThat(timeoutField.get(cancelledRpc)).isNull(); + } + private void waitForOpenRequestCount(int minCount, Duration timeout) throws InterruptedException { long deadline = System.currentTimeMillis() + timeout.toMillis(); while (System.currentTimeMillis() < deadline) { From c5f3794bbdabc084040fa84c5209438ac6a91c45 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Mon, 16 Nov 2026 16:29:33 +0000 Subject: [PATCH 3/3] refactor(bigtable): rename PoolSizer.maxIdleSessions to maxSessions --- .../data/v2/internal/session/PoolSizer.java | 10 ++++----- .../v2/internal/session/PoolSizerTest.java | 22 +++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java index 3aea1e6b7e70..9c6740e9a339 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java @@ -29,7 +29,7 @@ class PoolSizer { // 50% increase in latency (since that also increases requests-in-flight). private volatile float idlesSessionHeadRoom; private volatile int minIdleSessions; - private volatile int maxIdleSessions; + private volatile int maxSessions; private final PoolStats stats; private final Sized pendingRpcs; @@ -60,12 +60,12 @@ interface Sized { void updateConfig(SessionClientConfiguration.SessionPoolConfiguration poolConfiguration) { this.idlesSessionHeadRoom = poolConfiguration.getHeadroom(); this.minIdleSessions = poolConfiguration.getMinSessionCount(); - this.maxIdleSessions = poolConfiguration.getMaxSessionCount(); + this.maxSessions = poolConfiguration.getMaxSessionCount(); this.pendingVRpcsPerSession = poolConfiguration.getNewSessionQueueLength(); this.disabled = poolConfiguration.equals( SessionClientConfiguration.SessionPoolConfiguration.getDefaultInstance()) - || this.maxIdleSessions <= 0; + || this.maxSessions <= 0; } boolean isDisabled() { @@ -82,9 +82,9 @@ public int getScaleDelta() { int sessionsInUse = effectivePending + stats.getInUseCount(); int unboundedDesiredIdleSessions = (int) Math.ceil(sessionsInUse * idlesSessionHeadRoom); int desiredIdleSessions = - Math.max(Math.min(unboundedDesiredIdleSessions, maxIdleSessions), minIdleSessions); + Math.max(Math.min(unboundedDesiredIdleSessions, maxSessions), minIdleSessions); - int desiredCapacity = Math.min(sessionsInUse + desiredIdleSessions, maxIdleSessions); + int desiredCapacity = Math.min(sessionsInUse + desiredIdleSessions, maxSessions); int eventualCapacity = stats.getExpectedCapacity(); int immediateCapacity = eventualCapacity - stats.getStartingCount(); diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java index 897079f2244c..f7a6df9a4440 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java @@ -31,14 +31,14 @@ class PoolSizerTest { private static final float IDLE_SESSION_HEADROOM_RATIO = 0.5f; private static final int MIN_IDLE_SESSIONS = 5; - private static final int MAX_IDLE_SESSIONS = 400; + private static final int MAX_SESSIONS = 400; private static final int PENDING_RPCS_PER_SESSION = 10; private static final SessionClientConfiguration.SessionPoolConfiguration DEFAULT_CONFIG = SessionClientConfiguration.SessionPoolConfiguration.newBuilder() .setHeadroom(IDLE_SESSION_HEADROOM_RATIO) .setMinSessionCount(MIN_IDLE_SESSIONS) - .setMaxSessionCount(MAX_IDLE_SESSIONS) + .setMaxSessionCount(MAX_SESSIONS) .setNewSessionQueueLength(PENDING_RPCS_PER_SESSION) .build(); @@ -193,7 +193,7 @@ void testEquilibriumWhenStartingWithLoad() { int sessionsNeeded = forPending + stats.getInUseCount(); int desiredIdleSessions = (int) Math.ceil(sessionsNeeded * IDLE_SESSION_HEADROOM_RATIO); desiredIdleSessions = - Math.max(Math.min(desiredIdleSessions, MAX_IDLE_SESSIONS), MIN_IDLE_SESSIONS); + Math.max(Math.min(desiredIdleSessions, MAX_SESSIONS), MIN_IDLE_SESSIONS); int desiredCapacity = sessionsNeeded + desiredIdleSessions; assertThat(sizer.getScaleDelta()) @@ -251,22 +251,22 @@ void testEmptyConfig() { } @Test - void testDesiredCapacityCappedAtMaxIdleSessions() { + void testDesiredCapacityCappedAtMaxSessions() { PoolStats stats = new PoolStats(); AtomicInteger pendingCount = new AtomicInteger(0); PoolSizer sizer = new PoolSizer(stats, pendingCount::get, DEFAULT_CONFIG); - // MAX_IDLE_SESSIONS in use + 2 ready -> immediateCapacity = MAX_IDLE_SESSIONS + 2. - // Total desiredCapacity is capped at MAX_IDLE_SESSIONS, so scaleDelta is -2. - stats.getTestHelper().setInUseCount(MAX_IDLE_SESSIONS); + // MAX_SESSIONS in use + 2 ready -> immediateCapacity = MAX_SESSIONS + 2. + // Total desiredCapacity is capped at MAX_SESSIONS, so scaleDelta is -2. + stats.getTestHelper().setInUseCount(MAX_SESSIONS); stats.getTestHelper().setReadyCount(2); - stats.getTestHelper().setExpectedCapacity(MAX_IDLE_SESSIONS + 2); + stats.getTestHelper().setExpectedCapacity(MAX_SESSIONS + 2); assertThat(sizer.getScaleDelta()).isEqualTo(-2); - // When MAX_IDLE_SESSIONS are in use and 50 RPCs are queued, scaleDelta must be 0 - // rather than scaling above MAX_IDLE_SESSIONS. + // When MAX_SESSIONS are in use and 50 RPCs are queued, scaleDelta must be 0 + // rather than scaling above MAX_SESSIONS. stats.getTestHelper().setReadyCount(0); - stats.getTestHelper().setExpectedCapacity(MAX_IDLE_SESSIONS); + stats.getTestHelper().setExpectedCapacity(MAX_SESSIONS); pendingCount.set(50); assertThat(sizer.getScaleDelta()).isEqualTo(0); assertThat(sizer.handleNewCall()).isFalse();