Skip to content
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ class PoolSizer {
// 50% increase in latency (since that also increases requests-in-flight).
private volatile float idlesSessionHeadRoom;
private volatile int minIdleSessions;
private volatile int maxIdleSessions;
private volatile int maxSessions;

private final PoolStats stats;
private final Sized pendingRpcs;
Expand Down Expand Up @@ -60,12 +60,12 @@ interface Sized {
void updateConfig(SessionClientConfiguration.SessionPoolConfiguration poolConfiguration) {
this.idlesSessionHeadRoom = poolConfiguration.getHeadroom();
this.minIdleSessions = poolConfiguration.getMinSessionCount();
this.maxIdleSessions = poolConfiguration.getMaxSessionCount();
this.maxSessions = poolConfiguration.getMaxSessionCount();
this.pendingVRpcsPerSession = poolConfiguration.getNewSessionQueueLength();
this.disabled =
poolConfiguration.equals(
SessionClientConfiguration.SessionPoolConfiguration.getDefaultInstance())
|| this.maxIdleSessions <= 0;
|| this.maxSessions <= 0;
}

boolean isDisabled() {
Expand All @@ -82,9 +82,9 @@ public int getScaleDelta() {
int sessionsInUse = effectivePending + stats.getInUseCount();
int unboundedDesiredIdleSessions = (int) Math.ceil(sessionsInUse * idlesSessionHeadRoom);
int desiredIdleSessions =
Math.max(Math.min(unboundedDesiredIdleSessions, maxIdleSessions), minIdleSessions);
Math.max(Math.min(unboundedDesiredIdleSessions, maxSessions), minIdleSessions);

int desiredCapacity = sessionsInUse + desiredIdleSessions;
int desiredCapacity = Math.min(sessionsInUse + desiredIdleSessions, maxSessions);
int eventualCapacity = stats.getExpectedCapacity();
int immediateCapacity = eventualCapacity - stats.getStartingCount();

Expand All @@ -100,6 +100,14 @@ public int getScaleDelta() {
}
}

/** Returns the maximum number of pending vRPCs allowed in the pool queue. */
int getMaxPendingRpcs() {
if (isDisabled()) {
return 0;
}
return maxSessions * Math.max(1, pendingVRpcsPerSession);
}

/** Returns true if the session should be replaced */
@CheckReturnValue
public boolean handleGoAway(GoAwayResponse msg) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,6 @@ class SessionList {
// - starting sessions
// - closing sessions
private final Set<SessionHandle> allSessions = new HashSet<>();
private final Set<SessionHandle> inUseSessions = new HashSet<>();

// pool level statistics across all the afes
private final PoolStats poolStats = new PoolStats();
Expand Down Expand Up @@ -119,7 +118,6 @@ public Optional<SessionHandle> checkoutSession(AfeHandle afeHandle) {
handle -> {
poolStats.readyCount--;
poolStats.inUseCount++;
inUseSessions.add(handle);
if (handle.afe.get().sessions.isEmpty()) {
afesWithReadySessions.remove(afeHandle);
}
Expand Down Expand Up @@ -221,7 +219,6 @@ private void releaseToPool() {
AfeHandle afeHandle = this.afe.get();

poolStats.inUseCount--;
inUseSessions.remove(this);

if (!closing && session.getState() == SessionState.READY) {
poolStats.readyCount++;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,12 @@ private enum PoolState {
@GuardedBy("poolLock")
private int consecutiveFailures = 0;

@GuardedBy("poolLock")
private Status lastCloseStatus = Status.UNAVAILABLE;

@GuardedBy("poolLock")
private Metadata lastCloseTrailers = new Metadata();

/**
* When the client fallback to a non-session AFE session creation will return unimplemented
* errors. In which case the requests should fallback to classic client instead of waiting for an
Expand Down Expand Up @@ -729,6 +735,8 @@ private void onSessionClose(
// rather than session establishment failures and should not fail pending vRPCs.
if (prevState == SessionState.STARTING) {
consecutiveFailures++;
lastCloseStatus = status;
lastCloseTrailers = trailers;
Comment on lines +738 to +739

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

In gRPC, the trailers parameter in onClose can sometimes be null (e.g., in certain failure modes or unit tests). To prevent potential NullPointerException issues when referencing lastCloseTrailers later, we should defensively check if trailers is null and default to an empty Metadata instance.

Suggested change
lastCloseStatus = status;
lastCloseTrailers = trailers;
lastCloseStatus = status;
lastCloseTrailers = trailers != null ? trailers : new Metadata();

if (status.getCode() == Status.Code.UNIMPLEMENTED) {
consecutiveUnimplementedFailures++;
} else {
Expand Down Expand Up @@ -911,7 +919,7 @@ class PendingVRpc<ReqT extends Message, RespT extends Message> implements VRpc<R
private boolean isCancelled = false;
private VRpc<ReqT, RespT> realCall;

private BigtableTimer.Timeout deadlineTimeout;
private volatile BigtableTimer.Timeout deadlineTimeout;

public PendingVRpc(VRpcDescriptor<?, ReqT, RespT> desc) {
this.desc = desc;
Expand All @@ -922,7 +930,7 @@ public PendingVRpc(VRpcDescriptor<?, ReqT, RespT> desc) {
// region. Bounded acquisition on the request hot path; see poolLock.
@SuppressWarnings("GuardedBy")
public void start(ReqT req, VRpcCallContext ctx, VRpcListener<RespT> listener) {
Preconditions.checkState(this.req == null, "request is already started");
Preconditions.checkState(this.ctx == null, "request is already started");
Preconditions.checkNotNull(req, "request can't be null");
Preconditions.checkNotNull(ctx, "ctx can't be null");
Preconditions.checkNotNull(listener, "listener can't be null");
Expand All @@ -940,34 +948,52 @@ public void start(ReqT req, VRpcCallContext ctx, VRpcListener<RespT> listener) {
VRpcResult.createUncommitedError(
Status.UNAVAILABLE.withCause(
new IllegalStateException("SessionPool lock unavailable (pool wedged)")));
ctx.getExecutor()
.execute(
() -> {
if (isCancelled) return;
isCancelled = true;
listener.onClose(result);
});
cancelWithResult(result);
return;
}
try {
if (SessionPoolImpl.this.poolState != PoolState.STARTED) {
VRpcResult result =
VRpcResult.createUncommitedError(
Status.UNAVAILABLE.withCause(new IllegalStateException("SessionPool is closed")));
ctx.getExecutor().execute(() -> listener.onClose(result));
cancelWithResult(result);
return;
}
if (poolSizer.isDisabled()) {
VRpcResult result =
VRpcResult.createRejectedError(
Status.UNAVAILABLE.withDescription("SessionPool configuration disabled"));
ctx.getExecutor()
.execute(
() -> {
if (isCancelled) return;
isCancelled = true;
listener.onClose(result);
});
cancelWithResult(result);
return;
}
if (consecutiveFailures >= getMaxConsecutiveFailures(configManager)
&& sessions.getStats().getReadyCount() == 0) {
if (poolSizer.handleNewCall()) {
createSession(openParams);
}
VRpcResult result =
VRpcResult.createRejectedError(
Status.fromCode(lastCloseStatus.getCode())
.withDescription(
String.format(
"Session failed with consecutive failures. Most recent server status:"
+ " %s, metadata: %s.",
lastCloseStatus, lastCloseTrailers)));
Comment on lines +976 to +981

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

If lastCloseStatus is somehow Status.OK (which shouldn't normally happen for a session close failure, but is possible in mock tests or edge cases), calling Status.fromCode(lastCloseStatus.getCode()) will return Status.OK. Returning Status.OK for a rejected error is highly misleading to callers. We should defensively ensure that we map Status.Code.OK to an error status like Status.Code.UNAVAILABLE.

                  Status.fromCode(
                          lastCloseStatus.getCode() == Status.Code.OK
                              ? Status.Code.UNAVAILABLE
                              : lastCloseStatus.getCode())
                      .withDescription(
                          String.format(
                              "Session failed with consecutive failures. Most recent server status:"
                                  + " %s, metadata: %s.",
                              lastCloseStatus, lastCloseTrailers)))

cancelWithResult(result);
return;
}
int maxPendingRpcs = poolSizer.getMaxPendingRpcs();
if (maxPendingRpcs > 0 && pendingRpcs.size() >= maxPendingRpcs) {
if (poolSizer.handleNewCall()) {
createSession(openParams);
}
VRpcResult result =
VRpcResult.createRejectedError(
Status.RESOURCE_EXHAUSTED.withDescription(
String.format(
"SessionPool pending RPC queue is full (%d/%d)",
pendingRpcs.size(), maxPendingRpcs)));
cancelWithResult(result);
return;
}
// Only arm the deadline monitor after we've committed to queueing; otherwise the
Expand Down Expand Up @@ -1013,8 +1039,10 @@ public void cancel(@Nullable String message, @Nullable Throwable cause) {
// region below.
@SuppressWarnings("GuardedBy")
private void cancel(Status status, boolean onlyCancelPendingCall) {
if (deadlineTimeout != null) {
deadlineTimeout.cancel();
BigtableTimer.Timeout timeout = deadlineTimeout;
deadlineTimeout = null;
if (timeout != null) {
timeout.cancel();
}
// The eager removal is only an optimization to skip a wasted session pick in
// tryDrainPendingRpcs.
Expand All @@ -1037,26 +1065,36 @@ private void cancel(Status status, boolean onlyCancelPendingCall) {
() -> {
if (isCancelled) return;
isCancelled = true;
VRpcListener<RespT> pendingListener = listener;
req = null;
listener = null;
if (realCall != null) {
if (!onlyCancelPendingCall) {
realCall.cancel(status.getDescription(), status.getCause());
}
} else {
listener.onClose(VRpcResult.createRejectedError(status));
} else if (pendingListener != null) {
pendingListener.onClose(VRpcResult.createRejectedError(status));
}
});
}

void cancelWithResult(VRpcResult result) {
if (deadlineTimeout != null) {
deadlineTimeout.cancel();
BigtableTimer.Timeout timeout = deadlineTimeout;
deadlineTimeout = null;
if (timeout != null) {
timeout.cancel();
}
ctx.getExecutor()
.execute(
() -> {
if (isCancelled) return;
isCancelled = true;
listener.onClose(result);
VRpcListener<RespT> pendingListener = listener;
req = null;
listener = null;
if (pendingListener != null) {
pendingListener.onClose(result);
}
});
}
Comment on lines 1081 to 1099

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

Since ctx.getExecutor() is typically a multi-threaded executor (such as the gRPC callback executor or a user-provided thread pool), tasks submitted to it can run concurrently. Because isCancelled, req, and listener are mutated and read across cancel, cancelWithResult, and drainTo without synchronization, this introduces a data race and visibility issues. To ensure thread safety, we should synchronize on PendingVRpc.this when reading or writing these fields inside the executor tasks. Note that before refactoring to move the callback outside of the synchronized block to avoid potential deadlocks, we should verify if a true circular lock dependency exists, as nested lock acquisition is safe if the methods called within the locked block only perform thread-safe operations and do not acquire other locks or trigger callbacks that could reverse the lock acquisition order.

    void cancelWithResult(VRpcResult result) {
      BigtableTimer.Timeout timeout = deadlineTimeout;
      deadlineTimeout = null;
      if (timeout != null) {
        timeout.cancel();
      }
      ctx.getExecutor()
          .execute(
              () -> {
                VRpcListener<RespT> pendingListener;
                synchronized (PendingVRpc.this) {
                  if (isCancelled) return;
                  isCancelled = true;
                  pendingListener = listener;
                  req = null;
                  listener = null;
                }
                if (pendingListener != null) {
                  pendingListener.onClose(result);
                }
              });
    }
References
  1. Before refactoring code to avoid potential deadlocks by moving method calls outside of synchronized blocks, verify if a true circular lock dependency exists. Nested lock acquisition is safe if the methods called within the locked block only perform thread-safe operations (such as updating atomic variables or statistics) and do not acquire other locks or trigger callbacks that could reverse the lock acquisition order.


Expand All @@ -1077,8 +1115,10 @@ public void requestNext() {
}

private void drainTo(SessionHandle handle) {
if (deadlineTimeout != null) {
deadlineTimeout.cancel();
BigtableTimer.Timeout timeout = deadlineTimeout;
deadlineTimeout = null;
if (timeout != null) {
timeout.cancel();
}
ctx.getExecutor()
.execute(
Expand All @@ -1087,15 +1127,15 @@ private void drainTo(SessionHandle handle) {
SessionPoolImpl.this.onPendingVRpcCancelled(handle);
return;
}
ReqT pendingReq = req;
VRpcListener<RespT> pendingListener = listener;
req = null;
listener = null;
realCall = newRealCall(desc, handle);
realCall.start(req, ctx, listener);
realCall.start(pendingReq, ctx, pendingListener);
});
}

private VRpcListener<RespT> getListener() {
return listener;
}

private BigtableTimer.Timeout monitorDeadline(Deadline deadline) {
// Body runs on backgroundExecutor (off the tick thread). onlyCancelPendingCall=true avoids
// racing with a user cancel that already attached a real call.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,14 @@ class PoolSizerTest {

private static final float IDLE_SESSION_HEADROOM_RATIO = 0.5f;
private static final int MIN_IDLE_SESSIONS = 5;
private static final int MAX_IDLE_SESSIONS = 400;
private static final int MAX_SESSIONS = 400;
private static final int PENDING_RPCS_PER_SESSION = 10;

private static final SessionClientConfiguration.SessionPoolConfiguration DEFAULT_CONFIG =
SessionClientConfiguration.SessionPoolConfiguration.newBuilder()
.setHeadroom(IDLE_SESSION_HEADROOM_RATIO)
.setMinSessionCount(MIN_IDLE_SESSIONS)
.setMaxSessionCount(MAX_IDLE_SESSIONS)
.setMaxSessionCount(MAX_SESSIONS)
.setNewSessionQueueLength(PENDING_RPCS_PER_SESSION)
.build();

Expand Down Expand Up @@ -193,7 +193,7 @@ void testEquilibriumWhenStartingWithLoad() {
int sessionsNeeded = forPending + stats.getInUseCount();
int desiredIdleSessions = (int) Math.ceil(sessionsNeeded * IDLE_SESSION_HEADROOM_RATIO);
desiredIdleSessions =
Math.max(Math.min(desiredIdleSessions, MAX_IDLE_SESSIONS), MIN_IDLE_SESSIONS);
Math.max(Math.min(desiredIdleSessions, MAX_SESSIONS), MIN_IDLE_SESSIONS);
int desiredCapacity = sessionsNeeded + desiredIdleSessions;

assertThat(sizer.getScaleDelta())
Expand Down Expand Up @@ -249,4 +249,27 @@ void testEmptyConfig() {
assertThat(sizer.handleGoAway(GoAwayResponse.getDefaultInstance())).isFalse();
assertThat(sizer.handleSessionClose(Status.getDefaultInstance())).isFalse();
}

@Test
void testDesiredCapacityCappedAtMaxSessions() {
PoolStats stats = new PoolStats();
AtomicInteger pendingCount = new AtomicInteger(0);
PoolSizer sizer = new PoolSizer(stats, pendingCount::get, DEFAULT_CONFIG);

// MAX_SESSIONS in use + 2 ready -> immediateCapacity = MAX_SESSIONS + 2.
// Total desiredCapacity is capped at MAX_SESSIONS, so scaleDelta is -2.
stats.getTestHelper().setInUseCount(MAX_SESSIONS);
stats.getTestHelper().setReadyCount(2);
stats.getTestHelper().setExpectedCapacity(MAX_SESSIONS + 2);
assertThat(sizer.getScaleDelta()).isEqualTo(-2);

// When MAX_SESSIONS are in use and 50 RPCs are queued, scaleDelta must be 0
// rather than scaling above MAX_SESSIONS.
stats.getTestHelper().setReadyCount(0);
stats.getTestHelper().setExpectedCapacity(MAX_SESSIONS);
pendingCount.set(50);
assertThat(sizer.getScaleDelta()).isEqualTo(0);
assertThat(sizer.handleNewCall()).isFalse();
assertThat(sizer.getMaxPendingRpcs()).isEqualTo(MAX_SESSIONS * PENDING_RPCS_PER_SESSION);
}
}
Loading
Loading