Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ class PoolSizer {
// 50% increase in latency (since that also increases requests-in-flight).
private volatile float idlesSessionHeadRoom;
private volatile int minIdleSessions;
private volatile int maxIdleSessions;
private volatile int maxSessions;

private final PoolStats stats;
private final Sized pendingRpcs;
Expand Down Expand Up @@ -60,12 +60,12 @@ interface Sized {
void updateConfig(SessionClientConfiguration.SessionPoolConfiguration poolConfiguration) {
this.idlesSessionHeadRoom = poolConfiguration.getHeadroom();
this.minIdleSessions = poolConfiguration.getMinSessionCount();
this.maxIdleSessions = poolConfiguration.getMaxSessionCount();
this.maxSessions = poolConfiguration.getMaxSessionCount();
this.pendingVRpcsPerSession = poolConfiguration.getNewSessionQueueLength();
this.disabled =
poolConfiguration.equals(
SessionClientConfiguration.SessionPoolConfiguration.getDefaultInstance())
|| this.maxIdleSessions <= 0;
|| this.maxSessions <= 0;
}

boolean isDisabled() {
Expand All @@ -82,9 +82,9 @@ public int getScaleDelta() {
int sessionsInUse = effectivePending + stats.getInUseCount();
int unboundedDesiredIdleSessions = (int) Math.ceil(sessionsInUse * idlesSessionHeadRoom);
int desiredIdleSessions =
Math.max(Math.min(unboundedDesiredIdleSessions, maxIdleSessions), minIdleSessions);
Math.max(Math.min(unboundedDesiredIdleSessions, maxSessions), minIdleSessions);

int desiredCapacity = sessionsInUse + desiredIdleSessions;
int desiredCapacity = Math.min(sessionsInUse + desiredIdleSessions, maxSessions);
int eventualCapacity = stats.getExpectedCapacity();
int immediateCapacity = eventualCapacity - stats.getStartingCount();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,6 @@ class SessionList {
// - starting sessions
// - closing sessions
private final Set<SessionHandle> allSessions = new HashSet<>();
private final Set<SessionHandle> inUseSessions = new HashSet<>();

// pool level statistics across all the afes
private final PoolStats poolStats = new PoolStats();
Expand Down Expand Up @@ -119,7 +118,6 @@ public Optional<SessionHandle> checkoutSession(AfeHandle afeHandle) {
handle -> {
poolStats.readyCount--;
poolStats.inUseCount++;
inUseSessions.add(handle);
if (handle.afe.get().sessions.isEmpty()) {
afesWithReadySessions.remove(afeHandle);
}
Expand Down Expand Up @@ -221,7 +219,6 @@ private void releaseToPool() {
AfeHandle afeHandle = this.afe.get();

poolStats.inUseCount--;
inUseSessions.remove(this);

if (!closing && session.getState() == SessionState.READY) {
poolStats.readyCount++;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -911,7 +911,7 @@ class PendingVRpc<ReqT extends Message, RespT extends Message> implements VRpc<R
private boolean isCancelled = false;
private VRpc<ReqT, RespT> realCall;

private BigtableTimer.Timeout deadlineTimeout;
private volatile BigtableTimer.Timeout deadlineTimeout;

public PendingVRpc(VRpcDescriptor<?, ReqT, RespT> desc) {
this.desc = desc;
Expand All @@ -922,7 +922,7 @@ public PendingVRpc(VRpcDescriptor<?, ReqT, RespT> desc) {
// region. Bounded acquisition on the request hot path; see poolLock.
@SuppressWarnings("GuardedBy")
public void start(ReqT req, VRpcCallContext ctx, VRpcListener<RespT> listener) {
Preconditions.checkState(this.req == null, "request is already started");
Preconditions.checkState(this.ctx == null, "request is already started");
Preconditions.checkNotNull(req, "request can't be null");
Preconditions.checkNotNull(ctx, "ctx can't be null");
Preconditions.checkNotNull(listener, "listener can't be null");
Expand All @@ -940,34 +940,22 @@ public void start(ReqT req, VRpcCallContext ctx, VRpcListener<RespT> listener) {
VRpcResult.createUncommitedError(
Status.UNAVAILABLE.withCause(
new IllegalStateException("SessionPool lock unavailable (pool wedged)")));
ctx.getExecutor()
.execute(
() -> {
if (isCancelled) return;
isCancelled = true;
listener.onClose(result);
});
cancelWithResult(result);
return;
}
try {
if (SessionPoolImpl.this.poolState != PoolState.STARTED) {
VRpcResult result =
VRpcResult.createUncommitedError(
Status.UNAVAILABLE.withCause(new IllegalStateException("SessionPool is closed")));
ctx.getExecutor().execute(() -> listener.onClose(result));
cancelWithResult(result);
return;
}
if (poolSizer.isDisabled()) {
VRpcResult result =
VRpcResult.createRejectedError(
Status.UNAVAILABLE.withDescription("SessionPool configuration disabled"));
ctx.getExecutor()
.execute(
() -> {
if (isCancelled) return;
isCancelled = true;
listener.onClose(result);
});
cancelWithResult(result);
return;
}
// Only arm the deadline monitor after we've committed to queueing; otherwise the
Expand Down Expand Up @@ -1013,8 +1001,10 @@ public void cancel(@Nullable String message, @Nullable Throwable cause) {
// region below.
@SuppressWarnings("GuardedBy")
private void cancel(Status status, boolean onlyCancelPendingCall) {
if (deadlineTimeout != null) {
deadlineTimeout.cancel();
BigtableTimer.Timeout timeout = deadlineTimeout;
deadlineTimeout = null;
if (timeout != null) {
timeout.cancel();
}
// The eager removal is only an optimization to skip a wasted session pick in
// tryDrainPendingRpcs.
Expand All @@ -1037,26 +1027,36 @@ private void cancel(Status status, boolean onlyCancelPendingCall) {
() -> {
if (isCancelled) return;
isCancelled = true;
VRpcListener<RespT> pendingListener = listener;
req = null;
listener = null;
if (realCall != null) {
if (!onlyCancelPendingCall) {
realCall.cancel(status.getDescription(), status.getCause());
}
} else {
listener.onClose(VRpcResult.createRejectedError(status));
} else if (pendingListener != null) {
pendingListener.onClose(VRpcResult.createRejectedError(status));
}
});
Comment on lines 1027 to 1040

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

critical

There is a critical concurrency race condition in PendingVRpc between cancel, cancelWithResult, and drainTo.

The Issue

The tasks submitted to ctx.getExecutor() run asynchronously and potentially concurrently on a multi-threaded executor. Since the fields isCancelled, req, listener, and realCall are not volatile and are accessed/modified without any synchronization inside these tasks, this leads to data races and race conditions where the gRPC/VRpc contract of a single onClose call may be violated.

Suggested Fix

Use a synchronized(PendingVRpc.this) block inside the runnables to ensure atomic state transitions. Note that per repository rules, since we are using a common lock, explicit atomic state transitions (like compareAndSet) are not strictly necessary. Ensure BigtableTimer.Timeout is cancelled using cancel() without arguments as per repository standards.

    private void cancel(Status status, boolean onlyCancelPendingCall) {
      BigtableTimer.Timeout timeout = deadlineTimeout;
      deadlineTimeout = null;
      if (timeout != null) {
        timeout.cancel();
      }
      ctx.getExecutor()
          .execute(
              () -> {
                VRpcListener<RespT> pendingListener;
                VRpc<ReqT, RespT> call;
                boolean shouldCancelCall = false;
                synchronized (PendingVRpc.this) {
                  if (isCancelled) return;
                  isCancelled = true;
                  pendingListener = listener;
                  req = null;
                  listener = null;
                  call = realCall;
                  if (call != null) {
                    shouldCancelCall = !onlyCancelPendingCall;
                  }
                }
                if (call != null) {
                  if (shouldCancelCall) {
                    call.cancel(status.getDescription(), status.getCause());
                  }
                } else if (pendingListener != null) {
                  pendingListener.onClose(VRpcResult.createRejectedError(status));
                }
              });
    }
References
  1. When concurrent operations are protected by a common lock (e.g., synchronized (this)), atomic state transitions are not strictly necessary.
  2. Use cancel() without arguments when canceling a BigtableTimer.Timeout instance.

}

void cancelWithResult(VRpcResult result) {
if (deadlineTimeout != null) {
deadlineTimeout.cancel();
BigtableTimer.Timeout timeout = deadlineTimeout;
deadlineTimeout = null;
if (timeout != null) {
timeout.cancel();
}
ctx.getExecutor()
.execute(
() -> {
if (isCancelled) return;
isCancelled = true;
listener.onClose(result);
VRpcListener<RespT> pendingListener = listener;
req = null;
listener = null;
if (pendingListener != null) {
pendingListener.onClose(result);
}
});
}

Expand All @@ -1077,8 +1077,10 @@ public void requestNext() {
}

private void drainTo(SessionHandle handle) {
if (deadlineTimeout != null) {
deadlineTimeout.cancel();
BigtableTimer.Timeout timeout = deadlineTimeout;
deadlineTimeout = null;
if (timeout != null) {
timeout.cancel();
}
ctx.getExecutor()
.execute(
Expand All @@ -1087,15 +1089,15 @@ private void drainTo(SessionHandle handle) {
SessionPoolImpl.this.onPendingVRpcCancelled(handle);
return;
}
ReqT pendingReq = req;
VRpcListener<RespT> pendingListener = listener;
req = null;
listener = null;
realCall = newRealCall(desc, handle);
realCall.start(req, ctx, listener);
realCall.start(pendingReq, ctx, pendingListener);
});
}

private VRpcListener<RespT> getListener() {
return listener;
}

private BigtableTimer.Timeout monitorDeadline(Deadline deadline) {
// Body runs on backgroundExecutor (off the tick thread). onlyCancelPendingCall=true avoids
// racing with a user cancel that already attached a real call.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,14 @@ class PoolSizerTest {

private static final float IDLE_SESSION_HEADROOM_RATIO = 0.5f;
private static final int MIN_IDLE_SESSIONS = 5;
private static final int MAX_IDLE_SESSIONS = 400;
private static final int MAX_SESSIONS = 400;
private static final int PENDING_RPCS_PER_SESSION = 10;

private static final SessionClientConfiguration.SessionPoolConfiguration DEFAULT_CONFIG =
SessionClientConfiguration.SessionPoolConfiguration.newBuilder()
.setHeadroom(IDLE_SESSION_HEADROOM_RATIO)
.setMinSessionCount(MIN_IDLE_SESSIONS)
.setMaxSessionCount(MAX_IDLE_SESSIONS)
.setMaxSessionCount(MAX_SESSIONS)
.setNewSessionQueueLength(PENDING_RPCS_PER_SESSION)
.build();

Expand Down Expand Up @@ -193,7 +193,7 @@ void testEquilibriumWhenStartingWithLoad() {
int sessionsNeeded = forPending + stats.getInUseCount();
int desiredIdleSessions = (int) Math.ceil(sessionsNeeded * IDLE_SESSION_HEADROOM_RATIO);
desiredIdleSessions =
Math.max(Math.min(desiredIdleSessions, MAX_IDLE_SESSIONS), MIN_IDLE_SESSIONS);
Math.max(Math.min(desiredIdleSessions, MAX_SESSIONS), MIN_IDLE_SESSIONS);
int desiredCapacity = sessionsNeeded + desiredIdleSessions;

assertThat(sizer.getScaleDelta())
Expand Down Expand Up @@ -249,4 +249,26 @@ void testEmptyConfig() {
assertThat(sizer.handleGoAway(GoAwayResponse.getDefaultInstance())).isFalse();
assertThat(sizer.handleSessionClose(Status.getDefaultInstance())).isFalse();
}

@Test
void testDesiredCapacityCappedAtMaxSessions() {
PoolStats stats = new PoolStats();
AtomicInteger pendingCount = new AtomicInteger(0);
PoolSizer sizer = new PoolSizer(stats, pendingCount::get, DEFAULT_CONFIG);

// MAX_SESSIONS in use + 2 ready -> immediateCapacity = MAX_SESSIONS + 2.
// Total desiredCapacity is capped at MAX_SESSIONS, so scaleDelta is -2.
stats.getTestHelper().setInUseCount(MAX_SESSIONS);
stats.getTestHelper().setReadyCount(2);
stats.getTestHelper().setExpectedCapacity(MAX_SESSIONS + 2);
assertThat(sizer.getScaleDelta()).isEqualTo(-2);

// When MAX_SESSIONS are in use and 50 RPCs are queued, scaleDelta must be 0
// rather than scaling above MAX_SESSIONS.
stats.getTestHelper().setReadyCount(0);
stats.getTestHelper().setExpectedCapacity(MAX_SESSIONS);
pendingCount.set(50);
assertThat(sizer.getScaleDelta()).isEqualTo(0);
assertThat(sizer.handleNewCall()).isFalse();
}
Comment on lines +253 to +273

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

This test asserts that the total pool capacity is capped at maxIdleSessions. As discussed in the feedback for PoolSizer.java, this capping behavior is incorrect because maxIdleSessions should only limit the number of idle sessions, not the total sessions (active + idle). Once the capping logic in PoolSizer.java is corrected, this test should be updated or removed to reflect the correct scaling behavior.

}
Original file line number Diff line number Diff line change
Expand Up @@ -1448,6 +1448,46 @@ void configUpdateToEmptyClosesSessionsAndScalesBackUpWhenEnabled() throws Except
assertThat(sessionPool.hasSession()).isTrue();
}

@Test
void pendingVRpcClearsReferencesAfterDrainAndCancel() throws Exception {
sessionPool.start(OpenFakeSessionRequest.getDefaultInstance(), new Metadata());

// Create both PendingVRpcs before any session becomes READY so both return PendingVRpc
UnaryResponseFuture<SessionFakeScriptedResponse> drainedFuture = new UnaryResponseFuture<>();
VRpc<SessionFakeScriptedRequest, SessionFakeScriptedResponse> drainedRpc =
sessionPool.newCall(FakeDescriptor.SCRIPTED);
UnaryResponseFuture<SessionFakeScriptedResponse> cancelledFuture = new UnaryResponseFuture<>();
VRpc<SessionFakeScriptedRequest, SessionFakeScriptedResponse> cancelledRpc =
sessionPool.newCall(FakeDescriptor.SCRIPTED);

cancelledRpc.start(
SessionFakeScriptedRequest.getDefaultInstance(),
VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer),
cancelledFuture);
cancelledRpc.cancel("user cancelled", null);

drainedRpc.start(
SessionFakeScriptedRequest.getDefaultInstance(),
VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer),
drainedFuture);
drainedFuture.get(5, TimeUnit.SECONDS);

Field reqField = drainedRpc.getClass().getDeclaredField("req");
Field listenerField = drainedRpc.getClass().getDeclaredField("listener");
Field timeoutField = drainedRpc.getClass().getDeclaredField("deadlineTimeout");
reqField.setAccessible(true);
listenerField.setAccessible(true);
timeoutField.setAccessible(true);

assertThat(reqField.get(drainedRpc)).isNull();
assertThat(listenerField.get(drainedRpc)).isNull();
assertThat(timeoutField.get(drainedRpc)).isNull();

assertThat(reqField.get(cancelledRpc)).isNull();
assertThat(listenerField.get(cancelledRpc)).isNull();
assertThat(timeoutField.get(cancelledRpc)).isNull();
}

private void waitForOpenRequestCount(int minCount, Duration timeout) throws InterruptedException {
long deadline = System.currentTimeMillis() + timeout.toMillis();
while (System.currentTimeMillis() < deadline) {
Expand Down
Loading