Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,7 @@ public synchronized SessionStream newStream(
return new SessionStreamImpl(innerCall) {
// null until onBeforeSessionStart fires
@Nullable AfeId afeId = null;
boolean sessionEstablished = false;

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The sessionEstablished flag is read outside of the synchronized block in onMessage (line 332). To prevent thread visibility issues and ensure memory consistency across threads, this field should be declared volatile.

Suggested change
boolean sessionEstablished = false;
volatile boolean sessionEstablished = false;
References
  1. Ensure that fields used for cross-thread state checks, such as exceptions set by background threads and read by application threads, are declared as volatile to guarantee memory visibility and establish a proper happens-before relationship under the Java Memory Model.


@Override
public void start(Listener responseListener, Metadata headers) {
Expand All @@ -317,8 +318,6 @@ public void start(Listener responseListener, Metadata headers) {
public void onBeforeSessionStart(PeerInfo peerInfo) {
afeId = AfeId.extract(peerInfo);
synchronized (ChannelPoolDpImpl.this) {
fw.consecutiveFailures = 0;
recycleBackoff = INITIAL_RECYCLE_BACKOFF;
sessionsPerAfeId.add(afeId);
if (!fw.channel.isShutdown()) {
routeObservations.put(new RouteKey(fw.id, tenant), afeId);
Expand All @@ -328,13 +327,32 @@ public void onBeforeSessionStart(PeerInfo peerInfo) {
super.onBeforeSessionStart(peerInfo);
}

@Override
public void onMessage(SessionResponse msg) {
if (!sessionEstablished && msg.hasOpenSession()) {
synchronized (ChannelPoolDpImpl.this) {
sessionEstablished = true;
fw.consecutiveFailures = 0;
recycleBackoff = INITIAL_RECYCLE_BACKOFF;
}
}
super.onMessage(msg);
}
Comment on lines +331 to +340

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

To avoid redundant writes and ensure thread safety when resetting consecutive failures, we should use a double-checked locking pattern under the synchronized block after checking sessionEstablished.

              @Override
              public void onMessage(SessionResponse msg) {
                if (!sessionEstablished && msg.hasOpenSession()) {
                  synchronized (ChannelPoolDpImpl.this) {
                    if (!sessionEstablished) {
                      sessionEstablished = true;
                      fw.consecutiveFailures = 0;
                      recycleBackoff = INITIAL_RECYCLE_BACKOFF;
                    }
                  }
                }
                super.onMessage(msg);
              }


@Override
public void onClose(Status status, Metadata trailers) {
synchronized (ChannelPoolDpImpl.this) {
if (afeId != null) {
sessionsPerAfeId.remove(afeId);
} else if (!status.isOk() && status.getCode() != Code.CANCELLED) {
}
if (status.isOk()) {
fw.consecutiveFailures = 0;
recycleBackoff = INITIAL_RECYCLE_BACKOFF;
} else if (!sessionEstablished && status.getCode() != Code.CANCELLED) {
fw.consecutiveFailures++;
if (afeId != null) {
routeObservations.remove(new RouteKey(fw.id, tenant), afeId);
}
}
releaseChannel(fw, status);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ class PoolSizer {
// 50% increase in latency (since that also increases requests-in-flight).
private volatile float idlesSessionHeadRoom;
private volatile int minIdleSessions;
private volatile int maxIdleSessions;
private volatile int maxSessions;

private final PoolStats stats;
private final Sized pendingRpcs;
Expand Down Expand Up @@ -60,12 +60,12 @@ interface Sized {
void updateConfig(SessionClientConfiguration.SessionPoolConfiguration poolConfiguration) {
this.idlesSessionHeadRoom = poolConfiguration.getHeadroom();
this.minIdleSessions = poolConfiguration.getMinSessionCount();
this.maxIdleSessions = poolConfiguration.getMaxSessionCount();
this.maxSessions = poolConfiguration.getMaxSessionCount();
this.pendingVRpcsPerSession = poolConfiguration.getNewSessionQueueLength();
this.disabled =
poolConfiguration.equals(
SessionClientConfiguration.SessionPoolConfiguration.getDefaultInstance())
|| this.maxIdleSessions <= 0;
|| this.maxSessions <= 0;
}

boolean isDisabled() {
Expand All @@ -82,9 +82,9 @@ public int getScaleDelta() {
int sessionsInUse = effectivePending + stats.getInUseCount();
int unboundedDesiredIdleSessions = (int) Math.ceil(sessionsInUse * idlesSessionHeadRoom);
int desiredIdleSessions =
Math.max(Math.min(unboundedDesiredIdleSessions, maxIdleSessions), minIdleSessions);
Math.max(Math.min(unboundedDesiredIdleSessions, maxSessions), minIdleSessions);

int desiredCapacity = sessionsInUse + desiredIdleSessions;
int desiredCapacity = Math.min(sessionsInUse + desiredIdleSessions, maxSessions);
int eventualCapacity = stats.getExpectedCapacity();
int immediateCapacity = eventualCapacity - stats.getStartingCount();

Expand All @@ -100,6 +100,14 @@ public int getScaleDelta() {
}
}

/** Returns the maximum number of pending vRPCs allowed in the pool queue. */
int getMaxPendingRpcs() {
if (isDisabled()) {
return 0;
}
return maxSessions * Math.max(1, pendingVRpcsPerSession);
}

/** Returns true if the session should be replaced */
@CheckReturnValue
public boolean handleGoAway(GoAwayResponse msg) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,6 @@ class SessionList {
// - starting sessions
// - closing sessions
private final Set<SessionHandle> allSessions = new HashSet<>();
private final Set<SessionHandle> inUseSessions = new HashSet<>();

// pool level statistics across all the afes
private final PoolStats poolStats = new PoolStats();
Expand Down Expand Up @@ -119,7 +118,6 @@ public Optional<SessionHandle> checkoutSession(AfeHandle afeHandle) {
handle -> {
poolStats.readyCount--;
poolStats.inUseCount++;
inUseSessions.add(handle);
if (handle.afe.get().sessions.isEmpty()) {
afesWithReadySessions.remove(afeHandle);
}
Expand Down Expand Up @@ -221,7 +219,6 @@ private void releaseToPool() {
AfeHandle afeHandle = this.afe.get();

poolStats.inUseCount--;
inUseSessions.remove(this);

if (!closing && session.getState() == SessionState.READY) {
poolStats.readyCount++;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,12 @@ private enum PoolState {
@GuardedBy("poolLock")
private int consecutiveFailures = 0;

@GuardedBy("poolLock")
private Status lastCloseStatus = Status.UNAVAILABLE;

@GuardedBy("poolLock")
private Metadata lastCloseTrailers = new Metadata();

/**
* When the client fallback to a non-session AFE session creation will return unimplemented
* errors. In which case the requests should fallback to classic client instead of waiting for an
Expand Down Expand Up @@ -729,6 +735,8 @@ private void onSessionClose(
// rather than session establishment failures and should not fail pending vRPCs.
if (prevState == SessionState.STARTING) {
consecutiveFailures++;
lastCloseStatus = status;
lastCloseTrailers = trailers;
if (status.getCode() == Status.Code.UNIMPLEMENTED) {
consecutiveUnimplementedFailures++;
} else {
Expand Down Expand Up @@ -911,7 +919,7 @@ class PendingVRpc<ReqT extends Message, RespT extends Message> implements VRpc<R
private boolean isCancelled = false;
private VRpc<ReqT, RespT> realCall;

private BigtableTimer.Timeout deadlineTimeout;
private volatile BigtableTimer.Timeout deadlineTimeout;

public PendingVRpc(VRpcDescriptor<?, ReqT, RespT> desc) {
this.desc = desc;
Expand All @@ -922,7 +930,7 @@ public PendingVRpc(VRpcDescriptor<?, ReqT, RespT> desc) {
// region. Bounded acquisition on the request hot path; see poolLock.
@SuppressWarnings("GuardedBy")
public void start(ReqT req, VRpcCallContext ctx, VRpcListener<RespT> listener) {
Preconditions.checkState(this.req == null, "request is already started");
Preconditions.checkState(this.ctx == null, "request is already started");
Preconditions.checkNotNull(req, "request can't be null");
Preconditions.checkNotNull(ctx, "ctx can't be null");
Preconditions.checkNotNull(listener, "listener can't be null");
Expand All @@ -940,34 +948,52 @@ public void start(ReqT req, VRpcCallContext ctx, VRpcListener<RespT> listener) {
VRpcResult.createUncommitedError(
Status.UNAVAILABLE.withCause(
new IllegalStateException("SessionPool lock unavailable (pool wedged)")));
ctx.getExecutor()
.execute(
() -> {
if (isCancelled) return;
isCancelled = true;
listener.onClose(result);
});
cancelWithResult(result);
return;
}
try {
if (SessionPoolImpl.this.poolState != PoolState.STARTED) {
VRpcResult result =
VRpcResult.createUncommitedError(
Status.UNAVAILABLE.withCause(new IllegalStateException("SessionPool is closed")));
ctx.getExecutor().execute(() -> listener.onClose(result));
cancelWithResult(result);
return;
}
if (poolSizer.isDisabled()) {
VRpcResult result =
VRpcResult.createRejectedError(
Status.UNAVAILABLE.withDescription("SessionPool configuration disabled"));
ctx.getExecutor()
.execute(
() -> {
if (isCancelled) return;
isCancelled = true;
listener.onClose(result);
});
cancelWithResult(result);
return;
}
if (consecutiveFailures >= getMaxConsecutiveFailures(configManager)
&& sessions.getStats().getReadyCount() == 0) {
if (poolSizer.handleNewCall()) {
createSession(openParams);
}
VRpcResult result =
VRpcResult.createRejectedError(
Status.fromCode(lastCloseStatus.getCode())
.withDescription(
String.format(
"Session failed with consecutive failures. Most recent server status:"
+ " %s, metadata: %s.",
lastCloseStatus, lastCloseTrailers)));
cancelWithResult(result);
return;
}
int maxPendingRpcs = poolSizer.getMaxPendingRpcs();
if (maxPendingRpcs > 0 && pendingRpcs.size() >= maxPendingRpcs) {
if (poolSizer.handleNewCall()) {
createSession(openParams);
}
VRpcResult result =
VRpcResult.createRejectedError(
Status.RESOURCE_EXHAUSTED.withDescription(
String.format(
"SessionPool pending RPC queue is full (%d/%d)",
pendingRpcs.size(), maxPendingRpcs)));
cancelWithResult(result);
return;
}
// Only arm the deadline monitor after we've committed to queueing; otherwise the
Expand Down Expand Up @@ -1013,8 +1039,10 @@ public void cancel(@Nullable String message, @Nullable Throwable cause) {
// region below.
@SuppressWarnings("GuardedBy")
private void cancel(Status status, boolean onlyCancelPendingCall) {
if (deadlineTimeout != null) {
deadlineTimeout.cancel();
BigtableTimer.Timeout timeout = deadlineTimeout;
deadlineTimeout = null;
if (timeout != null) {
timeout.cancel();
}
// The eager removal is only an optimization to skip a wasted session pick in
// tryDrainPendingRpcs.
Expand All @@ -1037,26 +1065,36 @@ private void cancel(Status status, boolean onlyCancelPendingCall) {
() -> {
if (isCancelled) return;
isCancelled = true;
VRpcListener<RespT> pendingListener = listener;
req = null;
listener = null;
if (realCall != null) {
if (!onlyCancelPendingCall) {
realCall.cancel(status.getDescription(), status.getCause());
}
} else {
listener.onClose(VRpcResult.createRejectedError(status));
} else if (pendingListener != null) {
pendingListener.onClose(VRpcResult.createRejectedError(status));
}
});
Comment on lines 1065 to 1078

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

Since ctx.getExecutor() can be a multi-threaded executor, there is a potential check-then-act race condition on isCancelled and a data race on realCall, req, and listener. Synchronizing on PendingVRpc.this inside the executor task ensures thread safety and prevents duplicate calls to onClose or concurrent modification issues.

Suggested change
() -> {
if (isCancelled) return;
isCancelled = true;
VRpcListener<RespT> pendingListener = listener;
req = null;
listener = null;
if (realCall != null) {
if (!onlyCancelPendingCall) {
realCall.cancel(status.getDescription(), status.getCause());
}
} else {
listener.onClose(VRpcResult.createRejectedError(status));
} else if (pendingListener != null) {
pendingListener.onClose(VRpcResult.createRejectedError(status));
}
});
() -> {
VRpcListener<RespT> pendingListener;
VRpc<ReqT, RespT> localRealCall;
synchronized (PendingVRpc.this) {
if (isCancelled) return;
isCancelled = true;
pendingListener = listener;
req = null;
listener = null;
localRealCall = realCall;
}
if (localRealCall != null) {
if (!onlyCancelPendingCall) {
localRealCall.cancel(status.getDescription(), status.getCause());
}
} else if (pendingListener != null) {
pendingListener.onClose(VRpcResult.createRejectedError(status));
}
});

}

void cancelWithResult(VRpcResult result) {
if (deadlineTimeout != null) {
deadlineTimeout.cancel();
BigtableTimer.Timeout timeout = deadlineTimeout;
deadlineTimeout = null;
if (timeout != null) {
timeout.cancel();
}
ctx.getExecutor()
.execute(
() -> {
if (isCancelled) return;
isCancelled = true;
listener.onClose(result);
VRpcListener<RespT> pendingListener = listener;
req = null;
listener = null;
if (pendingListener != null) {
pendingListener.onClose(result);
}
});
Comment on lines 1087 to 1098

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

Similarly to the cancel method, cancelWithResult should synchronize on PendingVRpc.this to avoid race conditions on isCancelled, req, and listener when executed on a multi-threaded executor.

      ctx.getExecutor()
          .execute(
              () -> {
                VRpcListener<RespT> pendingListener;
                synchronized (PendingVRpc.this) {
                  if (isCancelled) return;
                  isCancelled = true;
                  pendingListener = listener;
                  req = null;
                  listener = null;
                }
                if (pendingListener != null) {
                  pendingListener.onClose(result);
                }
              });

}

Expand All @@ -1077,8 +1115,10 @@ public void requestNext() {
}

private void drainTo(SessionHandle handle) {
if (deadlineTimeout != null) {
deadlineTimeout.cancel();
BigtableTimer.Timeout timeout = deadlineTimeout;
deadlineTimeout = null;
if (timeout != null) {
timeout.cancel();
}
ctx.getExecutor()
.execute(
Expand All @@ -1087,15 +1127,15 @@ private void drainTo(SessionHandle handle) {
SessionPoolImpl.this.onPendingVRpcCancelled(handle);
return;
}
ReqT pendingReq = req;
VRpcListener<RespT> pendingListener = listener;
req = null;
listener = null;
realCall = newRealCall(desc, handle);
realCall.start(req, ctx, listener);
realCall.start(pendingReq, ctx, pendingListener);
});
}

private VRpcListener<RespT> getListener() {
return listener;
}

private BigtableTimer.Timeout monitorDeadline(Deadline deadline) {
// Body runs on backgroundExecutor (off the tick thread). onlyCancelPendingCall=true avoids
// racing with a user cancel that already attached a real call.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -632,4 +632,35 @@ void testRecycleChannelBackoff() {

pool.close();
}

@Test
void testPreOpenSessionFailureWithPeerInfoIncrementsFailures() {
when(channelSupplier.get()).thenReturn(channel);
when(channel.newCall(any(), any())).thenReturn(clientCall);
doNothing().when(clientCall).start(listener.capture(), any());
doReturn(Attributes.EMPTY).when(clientCall).getAttributes();

ChannelPoolDpImpl pool =
new ChannelPoolDpImpl(channelSupplier, defaultConfig, debugTagTracer, bgExecutor);

PeerInfo peerInfo = PeerInfo.newBuilder().setApplicationFrontendId(555).build();
Metadata headers = new Metadata();
headers.put(
SessionStreamImpl.PEER_INFO_KEY,
Base64.getEncoder().encodeToString(peerInfo.toByteArray()));

// 5 pre-OpenSessionResponse rejections (onHeaders with PeerInfo followed by onClose(!OK)
// without OpenSessionResponse) must count as consecutive failures and recycle the channel.
for (int i = 0; i < 5; i++) {
pool.newStream(FakeSessionGrpc.getOpenSessionMethod(), CallOptions.DEFAULT)
.start(mock(Listener.class), new Metadata());
listener.getValue().onHeaders(headers);
listener.getValue().onClose(Status.RESOURCE_EXHAUSTED, new Metadata());
}

verify(channel, times(1)).shutdown();
verify(channelSupplier, times(2)).get();

pool.close();
}
}
Loading
Loading