Repository navigation
fix(bigtable): bound SessionPool pendingRpcs and fast-fail during consecutive failures #15031
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
24d6574
86f6d01
8fdc29c
c5f3794
5110ab6
35030f8
65c90b2
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -144,6 +144,12 @@ private enum PoolState { | |
| @GuardedBy("poolLock") | ||
| private int consecutiveFailures = 0; | ||
|
|
||
| @GuardedBy("poolLock") | ||
| private Status lastCloseStatus = Status.UNAVAILABLE; | ||
|
|
||
| @GuardedBy("poolLock") | ||
| private Metadata lastCloseTrailers = new Metadata(); | ||
|
|
||
| /** | ||
| * When the client fallback to a non-session AFE session creation will return unimplemented | ||
| * errors. In which case the requests should fallback to classic client instead of waiting for an | ||
|
|
@@ -729,6 +735,8 @@ private void onSessionClose( | |
| // rather than session establishment failures and should not fail pending vRPCs. | ||
| if (prevState == SessionState.STARTING) { | ||
| consecutiveFailures++; | ||
| lastCloseStatus = status; | ||
| lastCloseTrailers = trailers; | ||
| if (status.getCode() == Status.Code.UNIMPLEMENTED) { | ||
| consecutiveUnimplementedFailures++; | ||
| } else { | ||
|
|
@@ -911,7 +919,7 @@ class PendingVRpc<ReqT extends Message, RespT extends Message> implements VRpc<R | |
| private boolean isCancelled = false; | ||
| private VRpc<ReqT, RespT> realCall; | ||
|
|
||
| private BigtableTimer.Timeout deadlineTimeout; | ||
| private volatile BigtableTimer.Timeout deadlineTimeout; | ||
|
|
||
| public PendingVRpc(VRpcDescriptor<?, ReqT, RespT> desc) { | ||
| this.desc = desc; | ||
|
|
@@ -922,7 +930,7 @@ public PendingVRpc(VRpcDescriptor<?, ReqT, RespT> desc) { | |
| // region. Bounded acquisition on the request hot path; see poolLock. | ||
| @SuppressWarnings("GuardedBy") | ||
| public void start(ReqT req, VRpcCallContext ctx, VRpcListener<RespT> listener) { | ||
| Preconditions.checkState(this.req == null, "request is already started"); | ||
| Preconditions.checkState(this.ctx == null, "request is already started"); | ||
| Preconditions.checkNotNull(req, "request can't be null"); | ||
| Preconditions.checkNotNull(ctx, "ctx can't be null"); | ||
| Preconditions.checkNotNull(listener, "listener can't be null"); | ||
|
|
@@ -940,34 +948,52 @@ public void start(ReqT req, VRpcCallContext ctx, VRpcListener<RespT> listener) { | |
| VRpcResult.createUncommitedError( | ||
| Status.UNAVAILABLE.withCause( | ||
| new IllegalStateException("SessionPool lock unavailable (pool wedged)"))); | ||
| ctx.getExecutor() | ||
| .execute( | ||
| () -> { | ||
| if (isCancelled) return; | ||
| isCancelled = true; | ||
| listener.onClose(result); | ||
| }); | ||
| cancelWithResult(result); | ||
| return; | ||
| } | ||
| try { | ||
| if (SessionPoolImpl.this.poolState != PoolState.STARTED) { | ||
| VRpcResult result = | ||
| VRpcResult.createUncommitedError( | ||
| Status.UNAVAILABLE.withCause(new IllegalStateException("SessionPool is closed"))); | ||
| ctx.getExecutor().execute(() -> listener.onClose(result)); | ||
| cancelWithResult(result); | ||
| return; | ||
| } | ||
| if (poolSizer.isDisabled()) { | ||
| VRpcResult result = | ||
| VRpcResult.createRejectedError( | ||
| Status.UNAVAILABLE.withDescription("SessionPool configuration disabled")); | ||
| ctx.getExecutor() | ||
| .execute( | ||
| () -> { | ||
| if (isCancelled) return; | ||
| isCancelled = true; | ||
| listener.onClose(result); | ||
| }); | ||
| cancelWithResult(result); | ||
| return; | ||
| } | ||
| if (consecutiveFailures >= getMaxConsecutiveFailures(configManager) | ||
| && sessions.getStats().getReadyCount() == 0) { | ||
| if (poolSizer.handleNewCall()) { | ||
| createSession(openParams); | ||
| } | ||
| VRpcResult result = | ||
| VRpcResult.createRejectedError( | ||
| Status.fromCode(lastCloseStatus.getCode()) | ||
| .withDescription( | ||
| String.format( | ||
| "Session failed with consecutive failures. Most recent server status:" | ||
| + " %s, metadata: %s.", | ||
| lastCloseStatus, lastCloseTrailers))); | ||
|
Comment on lines
+976
to
+981
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If Status.fromCode(
lastCloseStatus.getCode() == Status.Code.OK
? Status.Code.UNAVAILABLE
: lastCloseStatus.getCode())
.withDescription(
String.format(
"Session failed with consecutive failures. Most recent server status:"
+ " %s, metadata: %s.",
lastCloseStatus, lastCloseTrailers))) |
||
| cancelWithResult(result); | ||
| return; | ||
| } | ||
| int maxPendingRpcs = poolSizer.getMaxPendingRpcs(); | ||
| if (maxPendingRpcs > 0 && pendingRpcs.size() >= maxPendingRpcs) { | ||
| if (poolSizer.handleNewCall()) { | ||
| createSession(openParams); | ||
| } | ||
| VRpcResult result = | ||
| VRpcResult.createRejectedError( | ||
| Status.RESOURCE_EXHAUSTED.withDescription( | ||
| String.format( | ||
| "SessionPool pending RPC queue is full (%d/%d)", | ||
| pendingRpcs.size(), maxPendingRpcs))); | ||
| cancelWithResult(result); | ||
| return; | ||
| } | ||
| // Only arm the deadline monitor after we've committed to queueing; otherwise the | ||
|
|
@@ -1013,8 +1039,10 @@ public void cancel(@Nullable String message, @Nullable Throwable cause) { | |
| // region below. | ||
| @SuppressWarnings("GuardedBy") | ||
| private void cancel(Status status, boolean onlyCancelPendingCall) { | ||
| if (deadlineTimeout != null) { | ||
| deadlineTimeout.cancel(); | ||
| BigtableTimer.Timeout timeout = deadlineTimeout; | ||
| deadlineTimeout = null; | ||
| if (timeout != null) { | ||
| timeout.cancel(); | ||
| } | ||
| // The eager removal is only an optimization to skip a wasted session pick in | ||
| // tryDrainPendingRpcs. | ||
|
|
@@ -1037,26 +1065,36 @@ private void cancel(Status status, boolean onlyCancelPendingCall) { | |
| () -> { | ||
| if (isCancelled) return; | ||
| isCancelled = true; | ||
| VRpcListener<RespT> pendingListener = listener; | ||
| req = null; | ||
| listener = null; | ||
| if (realCall != null) { | ||
| if (!onlyCancelPendingCall) { | ||
| realCall.cancel(status.getDescription(), status.getCause()); | ||
| } | ||
| } else { | ||
| listener.onClose(VRpcResult.createRejectedError(status)); | ||
| } else if (pendingListener != null) { | ||
| pendingListener.onClose(VRpcResult.createRejectedError(status)); | ||
| } | ||
| }); | ||
| } | ||
|
|
||
| void cancelWithResult(VRpcResult result) { | ||
| if (deadlineTimeout != null) { | ||
| deadlineTimeout.cancel(); | ||
| BigtableTimer.Timeout timeout = deadlineTimeout; | ||
| deadlineTimeout = null; | ||
| if (timeout != null) { | ||
| timeout.cancel(); | ||
| } | ||
| ctx.getExecutor() | ||
| .execute( | ||
| () -> { | ||
| if (isCancelled) return; | ||
| isCancelled = true; | ||
| listener.onClose(result); | ||
| VRpcListener<RespT> pendingListener = listener; | ||
| req = null; | ||
| listener = null; | ||
| if (pendingListener != null) { | ||
| pendingListener.onClose(result); | ||
| } | ||
| }); | ||
| } | ||
|
Comment on lines
1081
to
1099
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Since void cancelWithResult(VRpcResult result) {
BigtableTimer.Timeout timeout = deadlineTimeout;
deadlineTimeout = null;
if (timeout != null) {
timeout.cancel();
}
ctx.getExecutor()
.execute(
() -> {
VRpcListener<RespT> pendingListener;
synchronized (PendingVRpc.this) {
if (isCancelled) return;
isCancelled = true;
pendingListener = listener;
req = null;
listener = null;
}
if (pendingListener != null) {
pendingListener.onClose(result);
}
});
}References
|
||
|
|
||
|
|
@@ -1077,8 +1115,10 @@ public void requestNext() { | |
| } | ||
|
|
||
| private void drainTo(SessionHandle handle) { | ||
| if (deadlineTimeout != null) { | ||
| deadlineTimeout.cancel(); | ||
| BigtableTimer.Timeout timeout = deadlineTimeout; | ||
| deadlineTimeout = null; | ||
| if (timeout != null) { | ||
| timeout.cancel(); | ||
| } | ||
| ctx.getExecutor() | ||
| .execute( | ||
|
|
@@ -1087,15 +1127,15 @@ private void drainTo(SessionHandle handle) { | |
| SessionPoolImpl.this.onPendingVRpcCancelled(handle); | ||
| return; | ||
| } | ||
| ReqT pendingReq = req; | ||
| VRpcListener<RespT> pendingListener = listener; | ||
| req = null; | ||
| listener = null; | ||
| realCall = newRealCall(desc, handle); | ||
| realCall.start(req, ctx, listener); | ||
| realCall.start(pendingReq, ctx, pendingListener); | ||
| }); | ||
| } | ||
|
|
||
| private VRpcListener<RespT> getListener() { | ||
| return listener; | ||
| } | ||
|
|
||
| private BigtableTimer.Timeout monitorDeadline(Deadline deadline) { | ||
| // Body runs on backgroundExecutor (off the tick thread). onlyCancelPendingCall=true avoids | ||
| // racing with a user cancel that already attached a real call. | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
In gRPC, the
trailersparameter inonClosecan sometimes benull(e.g., in certain failure modes or unit tests). To prevent potentialNullPointerExceptionissues when referencinglastCloseTrailerslater, we should defensively check iftrailersis null and default to an emptyMetadatainstance.