Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,10 @@ private enum PoolState {
@GuardedBy("poolLock")
private int consecutiveFailures = 0;

@GuardedBy("poolLock")
@Nullable
private VRpcResult lastFailureResult = null;

/**
* When the client fallback to a non-session AFE session creation will return unimplemented
* errors. In which case the requests should fallback to classic client instead of waiting for an
Expand Down Expand Up @@ -281,6 +285,7 @@ public SessionPoolImpl(
poolSizer.updateConfig(newConfig);
picker.updateConfig(newConfig);
if (poolSizer.isDisabled()) {
resetConsecutiveFailures();
if (retryCreateSessionTimeout != null) {
retryCreateSessionTimeout.cancel();
retryCreateSessionTimeout = null;
Expand Down Expand Up @@ -597,8 +602,7 @@ private void onSessionReady(SessionHandle handle, OpenSessionResponse ignored) {
"Session {0} in state {1}",
new Object[] {handle.getSession().getLogName(), handle.getSession().getState()});

consecutiveFailures = 0;
consecutiveUnimplementedFailures = 0;
resetConsecutiveFailures();

if (poolState != PoolState.STARTED) {
logger.fine(
Expand Down Expand Up @@ -688,6 +692,7 @@ private void onSessionClose(
SessionHandle handle, SessionState prevState, Status status, Metadata trailers) {

List<PendingVRpc<?, ?>> toBeClosed = new ArrayList<>();
VRpcResult failureResult = null;

poolLock.lock();
try {
Expand Down Expand Up @@ -729,13 +734,15 @@ private void onSessionClose(
// rather than session establishment failures and should not fail pending vRPCs.
if (prevState == SessionState.STARTING) {
consecutiveFailures++;
lastFailureResult = createConsecutiveFailureResult(status, trailers);
if (status.getCode() == Status.Code.UNIMPLEMENTED) {
consecutiveUnimplementedFailures++;
} else {
consecutiveUnimplementedFailures = 0;
}
// TODO: decide if max consecutive failures should be capped per client
if (consecutiveFailures >= getMaxConsecutiveFailures(configManager)) {
if (isFailingWithNoEstablishedSessions()) {
failureResult = lastFailureResult;
toBeClosed = popClosableRpcs();
}
}
Expand All @@ -757,24 +764,46 @@ private void onSessionClose(

if (!toBeClosed.isEmpty()) {
// vRPCs failing with consecutive failures should fail
VRpcResult result =
VRpcResult.createRejectedError(
Status.fromCode(status.getCode())
.withDescription(
String.format(
"Session failed with consecutive failures. Most recent server status: %s,"
+ " metadata: %s.",
status, trailers)));
for (PendingVRpc<?, ?> vrpc : toBeClosed) {
try {
vrpc.cancelWithResult(result);
vrpc.cancelWithResult(failureResult);
} catch (Throwable t) {
logger.log(Level.WARNING, "Exception dispatching close to op executor", t);
}
}
}
}

@GuardedBy("poolLock")
private void resetConsecutiveFailures() {
consecutiveFailures = 0;
lastFailureResult = null;
consecutiveUnimplementedFailures = 0;
}

@GuardedBy("poolLock")
private boolean hasEstablishedSession() {
return sessions.getStats().getReadyCount() + sessions.getStats().getInUseCount() > 0;
}

@GuardedBy("poolLock")
private boolean isFailingWithNoEstablishedSessions() {
return consecutiveFailures >= getMaxConsecutiveFailures(configManager)
&& lastFailureResult != null
&& !hasEstablishedSession();
}

private static VRpcResult createConsecutiveFailureResult(
Status status, @Nullable Metadata trailers) {
Status errorStatus = status.isOk() ? Status.UNAVAILABLE : Status.fromCode(status.getCode());
return VRpcResult.createRejectedError(
errorStatus.withDescription(
String.format(
"Session failed with consecutive failures. Most recent server status: %s,"
+ " metadata: %s.",
status, trailers)));
}

@GuardedBy("poolLock")
private void tryDrainPendingRpcs() {
while (!pendingRpcs.isEmpty()) {
Expand Down Expand Up @@ -895,7 +924,7 @@ public boolean hasSession() {
return false;
}
try {
return sessions.getStats().getReadyCount() + sessions.getStats().getInUseCount() > 0;
return hasEstablishedSession();
} finally {
poolLock.unlock();
}
Expand Down Expand Up @@ -970,6 +999,10 @@ public void start(ReqT req, VRpcCallContext ctx, VRpcListener<RespT> listener) {
});
return;
}
if (isFailingWithNoEstablishedSessions()) {
cancelWithResult(lastFailureResult);
return;
}
// Only arm the deadline monitor after we've committed to queueing; otherwise the
// fast-fail early return above would leak a timer that fires later and emits a phantom
// tracer.onAttemptFinish on the Active state's stale listener.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
import com.google.cloud.bigtable.data.v2.internal.middleware.VRpc;
import com.google.cloud.bigtable.data.v2.internal.middleware.VRpc.VRpcCallContext;
import com.google.cloud.bigtable.data.v2.internal.middleware.VRpc.VRpcResult;
import com.google.cloud.bigtable.data.v2.internal.session.Session.SessionState;
import com.google.cloud.bigtable.data.v2.internal.session.SessionList.SessionHandle;
import com.google.cloud.bigtable.data.v2.internal.session.fake.FakeClock;
import com.google.cloud.bigtable.data.v2.internal.session.fake.FakeServiceBuilder;
Expand Down Expand Up @@ -83,6 +84,7 @@
import io.grpc.Status;
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
Expand Down Expand Up @@ -1448,6 +1450,171 @@ void configUpdateToEmptyClosesSessionsAndScalesBackUpWhenEnabled() throws Except
assertThat(sessionPool.hasSession()).isTrue();
}

@Test
@SuppressWarnings("GuardedBy")
void consecutiveFailuresFastFailsDuringBudgetLockout() throws Exception {
com.google.rpc.Status errorStatus =
com.google.rpc.Status.newBuilder()
.setCode(Code.INTERNAL_VALUE)
.setMessage("fake internal error")
.build();
OpenFakeSessionRequest request =
OpenFakeSessionRequest.newBuilder()
.setStreamError(
StreamError.newBuilder()
.setStatus(errorStatus)
.setDelay(Durations.fromMillis(0))
.build())
.build();
sessionPool.start(request, new Metadata());

// Wait until all initial SessionCreationBudget slots have been exhausted and closed on the
// client
ReentrantLock poolLock = extractPoolLock(sessionPool);
Field consecutiveFailuresField = SessionPoolImpl.class.getDeclaredField("consecutiveFailures");
consecutiveFailuresField.setAccessible(true);
long deadline = System.currentTimeMillis() + 5000;
while (System.currentTimeMillis() < deadline) {
poolLock.lock();
try {
if (sessionPool.sessions.getAllSessions().isEmpty()
&& consecutiveFailuresField.getInt(sessionPool) >= 10) {
break;
}
} finally {
poolLock.unlock();
Comment on lines +1474 to +1485

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If this 5s polling loop ever times out on a slow runner before all 50 budget slots finish closing (allSessions is not empty yet), the loop exits silently and rpc.start(...) below could pass by being cancelled via a racing onSessionClose() rather than via PendingVRpc.start().

Could we assert after the loop (or inside poolLock) that sessionPool.sessions.getAllSessions().isEmpty() and consecutiveFailuresField.getInt(sessionPool) >= 10 were actually reached?

}
Thread.sleep(10);
}
poolLock.lock();
try {
assertThat(sessionPool.sessions.getAllSessions()).isEmpty();
assertThat(consecutiveFailuresField.getInt(sessionPool)).isAtLeast(10);
} finally {
poolLock.unlock();
}

// A new call arriving during the 60s budget lockout must fail immediately instead of queueing
UnaryResponseFuture<SessionFakeScriptedResponse> f = new UnaryResponseFuture<>();
VRpc<SessionFakeScriptedRequest, SessionFakeScriptedResponse> rpc =
sessionPool.newCall(FakeDescriptor.SCRIPTED);
rpc.start(
SessionFakeScriptedRequest.getDefaultInstance(),
VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer),
f);

ExecutionException ex =
assertThrows(ExecutionException.class, () -> f.get(1, TimeUnit.SECONDS));
assertThat(ex.getCause())
.hasMessageThat()
.contains("INTERNAL: Session failed with consecutive failures.");
}

@Test
@SuppressWarnings("GuardedBy")
void consecutiveFailuresPreservesPendingRpcsWhilePoolHasInUseSessions() throws Exception {
sessionPool.start(OpenFakeSessionRequest.getDefaultInstance(), new Metadata());
ReentrantLock poolLock = extractPoolLock(sessionPool);
long readyDeadline = System.currentTimeMillis() + 5000;
while (System.currentTimeMillis() < readyDeadline) {
poolLock.lock();
try {
if (sessionPool.sessions.getStats().getReadyCount() == 5) {
break;
}
} finally {
poolLock.unlock();
}
Thread.sleep(10);
}

Field consecutiveFailuresField = SessionPoolImpl.class.getDeclaredField("consecutiveFailures");
consecutiveFailuresField.setAccessible(true);
Field budgetField = SessionPoolImpl.class.getDeclaredField("budget");
budgetField.setAccessible(true);
Method onSessionCloseMethod =
SessionPoolImpl.class.getDeclaredMethod(
"onSessionClose",
SessionHandle.class,
SessionState.class,
Status.class,
Metadata.class);
onSessionCloseMethod.setAccessible(true);

UnaryResponseFuture<SessionFakeScriptedResponse> checkedOutFuture = new UnaryResponseFuture<>();
UnaryResponseFuture<SessionFakeScriptedResponse> queuedBeforeCloseFuture =
new UnaryResponseFuture<>();
UnaryResponseFuture<SessionFakeScriptedResponse> queuedAfterCloseFuture =
new UnaryResponseFuture<>();
VRpc<SessionFakeScriptedRequest, SessionFakeScriptedResponse> checkedOutRpc;
poolLock.lock();
try {
assertThat(sessionPool.sessions.getStats().getReadyCount()).isEqualTo(5);
// Exhaust the session creation budget so scale-up triggers in start()/onSessionClose() do not
// open new background sessions that could race with the single in-use session under test.
SessionCreationBudget budget = (SessionCreationBudget) budgetField.get(sessionPool);
while (budget.tryReserveSession()) {
budget.onSessionCreationFailure();
}

// Mark 4 of the 5 ready sessions closing, then check out the remaining 1 ready session via
// newCall() so readyCount == 0 and inUseCount == 1.
List<SessionHandle> handles = new ArrayList<>(sessionPool.sessions.getAllSessions());
for (int i = 1; i < handles.size(); i++) {
handles.get(i).onSessionClosing();
}
checkedOutRpc = sessionPool.newCall(FakeDescriptor.SCRIPTED);
assertThat(sessionPool.sessions.getStats().getReadyCount()).isEqualTo(0);
assertThat(sessionPool.sessions.getStats().getInUseCount()).isEqualTo(1);

// Queue a first PendingVRpc before the 10th STARTING failure arrives in onSessionClose.
consecutiveFailuresField.setInt(sessionPool, 9);
VRpc<SessionFakeScriptedRequest, SessionFakeScriptedResponse> queuedBeforeCloseRpc =
sessionPool.newCall(FakeDescriptor.SCRIPTED);
queuedBeforeCloseRpc.start(
SessionFakeScriptedRequest.newBuilder().setTag(0).build(),
VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer),
queuedBeforeCloseFuture);
assertThat(queuedBeforeCloseFuture.isDone()).isFalse();

Session mockStartingSession = mock(Session.class);
when(mockStartingSession.getLogName()).thenReturn("mock-starting");
SessionHandle failingStartingHandle = sessionPool.sessions.newHandle(mockStartingSession);

// Trigger the 10th STARTING failure via onSessionClose while inUseCount == 1;
// queuedBeforeCloseFuture must remain queued rather than being aborted.
onSessionCloseMethod.invoke(
sessionPool,
failingStartingHandle,
SessionState.STARTING,
Status.INTERNAL.withDescription("fake internal error"),
new Metadata());
assertThat(consecutiveFailuresField.getInt(sessionPool)).isEqualTo(10);
assertThat(queuedBeforeCloseFuture.isDone()).isFalse();

// A second PendingVRpc started after consecutiveFailures == 10 while inUseCount == 1 must
// also queue instead of failing fast.
VRpc<SessionFakeScriptedRequest, SessionFakeScriptedResponse> queuedAfterCloseRpc =
sessionPool.newCall(FakeDescriptor.SCRIPTED);
queuedAfterCloseRpc.start(
SessionFakeScriptedRequest.newBuilder().setTag(0).build(),
VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer),
queuedAfterCloseFuture);
assertThat(queuedAfterCloseFuture.isDone()).isFalse();
} finally {
poolLock.unlock();
}

// Complete the checked-out RPC so its session returns to the pool and drains both pending RPCs.
checkedOutRpc.start(
SessionFakeScriptedRequest.newBuilder().setTag(0).build(),
VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer),
checkedOutFuture);
assertThat(checkedOutFuture.get(5, TimeUnit.SECONDS)).isNotNull();
assertThat(queuedBeforeCloseFuture.get(5, TimeUnit.SECONDS)).isNotNull();
assertThat(queuedAfterCloseFuture.get(5, TimeUnit.SECONDS)).isNotNull();
}

private void waitForOpenRequestCount(int minCount, Duration timeout) throws InterruptedException {
long deadline = System.currentTimeMillis() + timeout.toMillis();
while (System.currentTimeMillis() < deadline) {
Expand Down
Loading