Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,21 @@ public synchronized SessionStream newStream(
return new SessionStreamImpl(innerCall) {
// null until onBeforeSessionStart fires
@Nullable AfeId afeId = null;
volatile boolean sessionEstablished = false;

@Override
public void afterSessionStart() {
// Only mark the session established and reset consecutiveFailures after
// OpenSessionResponse is processed. onBeforeSessionStart is not enough because the AFE
// sends initial PeerInfo response headers before completing OpenSession, so streams
// rejected during OpenSession (e.g. RESOURCE_EXHAUSTED) still fire
// onBeforeSessionStart before closing with an error.
sessionEstablished = true;
synchronized (ChannelPoolDpImpl.this) {
Comment thread
igorbernstein2 marked this conversation as resolved.
Resolved
fw.consecutiveFailures = 0;
recycleBackoff = INITIAL_RECYCLE_BACKOFF;
}
}

@Override
public void start(Listener responseListener, Metadata headers) {
Expand All @@ -317,8 +332,6 @@ public void start(Listener responseListener, Metadata headers) {
public void onBeforeSessionStart(PeerInfo peerInfo) {
afeId = AfeId.extract(peerInfo);
synchronized (ChannelPoolDpImpl.this) {
fw.consecutiveFailures = 0;
recycleBackoff = INITIAL_RECYCLE_BACKOFF;
sessionsPerAfeId.add(afeId);
if (!fw.channel.isShutdown()) {
routeObservations.put(new RouteKey(fw.id, tenant), afeId);
Expand All @@ -333,8 +346,12 @@ public void onClose(Status status, Metadata trailers) {
synchronized (ChannelPoolDpImpl.this) {
if (afeId != null) {
sessionsPerAfeId.remove(afeId);
} else if (!status.isOk() && status.getCode() != Code.CANCELLED) {
}
if (!sessionEstablished && !status.isOk() && status.getCode() != Code.CANCELLED) {
fw.consecutiveFailures++;
if (afeId != null) {
routeObservations.remove(new RouteKey(fw.id, tenant), afeId);
}
}
releaseChannel(fw, status);
}
Expand Down Expand Up @@ -572,8 +589,11 @@ enum State {
final ManagedChannel channel;
final Instant createdAt;
volatile State state = State.ACTIVE;
// Guarded by the enclosing ChannelPoolDpImpl instance.
int numOutstanding = 0;
// Guarded by the enclosing ChannelPoolDpImpl instance.
int consecutiveFailures = 0;
// Guarded by the enclosing ChannelPoolDpImpl instance.
Instant lastUsedAt;

ChannelWrapper(long id, ManagedChannel channel, Clock clock) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,11 @@ public void start(Listener responseListener, Metadata headers) {
responseListener.onClose(status, new Metadata());
}

@Override
public void afterSessionStart() {
// noop
}

@Override
public void sendMessage(SessionRequest request) {
// noop
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,11 @@ public void start(Listener responseListener, Metadata headers) {
delegate.start(responseListener, headers);
}

@Override
public void afterSessionStart() {
delegate.afterSessionStart();
}

@Override
public void sendMessage(SessionRequest request) {
delegate.sendMessage(request);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ public interface SessionStream {

public void start(Listener responseListener, Metadata headers);

public void afterSessionStart();

public void sendMessage(SessionRequest request);

public void halfClose();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,9 @@ public void onClose(Status status, Metadata trailers) {
call.request(Integer.MAX_VALUE);
}

@Override
public void afterSessionStart() {}

@Override
public void sendMessage(SessionRequest request) {
call.sendMessage(request);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -682,6 +682,7 @@ private void handleOpenSessionResponse(OpenSessionResponse openSession) {
}
PeerInfo localPeerInfo = stream.getPeerInfo();
updateState(SessionState.READY);
stream.afterSessionStart();
tracer.onOpen(localPeerInfo);
sessionListener.onReady(openSession);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -517,16 +517,18 @@ void testResetConsecutiveFailuresOnSuccess() {
}
verify(channel, times(0)).shutdown();

// A success: onHeaders (which calls onBeforeSessionStart)
pool.newStream(FakeSessionGrpc.getOpenSessionMethod(), CallOptions.DEFAULT)
.start(mock(Listener.class), new Metadata());
// A success: onHeaders (which calls onBeforeSessionStart) followed by afterSessionStart()
SessionStream stream =
pool.newStream(FakeSessionGrpc.getOpenSessionMethod(), CallOptions.DEFAULT);
stream.start(mock(Listener.class), new Metadata());

PeerInfo peerInfo = PeerInfo.newBuilder().setApplicationFrontendId(555).build();
Metadata headers = new Metadata();
headers.put(
SessionStreamImpl.PEER_INFO_KEY,
Base64.getEncoder().encodeToString(peerInfo.toByteArray()));
listener.getValue().onHeaders(headers);
stream.afterSessionStart();
listener.getValue().onClose(Status.OK, new Metadata());

// Another 4 failures - should still not recycle because counter was reset
Expand Down Expand Up @@ -632,4 +634,35 @@ void testRecycleChannelBackoff() {

pool.close();
}

@Test
void testPreOpenSessionFailureWithPeerInfoIncrementsFailures() {
when(channelSupplier.get()).thenReturn(channel);
when(channel.newCall(any(), any())).thenReturn(clientCall);
doNothing().when(clientCall).start(listener.capture(), any());
doReturn(Attributes.EMPTY).when(clientCall).getAttributes();

ChannelPoolDpImpl pool =
new ChannelPoolDpImpl(channelSupplier, defaultConfig, debugTagTracer, bgExecutor);

PeerInfo peerInfo = PeerInfo.newBuilder().setApplicationFrontendId(555).build();
Metadata headers = new Metadata();
headers.put(
SessionStreamImpl.PEER_INFO_KEY,
Base64.getEncoder().encodeToString(peerInfo.toByteArray()));

// 5 pre-OpenSessionResponse rejections (onHeaders with PeerInfo followed by onClose(!OK)
// without OpenSessionResponse) must count as consecutive failures and recycle the channel.
for (int i = 0; i < 5; i++) {
pool.newStream(FakeSessionGrpc.getOpenSessionMethod(), CallOptions.DEFAULT)
.start(mock(Listener.class), new Metadata());
listener.getValue().onHeaders(headers);
listener.getValue().onClose(Status.RESOURCE_EXHAUSTED, new Metadata());
}

verify(channel, times(1)).shutdown();
verify(channelSupplier, times(2)).get();

pool.close();
}
}
Loading