Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,10 @@ static boolean shouldCacheDialectError(Exception e) {

private void asyncCreateMultiplexedSession(
SettableApiFuture<SessionReference> sessionReferenceFuture, int remainingAttempts) {
if (isClosed || sessionReferenceFuture.isDone()) {
Comment thread
sakthivelmanii marked this conversation as resolved.
Resolved
sessionReferenceFuture.setException(newClosedException());
return;
}
this.sessionClient.asyncCreateMultiplexedSession(
new SessionConsumer() {
@Override
Expand Down Expand Up @@ -317,16 +321,37 @@ public void onSessionCreateFailure(Throwable t, int createFailureForSessionCount
MultiplexedSessionDatabaseClient.this.resourceNotFoundException.set(
(ResourceNotFoundException) spannerException);
}
// Set the exception to trigger an error for all waiters.
// Then retry the session creation if the error is (potentially) transient and the
// client has not been closed in the meantime.
sessionReferenceFuture.setException(t);
SettableApiFuture<SessionReference> retryFuture = null;
if (remainingAttempts > 1
&& !isClientClosed()
&& RETRYABLE_ERROR_CODES.contains(spannerException.getErrorCode())) {
final SettableApiFuture<SessionReference> future = SettableApiFuture.create();
MultiplexedSessionDatabaseClient.this.multiplexedSessionReference.set(future);
asyncCreateMultiplexedSession(future, remainingAttempts - 1);
final SettableApiFuture<SessionReference> nextFuture = SettableApiFuture.create();
if (MultiplexedSessionDatabaseClient.this.multiplexedSessionReference.compareAndSet(
sessionReferenceFuture, nextFuture)) {
retryFuture = nextFuture;
}
}

// 1. Unblock all waiters of the failed attempt first.
// This is executed OUTSIDE any lock to avoid deadlocks in foreign listener callbacks.
// Because the retryFuture was installed into multiplexedSessionReference above,
// any waking waiter that retries immediately is guaranteed to observe the retry future.
sessionReferenceFuture.setException(t);

// 2. If a retry was scheduled, initiate it:
if (retryFuture != null) {
// Guard against concurrent client close that happened while waking waiters.
if (isClientClosed() || retryFuture.isDone()) {
retryFuture.setException(newClosedException());
return;
}
try {
asyncCreateMultiplexedSession(retryFuture, remainingAttempts - 1);
} catch (Throwable e) {
// Failsafe: ensure waiting threads never hang if RPC scheduling throws
// synchronously.
retryFuture.setException(e);
}
}
}
});
Expand Down Expand Up @@ -424,14 +449,22 @@ public String getChannelPrimeSessionName() {

void close() {
boolean releaseChannelUsage = false;
ApiFuture<SessionReference> currentSessionFuture = null;
synchronized (this) {
if (!this.isClosed) {
this.isClosed = true;
this.maintainer.stop();
releaseChannelUsage = true;
currentSessionFuture = this.multiplexedSessionReference.get();
}
}
if (releaseChannelUsage) {
// If a session creation or retry is currently in flight, fail it immediately so
// transactions waiting on the closed client do not hang.
if (currentSessionFuture instanceof SettableApiFuture && !currentSessionFuture.isDone()) {
((SettableApiFuture<SessionReference>) currentSessionFuture)
.setException(newClosedException());
}
// The multiplexed session is no longer maintained, so this client must no longer be asked
// for a session when priming dynamic channel pool channels.
spanner.getRpc().unregisterChannelPrimeSessionSource(this);
Expand All @@ -447,6 +480,11 @@ void close() {
}
}

@VisibleForTesting
ApiFuture<SessionReference> getMultiplexedSessionFuture() {
return this.multiplexedSessionReference.get();
}

@VisibleForTesting
MultiplexedSessionMaintainer getMaintainer() {
return this.maintainer;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import com.google.cloud.spanner.Options.RpcPriority;
import com.google.cloud.spanner.TransactionRunnerImpl.TransactionContextImpl;
import com.google.cloud.spanner.connection.RandomResultSetGenerator;
import com.google.common.base.Stopwatch;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.MoreExecutors;
Expand All @@ -53,13 +54,16 @@
import java.time.Duration;
import java.util.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
Expand Down Expand Up @@ -102,7 +106,6 @@ public void createSpannerInstance() {
.getService();
}

@Ignore("Flaky test: b/579436726")
@Test
public void testCreateSessionDeadlineExceededWithNoSessionCreateWaitTime() throws Exception {
// Simulate a problem with the CreateSession RPC making it slow.
Expand Down Expand Up @@ -152,6 +155,120 @@ public void testCreateSessionDeadlineExceededWithNoSessionCreateWaitTime() throw
testSpanner.close();
}

@Test
public void testCloseUnblocksWaitingTransaction() throws Exception {
mockSpanner.freeze();

Spanner testSpanner =
SpannerOptions.newBuilder()
.setProjectId("test-project")
.setChannelProvider(channelProvider)
.setCredentials(NoCredentials.getInstance())
.build()
.getService();
ExecutorService executor = Executors.newFixedThreadPool(2);
try {
DatabaseClient client = testSpanner.getDatabaseClient(DatabaseId.of("p", "i", "d"));

// Wait until the initial CreateSession request is in flight and frozen on the server.
mockSpanner.waitForRequestsToContain(CreateSessionRequest.class, 5000);

// Start a query on a background thread so it blocks waiting for the session to be created.
try (ReadContext readContext = client.singleUse();
ResultSet resultSet = readContext.executeQuery(STATEMENT)) {
CountDownLatch queryStarted = new CountDownLatch(1);
Future<SpannerException> queryFailure =
executor.submit(
() -> {
queryStarted.countDown();
return assertThrows(SpannerException.class, resultSet::next);
});
assertTrue(queryStarted.await(5, TimeUnit.SECONDS));

// Close the client asynchronously so we can verify the blocked transaction fails before
// mockSpanner is unfrozen (GapicSpannerRpc.shutdown() otherwise waits up to 20s for the
// frozen CreateSession RPC to finish).
Future<?> closeFuture = executor.submit(testSpanner::close);

SpannerException exception = queryFailure.get(5, TimeUnit.SECONDS);
assertEquals(ErrorCode.FAILED_PRECONDITION, exception.getErrorCode());
assertTrue(exception.getMessage().contains("This client has been closed"));

mockSpanner.unfreeze();
closeFuture.get(5, TimeUnit.SECONDS);
}
} finally {
mockSpanner.unfreeze();
if (!testSpanner.isClosed()) {
testSpanner.close();
}
executor.shutdown();
assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS));
}
}

@Test
public void testCloseUnblocksTransactionWaitingOnRetry() throws Exception {
// Attempt 1 fails immediately with DEADLINE_EXCEEDED, and freezeAfter(1) atomically freezes
// the mock server as soon as the retry CreateSessionRequest (attempt 2) arrives.
mockSpanner.setCreateSessionExecutionTime(
SimulatedExecutionTime.ofException(Status.DEADLINE_EXCEEDED.asRuntimeException()));
mockSpanner.freezeAfter(1);

Spanner testSpanner =
SpannerOptions.newBuilder()
.setProjectId("test-project")
.setChannelProvider(channelProvider)
.setCredentials(NoCredentials.getInstance())
.build()
.getService();
ExecutorService executor = Executors.newFixedThreadPool(2);
try {
DatabaseClient client = testSpanner.getDatabaseClient(DatabaseId.of("p", "i", "d"));

// Wait until attempt 1 has failed and attempt 2 has reached the mock server and frozen.
Stopwatch watch = Stopwatch.createStarted();
while (mockSpanner.countRequestsOfType(CreateSessionRequest.class) < 2) {
if (watch.elapsed(TimeUnit.MILLISECONDS) > 5000) {
throw new TimeoutException(
"Timed out waiting for retry CreateSessionRequest to reach mock server");
}
Thread.sleep(1L);
}

// Start a query that binds to the pending retry future (attempt 2).
try (ReadContext readContext = client.singleUse();
ResultSet resultSet = readContext.executeQuery(STATEMENT)) {
CountDownLatch queryStarted = new CountDownLatch(1);
Future<SpannerException> queryFailure =
executor.submit(
() -> {
queryStarted.countDown();
return assertThrows(SpannerException.class, resultSet::next);
});
assertTrue(queryStarted.await(5, TimeUnit.SECONDS));

Future<?> closeFuture = executor.submit(testSpanner::close);

// The transaction must immediately fail with FAILED_PRECONDITION (client closed) while
// attempt 2 is still frozen.
SpannerException exception = queryFailure.get(5, TimeUnit.SECONDS);
assertEquals(ErrorCode.FAILED_PRECONDITION, exception.getErrorCode());
assertTrue(exception.getMessage().contains("This client has been closed"));

mockSpanner.unfreeze();
closeFuture.get(5, TimeUnit.SECONDS);
}
} finally {
mockSpanner.unfreeze();
if (!testSpanner.isClosed()) {
testSpanner.close();
}
executor.shutdown();
assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS));
}
}

@Test
public void testMultiUseReadOnlyTransactionUsesSameSession() {
// Execute two queries using the same transaction. Both queries should use the same
Expand Down
Loading
Loading