From ab941f696085147d5a57423356e3d552fcc979d2 Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian Date: Fri, 30 Oct 2026 10:16:11 +0000 Subject: [PATCH 1/8] test(spanner): unflake testCreateSessionDeadlineExceededWithNoSessionCreateWaitTime The initial future was completed with setException(t) before multiplexedSessionReference was updated to the retry future. Waiters waking up and polling the reference encountered the stale completed-with-error future. This fix installs the retry future before completing the initial future exceptionally to eliminate the race. --- .../MultiplexedSessionDatabaseClient.java | 14 +++- .../MultiplexedSessionDatabaseClientTest.java | 83 +++++++++++++++++++ 2 files changed, 93 insertions(+), 4 deletions(-) diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java index 03558aded997..e05ca7055f9b 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java @@ -324,16 +324,22 @@ public void onSessionCreateFailure(Throwable t, int createFailureForSessionCount MultiplexedSessionDatabaseClient.this.resourceNotFoundException.set( (ResourceNotFoundException) spannerException); } - // Set the exception to trigger an error for all waiters. - // Then retry the session creation if the error is (potentially) transient and the - // client has not been closed in the meantime. - sessionReferenceFuture.setException(t); + // Retry the session creation if the error is (potentially) transient and the client + // has not been closed in the meantime. The future for the retry is installed before + // the exception is set, so that transactions that are started after the waiters of + // this attempt have received the error, wait for the retry instead of failing with + // the error of this attempt. if (remainingAttempts > 1 && !isClientClosed() && RETRYABLE_ERROR_CODES.contains(spannerException.getErrorCode())) { final SettableApiFuture future = SettableApiFuture.create(); MultiplexedSessionDatabaseClient.this.multiplexedSessionReference.set(future); + // Set the exception to trigger an error for all waiters of this attempt. + sessionReferenceFuture.setException(t); asyncCreateMultiplexedSession(future, remainingAttempts - 1); + } else { + // Set the exception to trigger an error for all waiters. + sessionReferenceFuture.setException(t); } } }); diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java index dfbe267bfdcd..45c36d273496 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java @@ -39,6 +39,7 @@ import com.google.cloud.grpc.GrpcTransportOptions.ExecutorFactory; import com.google.cloud.spanner.SessionClient.SessionConsumer; import com.google.cloud.spanner.spi.v1.SpannerRpc; +import com.google.common.util.concurrent.MoreExecutors; import java.io.PrintWriter; import java.io.StringWriter; import java.lang.reflect.Field; @@ -247,6 +248,88 @@ public void testClosedClientIgnoresRefreshedSessionThatArrivesAfterClose() { assertEquals(sessionReference1, client.getCurrentSessionReference()); } + @Test + public void testRetryOfInitialSessionIsInstalledBeforeFailureIsPropagated() throws Exception { + Clock clock = mock(Clock.class); + when(clock.instant()).thenReturn(Instant.now()); + SessionClient sessionClient = mock(SessionClient.class); + SpannerImpl spanner = mock(SpannerImpl.class); + SpannerOptions spannerOptions = mock(SpannerOptions.class); + SessionPoolOptions sessionPoolOptions = mock(SessionPoolOptions.class); + when(sessionClient.getSpanner()).thenReturn(spanner); + when(sessionClient.getDatabaseId()).thenReturn(TEST_DATABASE_ID); + when(spanner.getRpc()).thenReturn(mock(SpannerRpc.class)); + when(spanner.getOptions()).thenReturn(spannerOptions); + when(spannerOptions.getSessionPoolOptions()).thenReturn(sessionPoolOptions); + when(sessionPoolOptions.getMultiplexedSessionMaintenanceDuration()) + .thenReturn(Duration.ofDays(7)); + when(sessionPoolOptions.getMultiplexedSessionMaintenanceLoopFrequency()) + .thenReturn(Duration.ofMinutes(10)); + SessionImpl session = mock(SessionImpl.class); + SessionReference sessionReference = mock(SessionReference.class); + when(session.getSessionReference()).thenReturn(sessionReference); + + // Capture the consumers of the initial attempt and of the retry. + AtomicReference initialConsumer = new AtomicReference<>(); + AtomicReference retryConsumer = new AtomicReference<>(); + doAnswer( + (Answer) + invocationOnMock -> { + initialConsumer.set(invocationOnMock.getArgument(0)); + return null; + }) + .doAnswer( + (Answer) + invocationOnMock -> { + retryConsumer.set(invocationOnMock.getArgument(0)); + return null; + }) + .when(sessionClient) + .asyncCreateMultiplexedSession(any(SessionConsumer.class)); + MultiplexedSessionDatabaseClient client = + new MultiplexedSessionDatabaseClient(sessionClient, clock); + assertNotNull(initialConsumer.get()); + + AtomicReference> currentReference = + getMultiplexedSessionReference(client); + ApiFuture initialFuture = currentReference.get(); + // A transaction that waits for the initial attempt is woken up as soon as that attempt fails. + // At that moment, a new transaction must already see the pending retry instead of the failed + // initial attempt, as it would otherwise fail with the (retryable) error of that attempt. + AtomicReference> futureSeenByNewTransaction = + new AtomicReference<>(); + initialFuture.addListener( + () -> futureSeenByNewTransaction.set(currentReference.get()), + MoreExecutors.directExecutor()); + + initialConsumer + .get() + .onSessionCreateFailure( + SpannerExceptionFactory.newSpannerException(ErrorCode.DEADLINE_EXCEEDED, "test"), 1); + + // Waiters for the initial attempt receive the error of that attempt. + SpannerException exception = + assertThrows(SpannerException.class, () -> SpannerApiFutures.get(initialFuture)); + assertEquals(ErrorCode.DEADLINE_EXCEEDED, exception.getErrorCode()); + // New transactions that started after the failure was propagated wait for the retry. + assertNotNull(futureSeenByNewTransaction.get()); + assertFalse(futureSeenByNewTransaction.get().isDone()); + assertNotNull(retryConsumer.get()); + retryConsumer.get().onSessionReady(session); + assertEquals(sessionReference, futureSeenByNewTransaction.get().get()); + assertEquals(sessionReference, client.getCurrentSessionReference()); + client.close(); + } + + @SuppressWarnings("unchecked") + private static AtomicReference> getMultiplexedSessionReference( + MultiplexedSessionDatabaseClient client) throws Exception { + Field field = + MultiplexedSessionDatabaseClient.class.getDeclaredField("multiplexedSessionReference"); + field.setAccessible(true); + return (AtomicReference>) field.get(client); + } + @Test public void testDisableMultiplexedSessionEnvVar() throws Exception { assumeTrue(isJava8() && !isWindows()); From a506f9eeb1a7dd92a75d6d0a7f47d0c65dd610b8 Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian Date: Fri, 30 Oct 2026 12:35:07 +0000 Subject: [PATCH 2/8] Addressed comments --- .../MultiplexedSessionDatabaseClient.java | 16 +++-- .../MultiplexedSessionDatabaseClientTest.java | 65 ++++++++++--------- 2 files changed, 43 insertions(+), 38 deletions(-) diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java index e05ca7055f9b..79bdbf2a50d3 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java @@ -333,14 +333,16 @@ public void onSessionCreateFailure(Throwable t, int createFailureForSessionCount && !isClientClosed() && RETRYABLE_ERROR_CODES.contains(spannerException.getErrorCode())) { final SettableApiFuture future = SettableApiFuture.create(); - MultiplexedSessionDatabaseClient.this.multiplexedSessionReference.set(future); - // Set the exception to trigger an error for all waiters of this attempt. - sessionReferenceFuture.setException(t); - asyncCreateMultiplexedSession(future, remainingAttempts - 1); - } else { - // Set the exception to trigger an error for all waiters. - sessionReferenceFuture.setException(t); + if (MultiplexedSessionDatabaseClient.this.multiplexedSessionReference.compareAndSet( + sessionReferenceFuture, future)) { + // Set the exception to trigger an error for all waiters of this attempt. + sessionReferenceFuture.setException(t); + asyncCreateMultiplexedSession(future, remainingAttempts - 1); + return; + } } + // Set the exception to trigger an error for all waiters. + sessionReferenceFuture.setException(t); } }); } diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java index 45c36d273496..20a09c194141 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java @@ -288,37 +288,40 @@ public void testRetryOfInitialSessionIsInstalledBeforeFailureIsPropagated() thro .asyncCreateMultiplexedSession(any(SessionConsumer.class)); MultiplexedSessionDatabaseClient client = new MultiplexedSessionDatabaseClient(sessionClient, clock); - assertNotNull(initialConsumer.get()); - - AtomicReference> currentReference = - getMultiplexedSessionReference(client); - ApiFuture initialFuture = currentReference.get(); - // A transaction that waits for the initial attempt is woken up as soon as that attempt fails. - // At that moment, a new transaction must already see the pending retry instead of the failed - // initial attempt, as it would otherwise fail with the (retryable) error of that attempt. - AtomicReference> futureSeenByNewTransaction = - new AtomicReference<>(); - initialFuture.addListener( - () -> futureSeenByNewTransaction.set(currentReference.get()), - MoreExecutors.directExecutor()); - - initialConsumer - .get() - .onSessionCreateFailure( - SpannerExceptionFactory.newSpannerException(ErrorCode.DEADLINE_EXCEEDED, "test"), 1); - - // Waiters for the initial attempt receive the error of that attempt. - SpannerException exception = - assertThrows(SpannerException.class, () -> SpannerApiFutures.get(initialFuture)); - assertEquals(ErrorCode.DEADLINE_EXCEEDED, exception.getErrorCode()); - // New transactions that started after the failure was propagated wait for the retry. - assertNotNull(futureSeenByNewTransaction.get()); - assertFalse(futureSeenByNewTransaction.get().isDone()); - assertNotNull(retryConsumer.get()); - retryConsumer.get().onSessionReady(session); - assertEquals(sessionReference, futureSeenByNewTransaction.get().get()); - assertEquals(sessionReference, client.getCurrentSessionReference()); - client.close(); + try { + assertNotNull(initialConsumer.get()); + + AtomicReference> currentReference = + getMultiplexedSessionReference(client); + ApiFuture initialFuture = currentReference.get(); + // A transaction that waits for the initial attempt is woken up as soon as that attempt fails. + // At that moment, a new transaction must already see the pending retry instead of the failed + // initial attempt, as it would otherwise fail with the (retryable) error of that attempt. + AtomicReference> futureSeenByNewTransaction = + new AtomicReference<>(); + initialFuture.addListener( + () -> futureSeenByNewTransaction.set(currentReference.get()), + MoreExecutors.directExecutor()); + + initialConsumer + .get() + .onSessionCreateFailure( + SpannerExceptionFactory.newSpannerException(ErrorCode.DEADLINE_EXCEEDED, "test"), 1); + + // Waiters for the initial attempt receive the error of that attempt. + SpannerException exception = + assertThrows(SpannerException.class, () -> SpannerApiFutures.get(initialFuture)); + assertEquals(ErrorCode.DEADLINE_EXCEEDED, exception.getErrorCode()); + // New transactions that started after the failure was propagated wait for the retry. + assertNotNull(futureSeenByNewTransaction.get()); + assertFalse(futureSeenByNewTransaction.get().isDone()); + assertNotNull(retryConsumer.get()); + retryConsumer.get().onSessionReady(session); + assertEquals(sessionReference, futureSeenByNewTransaction.get().get()); + assertEquals(sessionReference, client.getCurrentSessionReference()); + } finally { + client.close(); + } } @SuppressWarnings("unchecked") From 210c240b0044fc95fd739b798d0b7d21876bc91d Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian Date: Fri, 30 Oct 2026 12:47:56 +0000 Subject: [PATCH 3/8] Addressed comments --- .../cloud/spanner/MultiplexedSessionDatabaseClient.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java index 79bdbf2a50d3..4b4c783b7d23 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java @@ -337,7 +337,11 @@ public void onSessionCreateFailure(Throwable t, int createFailureForSessionCount sessionReferenceFuture, future)) { // Set the exception to trigger an error for all waiters of this attempt. sessionReferenceFuture.setException(t); - asyncCreateMultiplexedSession(future, remainingAttempts - 1); + try { + asyncCreateMultiplexedSession(future, remainingAttempts - 1); + } catch (Throwable e) { + future.setException(e); + } return; } } From 53c1676020f964d33ef5bc7046c2d8d417e73bfb Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian Date: Thu, 5 Nov 2026 19:45:35 +0000 Subject: [PATCH 4/8] Addressed comments --- .../MultiplexedSessionDatabaseClient.java | 25 ++++++++++++++----- .../MultiplexedSessionDatabaseClientTest.java | 16 +++--------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java index 4b4c783b7d23..7b61b9c42dfd 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java @@ -330,20 +330,24 @@ public void onSessionCreateFailure(Throwable t, int createFailureForSessionCount // this attempt have received the error, wait for the retry instead of failing with // the error of this attempt. if (remainingAttempts > 1 - && !isClientClosed() && RETRYABLE_ERROR_CODES.contains(spannerException.getErrorCode())) { final SettableApiFuture future = SettableApiFuture.create(); - if (MultiplexedSessionDatabaseClient.this.multiplexedSessionReference.compareAndSet( - sessionReferenceFuture, future)) { - // Set the exception to trigger an error for all waiters of this attempt. - sessionReferenceFuture.setException(t); + boolean replaced = MultiplexedSessionDatabaseClient.this.multiplexedSessionReference.compareAndSet( + sessionReferenceFuture, future); + // Set the exception to trigger an error for all waiters of this attempt. + sessionReferenceFuture.setException(t); + if (replaced) { + if (isClientClosed()) { + future.setException(newClosedException()); + return; + } try { asyncCreateMultiplexedSession(future, remainingAttempts - 1); } catch (Throwable e) { future.setException(e); } - return; } + return; } // Set the exception to trigger an error for all waiters. sessionReferenceFuture.setException(t); @@ -450,6 +454,10 @@ void close() { releaseChannelUsage = true; } } + ApiFuture future = this.multiplexedSessionReference.get(); + if (future instanceof SettableApiFuture) { + ((SettableApiFuture) future).setException(newClosedException()); + } if (releaseChannelUsage) { // The multiplexed session is no longer maintained, so this client must no longer be asked // for a session when priming dynamic channel pool channels. @@ -466,6 +474,11 @@ void close() { } } + @VisibleForTesting + AtomicReference> getMultiplexedSessionReference() { + return this.multiplexedSessionReference; + } + @VisibleForTesting MultiplexedSessionMaintainer getMaintainer() { return this.maintainer; diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java index 20a09c194141..e93cb6ed05e4 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java @@ -291,16 +291,11 @@ public void testRetryOfInitialSessionIsInstalledBeforeFailureIsPropagated() thro try { assertNotNull(initialConsumer.get()); - AtomicReference> currentReference = - getMultiplexedSessionReference(client); - ApiFuture initialFuture = currentReference.get(); - // A transaction that waits for the initial attempt is woken up as soon as that attempt fails. - // At that moment, a new transaction must already see the pending retry instead of the failed - // initial attempt, as it would otherwise fail with the (retryable) error of that attempt. + ApiFuture initialFuture = client.getMultiplexedSessionFuture(); AtomicReference> futureSeenByNewTransaction = new AtomicReference<>(); initialFuture.addListener( - () -> futureSeenByNewTransaction.set(currentReference.get()), + () -> futureSeenByNewTransaction.set(client.getMultiplexedSessionFuture()), MoreExecutors.directExecutor()); initialConsumer @@ -326,11 +321,8 @@ public void testRetryOfInitialSessionIsInstalledBeforeFailureIsPropagated() thro @SuppressWarnings("unchecked") private static AtomicReference> getMultiplexedSessionReference( - MultiplexedSessionDatabaseClient client) throws Exception { - Field field = - MultiplexedSessionDatabaseClient.class.getDeclaredField("multiplexedSessionReference"); - field.setAccessible(true); - return (AtomicReference>) field.get(client); + MultiplexedSessionDatabaseClient client) { + return client.getMultiplexedSessionReference(); } @Test From 58d70f26d6002872e54f3cc631340c8a42b186b3 Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian Date: Thu, 5 Nov 2026 20:49:57 +0000 Subject: [PATCH 5/8] Fix test compilation --- .../spanner/MultiplexedSessionDatabaseClientTest.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java index e93cb6ed05e4..6be3d8e5be6c 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java @@ -291,11 +291,16 @@ public void testRetryOfInitialSessionIsInstalledBeforeFailureIsPropagated() thro try { assertNotNull(initialConsumer.get()); - ApiFuture initialFuture = client.getMultiplexedSessionFuture(); + AtomicReference> currentReference = + client.getMultiplexedSessionReference(); + ApiFuture initialFuture = currentReference.get(); + // A transaction that waits for the initial attempt is woken up as soon as that attempt fails. + // At that moment, a new transaction must already see the pending retry instead of the failed + // initial attempt, as it would otherwise fail with the (retryable) error of that attempt. AtomicReference> futureSeenByNewTransaction = new AtomicReference<>(); initialFuture.addListener( - () -> futureSeenByNewTransaction.set(client.getMultiplexedSessionFuture()), + () -> futureSeenByNewTransaction.set(currentReference.get()), MoreExecutors.directExecutor()); initialConsumer From 7775cba69e0d981268d9c0498bcecad4c98860f9 Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian Date: Fri, 6 Nov 2026 09:52:36 +0000 Subject: [PATCH 6/8] Addressed comments --- .../MultiplexedSessionDatabaseClient.java | 65 +++++++++++-------- .../MultiplexedSessionDatabaseClientTest.java | 20 +++--- 2 files changed, 45 insertions(+), 40 deletions(-) diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java index 7b61b9c42dfd..34a7b94eb977 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java @@ -324,33 +324,38 @@ public void onSessionCreateFailure(Throwable t, int createFailureForSessionCount MultiplexedSessionDatabaseClient.this.resourceNotFoundException.set( (ResourceNotFoundException) spannerException); } - // Retry the session creation if the error is (potentially) transient and the client - // has not been closed in the meantime. The future for the retry is installed before - // the exception is set, so that transactions that are started after the waiters of - // this attempt have received the error, wait for the retry instead of failing with - // the error of this attempt. + SettableApiFuture retryFuture = null; if (remainingAttempts > 1 + && !isClientClosed() && RETRYABLE_ERROR_CODES.contains(spannerException.getErrorCode())) { - final SettableApiFuture future = SettableApiFuture.create(); - boolean replaced = MultiplexedSessionDatabaseClient.this.multiplexedSessionReference.compareAndSet( - sessionReferenceFuture, future); - // Set the exception to trigger an error for all waiters of this attempt. - sessionReferenceFuture.setException(t); - if (replaced) { - if (isClientClosed()) { - future.setException(newClosedException()); - return; - } - try { - asyncCreateMultiplexedSession(future, remainingAttempts - 1); - } catch (Throwable e) { - future.setException(e); - } + final SettableApiFuture nextFuture = SettableApiFuture.create(); + if (MultiplexedSessionDatabaseClient.this.multiplexedSessionReference.compareAndSet( + sessionReferenceFuture, nextFuture)) { + retryFuture = nextFuture; } - return; } - // Set the exception to trigger an error for all waiters. + + // 1. Unblock all waiters of the failed attempt first. + // This is executed OUTSIDE any lock to avoid deadlocks in foreign listener callbacks. + // Because the retryFuture was installed into multiplexedSessionReference above, + // any waking waiter that retries immediately is guaranteed to observe the retry future. sessionReferenceFuture.setException(t); + + // 2. If a retry was scheduled, initiate it: + if (retryFuture != null) { + // Guard against concurrent client close that happened while waking waiters. + if (isClientClosed()) { + retryFuture.setException(newClosedException()); + return; + } + try { + asyncCreateMultiplexedSession(retryFuture, remainingAttempts - 1); + } catch (Throwable e) { + // Failsafe: ensure waiting threads never hang if RPC scheduling throws + // synchronously. + retryFuture.setException(e); + } + } } }); } @@ -447,18 +452,22 @@ public String getChannelPrimeSessionName() { void close() { boolean releaseChannelUsage = false; + ApiFuture currentSessionFuture = null; synchronized (this) { if (!this.isClosed) { this.isClosed = true; this.maintainer.stop(); releaseChannelUsage = true; + currentSessionFuture = this.multiplexedSessionReference.get(); } } - ApiFuture future = this.multiplexedSessionReference.get(); - if (future instanceof SettableApiFuture) { - ((SettableApiFuture) future).setException(newClosedException()); - } if (releaseChannelUsage) { + // If a session creation or retry is currently in flight, fail it immediately so + // transactions waiting on the closed client do not hang. + if (currentSessionFuture instanceof SettableApiFuture && !currentSessionFuture.isDone()) { + ((SettableApiFuture) currentSessionFuture) + .setException(newClosedException()); + } // The multiplexed session is no longer maintained, so this client must no longer be asked // for a session when priming dynamic channel pool channels. spanner.getRpc().unregisterChannelPrimeSessionSource(this); @@ -475,8 +484,8 @@ void close() { } @VisibleForTesting - AtomicReference> getMultiplexedSessionReference() { - return this.multiplexedSessionReference; + ApiFuture getMultiplexedSessionFuture() { + return this.multiplexedSessionReference.get(); } @VisibleForTesting diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java index 6be3d8e5be6c..53fb3e519821 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java @@ -250,6 +250,7 @@ public void testClosedClientIgnoresRefreshedSessionThatArrivesAfterClose() { @Test public void testRetryOfInitialSessionIsInstalledBeforeFailureIsPropagated() throws Exception { + assumeTrue(isJava8()); Clock clock = mock(Clock.class); when(clock.instant()).thenReturn(Instant.now()); SessionClient sessionClient = mock(SessionClient.class); @@ -291,16 +292,14 @@ public void testRetryOfInitialSessionIsInstalledBeforeFailureIsPropagated() thro try { assertNotNull(initialConsumer.get()); - AtomicReference> currentReference = - client.getMultiplexedSessionReference(); - ApiFuture initialFuture = currentReference.get(); + ApiFuture initialFuture = client.getMultiplexedSessionFuture(); // A transaction that waits for the initial attempt is woken up as soon as that attempt fails. // At that moment, a new transaction must already see the pending retry instead of the failed // initial attempt, as it would otherwise fail with the (retryable) error of that attempt. AtomicReference> futureSeenByNewTransaction = new AtomicReference<>(); initialFuture.addListener( - () -> futureSeenByNewTransaction.set(currentReference.get()), + () -> futureSeenByNewTransaction.set(client.getMultiplexedSessionFuture()), MoreExecutors.directExecutor()); initialConsumer @@ -308,14 +307,17 @@ public void testRetryOfInitialSessionIsInstalledBeforeFailureIsPropagated() thro .onSessionCreateFailure( SpannerExceptionFactory.newSpannerException(ErrorCode.DEADLINE_EXCEEDED, "test"), 1); - // Waiters for the initial attempt receive the error of that attempt. + // 1. Waiters for the initial attempt receive the error of that attempt. SpannerException exception = assertThrows(SpannerException.class, () -> SpannerApiFutures.get(initialFuture)); assertEquals(ErrorCode.DEADLINE_EXCEEDED, exception.getErrorCode()); - // New transactions that started after the failure was propagated wait for the retry. + + // 2. New transactions arriving after the failure observe the retry future. assertNotNull(futureSeenByNewTransaction.get()); assertFalse(futureSeenByNewTransaction.get().isDone()); assertNotNull(retryConsumer.get()); + + // 3. Completing the retry succeeds the pending future. retryConsumer.get().onSessionReady(session); assertEquals(sessionReference, futureSeenByNewTransaction.get().get()); assertEquals(sessionReference, client.getCurrentSessionReference()); @@ -324,12 +326,6 @@ public void testRetryOfInitialSessionIsInstalledBeforeFailureIsPropagated() thro } } - @SuppressWarnings("unchecked") - private static AtomicReference> getMultiplexedSessionReference( - MultiplexedSessionDatabaseClient client) { - return client.getMultiplexedSessionReference(); - } - @Test public void testDisableMultiplexedSessionEnvVar() throws Exception { assumeTrue(isJava8() && !isWindows()); From 8f858ce1133f8513a8e616d4a92cacfb8082c3e8 Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian Date: Mon, 9 Nov 2026 19:17:28 +0000 Subject: [PATCH 7/8] Addressed comments --- .../MultiplexedSessionDatabaseClient.java | 6 +- ...edSessionDatabaseClientMockServerTest.java | 87 +++++++++++++++++++ .../MultiplexedSessionDatabaseClientTest.java | 7 +- 3 files changed, 98 insertions(+), 2 deletions(-) diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java index 34a7b94eb977..f2f40ac271a2 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java @@ -286,6 +286,10 @@ static boolean shouldCacheDialectError(Exception e) { private void asyncCreateMultiplexedSession( SettableApiFuture sessionReferenceFuture, int remainingAttempts) { + if (isClosed || sessionReferenceFuture.isDone()) { + sessionReferenceFuture.setException(newClosedException()); + return; + } this.sessionClient.asyncCreateMultiplexedSession( new SessionConsumer() { @Override @@ -344,7 +348,7 @@ public void onSessionCreateFailure(Throwable t, int createFailureForSessionCount // 2. If a retry was scheduled, initiate it: if (retryFuture != null) { // Guard against concurrent client close that happened while waking waiters. - if (isClientClosed()) { + if (isClientClosed() || retryFuture.isDone()) { retryFuture.setException(newClosedException()); return; } diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientMockServerTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientMockServerTest.java index 33f64aa636c4..41ed3079685d 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientMockServerTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientMockServerTest.java @@ -42,6 +42,7 @@ import com.google.cloud.spanner.Options.RpcPriority; import com.google.cloud.spanner.TransactionRunnerImpl.TransactionContextImpl; import com.google.cloud.spanner.connection.RandomResultSetGenerator; +import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.util.concurrent.MoreExecutors; @@ -54,6 +55,7 @@ import java.util.*; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -150,6 +152,91 @@ public void testCreateSessionDeadlineExceededWithNoSessionCreateWaitTime() throw testSpanner.close(); } + @Test + public void testCloseUnblocksWaitingTransaction() throws Exception { + mockSpanner.freeze(); + + Spanner testSpanner = + SpannerOptions.newBuilder() + .setProjectId("test-project") + .setChannelProvider(channelProvider) + .setCredentials(NoCredentials.getInstance()) + .build() + .getService(); + try { + DatabaseClient client = testSpanner.getDatabaseClient(DatabaseId.of("p", "i", "d")); + + // Wait until the initial CreateSession request is in flight and frozen on the server. + mockSpanner.waitForRequestsToContain(CreateSessionRequest.class, 5000); + + // Start a query that blocks waiting for the session to be created. + try (ReadContext readContext = client.singleUse(); + ResultSet resultSet = readContext.executeQuery(STATEMENT)) { + // Close the client while the transaction is waiting for session creation. + testSpanner.close(); + + // The transaction must immediately fail with FAILED_PRECONDITION (client closed), + // rather than hanging indefinitely waiting for mockSpanner to unfreeze. + SpannerException exception = assertThrows(SpannerException.class, resultSet::next); + assertEquals(ErrorCode.FAILED_PRECONDITION, exception.getErrorCode()); + assertTrue(exception.getMessage().contains("This client has been closed")); + } + } finally { + mockSpanner.unfreeze(); + if (!testSpanner.isClosed()) { + testSpanner.close(); + } + } + } + + @Test + public void testCloseUnblocksTransactionWaitingOnRetry() throws Exception { + // Attempt 1 fails immediately with DEADLINE_EXCEEDED, and freezeAfter(1) atomically freezes + // the mock server as soon as the retry CreateSessionRequest (attempt 2) arrives. + mockSpanner.setCreateSessionExecutionTime( + SimulatedExecutionTime.ofException(Status.DEADLINE_EXCEEDED.asRuntimeException())); + mockSpanner.freezeAfter(1); + + Spanner testSpanner = + SpannerOptions.newBuilder() + .setProjectId("test-project") + .setChannelProvider(channelProvider) + .setCredentials(NoCredentials.getInstance()) + .build() + .getService(); + try { + DatabaseClient client = testSpanner.getDatabaseClient(DatabaseId.of("p", "i", "d")); + + // Wait until attempt 1 has failed and attempt 2 has reached the mock server and frozen. + Stopwatch watch = Stopwatch.createStarted(); + while (mockSpanner.countRequestsOfType(CreateSessionRequest.class) < 2) { + if (watch.elapsed(TimeUnit.MILLISECONDS) > 5000) { + throw new TimeoutException( + "Timed out waiting for retry CreateSessionRequest to reach mock server"); + } + Thread.sleep(1L); + } + + // Start a query that binds to the pending retry future (attempt 2). + try (ReadContext readContext = client.singleUse(); + ResultSet resultSet = readContext.executeQuery(STATEMENT)) { + // Close the client while the query is waiting for attempt 2. + testSpanner.close(); + + // The transaction must immediately fail with FAILED_PRECONDITION (client closed), + // instead of hanging while attempt 2 is frozen. + SpannerException exception = assertThrows(SpannerException.class, resultSet::next); + assertEquals(ErrorCode.FAILED_PRECONDITION, exception.getErrorCode()); + assertTrue(exception.getMessage().contains("This client has been closed")); + } + } finally { + mockSpanner.unfreeze(); + if (!testSpanner.isClosed()) { + testSpanner.close(); + } + } + } + @Test public void testMultiUseReadOnlyTransactionUsesSameSession() { // Execute two queries using the same transaction. Both queries should use the same diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java index 53fb3e519821..48651454c6ba 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java @@ -250,7 +250,8 @@ public void testClosedClientIgnoresRefreshedSessionThatArrivesAfterClose() { @Test public void testRetryOfInitialSessionIsInstalledBeforeFailureIsPropagated() throws Exception { - assumeTrue(isJava8()); + // Mockito dynamic mock creation is not supported in GraalVM native-image tests. + assumeFalse(isNativeImage()); Clock clock = mock(Clock.class); when(clock.instant()).thenReturn(Instant.now()); SessionClient sessionClient = mock(SessionClient.class); @@ -670,6 +671,10 @@ private boolean isJava8() { return JavaVersionUtil.getJavaMajorVersion() == 8; } + private boolean isNativeImage() { + return System.getProperty("org.graalvm.nativeimage.imagecode") != null; + } + private boolean isWindows() { return System.getProperty("os.name").toLowerCase().contains("windows"); } From 1228b5a116525241f47049b69caac1d5944a56ab Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian Date: Mon, 16 Nov 2026 09:29:55 +0000 Subject: [PATCH 8/8] Addressed comments --- ...edSessionDatabaseClientMockServerTest.java | 54 ++++++-- .../MultiplexedSessionDatabaseClientTest.java | 117 ++++++++++++++++++ 2 files changed, 160 insertions(+), 11 deletions(-) diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientMockServerTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientMockServerTest.java index 41ed3079685d..ff257414a0be 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientMockServerTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientMockServerTest.java @@ -54,6 +54,9 @@ import java.time.Duration; import java.util.*; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; @@ -163,29 +166,44 @@ public void testCloseUnblocksWaitingTransaction() throws Exception { .setCredentials(NoCredentials.getInstance()) .build() .getService(); + ExecutorService executor = Executors.newFixedThreadPool(2); try { DatabaseClient client = testSpanner.getDatabaseClient(DatabaseId.of("p", "i", "d")); // Wait until the initial CreateSession request is in flight and frozen on the server. mockSpanner.waitForRequestsToContain(CreateSessionRequest.class, 5000); - // Start a query that blocks waiting for the session to be created. + // Start a query on a background thread so it blocks waiting for the session to be created. try (ReadContext readContext = client.singleUse(); ResultSet resultSet = readContext.executeQuery(STATEMENT)) { - // Close the client while the transaction is waiting for session creation. - testSpanner.close(); + CountDownLatch queryStarted = new CountDownLatch(1); + Future queryFailure = + executor.submit( + () -> { + queryStarted.countDown(); + return assertThrows(SpannerException.class, resultSet::next); + }); + assertTrue(queryStarted.await(5, TimeUnit.SECONDS)); - // The transaction must immediately fail with FAILED_PRECONDITION (client closed), - // rather than hanging indefinitely waiting for mockSpanner to unfreeze. - SpannerException exception = assertThrows(SpannerException.class, resultSet::next); + // Close the client asynchronously so we can verify the blocked transaction fails before + // mockSpanner is unfrozen (GapicSpannerRpc.shutdown() otherwise waits up to 20s for the + // frozen CreateSession RPC to finish). + Future closeFuture = executor.submit(testSpanner::close); + + SpannerException exception = queryFailure.get(5, TimeUnit.SECONDS); assertEquals(ErrorCode.FAILED_PRECONDITION, exception.getErrorCode()); assertTrue(exception.getMessage().contains("This client has been closed")); + + mockSpanner.unfreeze(); + closeFuture.get(5, TimeUnit.SECONDS); } } finally { mockSpanner.unfreeze(); if (!testSpanner.isClosed()) { testSpanner.close(); } + executor.shutdown(); + assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS)); } } @@ -204,6 +222,7 @@ public void testCloseUnblocksTransactionWaitingOnRetry() throws Exception { .setCredentials(NoCredentials.getInstance()) .build() .getService(); + ExecutorService executor = Executors.newFixedThreadPool(2); try { DatabaseClient client = testSpanner.getDatabaseClient(DatabaseId.of("p", "i", "d")); @@ -220,20 +239,33 @@ public void testCloseUnblocksTransactionWaitingOnRetry() throws Exception { // Start a query that binds to the pending retry future (attempt 2). try (ReadContext readContext = client.singleUse(); ResultSet resultSet = readContext.executeQuery(STATEMENT)) { - // Close the client while the query is waiting for attempt 2. - testSpanner.close(); + CountDownLatch queryStarted = new CountDownLatch(1); + Future queryFailure = + executor.submit( + () -> { + queryStarted.countDown(); + return assertThrows(SpannerException.class, resultSet::next); + }); + assertTrue(queryStarted.await(5, TimeUnit.SECONDS)); - // The transaction must immediately fail with FAILED_PRECONDITION (client closed), - // instead of hanging while attempt 2 is frozen. - SpannerException exception = assertThrows(SpannerException.class, resultSet::next); + Future closeFuture = executor.submit(testSpanner::close); + + // The transaction must immediately fail with FAILED_PRECONDITION (client closed) while + // attempt 2 is still frozen. + SpannerException exception = queryFailure.get(5, TimeUnit.SECONDS); assertEquals(ErrorCode.FAILED_PRECONDITION, exception.getErrorCode()); assertTrue(exception.getMessage().contains("This client has been closed")); + + mockSpanner.unfreeze(); + closeFuture.get(5, TimeUnit.SECONDS); } } finally { mockSpanner.unfreeze(); if (!testSpanner.isClosed()) { testSpanner.close(); } + executor.shutdown(); + assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS)); } } diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java index 48651454c6ba..63245a32a226 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClientTest.java @@ -327,6 +327,123 @@ public void testRetryOfInitialSessionIsInstalledBeforeFailureIsPropagated() thro } } + @Test + public void testCloseDuringFailureNotificationCancelsScheduledRetryWithoutDispatchingRpc() { + assumeFalse(isNativeImage()); + Clock clock = mock(Clock.class); + when(clock.instant()).thenReturn(Instant.now()); + SessionClient sessionClient = mock(SessionClient.class); + SpannerImpl spanner = mock(SpannerImpl.class); + SpannerOptions spannerOptions = mock(SpannerOptions.class); + SessionPoolOptions sessionPoolOptions = mock(SessionPoolOptions.class); + when(sessionClient.getSpanner()).thenReturn(spanner); + when(sessionClient.getDatabaseId()).thenReturn(TEST_DATABASE_ID); + when(spanner.getRpc()).thenReturn(mock(SpannerRpc.class)); + when(spanner.getOptions()).thenReturn(spannerOptions); + when(spannerOptions.getSessionPoolOptions()).thenReturn(sessionPoolOptions); + when(sessionPoolOptions.getMultiplexedSessionMaintenanceDuration()) + .thenReturn(Duration.ofDays(7)); + when(sessionPoolOptions.getMultiplexedSessionMaintenanceLoopFrequency()) + .thenReturn(Duration.ofMinutes(10)); + + AtomicReference initialConsumer = new AtomicReference<>(); + doAnswer( + (Answer) + invocationOnMock -> { + initialConsumer.set(invocationOnMock.getArgument(0)); + return null; + }) + .when(sessionClient) + .asyncCreateMultiplexedSession(any(SessionConsumer.class)); + + MultiplexedSessionDatabaseClient client = + new MultiplexedSessionDatabaseClient(sessionClient, clock); + try { + assertNotNull(initialConsumer.get()); + ApiFuture initialFuture = client.getMultiplexedSessionFuture(); + + // Simulate a waiter/listener closing the client while initialFuture is failing. + AtomicReference> retryFutureRef = new AtomicReference<>(); + initialFuture.addListener( + () -> { + retryFutureRef.set(client.getMultiplexedSessionFuture()); + client.close(); + }, + MoreExecutors.directExecutor()); + + initialConsumer + .get() + .onSessionCreateFailure( + SpannerExceptionFactory.newSpannerException(ErrorCode.DEADLINE_EXCEEDED, "test"), 1); + + SpannerException initialException = + assertThrows(SpannerException.class, () -> SpannerApiFutures.get(initialFuture)); + assertEquals(ErrorCode.DEADLINE_EXCEEDED, initialException.getErrorCode()); + + ApiFuture retryFuture = retryFutureRef.get(); + assertNotNull(retryFuture); + SpannerException retryException = + assertThrows(SpannerException.class, () -> SpannerApiFutures.get(retryFuture)); + assertEquals(ErrorCode.FAILED_PRECONDITION, retryException.getErrorCode()); + assertTrue(retryException.getMessage().contains("This client has been closed")); + + verify(sessionClient, times(1)).asyncCreateMultiplexedSession(any(SessionConsumer.class)); + } finally { + client.close(); + } + } + + @Test + public void testFailureAfterCloseDoesNotScheduleRetry() { + assumeFalse(isNativeImage()); + Clock clock = mock(Clock.class); + when(clock.instant()).thenReturn(Instant.now()); + SessionClient sessionClient = mock(SessionClient.class); + SpannerImpl spanner = mock(SpannerImpl.class); + SpannerOptions spannerOptions = mock(SpannerOptions.class); + SessionPoolOptions sessionPoolOptions = mock(SessionPoolOptions.class); + when(sessionClient.getSpanner()).thenReturn(spanner); + when(sessionClient.getDatabaseId()).thenReturn(TEST_DATABASE_ID); + when(spanner.getRpc()).thenReturn(mock(SpannerRpc.class)); + when(spanner.getOptions()).thenReturn(spannerOptions); + when(spannerOptions.getSessionPoolOptions()).thenReturn(sessionPoolOptions); + when(sessionPoolOptions.getMultiplexedSessionMaintenanceDuration()) + .thenReturn(Duration.ofDays(7)); + when(sessionPoolOptions.getMultiplexedSessionMaintenanceLoopFrequency()) + .thenReturn(Duration.ofMinutes(10)); + + AtomicReference initialConsumer = new AtomicReference<>(); + doAnswer( + (Answer) + invocationOnMock -> { + initialConsumer.set(invocationOnMock.getArgument(0)); + return null; + }) + .when(sessionClient) + .asyncCreateMultiplexedSession(any(SessionConsumer.class)); + + MultiplexedSessionDatabaseClient client = + new MultiplexedSessionDatabaseClient(sessionClient, clock); + try { + assertNotNull(initialConsumer.get()); + ApiFuture initialFuture = client.getMultiplexedSessionFuture(); + + client.close(); + initialConsumer + .get() + .onSessionCreateFailure( + SpannerExceptionFactory.newSpannerException(ErrorCode.DEADLINE_EXCEEDED, "test"), 1); + + assertEquals(initialFuture, client.getMultiplexedSessionFuture()); + SpannerException exception = + assertThrows(SpannerException.class, () -> SpannerApiFutures.get(initialFuture)); + assertEquals(ErrorCode.FAILED_PRECONDITION, exception.getErrorCode()); + verify(sessionClient, times(1)).asyncCreateMultiplexedSession(any(SessionConsumer.class)); + } finally { + client.close(); + } + } + @Test public void testDisableMultiplexedSessionEnvVar() throws Exception { assumeTrue(isJava8() && !isWindows());