From ee14c0f7fda62eee579d7be8fcb6d748abc9c2d6 Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian <179120858+sakthivelmanii@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:37:42 +0000 Subject: [PATCH 1/2] chore(spanner): restore active transaction state before retrying aborted commit ### Summary Restores active read/write transaction state when `Connection.runTransaction(...)` retries after `commit()` encounters an `AbortedException`, ensuring retry attempts execute atomically within a single read/write transaction rather than falling back to autocommit mode. ### What Changed & Why - **Preserve transactional atomicity across commit-abort retries**: Previously, an `AbortedException` during `commit()` cleared the connection's active transaction markers before entering the retry loop. When `autocommit` was enabled, statements in the next retry attempt executed as individual autocommit operations instead of a single atomic read/write transaction, and the retry commit failed with `FAILED_PRECONDITION`. Retrying now restores active transaction state, rolls back uncommitted transaction-local connection properties, clears buffered mutations and savepoints from the aborted attempt, and reuses the underlying transaction session and tracing span across retries. - **Clean failure and cancellation teardown**: Non-abort commit errors, swallowed statement aborts, and thread interruptions during commit or retry backoff now clean up connection state, active batches, session resources, and OpenTelemetry spans without masking the original error with a secondary `FAILED_PRECONDITION`. - **Behavior change for aborts without server retry metadata**: If a transaction callable throws a synthetic `AbortedException` (or an abort arrives without `RetryInfo` metadata), `Connection.runTransaction(...)` now retries using jittered exponential backoff (capped at 32 seconds) instead of failing with an invalid sleep delay or null transaction state. --- .../AbstractMultiUseTransaction.java | 5 + .../spanner/connection/ConnectionImpl.java | 93 ++- .../connection/ReadWriteTransaction.java | 189 ++++- .../connection/TransactionRunnerImpl.java | 43 +- .../RunTransactionMockServerTest.java | 772 ++++++++++++++++++ 5 files changed, 1063 insertions(+), 39 deletions(-) diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/AbstractMultiUseTransaction.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/AbstractMultiUseTransaction.java index ca78c3e5aeaf..19b4cff69449 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/AbstractMultiUseTransaction.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/AbstractMultiUseTransaction.java @@ -174,6 +174,11 @@ ImmutableList getSavepoints() { return ImmutableList.copyOf(savepoints); } + /** Clears all in-memory savepoints recorded during the current transaction attempt. */ + void clearSavepoints() { + savepoints.clear(); + } + @Override public void savepoint(@Nonnull String name, @Nonnull Dialect dialect) { if (dialect != Dialect.POSTGRESQL) { diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionImpl.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionImpl.java index cadd63757398..5c33f5db572d 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionImpl.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionImpl.java @@ -92,6 +92,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Suppliers; import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.Uninterruptibles; import com.google.spanner.v1.DirectedReadOptions; import com.google.spanner.v1.ExecuteSqlRequest.QueryOptions; import com.google.spanner.v1.RequestOptions; @@ -1421,9 +1422,96 @@ public T runTransaction(TransactionCallable callable) { } } + private static final EndTransactionCallback NO_OP_END_TRANSACTION_CALLBACK = + new EndTransactionCallback() { + @Override + public void onSuccess() {} + + @Override + public void onFailure() {} + }; + + /** + * Restores active transaction state on this connection and resets {@code retryUnitOfWork} before + * {@link TransactionRunnerImpl} executes a retry attempt. + */ void resetForRetry(UnitOfWork retryUnitOfWork) { - retryUnitOfWork.resetForRetry(); - this.currentUnitOfWork = retryUnitOfWork; + if (isBatchActive()) { + abortBatch(); + } + if (this.currentUnitOfWork != null && this.currentUnitOfWork != retryUnitOfWork) { + closeUnitOfWorkAfterRunnerFailure(this.currentUnitOfWork); + } + this.connectionState.rollback(); + this.statementTag = null; + this.transactionBeginMarked = true; + clearLastTransactionAndSetDefaultTransactionOptions(getDefaultIsolationLevel()); + if (isAutocommit()) { + this.inTransaction = true; + } + // Re-apply local RETRY_ABORTS_INTERNALLY = false while currentUnitOfWork is null (cleared + // above) and isInTransaction() is true, before re-attaching retryUnitOfWork. + setLocalConnectionPropertyValue(RETRY_ABORTS_INTERNALLY, false); + try { + retryUnitOfWork.resetForRetry(); + } finally { + this.currentUnitOfWork = retryUnitOfWork; + } + } + + /** + * Cleans up connection transaction state and closes any active or kept-open aborted unit of work + * when {@link TransactionRunnerImpl} exits with a non-retryable failure or interruption. + */ + void rollbackAfterAbort(UnitOfWork transaction) { + if (isBatchActive()) { + try { + abortBatch(); + } catch (Throwable ignored) { + // Best-effort batch cleanup so the primary failure is never masked. + } + } + UnitOfWork current = this.currentUnitOfWork; + this.connectionState.rollback(); + this.statementTag = null; + this.transactionBeginMarked = false; + clearLastTransactionAndSetDefaultTransactionOptions(getDefaultIsolationLevel()); + if (isAutocommit()) { + this.inTransaction = false; + } + closeUnitOfWorkAfterRunnerFailure(transaction); + if (current != null && current != transaction) { + closeUnitOfWorkAfterRunnerFailure(current); + } + } + + private void closeUnitOfWorkAfterRunnerFailure(@Nullable UnitOfWork uow) { + if (uow instanceof AbstractMultiUseTransaction) { + boolean wasInterrupted = Thread.interrupted(); + try { + // Drain any in-flight statement or commit task on the single-threaded statementExecutor so + // COMMITTING settles to COMMITTED, COMMIT_FAILED, or ABORTED before inspecting state. + try { + Uninterruptibles.getUninterruptibly( + ((AbstractBaseUnitOfWork) uow).getStatementExecutor().submit(() -> null)); + } catch (Throwable ignored) { + // Best-effort barrier; proceed to state check. + } + UnitOfWorkState state = uow.getState(); + if (state == UnitOfWorkState.STARTED || state == UnitOfWorkState.ABORTED) { + try { + Uninterruptibles.getUninterruptibly( + uow.rollbackAsync(CallType.SYNC, NO_OP_END_TRANSACTION_CALLBACK)); + } catch (Throwable ignored) { + // Best-effort rollback/close so the caller's primary exception is never masked. + } + } + } finally { + if (wasInterrupted) { + Thread.currentThread().interrupt(); + } + } + } } @Override @@ -2358,6 +2446,7 @@ UnitOfWork createNewUnitOfWork( getConnectionPropertyValue(DELAY_TRANSACTION_START_UNTIL_FIRST_WRITE)) .setKeepTransactionAlive(getConnectionPropertyValue(KEEP_TRANSACTION_ALIVE)) .setRetryAbortsInternally(getConnectionPropertyValue(RETRY_ABORTS_INTERNALLY)) + .setKeepAbortedTransactionOpen(this.transactionRunnerActive) .setSavepointSupport(getConnectionPropertyValue(SAVEPOINT_SUPPORT)) .setReturnCommitStats(getConnectionPropertyValue(RETURN_COMMIT_STATS)) .setMaxCommitDelay(getConnectionPropertyValue(MAX_COMMIT_DELAY)) diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ReadWriteTransaction.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ReadWriteTransaction.java index 246c203c8674..d2dd228fc00d 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ReadWriteTransaction.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ReadWriteTransaction.java @@ -50,6 +50,7 @@ import com.google.cloud.spanner.ThreadFactoryUtil; import com.google.cloud.spanner.TransactionContext; import com.google.cloud.spanner.TransactionManager; +import com.google.cloud.spanner.TransactionManager.TransactionState; import com.google.cloud.spanner.connection.AbstractStatementParser.ParsedStatement; import com.google.cloud.spanner.connection.AbstractStatementParser.StatementType; import com.google.cloud.spanner.connection.TransactionRetryListener.RetryResult; @@ -164,6 +165,7 @@ static ScheduledThreadPoolExecutor getKeepAliveService() { private final TransactionOption[] transactionOptions; private TransactionManager txManager; private final boolean retryAbortsInternally; + private final boolean keepAbortedTransactionOpen; private final boolean delayTransactionStartUntilFirstWrite; private final boolean keepTransactionAlive; private final long keepAliveIntervalMillis; @@ -218,6 +220,7 @@ static class Builder extends AbstractMultiUseTransaction.Builder executeQueryAsync( () -> { checkTimedOut(); checkAborted(); - return DirectExecuteResultSet.ofResultSet( - internalExecuteQuery(statement, analyzeMode, options)); + try { + return DirectExecuteResultSet.ofResultSet( + internalExecuteQuery(statement, analyzeMode, options)); + } catch (AbortedException aborted) { + if (keepAbortedTransactionOpen) { + this.state = UnitOfWorkState.ABORTED; + this.abortedException = aborted; + } + throw aborted; + } }, SpannerGrpc.getExecuteStreamingSqlMethod()); } @@ -861,15 +893,23 @@ private ApiFuture> internalExecuteUpdateAsync( () -> { checkTimedOut(); checkAborted(); - if (analyzeMode == AnalyzeMode.NONE) { - return Tuple.of( - get(txContextFuture).executeUpdate(update.getStatement(), options), null); + try { + if (analyzeMode == AnalyzeMode.NONE) { + return Tuple.of( + get(txContextFuture).executeUpdate(update.getStatement(), options), null); + } + ResultSet resultSet = + get(txContextFuture) + .analyzeUpdateStatement( + update.getStatement(), analyzeMode.getQueryAnalyzeMode(), options); + return Tuple.of(null, resultSet); + } catch (AbortedException aborted) { + if (keepAbortedTransactionOpen) { + this.state = UnitOfWorkState.ABORTED; + this.abortedException = aborted; + } + throw aborted; } - ResultSet resultSet = - get(txContextFuture) - .analyzeUpdateStatement( - update.getStatement(), analyzeMode.getQueryAnalyzeMode(), options); - return Tuple.of(null, resultSet); }, SpannerGrpc.getExecuteSqlMethod()); } @@ -929,7 +969,15 @@ public ApiFuture executeBatchUpdateAsync( () -> { checkTimedOut(); checkAborted(); - return get(txContextFuture).batchUpdate(updateStatements); + try { + return get(txContextFuture).batchUpdate(updateStatements); + } catch (AbortedException aborted) { + if (keepAbortedTransactionOpen) { + this.state = UnitOfWorkState.ABORTED; + this.abortedException = aborted; + } + throw aborted; + } }, SpannerGrpc.getExecuteBatchDmlMethod()); } @@ -987,6 +1035,7 @@ public ApiFuture commitAsync( state = UnitOfWorkState.COMMITTED; res = SettableApiFuture.create(); ((SettableApiFuture) res).set(null); + asyncEndUnitOfWorkSpan(); } else if (retryAbortsInternally) { res = executeStatementAsync( @@ -1021,32 +1070,45 @@ public ApiFuture commitAsync( }, InterceptorsUsage.IGNORE_INTERCEPTORS, ImmutableList.of(SpannerGrpc.getCommitMethod())); + asyncEndUnitOfWorkSpan(); } else { res = executeStatementAsync( callType, COMMIT_STATEMENT, () -> { - checkTimedOut(); try { + checkTimedOut(); Void result = commitCallable.call(); callback.onSuccess(); + if (span != null) { + span.end(); + } return result; } catch (Throwable t) { commitResponseFuture.setException(t); callback.onFailure(); - state = UnitOfWorkState.COMMIT_FAILED; - try { - txManager.close(); - } catch (Throwable t2) { - // Ignore. + if (keepAbortedTransactionOpen && t instanceof AbortedException) { + // Under TransactionRunnerImpl, leave txManager and unit-of-work span open so + // resetForRetry() can reuse them or rollbackAfterAbort() can close them once. + state = UnitOfWorkState.ABORTED; + abortedException = (AbortedException) t; + } else { + state = UnitOfWorkState.COMMIT_FAILED; + try { + txManager.close(); + } catch (Throwable t2) { + // Ignore. + } + if (span != null) { + span.end(); + } } throw t; } }, SpannerGrpc.getCommitMethod()); } - asyncEndUnitOfWorkSpan(); return res; } } @@ -1276,10 +1338,13 @@ private void handleAborted(AbortedException aborted) { } } } else { - try { - txManager.close(); - } catch (Throwable t) { - // ignore + if (!keepAbortedTransactionOpen && txManager != null) { + try { + txManager.close(); + } catch (Throwable t) { + // ignore + } + txManager = null; } // Internal retry is not enabled. this.state = UnitOfWorkState.ABORTED; @@ -1356,21 +1421,49 @@ private ApiFuture rollbackAsync(CallType callType, boolean updateStatusAnd state == UnitOfWorkState.STARTED || state == UnitOfWorkState.ABORTED, "This transaction has status " + state.name()); cancelScheduledKeepAlivePing(); + boolean wasAborted = state == UnitOfWorkState.ABORTED; if (updateStatusAndEndSpan) { state = UnitOfWorkState.ROLLED_BACK; } - if (txContextFuture != null && state != UnitOfWorkState.ABORTED) { - ApiFuture result = - executeStatementAsync( - callType, ROLLBACK_STATEMENT, rollbackCallable, SpannerGrpc.getRollbackMethod()); + if (wasAborted) { if (updateStatusAndEndSpan) { - // Note: We end the transaction span after executing the rollback to include the rollback in - // the transaction span. Even though both methods are executed asynchronously, they are both - // executed using the same single-threaded executor, meaning that the span will only be - // ended after the rollback has finished. - asyncEndUnitOfWorkSpan(); + // TransactionManagerImpl.rollback() requires STARTED; close any kept-open aborted txManager + // to release its session and span, then end the unit-of-work span before completing. + return getStatementExecutor() + .submit( + () -> { + if (txManager != null) { + try { + txManager.close(); + } catch (Throwable ignored) { + // Ignore cleanup errors on an already-aborted transaction manager. + } + txManager = null; + } + if (span != null) { + span.end(); + } + return null; + }); } - return result; + return ApiFutures.immediateFuture(null); + } + if (txContextFuture != null) { + return executeStatementAsync( + callType, + ROLLBACK_STATEMENT, + () -> { + try { + return rollbackCallable.call(); + } finally { + if (updateStatusAndEndSpan && span != null) { + // End the unit-of-work span inside the rollback task so callers awaiting the + // returned future observe deterministic span completion. + span.end(); + } + } + }, + SpannerGrpc.getRollbackMethod()); } else if (updateStatusAndEndSpan) { return asyncEndUnitOfWorkSpan(); } else { @@ -1378,9 +1471,39 @@ private ApiFuture rollbackAsync(CallType callType, boolean updateStatusAnd } } + /** + * Resets this read/write transaction for a new attempt by {@link TransactionRunnerImpl}, reusing + * an aborted {@link TransactionManager} when available or closing an un-aborted manager, and + * clearing buffered per-attempt collections and state. + */ @Override public void resetForRetry() { - txContextFuture = ApiFutures.immediateFuture(txManager.resetForRetry()); + cancelScheduledKeepAlivePing(); + if (txManager != null) { + if (state == UnitOfWorkState.ABORTED || txManager.getState() == TransactionState.ABORTED) { + txContextFuture = ApiFutures.immediateFuture(txManager.resetForRetry()); + } else { + try { + txManager.close(); + } catch (Throwable ignored) { + // Ignore cleanup errors when closing an un-aborted transaction manager. + } + txManager = null; + txContextFuture = null; + } + } else { + txContextFuture = null; + } + canUseSingleUseRead = false; + commitResponseFuture = null; + abortedException = null; + rolledBackToSavepointException = null; + autoSavepoint = null; + transactionStarted = null; + statements.clear(); + mutations.clear(); + clearSavepoints(); + state = UnitOfWorkState.STARTED; } @Override diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/TransactionRunnerImpl.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/TransactionRunnerImpl.java index 504b084dba32..39a43a6a40bb 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/TransactionRunnerImpl.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/TransactionRunnerImpl.java @@ -23,38 +23,73 @@ import com.google.cloud.spanner.connection.Connection.TransactionCallable; import com.google.cloud.spanner.connection.ConnectionImpl.Caller; import com.google.cloud.spanner.connection.UnitOfWork.CallType; +import com.google.cloud.spanner.connection.UnitOfWork.UnitOfWorkState; +import java.util.concurrent.ThreadLocalRandom; class TransactionRunnerImpl { + private static final long MAX_FALLBACK_BACKOFF_MILLIS = 32_000L; + private final ConnectionImpl connection; TransactionRunnerImpl(ConnectionImpl connection) { this.connection = connection; } + /** + * Computes jittered exponential backoff in milliseconds when an {@link AbortedException} does not + * carry a {@code RetryInfo} delay ({@code getRetryDelayInMillis() < 0}), preventing a tight 0ms + * CPU spin loop on repeated synthetic aborts while keeping initial retry latency minimal. + */ + static long nextFallbackBackoffMillis(int fallbackAttempt) { + long maxWindowMillis = + Math.min(MAX_FALLBACK_BACKOFF_MILLIS, 1L << Math.min(fallbackAttempt + 1, 15)); + long halfWindowMillis = Math.max(1L, maxWindowMillis / 2L); + return halfWindowMillis + ThreadLocalRandom.current().nextLong(halfWindowMillis + 1L); + } + T run(TransactionCallable callable) { connection.beginTransaction(); // Disable internal retries during this transaction. connection.setRetryAbortsInternally(/* retryAbortsInternally= */ false, /* local= */ true); UnitOfWork transaction = connection.getCurrentUnitOfWorkOrStartNewUnitOfWork(); + int fallbackAttempts = 0; while (true) { try { T result = callable.run(connection); + if (transaction.getState() == UnitOfWorkState.ABORTED + && transaction instanceof ReadWriteTransaction) { + // Re-surface a caught mid-callable abort before commitAsync so an orphaned replacement + // ReadWriteTransaction is never silently committed, preserving original RetryInfo. + AbortedException storedAbort = ((ReadWriteTransaction) transaction).getAbortedException(); + if (storedAbort != null) { + throw storedAbort; + } + ((ReadWriteTransaction) transaction).checkAborted(); + } get(connection.commitAsync(CallType.SYNC, Caller.TRANSACTION_RUNNER)); return result; } catch (AbortedException abortedException) { try { + long retryDelayMillis = abortedException.getRetryDelayInMillis(); + long sleepMillis; + if (retryDelayMillis >= 0L) { + fallbackAttempts = 0; + sleepMillis = retryDelayMillis; + } else { + sleepMillis = nextFallbackBackoffMillis(fallbackAttempts++); + } //noinspection BusyWait - Thread.sleep(abortedException.getRetryDelayInMillis()); + Thread.sleep(sleepMillis); connection.resetForRetry(transaction); } catch (InterruptedException interruptedException) { - connection.rollbackAsync(CallType.SYNC, Caller.TRANSACTION_RUNNER); + connection.rollbackAfterAbort(transaction); throw SpannerExceptionFactory.propagateInterrupt(interruptedException); } catch (Throwable t) { - connection.rollbackAsync(CallType.SYNC, Caller.TRANSACTION_RUNNER); + connection.rollbackAfterAbort(transaction); throw t; } } catch (Throwable t) { - connection.rollbackAsync(CallType.SYNC, Caller.TRANSACTION_RUNNER); + connection.rollbackAfterAbort(transaction); throw t; } } diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/RunTransactionMockServerTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/RunTransactionMockServerTest.java index d4af5dd4e7e8..6d5212ef8055 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/RunTransactionMockServerTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/RunTransactionMockServerTest.java @@ -18,24 +18,50 @@ import static com.google.cloud.spanner.connection.ConnectionProperties.DEFAULT_ISOLATION_LEVEL; import static com.google.cloud.spanner.connection.ConnectionProperties.READ_LOCK_MODE; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; +import com.google.api.core.ApiFuture; +import com.google.cloud.Timestamp; +import com.google.cloud.spanner.AbortedException; import com.google.cloud.spanner.ErrorCode; import com.google.cloud.spanner.MockSpannerServiceImpl.SimulatedExecutionTime; +import com.google.cloud.spanner.Mutation; import com.google.cloud.spanner.ResultSet; +import com.google.cloud.spanner.SpannerApiFutures; import com.google.cloud.spanner.SpannerException; +import com.google.cloud.spanner.SpannerExceptionFactory; +import com.google.cloud.spanner.SpannerOptions; +import com.google.cloud.spanner.SpannerOptionsTestHelper; +import com.google.cloud.spanner.connection.UnitOfWork.UnitOfWorkState; +import com.google.rpc.RetryInfo; import com.google.spanner.v1.BeginTransactionRequest; import com.google.spanner.v1.CommitRequest; +import com.google.spanner.v1.ExecuteBatchDmlRequest; import com.google.spanner.v1.ExecuteSqlRequest; import com.google.spanner.v1.RollbackRequest; import com.google.spanner.v1.TransactionOptions; import com.google.spanner.v1.TransactionOptions.IsolationLevel; import com.google.spanner.v1.TransactionOptions.ReadWrite.ReadLockMode; +import io.grpc.Metadata; import io.grpc.Status; +import io.grpc.protobuf.ProtoUtils; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.util.List; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -279,4 +305,750 @@ public void testRollbackInRunTransaction() { assertEquals(1, mockSpanner.countRequestsOfType(CommitRequest.class)); assertEquals(0, mockSpanner.countRequestsOfType(RollbackRequest.class)); } + + @Test + public void testRunTransactionCommitAbortedWithAutocommit() { + for (IsolationLevel isolationLevel : DEFAULT_ISOLATION_LEVEL.getValidValues()) { + for (ReadLockMode readLockMode : READ_LOCK_MODE.getValidValues()) { + final AtomicInteger attempts = new AtomicInteger(); + final AtomicReference firstAttemptUow = new AtomicReference<>(); + try (Connection connection = createConnection()) { + connection.setAutocommit(true); + connection.setDefaultIsolationLevel(isolationLevel); + connection.setReadLockMode(readLockMode); + connection.runTransaction( + transaction -> { + int attempt = attempts.incrementAndGet(); + assertTrue(transaction.isInTransaction()); + assertFalse(transaction.isRetryAbortsInternally()); + UnitOfWork currentUow = + ((ConnectionImpl) transaction).getCurrentUnitOfWorkOrStartNewUnitOfWork(); + if (attempt == 1) { + firstAttemptUow.set(currentUow); + } else { + assertSame(firstAttemptUow.get(), currentUow); + } + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + if (attempt == 1) { + mockSpanner.abortNextStatement(); + } + return null; + }); + assertTrue(connection.isAutocommit()); + assertFalse(connection.isInTransaction()); + assertTrue(connection.isRetryAbortsInternally()); + } + assertEquals(2, attempts.get()); + List sqlRequests = + mockSpanner.getRequestsOfType(ExecuteSqlRequest.class); + assertEquals(4, sqlRequests.size()); + assertEquals(2, mockSpanner.countRequestsOfType(CommitRequest.class)); + assertEquals(0, mockSpanner.countRequestsOfType(BeginTransactionRequest.class)); + + for (ExecuteSqlRequest request : sqlRequests) { + assertTrue(request.hasTransaction()); + assertFalse(request.getTransaction().hasSingleUse()); + } + assertTrue(sqlRequests.get(0).getTransaction().hasBegin()); + assertTrue(sqlRequests.get(1).getTransaction().hasId()); + assertTrue(sqlRequests.get(2).getTransaction().hasBegin()); + assertTrue(sqlRequests.get(3).getTransaction().hasId()); + for (int i : new int[] {0, 2}) { + TransactionOptions options = sqlRequests.get(i).getTransaction().getBegin(); + assertEquals(isolationLevel, options.getIsolationLevel()); + assertEquals(readLockMode, options.getReadWrite().getReadLockMode()); + } + + mockSpanner.clearRequests(); + } + } + } + + @Test + public void testRunTransactionCommitAbortedThenCallableFailsWithAutocommit() { + final AtomicInteger attempts = new AtomicInteger(); + try (Connection connection = createConnection()) { + connection.setAutocommit(true); + IllegalStateException exception = + assertThrows( + IllegalStateException.class, + () -> + connection.runTransaction( + transaction -> { + int attempt = attempts.incrementAndGet(); + assertTrue(transaction.isInTransaction()); + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + if (attempt == 1) { + mockSpanner.abortNextStatement(); + return null; + } + throw new IllegalStateException("callable failure on retry"); + })); + assertEquals("callable failure on retry", exception.getMessage()); + assertTrue(connection.isAutocommit()); + assertFalse(connection.isInTransaction()); + assertFalse(connection.isTransactionStarted()); + assertTrue(connection.isRetryAbortsInternally()); + } + assertEquals(2, attempts.get()); + List sqlRequests = mockSpanner.getRequestsOfType(ExecuteSqlRequest.class); + assertEquals(2, sqlRequests.size()); + for (ExecuteSqlRequest request : sqlRequests) { + assertTrue(request.hasTransaction()); + assertTrue(request.getTransaction().hasBegin()); + assertFalse(request.getTransaction().hasSingleUse()); + } + assertEquals(1, mockSpanner.countRequestsOfType(CommitRequest.class)); + assertEquals(1, mockSpanner.countRequestsOfType(RollbackRequest.class)); + } + + @Test + public void testRunTransactionCommitAbortedResetsSavepointsMutationsAndLocalState() { + final AtomicInteger retryListenerCalls = new AtomicInteger(); + TransactionRetryListener listener = + new TransactionRetryListener() { + @Override + public void retryStarting( + Timestamp transactionStarted, long transactionId, int retryAttempt) { + retryListenerCalls.incrementAndGet(); + } + + @Override + public void retryFinished( + Timestamp transactionStarted, + long transactionId, + int retryAttempt, + RetryResult result) { + retryListenerCalls.incrementAndGet(); + } + }; + + try (Connection connection = createConnection()) { + connection.addTransactionRetryListener(listener); + + // 1. Statement ABORTED retry (autocommit=false): savepoints, mutations, and local state + // reset. + final AtomicInteger statementAbortAttempts = new AtomicInteger(); + final AtomicReference firstUow = new AtomicReference<>(); + connection.runTransaction( + transaction -> { + int attempt = statementAbortAttempts.incrementAndGet(); + ConnectionImpl impl = (ConnectionImpl) transaction; + AbstractMultiUseTransaction uow = + (AbstractMultiUseTransaction) impl.getCurrentUnitOfWorkOrStartNewUnitOfWork(); + assertFalse(transaction.isRetryAbortsInternally()); + if (attempt == 1) { + firstUow.set(uow); + assertFalse(transaction.isReturnCommitStats()); + impl.setReturnCommitStats(true, /* local= */ true); + assertTrue(transaction.isReturnCommitStats()); + transaction.savepoint("sp_stmt"); + transaction.bufferedWrite(Mutation.newInsertBuilder("TEST").set("ID").to(1L).build()); + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + mockSpanner.abortNextStatement(); + transaction.executeUpdate(INSERT_STATEMENT); + return null; + } + assertSame(firstUow.get(), uow); + assertFalse(transaction.isReturnCommitStats()); + assertTrue(uow.getSavepoints().isEmpty()); + transaction.savepoint("sp_stmt"); + transaction.bufferedWrite(Mutation.newInsertBuilder("TEST").set("ID").to(2L).build()); + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + return null; + }); + assertEquals(2, statementAbortAttempts.get()); + assertEquals(0, retryListenerCalls.get()); + List stmtCommitRequests = mockSpanner.getRequestsOfType(CommitRequest.class); + assertEquals(1, stmtCommitRequests.size()); + assertFalse(stmtCommitRequests.get(0).getReturnCommitStats()); + assertEquals(1, stmtCommitRequests.get(0).getMutationsCount()); + assertEquals( + "2", + stmtCommitRequests + .get(0) + .getMutations(0) + .getInsert() + .getValues(0) + .getValues(0) + .getStringValue()); + + // 2. Commit ABORTED retry (autocommit=true): savepoints, mutations, and local state reset. + mockSpanner.clearRequests(); + connection.setAutocommit(true); + final AtomicInteger commitAbortAttempts = new AtomicInteger(); + final AtomicReference firstCommitUow = new AtomicReference<>(); + connection.runTransaction( + transaction -> { + int attempt = commitAbortAttempts.incrementAndGet(); + ConnectionImpl impl = (ConnectionImpl) transaction; + AbstractMultiUseTransaction uow = + (AbstractMultiUseTransaction) impl.getCurrentUnitOfWorkOrStartNewUnitOfWork(); + assertFalse(transaction.isRetryAbortsInternally()); + if (attempt == 1) { + firstCommitUow.set(uow); + assertFalse(transaction.isReturnCommitStats()); + impl.setReturnCommitStats(true, /* local= */ true); + assertTrue(transaction.isReturnCommitStats()); + transaction.savepoint("sp_commit"); + transaction.write(Mutation.newInsertBuilder("TEST").set("ID").to(10L).build()); + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + mockSpanner.abortNextStatement(); + return null; + } + assertSame(firstCommitUow.get(), uow); + assertFalse(transaction.isReturnCommitStats()); + assertTrue(uow.getSavepoints().isEmpty()); + transaction.savepoint("sp_commit"); + transaction.write(Mutation.newInsertBuilder("TEST").set("ID").to(20L).build()); + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + return null; + }); + assertEquals(2, commitAbortAttempts.get()); + assertEquals(0, retryListenerCalls.get()); + List commitRequests = mockSpanner.getRequestsOfType(CommitRequest.class); + assertEquals(2, commitRequests.size()); + assertFalse(commitRequests.get(0).getReturnCommitStats()); + assertFalse(commitRequests.get(1).getReturnCommitStats()); + assertEquals(1, commitRequests.get(1).getMutationsCount()); + assertEquals( + "20", + commitRequests + .get(1) + .getMutations(0) + .getInsert() + .getValues(0) + .getValues(0) + .getStringValue()); + } + } + + @Test + public void testRunTransactionNonAbortFailuresAndInterruptedBackoff() { + try (Connection connection = createConnection()) { + connection.setAutocommit(true); + + // 1. Non-abort Commit RPC failure propagates original exception and resets state. + mockSpanner.setCommitExecutionTime( + SimulatedExecutionTime.ofException( + Status.PERMISSION_DENIED.withDescription("commit denied").asRuntimeException())); + SpannerException commitFailure = + assertThrows( + SpannerException.class, + () -> + connection.runTransaction( + transaction -> { + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + return null; + })); + assertEquals(ErrorCode.PERMISSION_DENIED, commitFailure.getErrorCode()); + assertTrue(commitFailure.getMessage(), commitFailure.getMessage().contains("commit denied")); + assertFalse(connection.isInTransaction()); + assertFalse(connection.isTransactionStarted()); + assertTrue(connection.isRetryAbortsInternally()); + mockSpanner.removeAllExecutionTimes(); + mockSpanner.clearRequests(); + + // 2a. Callable catches statement AbortedException and throws non-abort exception. + IllegalArgumentException caughtAbortFailure = + assertThrows( + IllegalArgumentException.class, + () -> + connection.runTransaction( + transaction -> { + mockSpanner.abortNextStatement(); + try { + transaction.executeUpdate(INSERT_STATEMENT); + } catch (SpannerException ignored) { + // Intentionally swallow abort and throw application error. + } + throw new IllegalArgumentException("failed after caught abort"); + })); + assertEquals("failed after caught abort", caughtAbortFailure.getMessage()); + assertEquals(0, mockSpanner.countRequestsOfType(RollbackRequest.class)); + assertFalse(connection.isInTransaction()); + assertTrue(connection.isRetryAbortsInternally()); + mockSpanner.clearRequests(); + + // 2b. Callable catches statement AbortedException and returns normally: runner detects + // aborted transaction before commit and retries cleanly. + final AtomicInteger swallowedAbortAttempts = new AtomicInteger(); + String recovered = + connection.runTransaction( + transaction -> { + if (swallowedAbortAttempts.incrementAndGet() == 1) { + mockSpanner.abortNextStatement(); + try { + transaction.executeUpdate(INSERT_STATEMENT); + } catch (SpannerException ignored) { + // Intentionally swallow abort and return normally. + } + return "stale"; + } + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + return "recovered"; + }); + assertEquals("recovered", recovered); + assertEquals(2, swallowedAbortAttempts.get()); + assertEquals(1, mockSpanner.countRequestsOfType(CommitRequest.class)); + mockSpanner.clearRequests(); + + // 3. Interrupted commit during Commit abort cleanup propagates CANCELLED without + // FAILED_PRECONDITION and leaves connection usable. + try { + SpannerException interrupted = + assertThrows( + SpannerException.class, + () -> + connection.runTransaction( + transaction -> { + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + mockSpanner.abortNextStatement(); + Thread.currentThread().interrupt(); + return null; + })); + assertEquals(ErrorCode.CANCELLED, interrupted.getErrorCode()); + assertTrue(Thread.interrupted()); + } finally { + Thread.interrupted(); + } + assertFalse(connection.isInTransaction()); + assertFalse(connection.isTransactionStarted()); + assertTrue(connection.isRetryAbortsInternally()); + + mockSpanner.clearRequests(); + connection.runTransaction( + transaction -> { + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + return null; + }); + assertEquals(1, mockSpanner.countRequestsOfType(CommitRequest.class)); + } + } + + @Test + public void testRunTransactionSpanAndSessionLifecycleOnAborts() { + SpannerOptionsTestHelper.resetActiveTracingFramework(); + SpannerOptions.enableOpenTelemetryTraces(); + InMemorySpanExporter spanExporter = InMemorySpanExporter.create(); + SdkTracerProvider tracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(spanExporter)) + .build(); + OpenTelemetrySdk openTelemetry = + OpenTelemetrySdk.builder() + .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance())) + .setTracerProvider(tracerProvider) + .build(); + + try { + try (Connection connection = createTracingConnection(openTelemetry, ";autocommit=true")) { + // 1. Commit-abort retry: 0 finished transaction spans mid-retry, 1 of each after commit. + final AtomicInteger commitRetryAttempts = new AtomicInteger(); + connection.runTransaction( + transaction -> { + int attempt = commitRetryAttempts.incrementAndGet(); + if (attempt == 2) { + assertEquals( + 0L, countFinishedSpans(spanExporter, "CloudSpannerJdbc.ReadWriteTransaction")); + assertEquals( + 0L, countFinishedSpans(spanExporter, "CloudSpanner.ReadWriteTransaction")); + } + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + if (attempt == 1) { + mockSpanner.abortNextStatement(); + } + return null; + }); + assertEquals(2, commitRetryAttempts.get()); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpannerJdbc.ReadWriteTransaction")); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpanner.ReadWriteTransaction")); + + // 2. Statement-abort retry: 0 finished transaction spans mid-retry, 1 of each after commit. + spanExporter.reset(); + mockSpanner.clearRequests(); + final AtomicInteger stmtRetryAttempts = new AtomicInteger(); + connection.runTransaction( + transaction -> { + int attempt = stmtRetryAttempts.incrementAndGet(); + if (attempt == 2) { + assertEquals( + 0L, countFinishedSpans(spanExporter, "CloudSpannerJdbc.ReadWriteTransaction")); + assertEquals( + 0L, countFinishedSpans(spanExporter, "CloudSpanner.ReadWriteTransaction")); + } + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + if (attempt == 1) { + mockSpanner.abortNextStatement(); + } + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + return null; + }); + assertEquals(2, stmtRetryAttempts.get()); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpannerJdbc.ReadWriteTransaction")); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpanner.ReadWriteTransaction")); + + // 3. Caught-abort failure: exactly 1 finished unit-of-work and TransactionManager span. + spanExporter.reset(); + mockSpanner.clearRequests(); + assertThrows( + IllegalStateException.class, + () -> + connection.runTransaction( + transaction -> { + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + mockSpanner.abortNextStatement(); + try { + transaction.executeUpdate(INSERT_STATEMENT); + } catch (SpannerException ignored) { + // Swallow abort and fail with non-abort exception. + } + throw new IllegalStateException("caught abort failure"); + })); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpannerJdbc.ReadWriteTransaction")); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpanner.ReadWriteTransaction")); + + // 4. Interrupted commit: exactly 1 finished unit-of-work and TransactionManager span. + spanExporter.reset(); + mockSpanner.clearRequests(); + try { + SpannerException interrupted = + assertThrows( + SpannerException.class, + () -> + connection.runTransaction( + transaction -> { + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + mockSpanner.abortNextStatement(); + Thread.currentThread().interrupt(); + return null; + })); + assertEquals(ErrorCode.CANCELLED, interrupted.getErrorCode()); + } finally { + Thread.interrupted(); + } + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpannerJdbc.ReadWriteTransaction")); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpanner.ReadWriteTransaction")); + } + } finally { + SpannerPool.closeSpannerPool(); + openTelemetry.close(); + SpannerOptionsTestHelper.resetActiveTracingFramework(); + } + } + + @Test + public void testRunTransactionSyntheticAbortDelayedStartAndActiveDmlBatch() { + try (Connection connection = createConnection()) { + connection.setAutocommit(true); + + // 1. Pre-statement synthetic AbortedException (txManager == null). + final AtomicInteger preStmtAttempts = new AtomicInteger(); + String preStmtResult = + connection.runTransaction( + transaction -> { + if (preStmtAttempts.incrementAndGet() == 1) { + throw SpannerExceptionFactory.newSpannerException( + ErrorCode.ABORTED, "synthetic pre-statement abort"); + } + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + return "pre-stmt-ok"; + }); + assertEquals("pre-stmt-ok", preStmtResult); + assertEquals(2, preStmtAttempts.get()); + assertEquals(1, mockSpanner.countRequestsOfType(ExecuteSqlRequest.class)); + assertEquals(1, mockSpanner.countRequestsOfType(CommitRequest.class)); + assertEquals(0, mockSpanner.countRequestsOfType(RollbackRequest.class)); + mockSpanner.clearRequests(); + + // 2. Synthetic AbortedException thrown by callable after INSERT (rolls back STARTED + // txManager). + final AtomicInteger postInsertAttempts = new AtomicInteger(); + connection.runTransaction( + transaction -> { + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + if (postInsertAttempts.incrementAndGet() == 1) { + throw SpannerExceptionFactory.newSpannerException( + ErrorCode.ABORTED, "synthetic post-insert abort"); + } + return null; + }); + assertEquals(2, postInsertAttempts.get()); + assertEquals(2, mockSpanner.countRequestsOfType(ExecuteSqlRequest.class)); + assertEquals(1, mockSpanner.countRequestsOfType(RollbackRequest.class)); + assertEquals(1, mockSpanner.countRequestsOfType(CommitRequest.class)); + mockSpanner.clearRequests(); + + // 3. delayTransactionStartUntilFirstWrite=true read-then-abort (txManager remains null on + // Attempt 1). + connection.setDelayTransactionStartUntilFirstWrite(true); + try { + final AtomicInteger delayedAttempts = new AtomicInteger(); + connection.runTransaction( + transaction -> { + try (ResultSet rs = transaction.executeQuery(SELECT1_STATEMENT)) { + assertTrue(rs.next()); + } + if (delayedAttempts.incrementAndGet() == 1) { + throw SpannerExceptionFactory.newSpannerException( + ErrorCode.ABORTED, "synthetic delayed-start abort"); + } + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + return null; + }); + assertEquals(2, delayedAttempts.get()); + assertEquals(1, mockSpanner.countRequestsOfType(CommitRequest.class)); + assertEquals(0, mockSpanner.countRequestsOfType(RollbackRequest.class)); + } finally { + connection.setDelayTransactionStartUntilFirstWrite(false); + } + mockSpanner.clearRequests(); + + // 4a. Abort while DML batch is active resets batch state and succeeds on retry. + final AtomicInteger batchAbortAttempts = new AtomicInteger(); + connection.runTransaction( + transaction -> { + transaction.startBatchDml(); + transaction.executeUpdate(INSERT_STATEMENT); + if (batchAbortAttempts.incrementAndGet() == 1) { + throw SpannerExceptionFactory.newSpannerException( + ErrorCode.ABORTED, "abort during active batch"); + } + assertArrayEquals(new long[] {1L}, transaction.runBatch()); + return null; + }); + assertFalse(connection.isDmlBatchActive()); + assertEquals(2, batchAbortAttempts.get()); + assertEquals(1, mockSpanner.countRequestsOfType(ExecuteBatchDmlRequest.class)); + assertEquals(1, mockSpanner.countRequestsOfType(CommitRequest.class)); + mockSpanner.clearRequests(); + + // 4b. Non-abort failure while DML batch is active aborts batch and rolls back transaction. + IllegalStateException batchFailure = + assertThrows( + IllegalStateException.class, + () -> + connection.runTransaction( + transaction -> { + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + transaction.startBatchDml(); + transaction.executeUpdate(INSERT_STATEMENT); + throw new IllegalStateException("failure during active batch"); + })); + assertEquals("failure during active batch", batchFailure.getMessage()); + assertFalse(connection.isDmlBatchActive()); + assertFalse(connection.isInTransaction()); + assertEquals(1, mockSpanner.countRequestsOfType(RollbackRequest.class)); + } + } + + @Test + public void testApplicationCommitAbortClosesTransaction() { + SpannerOptionsTestHelper.resetActiveTracingFramework(); + SpannerOptions.enableOpenTelemetryTraces(); + InMemorySpanExporter spanExporter = InMemorySpanExporter.create(); + SdkTracerProvider tracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(spanExporter)) + .build(); + OpenTelemetrySdk openTelemetry = + OpenTelemetrySdk.builder() + .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance())) + .setTracerProvider(tracerProvider) + .build(); + + try { + try (Connection connection = createTracingConnection(openTelemetry, "")) { + connection.setRetryAbortsInternally(false); + assertFalse(connection.isRetryAbortsInternally()); + // 1. Direct application commit() outside runTransaction transitions to COMMIT_FAILED and + // closes txManager + unit-of-work span. + assertEquals(1L, connection.executeUpdate(INSERT_STATEMENT)); + UnitOfWork directUowBefore = + ((ConnectionImpl) connection).getCurrentUnitOfWorkOrStartNewUnitOfWork(); + mockSpanner.abortNextStatement(); + assertThrows(AbortedException.class, connection::commit); + assertEquals(UnitOfWorkState.COMMIT_FAILED, directUowBefore.getState()); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpannerJdbc.ReadWriteTransaction")); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpanner.ReadWriteTransaction")); + + // 2. Run runTransaction with commit abort on the same connection, then verify subsequent + // direct application commit() still closes transaction with COMMIT_FAILED. + spanExporter.reset(); + mockSpanner.clearRequests(); + connection.setAutocommit(true); + final AtomicInteger runnerAttempts = new AtomicInteger(); + connection.runTransaction( + transaction -> { + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + if (runnerAttempts.incrementAndGet() == 1) { + mockSpanner.abortNextStatement(); + } + return null; + }); + assertEquals(2, runnerAttempts.get()); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpannerJdbc.ReadWriteTransaction")); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpanner.ReadWriteTransaction")); + + spanExporter.reset(); + mockSpanner.clearRequests(); + connection.beginTransaction(); + assertFalse(connection.isRetryAbortsInternally()); + assertEquals(1L, connection.executeUpdate(INSERT_STATEMENT)); + UnitOfWork directUowAfter = + ((ConnectionImpl) connection).getCurrentUnitOfWorkOrStartNewUnitOfWork(); + mockSpanner.abortNextStatement(); + assertThrows(AbortedException.class, connection::commit); + assertEquals(UnitOfWorkState.COMMIT_FAILED, directUowAfter.getState()); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpannerJdbc.ReadWriteTransaction")); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpanner.ReadWriteTransaction")); + } + } finally { + SpannerPool.closeSpannerPool(); + openTelemetry.close(); + SpannerOptionsTestHelper.resetActiveTracingFramework(); + } + } + + @Test + public void testRunTransactionInterruptedDuringBackoffAfterCommitAbort() throws Exception { + Metadata trailers = new Metadata(); + trailers.put( + ProtoUtils.keyForProto(RetryInfo.getDefaultInstance()), + RetryInfo.newBuilder() + .setRetryDelay(com.google.protobuf.Duration.newBuilder().setSeconds(30L)) + .build()); + mockSpanner.setCommitExecutionTime( + SimulatedExecutionTime.ofException( + Status.ABORTED.withDescription("commit aborted").asRuntimeException(trailers))); + Thread runnerThread = Thread.currentThread(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30L); + // The 30s RetryInfo delay is the only timed wait the runner thread enters; interrupt it there. + Thread interrupter = + new Thread( + () -> { + while (runnerThread.getState() != Thread.State.TIMED_WAITING + && System.nanoTime() < deadline) { + Thread.yield(); + } + runnerThread.interrupt(); + }); + try (Connection connection = createConnection()) { + connection.setAutocommit(true); + interrupter.start(); + SpannerException exception = + assertThrows( + SpannerException.class, + () -> + connection.runTransaction( + transaction -> { + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + return null; + })); + assertEquals(ErrorCode.CANCELLED, exception.getErrorCode()); + assertTrue(Thread.interrupted()); + assertFalse(connection.isInTransaction()); + assertFalse(connection.isTransactionStarted()); + assertTrue(connection.isRetryAbortsInternally()); + } finally { + interrupter.join(); + Thread.interrupted(); + mockSpanner.removeAllExecutionTimes(); + } + assertEquals(1, mockSpanner.countRequestsOfType(CommitRequest.class)); + assertEquals(0, mockSpanner.countRequestsOfType(RollbackRequest.class)); + } + + @Test + public void testTimedOutAsyncStatementBeforeCommitEndsSpanAndClosesTransaction() { + SpannerOptionsTestHelper.resetActiveTracingFramework(); + SpannerOptions.enableOpenTelemetryTraces(); + InMemorySpanExporter spanExporter = InMemorySpanExporter.create(); + SdkTracerProvider tracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(spanExporter)) + .build(); + OpenTelemetrySdk openTelemetry = + OpenTelemetrySdk.builder() + .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance())) + .setTracerProvider(tracerProvider) + .build(); + + try { + try (Connection connection = createTracingConnection(openTelemetry, "")) { + connection.setRetryAbortsInternally(false); + assertEquals(1L, connection.executeUpdate(INSERT_STATEMENT)); + UnitOfWork uow = ((ConnectionImpl) connection).getCurrentUnitOfWorkOrStartNewUnitOfWork(); + + connection.setStatementTimeout(15L, TimeUnit.MILLISECONDS); + mockSpanner.freeze(); + ApiFuture timedOutDml; + ApiFuture queuedCommit; + try { + timedOutDml = connection.executeUpdateAsync(INSERT_STATEMENT); + queuedCommit = connection.commitAsync(); + SpannerException dmlTimeout = + assertThrows(SpannerException.class, () -> SpannerApiFutures.get(timedOutDml)); + assertEquals(ErrorCode.DEADLINE_EXCEEDED, dmlTimeout.getErrorCode()); + } finally { + mockSpanner.unfreeze(); + } + SpannerException commitFailure = + assertThrows(SpannerException.class, () -> SpannerApiFutures.get(queuedCommit)); + assertEquals(ErrorCode.FAILED_PRECONDITION, commitFailure.getErrorCode()); + assertEquals(UnitOfWorkState.COMMIT_FAILED, uow.getState()); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpannerJdbc.ReadWriteTransaction")); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpanner.ReadWriteTransaction")); + } + } finally { + mockSpanner.unfreeze(); + SpannerPool.closeSpannerPool(); + openTelemetry.close(); + SpannerOptionsTestHelper.resetActiveTracingFramework(); + } + } + + @Test + public void testSyntheticAbortWithoutRetryInfoAppliesExponentialBackoff() { + try (Connection connection = createConnection()) { + connection.setAutocommit(true); + final AtomicInteger attempts = new AtomicInteger(); + long startNanos = System.nanoTime(); + String result = + connection.runTransaction( + transaction -> { + if (attempts.incrementAndGet() <= 6) { + throw SpannerExceptionFactory.newSpannerException( + ErrorCode.ABORTED, "synthetic abort without retry info"); + } + return "done"; + }); + long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos); + assertEquals("done", result); + assertEquals(7, attempts.get()); + assertTrue( + "Expected exponential fallback backoff (>= 50ms across 6 retries), got " + + elapsedMillis + + "ms", + elapsedMillis >= 50L); + } + } + + private Connection createTracingConnection( + OpenTelemetry openTelemetry, String additionalUrlOptions) { + return ConnectionOptions.newBuilder() + .setTracingPrefix("CloudSpannerJdbc") + .setOpenTelemetry(openTelemetry) + .setUri(getBaseUrl() + additionalUrlOptions) + .build() + .getConnection(); + } + + private static long countFinishedSpans(InMemorySpanExporter exporter, String spanName) { + assertEquals(CompletableResultCode.ofSuccess(), exporter.flush()); + return exporter.getFinishedSpanItems().stream() + .filter(span -> span.getName().equals(spanName)) + .count(); + } } From b73f26a136bf574de045695332d603112253d4ee Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian <179120858+sakthivelmanii@users.noreply.github.com> Date: Thu, 19 Nov 2026 14:38:20 +0000 Subject: [PATCH 2/2] Addressed comments --- .../connection/ReadWriteTransaction.java | 35 ++- .../RunTransactionMockServerTest.java | 294 +++++++++++++++--- 2 files changed, 270 insertions(+), 59 deletions(-) diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ReadWriteTransaction.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ReadWriteTransaction.java index 1f3248b3d9aa..c3820ea538b3 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ReadWriteTransaction.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ReadWriteTransaction.java @@ -784,16 +784,22 @@ public ApiFuture executeQueryAsync( () -> { checkTimedOut(); checkAborted(); - try { - return DirectExecuteResultSet.ofResultSet( - internalExecuteQuery(statement, analyzeMode, options)); - } catch (AbortedException aborted) { - if (keepAbortedTransactionOpen) { - this.state = UnitOfWorkState.ABORTED; - this.abortedException = aborted; - } - throw aborted; + ResultSet queryResultSet = internalExecuteQuery(statement, analyzeMode, options); + if (keepAbortedTransactionOpen) { + queryResultSet = + new ReplaceableForwardingResultSet(queryResultSet) { + @Override + public boolean next() throws SpannerException { + try { + return super.next(); + } catch (AbortedException aborted) { + markAborted(aborted); + throw aborted; + } + } + }; } + return DirectExecuteResultSet.ofResultSet(queryResultSet); }, SpannerGrpc.getExecuteStreamingSqlMethod()); } @@ -1500,10 +1506,13 @@ public void resetForRetry() { } } if (!resetSucceeded) { - try { - txManager.close(); - } catch (Throwable ignored) { - // Ignore cleanup errors when closing an un-aborted transaction manager. + TransactionState managerState = txManager.getState(); + if (managerState == TransactionState.STARTED || managerState == TransactionState.ABORTED) { + try { + txManager.close(); + } catch (Throwable ignored) { + // Ignore cleanup errors when closing an un-aborted transaction manager. + } } txManager = null; txContextFuture = null; diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/RunTransactionMockServerTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/RunTransactionMockServerTest.java index 6d5212ef8055..15a8b897f1a0 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/RunTransactionMockServerTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/RunTransactionMockServerTest.java @@ -21,6 +21,9 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; @@ -30,6 +33,7 @@ import com.google.cloud.spanner.AbortedException; import com.google.cloud.spanner.ErrorCode; import com.google.cloud.spanner.MockSpannerServiceImpl.SimulatedExecutionTime; +import com.google.cloud.spanner.MockSpannerServiceImpl.StatementResult; import com.google.cloud.spanner.Mutation; import com.google.cloud.spanner.ResultSet; import com.google.cloud.spanner.SpannerApiFutures; @@ -38,6 +42,7 @@ import com.google.cloud.spanner.SpannerOptions; import com.google.cloud.spanner.SpannerOptionsTestHelper; import com.google.cloud.spanner.connection.UnitOfWork.UnitOfWorkState; +import com.google.protobuf.ByteString; import com.google.rpc.RetryInfo; import com.google.spanner.v1.BeginTransactionRequest; import com.google.spanner.v1.CommitRequest; @@ -912,31 +917,11 @@ public void testApplicationCommitAbortClosesTransaction() { } @Test - public void testRunTransactionInterruptedDuringBackoffAfterCommitAbort() throws Exception { - Metadata trailers = new Metadata(); - trailers.put( - ProtoUtils.keyForProto(RetryInfo.getDefaultInstance()), - RetryInfo.newBuilder() - .setRetryDelay(com.google.protobuf.Duration.newBuilder().setSeconds(30L)) - .build()); - mockSpanner.setCommitExecutionTime( - SimulatedExecutionTime.ofException( - Status.ABORTED.withDescription("commit aborted").asRuntimeException(trailers))); - Thread runnerThread = Thread.currentThread(); - long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30L); - // The 30s RetryInfo delay is the only timed wait the runner thread enters; interrupt it there. - Thread interrupter = - new Thread( - () -> { - while (runnerThread.getState() != Thread.State.TIMED_WAITING - && System.nanoTime() < deadline) { - Thread.yield(); - } - runnerThread.interrupt(); - }); + public void testRunTransactionInterruptedDuringBackoffAfterAbort() { + // 1. Statement abort with pre-set interrupt closes kept-open aborted txManager without sending + // a Rollback RPC. try (Connection connection = createConnection()) { connection.setAutocommit(true); - interrupter.start(); SpannerException exception = assertThrows( SpannerException.class, @@ -944,7 +929,16 @@ public void testRunTransactionInterruptedDuringBackoffAfterCommitAbort() throws connection.runTransaction( transaction -> { assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); - return null; + mockSpanner.abortNextStatement(); + AbortedException aborted = + assertThrows( + AbortedException.class, + () -> transaction.executeUpdate(INSERT_STATEMENT)); + // Pre-setting the interrupt flag ensures Thread.sleep(...) in the runner's + // catch (AbortedException) block deterministically throws + // InterruptedException immediately without sleeping or background threads. + Thread.currentThread().interrupt(); + throw aborted; })); assertEquals(ErrorCode.CANCELLED, exception.getErrorCode()); assertTrue(Thread.interrupted()); @@ -952,12 +946,44 @@ public void testRunTransactionInterruptedDuringBackoffAfterCommitAbort() throws assertFalse(connection.isTransactionStarted()); assertTrue(connection.isRetryAbortsInternally()); } finally { - interrupter.join(); Thread.interrupted(); - mockSpanner.removeAllExecutionTimes(); } - assertEquals(1, mockSpanner.countRequestsOfType(CommitRequest.class)); + assertEquals(2, mockSpanner.countRequestsOfType(ExecuteSqlRequest.class)); + assertEquals(0, mockSpanner.countRequestsOfType(CommitRequest.class)); assertEquals(0, mockSpanner.countRequestsOfType(RollbackRequest.class)); + mockSpanner.clearRequests(); + + // 2. Active DML batch + pre-set interrupt during post-abort backoff exercises + // rollbackAfterAbort() (whereas rollbackAsync() would fail with FAILED_PRECONDITION due to the + // active batch and mask CANCELLED). + try (Connection connection = createConnection()) { + connection.setAutocommit(true); + SpannerException exception = + assertThrows( + SpannerException.class, + () -> + connection.runTransaction( + transaction -> { + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + transaction.startBatchDml(); + transaction.executeUpdate(INSERT_STATEMENT); + Thread.currentThread().interrupt(); + throw SpannerExceptionFactory.newSpannerException( + ErrorCode.ABORTED, "abort during active batch"); + })); + assertEquals(ErrorCode.CANCELLED, exception.getErrorCode()); + assertTrue(Thread.interrupted()); + assertFalse(connection.isDmlBatchActive()); + assertFalse(connection.isInTransaction()); + assertFalse(connection.isTransactionStarted()); + assertTrue(connection.isRetryAbortsInternally()); + } finally { + Thread.interrupted(); + } + assertEquals(1, mockSpanner.countRequestsOfType(ExecuteSqlRequest.class)); + assertEquals(0, mockSpanner.countRequestsOfType(ExecuteBatchDmlRequest.class)); + assertEquals(0, mockSpanner.countRequestsOfType(CommitRequest.class)); + assertEquals(1, mockSpanner.countRequestsOfType(RollbackRequest.class)); } @Test @@ -1011,27 +1037,203 @@ public void testTimedOutAsyncStatementBeforeCommitEndsSpanAndClosesTransaction() @Test public void testSyntheticAbortWithoutRetryInfoAppliesExponentialBackoff() { + for (int attempt = 0; attempt <= 25; attempt++) { + long expectedMax = Math.min(32_000L, 1L << Math.min(attempt + 1, 15)); + long expectedMin = Math.max(1L, expectedMax / 2L); + for (int sample = 0; sample < 20; sample++) { + long delayMillis = TransactionRunnerImpl.nextFallbackBackoffMillis(attempt); + assertTrue( + "Attempt " + + attempt + + " delay " + + delayMillis + + "ms outside [" + + expectedMin + + ", " + + expectedMax + + "]", + delayMillis >= expectedMin && delayMillis <= expectedMax); + } + } + } + + @Test + public void testResetForRetryHandlesNonAbortedManagerAndPreservesPreviousTransactionId() { + // 1. When internal retry fails with concurrent modification, ReadWriteTransaction rolls back + // txManager (ROLLED_BACK) and marks itself ABORTED; resetForRetry() must fall back cleanly + // instead of throwing IllegalStateException from txManager.resetForRetry(). try (Connection connection = createConnection()) { + connection.setAutocommit(false); + connection.setRetryAbortsInternally(true); + ReadWriteTransaction uow = + (ReadWriteTransaction) + ((ConnectionImpl) connection).getCurrentUnitOfWorkOrStartNewUnitOfWork(); + try { + try (ResultSet rs = connection.executeQuery(SELECT_COUNT_STATEMENT)) { + assertTrue(rs.next()); + assertEquals(COUNT_BEFORE_INSERT, rs.getLong(0)); + } + mockSpanner.putStatementResult( + StatementResult.query(SELECT_COUNT_STATEMENT, SELECT_COUNT_RESULTSET_AFTER_INSERT)); + mockSpanner.abortNextStatement(); + assertThrows(AbortedException.class, () -> connection.executeUpdate(INSERT_STATEMENT)); + } finally { + mockSpanner.putStatementResult( + StatementResult.query(SELECT_COUNT_STATEMENT, SELECT_COUNT_RESULTSET_BEFORE_INSERT)); + } + assertEquals(UnitOfWorkState.ABORTED, uow.getState()); + assertNotNull(uow.getAbortedException()); + uow.resetForRetry(); + assertEquals(UnitOfWorkState.STARTED, uow.getState()); + assertNull(uow.getAbortedException()); + assertEquals(1L, connection.executeUpdate(INSERT_STATEMENT)); + connection.commit(); + mockSpanner.clearRequests(); + + // 2. Statement-level backend abort leaves txManager.getState() as STARTED while + // txn.isAborted() is true; resetForRetry() must still invoke txManager.resetForRetry() so + // multiplexed sessions forward the aborted attempt's transaction ID on retry. connection.setAutocommit(true); - final AtomicInteger attempts = new AtomicInteger(); - long startNanos = System.nanoTime(); - String result = - connection.runTransaction( - transaction -> { - if (attempts.incrementAndGet() <= 6) { - throw SpannerExceptionFactory.newSpannerException( - ErrorCode.ABORTED, "synthetic abort without retry info"); - } - return "done"; - }); - long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos); - assertEquals("done", result); - assertEquals(7, attempts.get()); - assertTrue( - "Expected exponential fallback backoff (>= 50ms across 6 retries), got " - + elapsedMillis - + "ms", - elapsedMillis >= 50L); + final AtomicInteger stmtAbortAttempts = new AtomicInteger(); + connection.runTransaction( + transaction -> { + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + if (stmtAbortAttempts.incrementAndGet() == 1) { + mockSpanner.abortNextStatement(); + } + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + return null; + }); + assertEquals(2, stmtAbortAttempts.get()); + List sqlRequests = mockSpanner.getRequestsOfType(ExecuteSqlRequest.class); + assertEquals(4, sqlRequests.size()); + assertTrue(sqlRequests.get(1).getTransaction().hasId()); + assertNotEquals(ByteString.EMPTY, sqlRequests.get(1).getTransaction().getId()); + assertEquals( + sqlRequests.get(1).getTransaction().getId(), + sqlRequests + .get(2) + .getTransaction() + .getBegin() + .getReadWrite() + .getMultiplexedSessionPreviousTransactionId()); + } + } + + @Test + public void testRunTransactionMidStreamQueryAbort() { + SpannerOptionsTestHelper.resetActiveTracingFramework(); + SpannerOptions.enableOpenTelemetryTraces(); + InMemorySpanExporter spanExporter = InMemorySpanExporter.create(); + SdkTracerProvider tracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(spanExporter)) + .build(); + OpenTelemetrySdk openTelemetry = + OpenTelemetrySdk.builder() + .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance())) + .setTracerProvider(tracerProvider) + .build(); + + try { + try (Connection connection = createTracingConnection(openTelemetry, ";autocommit=true")) { + Metadata trailers = new Metadata(); + trailers.put( + ProtoUtils.keyForProto(RetryInfo.getDefaultInstance()), + RetryInfo.newBuilder() + .setRetryDelay(com.google.protobuf.Duration.newBuilder().setNanos(1_000_000)) + .build()); + + // 1. Uncaught mid-stream abort on ResultSet.next() (after row 0 succeeded) must preserve + // a single CloudSpanner.ReadWriteTransaction span and forward previousTransactionId. + mockSpanner.setExecuteStreamingSqlExecutionTime( + SimulatedExecutionTime.ofStreamException( + Status.ABORTED.withDescription("mid-stream abort").asRuntimeException(trailers), + 2L)); + final AtomicInteger attempts = new AtomicInteger(); + int rowCount = + connection.runTransaction( + transaction -> { + attempts.incrementAndGet(); + int rows = 0; + try (ResultSet resultSet = transaction.executeQuery(SELECT_RANDOM_STATEMENT)) { + while (resultSet.next()) { + rows++; + } + } + return rows; + }); + assertEquals(RANDOM_RESULT_SET_ROW_COUNT, rowCount); + assertEquals(2, attempts.get()); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpannerJdbc.ReadWriteTransaction")); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpanner.ReadWriteTransaction")); + List streamRequests = + mockSpanner.getRequestsOfType(ExecuteSqlRequest.class); + assertEquals(2, streamRequests.size()); + assertNotEquals( + ByteString.EMPTY, + streamRequests + .get(1) + .getTransaction() + .getBegin() + .getReadWrite() + .getMultiplexedSessionPreviousTransactionId()); + + // 2. Caught mid-stream abort on ResultSet.next() + thrown non-abort exception must not + // send an unnecessary Rollback RPC on the already-aborted transaction. + mockSpanner.clearRequests(); + mockSpanner.setExecuteStreamingSqlExecutionTime( + SimulatedExecutionTime.ofStreamException( + Status.ABORTED.withDescription("mid-stream abort").asRuntimeException(trailers), + 2L)); + assertThrows( + IllegalStateException.class, + () -> + connection.runTransaction( + transaction -> { + try (ResultSet resultSet = + transaction.executeQuery(SELECT_RANDOM_STATEMENT)) { + while (resultSet.next()) { + // Consume until mid-stream abort. + } + } catch (AbortedException ignored) { + // Swallow abort and fail with non-abort exception. + } + throw new IllegalStateException("failed after caught mid-stream abort"); + })); + assertEquals(0, mockSpanner.countRequestsOfType(RollbackRequest.class)); + + // 3. Caught mid-stream abort on ResultSet.next() + normal return must re-surface the + // stored AbortedException before commitAsync and retry cleanly. + mockSpanner.clearRequests(); + mockSpanner.setExecuteStreamingSqlExecutionTime( + SimulatedExecutionTime.ofStreamException( + Status.ABORTED.withDescription("mid-stream abort").asRuntimeException(trailers), + 2L)); + final AtomicInteger swallowedMidStreamAttempts = new AtomicInteger(); + int recoveredRows = + connection.runTransaction( + transaction -> { + int attempt = swallowedMidStreamAttempts.incrementAndGet(); + int rows = 0; + try (ResultSet resultSet = transaction.executeQuery(SELECT_RANDOM_STATEMENT)) { + while (resultSet.next()) { + rows++; + } + } catch (AbortedException ignored) { + assertEquals(1, attempt); + return -1; + } + return rows; + }); + assertEquals(RANDOM_RESULT_SET_ROW_COUNT, recoveredRows); + assertEquals(2, swallowedMidStreamAttempts.get()); + } + } finally { + mockSpanner.removeAllExecutionTimes(); + SpannerPool.closeSpannerPool(); + openTelemetry.close(); + SpannerOptionsTestHelper.resetActiveTracingFramework(); } }