diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/AbstractMultiUseTransaction.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/AbstractMultiUseTransaction.java index ca78c3e5aeaf..19b4cff69449 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/AbstractMultiUseTransaction.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/AbstractMultiUseTransaction.java @@ -174,6 +174,11 @@ ImmutableList getSavepoints() { return ImmutableList.copyOf(savepoints); } + /** Clears all in-memory savepoints recorded during the current transaction attempt. */ + void clearSavepoints() { + savepoints.clear(); + } + @Override public void savepoint(@Nonnull String name, @Nonnull Dialect dialect) { if (dialect != Dialect.POSTGRESQL) { diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionImpl.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionImpl.java index 2f9a5de2ea2a..623e734626c5 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionImpl.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionImpl.java @@ -91,6 +91,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Suppliers; import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.Uninterruptibles; import com.google.spanner.v1.DirectedReadOptions; import com.google.spanner.v1.ExecuteSqlRequest.QueryOptions; import com.google.spanner.v1.RequestOptions; @@ -1397,9 +1398,96 @@ public T runTransaction(TransactionCallable callable) { } } + private static final EndTransactionCallback NO_OP_END_TRANSACTION_CALLBACK = + new EndTransactionCallback() { + @Override + public void onSuccess() {} + + @Override + public void onFailure() {} + }; + + /** + * Restores active transaction state on this connection and resets {@code retryUnitOfWork} before + * {@link TransactionRunnerImpl} executes a retry attempt. + */ void resetForRetry(UnitOfWork retryUnitOfWork) { - retryUnitOfWork.resetForRetry(); - this.currentUnitOfWork = retryUnitOfWork; + if (isBatchActive()) { + abortBatch(); + } + if (this.currentUnitOfWork != null && this.currentUnitOfWork != retryUnitOfWork) { + closeUnitOfWorkAfterRunnerFailure(this.currentUnitOfWork); + } + this.connectionState.rollback(); + this.statementTag = null; + this.transactionBeginMarked = true; + clearLastTransactionAndSetDefaultTransactionOptions(getDefaultIsolationLevel()); + if (isAutocommit()) { + this.inTransaction = true; + } + // Re-apply local RETRY_ABORTS_INTERNALLY = false while currentUnitOfWork is null (cleared + // above) and isInTransaction() is true, before re-attaching retryUnitOfWork. + setLocalConnectionPropertyValue(RETRY_ABORTS_INTERNALLY, false); + try { + retryUnitOfWork.resetForRetry(); + } finally { + this.currentUnitOfWork = retryUnitOfWork; + } + } + + /** + * Cleans up connection transaction state and closes any active or kept-open aborted unit of work + * when {@link TransactionRunnerImpl} exits with a non-retryable failure or interruption. + */ + void rollbackAfterAbort(UnitOfWork transaction) { + if (isBatchActive()) { + try { + abortBatch(); + } catch (Throwable ignored) { + // Best-effort batch cleanup so the primary failure is never masked. + } + } + UnitOfWork current = this.currentUnitOfWork; + this.connectionState.rollback(); + this.statementTag = null; + this.transactionBeginMarked = false; + clearLastTransactionAndSetDefaultTransactionOptions(getDefaultIsolationLevel()); + if (isAutocommit()) { + this.inTransaction = false; + } + closeUnitOfWorkAfterRunnerFailure(transaction); + if (current != null && current != transaction) { + closeUnitOfWorkAfterRunnerFailure(current); + } + } + + private void closeUnitOfWorkAfterRunnerFailure(@Nullable UnitOfWork uow) { + if (uow instanceof AbstractMultiUseTransaction) { + boolean wasInterrupted = Thread.interrupted(); + try { + // Drain any in-flight statement or commit task on the single-threaded statementExecutor so + // COMMITTING settles to COMMITTED, COMMIT_FAILED, or ABORTED before inspecting state. + try { + Uninterruptibles.getUninterruptibly( + ((AbstractBaseUnitOfWork) uow).getStatementExecutor().submit(() -> null)); + } catch (Throwable ignored) { + // Best-effort barrier; proceed to state check. + } + UnitOfWorkState state = uow.getState(); + if (state == UnitOfWorkState.STARTED || state == UnitOfWorkState.ABORTED) { + try { + Uninterruptibles.getUninterruptibly( + uow.rollbackAsync(CallType.SYNC, NO_OP_END_TRANSACTION_CALLBACK)); + } catch (Throwable ignored) { + // Best-effort rollback/close so the caller's primary exception is never masked. + } + } + } finally { + if (wasInterrupted) { + Thread.currentThread().interrupt(); + } + } + } } @Override @@ -2334,6 +2422,7 @@ UnitOfWork createNewUnitOfWork( getConnectionPropertyValue(DELAY_TRANSACTION_START_UNTIL_FIRST_WRITE)) .setKeepTransactionAlive(getConnectionPropertyValue(KEEP_TRANSACTION_ALIVE)) .setRetryAbortsInternally(getConnectionPropertyValue(RETRY_ABORTS_INTERNALLY)) + .setKeepAbortedTransactionOpen(this.transactionRunnerActive) .setSavepointSupport(getConnectionPropertyValue(SAVEPOINT_SUPPORT)) .setReturnCommitStats(getConnectionPropertyValue(RETURN_COMMIT_STATS)) .setMaxCommitDelay(getConnectionPropertyValue(MAX_COMMIT_DELAY)) diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ReadWriteTransaction.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ReadWriteTransaction.java index 594e8c6f6dde..1f3248b3d9aa 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ReadWriteTransaction.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ReadWriteTransaction.java @@ -50,6 +50,7 @@ import com.google.cloud.spanner.ThreadFactoryUtil; import com.google.cloud.spanner.TransactionContext; import com.google.cloud.spanner.TransactionManager; +import com.google.cloud.spanner.TransactionManager.TransactionState; import com.google.cloud.spanner.connection.AbstractStatementParser.ParsedStatement; import com.google.cloud.spanner.connection.AbstractStatementParser.StatementType; import com.google.cloud.spanner.connection.TransactionRetryListener.RetryResult; @@ -164,6 +165,7 @@ static ScheduledThreadPoolExecutor getKeepAliveService() { private final TransactionOption[] transactionOptions; private TransactionManager txManager; private final boolean retryAbortsInternally; + private final boolean keepAbortedTransactionOpen; private final boolean delayTransactionStartUntilFirstWrite; private final boolean keepTransactionAlive; private final long keepAliveIntervalMillis; @@ -218,6 +220,7 @@ static class Builder extends AbstractMultiUseTransaction.Builder executeQueryAsync( () -> { checkTimedOut(); checkAborted(); - return DirectExecuteResultSet.ofResultSet( - internalExecuteQuery(statement, analyzeMode, options)); + try { + return DirectExecuteResultSet.ofResultSet( + internalExecuteQuery(statement, analyzeMode, options)); + } catch (AbortedException aborted) { + if (keepAbortedTransactionOpen) { + this.state = UnitOfWorkState.ABORTED; + this.abortedException = aborted; + } + throw aborted; + } }, SpannerGrpc.getExecuteStreamingSqlMethod()); } @@ -864,15 +896,23 @@ private ApiFuture> internalExecuteUpdateAsync( () -> { checkTimedOut(); checkAborted(); - if (analyzeMode == AnalyzeMode.NONE) { - return Tuple.of( - get(txContextFuture).executeUpdate(update.getStatement(), options), null); + try { + if (analyzeMode == AnalyzeMode.NONE) { + return Tuple.of( + get(txContextFuture).executeUpdate(update.getStatement(), options), null); + } + ResultSet resultSet = + get(txContextFuture) + .analyzeUpdateStatement( + update.getStatement(), analyzeMode.getQueryAnalyzeMode(), options); + return Tuple.of(null, resultSet); + } catch (AbortedException aborted) { + if (keepAbortedTransactionOpen) { + this.state = UnitOfWorkState.ABORTED; + this.abortedException = aborted; + } + throw aborted; } - ResultSet resultSet = - get(txContextFuture) - .analyzeUpdateStatement( - update.getStatement(), analyzeMode.getQueryAnalyzeMode(), options); - return Tuple.of(null, resultSet); }, SpannerGrpc.getExecuteSqlMethod()); } @@ -932,7 +972,15 @@ public ApiFuture executeBatchUpdateAsync( () -> { checkTimedOut(); checkAborted(); - return get(txContextFuture).batchUpdate(updateStatements); + try { + return get(txContextFuture).batchUpdate(updateStatements); + } catch (AbortedException aborted) { + if (keepAbortedTransactionOpen) { + this.state = UnitOfWorkState.ABORTED; + this.abortedException = aborted; + } + throw aborted; + } }, SpannerGrpc.getExecuteBatchDmlMethod()); } @@ -990,6 +1038,7 @@ public ApiFuture commitAsync( state = UnitOfWorkState.COMMITTED; res = SettableApiFuture.create(); ((SettableApiFuture) res).set(null); + asyncEndUnitOfWorkSpan(); } else if (retryAbortsInternally) { res = executeStatementAsync( @@ -1024,32 +1073,45 @@ public ApiFuture commitAsync( }, InterceptorsUsage.IGNORE_INTERCEPTORS, ImmutableList.of(SpannerGrpc.getCommitMethod())); + asyncEndUnitOfWorkSpan(); } else { res = executeStatementAsync( callType, COMMIT_STATEMENT, () -> { - checkTimedOut(); try { + checkTimedOut(); Void result = commitCallable.call(); callback.onSuccess(); + if (span != null) { + span.end(); + } return result; } catch (Throwable t) { commitResponseFuture.setException(t); callback.onFailure(); - state = UnitOfWorkState.COMMIT_FAILED; - try { - txManager.close(); - } catch (Throwable t2) { - // Ignore. + if (keepAbortedTransactionOpen && t instanceof AbortedException) { + // Under TransactionRunnerImpl, leave txManager and unit-of-work span open so + // resetForRetry() can reuse them or rollbackAfterAbort() can close them once. + state = UnitOfWorkState.ABORTED; + abortedException = (AbortedException) t; + } else { + state = UnitOfWorkState.COMMIT_FAILED; + try { + txManager.close(); + } catch (Throwable t2) { + // Ignore. + } + if (span != null) { + span.end(); + } } throw t; } }, SpannerGrpc.getCommitMethod()); } - asyncEndUnitOfWorkSpan(); return res; } } @@ -1277,10 +1339,13 @@ private void handleAborted(AbortedException aborted) { } } } else { - try { - txManager.close(); - } catch (Throwable t) { - // ignore + if (!keepAbortedTransactionOpen && txManager != null) { + try { + txManager.close(); + } catch (Throwable t) { + // ignore + } + txManager = null; } // Internal retry is not enabled. markAborted(aborted); @@ -1368,18 +1433,45 @@ private ApiFuture rollbackAsync(CallType callType, boolean updateStatusAnd if (updateStatusAndEndSpan) { state = UnitOfWorkState.ROLLED_BACK; } - if (txContextFuture != null && !isAborted) { - ApiFuture result = - executeStatementAsync( - callType, ROLLBACK_STATEMENT, rollbackCallable, SpannerGrpc.getRollbackMethod()); + if (isAborted) { if (updateStatusAndEndSpan) { - // Note: We end the transaction span after executing the rollback to include the rollback in - // the transaction span. Even though both methods are executed asynchronously, they are both - // executed using the same single-threaded executor, meaning that the span will only be - // ended after the rollback has finished. - asyncEndUnitOfWorkSpan(); + // TransactionManagerImpl.rollback() requires STARTED; close any kept-open aborted txManager + // to release its session and span, then end the unit-of-work span before completing. + return getStatementExecutor() + .submit( + () -> { + if (txManager != null) { + try { + txManager.close(); + } catch (Throwable ignored) { + // Ignore cleanup errors on an already-aborted transaction manager. + } + txManager = null; + } + if (span != null) { + span.end(); + } + return null; + }); } - return result; + return ApiFutures.immediateFuture(null); + } + if (txContextFuture != null) { + return executeStatementAsync( + callType, + ROLLBACK_STATEMENT, + () -> { + try { + return rollbackCallable.call(); + } finally { + if (updateStatusAndEndSpan && span != null) { + // End the unit-of-work span inside the rollback task so callers awaiting the + // returned future observe deterministic span completion. + span.end(); + } + } + }, + SpannerGrpc.getRollbackMethod()); } else if (updateStatusAndEndSpan) { return asyncEndUnitOfWorkSpan(); } else { @@ -1387,9 +1479,48 @@ private ApiFuture rollbackAsync(CallType callType, boolean updateStatusAnd } } + /** + * Resets this read/write transaction for a new attempt by {@link TransactionRunnerImpl}, reusing + * an aborted {@link TransactionManager} when available or closing an un-aborted manager, and + * clearing buffered per-attempt collections and state. + */ @Override public void resetForRetry() { - txContextFuture = ApiFutures.immediateFuture(txManager.resetForRetry()); + cancelScheduledKeepAlivePing(); + if (txManager != null) { + boolean resetSucceeded = false; + if (state == UnitOfWorkState.ABORTED + || this.abortedException != null + || txManager.getState() == TransactionState.ABORTED) { + try { + txContextFuture = ApiFutures.immediateFuture(txManager.resetForRetry()); + resetSucceeded = true; + } catch (IllegalStateException ignored) { + // Synthetic aborts may mark ReadWriteTransaction as ABORTED without aborting txManager. + } + } + if (!resetSucceeded) { + try { + txManager.close(); + } catch (Throwable ignored) { + // Ignore cleanup errors when closing an un-aborted transaction manager. + } + txManager = null; + txContextFuture = null; + } + } else { + txContextFuture = null; + } + canUseSingleUseRead = false; + commitResponseFuture = null; + abortedException = null; + rolledBackToSavepointException = null; + autoSavepoint = null; + transactionStarted = null; + statements.clear(); + mutations.clear(); + clearSavepoints(); + state = UnitOfWorkState.STARTED; } @Override diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/TransactionRunnerImpl.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/TransactionRunnerImpl.java index 504b084dba32..39a43a6a40bb 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/TransactionRunnerImpl.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/TransactionRunnerImpl.java @@ -23,38 +23,73 @@ import com.google.cloud.spanner.connection.Connection.TransactionCallable; import com.google.cloud.spanner.connection.ConnectionImpl.Caller; import com.google.cloud.spanner.connection.UnitOfWork.CallType; +import com.google.cloud.spanner.connection.UnitOfWork.UnitOfWorkState; +import java.util.concurrent.ThreadLocalRandom; class TransactionRunnerImpl { + private static final long MAX_FALLBACK_BACKOFF_MILLIS = 32_000L; + private final ConnectionImpl connection; TransactionRunnerImpl(ConnectionImpl connection) { this.connection = connection; } + /** + * Computes jittered exponential backoff in milliseconds when an {@link AbortedException} does not + * carry a {@code RetryInfo} delay ({@code getRetryDelayInMillis() < 0}), preventing a tight 0ms + * CPU spin loop on repeated synthetic aborts while keeping initial retry latency minimal. + */ + static long nextFallbackBackoffMillis(int fallbackAttempt) { + long maxWindowMillis = + Math.min(MAX_FALLBACK_BACKOFF_MILLIS, 1L << Math.min(fallbackAttempt + 1, 15)); + long halfWindowMillis = Math.max(1L, maxWindowMillis / 2L); + return halfWindowMillis + ThreadLocalRandom.current().nextLong(halfWindowMillis + 1L); + } + T run(TransactionCallable callable) { connection.beginTransaction(); // Disable internal retries during this transaction. connection.setRetryAbortsInternally(/* retryAbortsInternally= */ false, /* local= */ true); UnitOfWork transaction = connection.getCurrentUnitOfWorkOrStartNewUnitOfWork(); + int fallbackAttempts = 0; while (true) { try { T result = callable.run(connection); + if (transaction.getState() == UnitOfWorkState.ABORTED + && transaction instanceof ReadWriteTransaction) { + // Re-surface a caught mid-callable abort before commitAsync so an orphaned replacement + // ReadWriteTransaction is never silently committed, preserving original RetryInfo. + AbortedException storedAbort = ((ReadWriteTransaction) transaction).getAbortedException(); + if (storedAbort != null) { + throw storedAbort; + } + ((ReadWriteTransaction) transaction).checkAborted(); + } get(connection.commitAsync(CallType.SYNC, Caller.TRANSACTION_RUNNER)); return result; } catch (AbortedException abortedException) { try { + long retryDelayMillis = abortedException.getRetryDelayInMillis(); + long sleepMillis; + if (retryDelayMillis >= 0L) { + fallbackAttempts = 0; + sleepMillis = retryDelayMillis; + } else { + sleepMillis = nextFallbackBackoffMillis(fallbackAttempts++); + } //noinspection BusyWait - Thread.sleep(abortedException.getRetryDelayInMillis()); + Thread.sleep(sleepMillis); connection.resetForRetry(transaction); } catch (InterruptedException interruptedException) { - connection.rollbackAsync(CallType.SYNC, Caller.TRANSACTION_RUNNER); + connection.rollbackAfterAbort(transaction); throw SpannerExceptionFactory.propagateInterrupt(interruptedException); } catch (Throwable t) { - connection.rollbackAsync(CallType.SYNC, Caller.TRANSACTION_RUNNER); + connection.rollbackAfterAbort(transaction); throw t; } } catch (Throwable t) { - connection.rollbackAsync(CallType.SYNC, Caller.TRANSACTION_RUNNER); + connection.rollbackAfterAbort(transaction); throw t; } } diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/RunTransactionMockServerTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/RunTransactionMockServerTest.java index d4af5dd4e7e8..6d5212ef8055 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/RunTransactionMockServerTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/RunTransactionMockServerTest.java @@ -18,24 +18,50 @@ import static com.google.cloud.spanner.connection.ConnectionProperties.DEFAULT_ISOLATION_LEVEL; import static com.google.cloud.spanner.connection.ConnectionProperties.READ_LOCK_MODE; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; +import com.google.api.core.ApiFuture; +import com.google.cloud.Timestamp; +import com.google.cloud.spanner.AbortedException; import com.google.cloud.spanner.ErrorCode; import com.google.cloud.spanner.MockSpannerServiceImpl.SimulatedExecutionTime; +import com.google.cloud.spanner.Mutation; import com.google.cloud.spanner.ResultSet; +import com.google.cloud.spanner.SpannerApiFutures; import com.google.cloud.spanner.SpannerException; +import com.google.cloud.spanner.SpannerExceptionFactory; +import com.google.cloud.spanner.SpannerOptions; +import com.google.cloud.spanner.SpannerOptionsTestHelper; +import com.google.cloud.spanner.connection.UnitOfWork.UnitOfWorkState; +import com.google.rpc.RetryInfo; import com.google.spanner.v1.BeginTransactionRequest; import com.google.spanner.v1.CommitRequest; +import com.google.spanner.v1.ExecuteBatchDmlRequest; import com.google.spanner.v1.ExecuteSqlRequest; import com.google.spanner.v1.RollbackRequest; import com.google.spanner.v1.TransactionOptions; import com.google.spanner.v1.TransactionOptions.IsolationLevel; import com.google.spanner.v1.TransactionOptions.ReadWrite.ReadLockMode; +import io.grpc.Metadata; import io.grpc.Status; +import io.grpc.protobuf.ProtoUtils; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.util.List; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -279,4 +305,750 @@ public void testRollbackInRunTransaction() { assertEquals(1, mockSpanner.countRequestsOfType(CommitRequest.class)); assertEquals(0, mockSpanner.countRequestsOfType(RollbackRequest.class)); } + + @Test + public void testRunTransactionCommitAbortedWithAutocommit() { + for (IsolationLevel isolationLevel : DEFAULT_ISOLATION_LEVEL.getValidValues()) { + for (ReadLockMode readLockMode : READ_LOCK_MODE.getValidValues()) { + final AtomicInteger attempts = new AtomicInteger(); + final AtomicReference firstAttemptUow = new AtomicReference<>(); + try (Connection connection = createConnection()) { + connection.setAutocommit(true); + connection.setDefaultIsolationLevel(isolationLevel); + connection.setReadLockMode(readLockMode); + connection.runTransaction( + transaction -> { + int attempt = attempts.incrementAndGet(); + assertTrue(transaction.isInTransaction()); + assertFalse(transaction.isRetryAbortsInternally()); + UnitOfWork currentUow = + ((ConnectionImpl) transaction).getCurrentUnitOfWorkOrStartNewUnitOfWork(); + if (attempt == 1) { + firstAttemptUow.set(currentUow); + } else { + assertSame(firstAttemptUow.get(), currentUow); + } + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + if (attempt == 1) { + mockSpanner.abortNextStatement(); + } + return null; + }); + assertTrue(connection.isAutocommit()); + assertFalse(connection.isInTransaction()); + assertTrue(connection.isRetryAbortsInternally()); + } + assertEquals(2, attempts.get()); + List sqlRequests = + mockSpanner.getRequestsOfType(ExecuteSqlRequest.class); + assertEquals(4, sqlRequests.size()); + assertEquals(2, mockSpanner.countRequestsOfType(CommitRequest.class)); + assertEquals(0, mockSpanner.countRequestsOfType(BeginTransactionRequest.class)); + + for (ExecuteSqlRequest request : sqlRequests) { + assertTrue(request.hasTransaction()); + assertFalse(request.getTransaction().hasSingleUse()); + } + assertTrue(sqlRequests.get(0).getTransaction().hasBegin()); + assertTrue(sqlRequests.get(1).getTransaction().hasId()); + assertTrue(sqlRequests.get(2).getTransaction().hasBegin()); + assertTrue(sqlRequests.get(3).getTransaction().hasId()); + for (int i : new int[] {0, 2}) { + TransactionOptions options = sqlRequests.get(i).getTransaction().getBegin(); + assertEquals(isolationLevel, options.getIsolationLevel()); + assertEquals(readLockMode, options.getReadWrite().getReadLockMode()); + } + + mockSpanner.clearRequests(); + } + } + } + + @Test + public void testRunTransactionCommitAbortedThenCallableFailsWithAutocommit() { + final AtomicInteger attempts = new AtomicInteger(); + try (Connection connection = createConnection()) { + connection.setAutocommit(true); + IllegalStateException exception = + assertThrows( + IllegalStateException.class, + () -> + connection.runTransaction( + transaction -> { + int attempt = attempts.incrementAndGet(); + assertTrue(transaction.isInTransaction()); + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + if (attempt == 1) { + mockSpanner.abortNextStatement(); + return null; + } + throw new IllegalStateException("callable failure on retry"); + })); + assertEquals("callable failure on retry", exception.getMessage()); + assertTrue(connection.isAutocommit()); + assertFalse(connection.isInTransaction()); + assertFalse(connection.isTransactionStarted()); + assertTrue(connection.isRetryAbortsInternally()); + } + assertEquals(2, attempts.get()); + List sqlRequests = mockSpanner.getRequestsOfType(ExecuteSqlRequest.class); + assertEquals(2, sqlRequests.size()); + for (ExecuteSqlRequest request : sqlRequests) { + assertTrue(request.hasTransaction()); + assertTrue(request.getTransaction().hasBegin()); + assertFalse(request.getTransaction().hasSingleUse()); + } + assertEquals(1, mockSpanner.countRequestsOfType(CommitRequest.class)); + assertEquals(1, mockSpanner.countRequestsOfType(RollbackRequest.class)); + } + + @Test + public void testRunTransactionCommitAbortedResetsSavepointsMutationsAndLocalState() { + final AtomicInteger retryListenerCalls = new AtomicInteger(); + TransactionRetryListener listener = + new TransactionRetryListener() { + @Override + public void retryStarting( + Timestamp transactionStarted, long transactionId, int retryAttempt) { + retryListenerCalls.incrementAndGet(); + } + + @Override + public void retryFinished( + Timestamp transactionStarted, + long transactionId, + int retryAttempt, + RetryResult result) { + retryListenerCalls.incrementAndGet(); + } + }; + + try (Connection connection = createConnection()) { + connection.addTransactionRetryListener(listener); + + // 1. Statement ABORTED retry (autocommit=false): savepoints, mutations, and local state + // reset. + final AtomicInteger statementAbortAttempts = new AtomicInteger(); + final AtomicReference firstUow = new AtomicReference<>(); + connection.runTransaction( + transaction -> { + int attempt = statementAbortAttempts.incrementAndGet(); + ConnectionImpl impl = (ConnectionImpl) transaction; + AbstractMultiUseTransaction uow = + (AbstractMultiUseTransaction) impl.getCurrentUnitOfWorkOrStartNewUnitOfWork(); + assertFalse(transaction.isRetryAbortsInternally()); + if (attempt == 1) { + firstUow.set(uow); + assertFalse(transaction.isReturnCommitStats()); + impl.setReturnCommitStats(true, /* local= */ true); + assertTrue(transaction.isReturnCommitStats()); + transaction.savepoint("sp_stmt"); + transaction.bufferedWrite(Mutation.newInsertBuilder("TEST").set("ID").to(1L).build()); + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + mockSpanner.abortNextStatement(); + transaction.executeUpdate(INSERT_STATEMENT); + return null; + } + assertSame(firstUow.get(), uow); + assertFalse(transaction.isReturnCommitStats()); + assertTrue(uow.getSavepoints().isEmpty()); + transaction.savepoint("sp_stmt"); + transaction.bufferedWrite(Mutation.newInsertBuilder("TEST").set("ID").to(2L).build()); + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + return null; + }); + assertEquals(2, statementAbortAttempts.get()); + assertEquals(0, retryListenerCalls.get()); + List stmtCommitRequests = mockSpanner.getRequestsOfType(CommitRequest.class); + assertEquals(1, stmtCommitRequests.size()); + assertFalse(stmtCommitRequests.get(0).getReturnCommitStats()); + assertEquals(1, stmtCommitRequests.get(0).getMutationsCount()); + assertEquals( + "2", + stmtCommitRequests + .get(0) + .getMutations(0) + .getInsert() + .getValues(0) + .getValues(0) + .getStringValue()); + + // 2. Commit ABORTED retry (autocommit=true): savepoints, mutations, and local state reset. + mockSpanner.clearRequests(); + connection.setAutocommit(true); + final AtomicInteger commitAbortAttempts = new AtomicInteger(); + final AtomicReference firstCommitUow = new AtomicReference<>(); + connection.runTransaction( + transaction -> { + int attempt = commitAbortAttempts.incrementAndGet(); + ConnectionImpl impl = (ConnectionImpl) transaction; + AbstractMultiUseTransaction uow = + (AbstractMultiUseTransaction) impl.getCurrentUnitOfWorkOrStartNewUnitOfWork(); + assertFalse(transaction.isRetryAbortsInternally()); + if (attempt == 1) { + firstCommitUow.set(uow); + assertFalse(transaction.isReturnCommitStats()); + impl.setReturnCommitStats(true, /* local= */ true); + assertTrue(transaction.isReturnCommitStats()); + transaction.savepoint("sp_commit"); + transaction.write(Mutation.newInsertBuilder("TEST").set("ID").to(10L).build()); + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + mockSpanner.abortNextStatement(); + return null; + } + assertSame(firstCommitUow.get(), uow); + assertFalse(transaction.isReturnCommitStats()); + assertTrue(uow.getSavepoints().isEmpty()); + transaction.savepoint("sp_commit"); + transaction.write(Mutation.newInsertBuilder("TEST").set("ID").to(20L).build()); + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + return null; + }); + assertEquals(2, commitAbortAttempts.get()); + assertEquals(0, retryListenerCalls.get()); + List commitRequests = mockSpanner.getRequestsOfType(CommitRequest.class); + assertEquals(2, commitRequests.size()); + assertFalse(commitRequests.get(0).getReturnCommitStats()); + assertFalse(commitRequests.get(1).getReturnCommitStats()); + assertEquals(1, commitRequests.get(1).getMutationsCount()); + assertEquals( + "20", + commitRequests + .get(1) + .getMutations(0) + .getInsert() + .getValues(0) + .getValues(0) + .getStringValue()); + } + } + + @Test + public void testRunTransactionNonAbortFailuresAndInterruptedBackoff() { + try (Connection connection = createConnection()) { + connection.setAutocommit(true); + + // 1. Non-abort Commit RPC failure propagates original exception and resets state. + mockSpanner.setCommitExecutionTime( + SimulatedExecutionTime.ofException( + Status.PERMISSION_DENIED.withDescription("commit denied").asRuntimeException())); + SpannerException commitFailure = + assertThrows( + SpannerException.class, + () -> + connection.runTransaction( + transaction -> { + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + return null; + })); + assertEquals(ErrorCode.PERMISSION_DENIED, commitFailure.getErrorCode()); + assertTrue(commitFailure.getMessage(), commitFailure.getMessage().contains("commit denied")); + assertFalse(connection.isInTransaction()); + assertFalse(connection.isTransactionStarted()); + assertTrue(connection.isRetryAbortsInternally()); + mockSpanner.removeAllExecutionTimes(); + mockSpanner.clearRequests(); + + // 2a. Callable catches statement AbortedException and throws non-abort exception. + IllegalArgumentException caughtAbortFailure = + assertThrows( + IllegalArgumentException.class, + () -> + connection.runTransaction( + transaction -> { + mockSpanner.abortNextStatement(); + try { + transaction.executeUpdate(INSERT_STATEMENT); + } catch (SpannerException ignored) { + // Intentionally swallow abort and throw application error. + } + throw new IllegalArgumentException("failed after caught abort"); + })); + assertEquals("failed after caught abort", caughtAbortFailure.getMessage()); + assertEquals(0, mockSpanner.countRequestsOfType(RollbackRequest.class)); + assertFalse(connection.isInTransaction()); + assertTrue(connection.isRetryAbortsInternally()); + mockSpanner.clearRequests(); + + // 2b. Callable catches statement AbortedException and returns normally: runner detects + // aborted transaction before commit and retries cleanly. + final AtomicInteger swallowedAbortAttempts = new AtomicInteger(); + String recovered = + connection.runTransaction( + transaction -> { + if (swallowedAbortAttempts.incrementAndGet() == 1) { + mockSpanner.abortNextStatement(); + try { + transaction.executeUpdate(INSERT_STATEMENT); + } catch (SpannerException ignored) { + // Intentionally swallow abort and return normally. + } + return "stale"; + } + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + return "recovered"; + }); + assertEquals("recovered", recovered); + assertEquals(2, swallowedAbortAttempts.get()); + assertEquals(1, mockSpanner.countRequestsOfType(CommitRequest.class)); + mockSpanner.clearRequests(); + + // 3. Interrupted commit during Commit abort cleanup propagates CANCELLED without + // FAILED_PRECONDITION and leaves connection usable. + try { + SpannerException interrupted = + assertThrows( + SpannerException.class, + () -> + connection.runTransaction( + transaction -> { + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + mockSpanner.abortNextStatement(); + Thread.currentThread().interrupt(); + return null; + })); + assertEquals(ErrorCode.CANCELLED, interrupted.getErrorCode()); + assertTrue(Thread.interrupted()); + } finally { + Thread.interrupted(); + } + assertFalse(connection.isInTransaction()); + assertFalse(connection.isTransactionStarted()); + assertTrue(connection.isRetryAbortsInternally()); + + mockSpanner.clearRequests(); + connection.runTransaction( + transaction -> { + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + return null; + }); + assertEquals(1, mockSpanner.countRequestsOfType(CommitRequest.class)); + } + } + + @Test + public void testRunTransactionSpanAndSessionLifecycleOnAborts() { + SpannerOptionsTestHelper.resetActiveTracingFramework(); + SpannerOptions.enableOpenTelemetryTraces(); + InMemorySpanExporter spanExporter = InMemorySpanExporter.create(); + SdkTracerProvider tracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(spanExporter)) + .build(); + OpenTelemetrySdk openTelemetry = + OpenTelemetrySdk.builder() + .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance())) + .setTracerProvider(tracerProvider) + .build(); + + try { + try (Connection connection = createTracingConnection(openTelemetry, ";autocommit=true")) { + // 1. Commit-abort retry: 0 finished transaction spans mid-retry, 1 of each after commit. + final AtomicInteger commitRetryAttempts = new AtomicInteger(); + connection.runTransaction( + transaction -> { + int attempt = commitRetryAttempts.incrementAndGet(); + if (attempt == 2) { + assertEquals( + 0L, countFinishedSpans(spanExporter, "CloudSpannerJdbc.ReadWriteTransaction")); + assertEquals( + 0L, countFinishedSpans(spanExporter, "CloudSpanner.ReadWriteTransaction")); + } + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + if (attempt == 1) { + mockSpanner.abortNextStatement(); + } + return null; + }); + assertEquals(2, commitRetryAttempts.get()); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpannerJdbc.ReadWriteTransaction")); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpanner.ReadWriteTransaction")); + + // 2. Statement-abort retry: 0 finished transaction spans mid-retry, 1 of each after commit. + spanExporter.reset(); + mockSpanner.clearRequests(); + final AtomicInteger stmtRetryAttempts = new AtomicInteger(); + connection.runTransaction( + transaction -> { + int attempt = stmtRetryAttempts.incrementAndGet(); + if (attempt == 2) { + assertEquals( + 0L, countFinishedSpans(spanExporter, "CloudSpannerJdbc.ReadWriteTransaction")); + assertEquals( + 0L, countFinishedSpans(spanExporter, "CloudSpanner.ReadWriteTransaction")); + } + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + if (attempt == 1) { + mockSpanner.abortNextStatement(); + } + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + return null; + }); + assertEquals(2, stmtRetryAttempts.get()); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpannerJdbc.ReadWriteTransaction")); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpanner.ReadWriteTransaction")); + + // 3. Caught-abort failure: exactly 1 finished unit-of-work and TransactionManager span. + spanExporter.reset(); + mockSpanner.clearRequests(); + assertThrows( + IllegalStateException.class, + () -> + connection.runTransaction( + transaction -> { + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + mockSpanner.abortNextStatement(); + try { + transaction.executeUpdate(INSERT_STATEMENT); + } catch (SpannerException ignored) { + // Swallow abort and fail with non-abort exception. + } + throw new IllegalStateException("caught abort failure"); + })); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpannerJdbc.ReadWriteTransaction")); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpanner.ReadWriteTransaction")); + + // 4. Interrupted commit: exactly 1 finished unit-of-work and TransactionManager span. + spanExporter.reset(); + mockSpanner.clearRequests(); + try { + SpannerException interrupted = + assertThrows( + SpannerException.class, + () -> + connection.runTransaction( + transaction -> { + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + mockSpanner.abortNextStatement(); + Thread.currentThread().interrupt(); + return null; + })); + assertEquals(ErrorCode.CANCELLED, interrupted.getErrorCode()); + } finally { + Thread.interrupted(); + } + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpannerJdbc.ReadWriteTransaction")); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpanner.ReadWriteTransaction")); + } + } finally { + SpannerPool.closeSpannerPool(); + openTelemetry.close(); + SpannerOptionsTestHelper.resetActiveTracingFramework(); + } + } + + @Test + public void testRunTransactionSyntheticAbortDelayedStartAndActiveDmlBatch() { + try (Connection connection = createConnection()) { + connection.setAutocommit(true); + + // 1. Pre-statement synthetic AbortedException (txManager == null). + final AtomicInteger preStmtAttempts = new AtomicInteger(); + String preStmtResult = + connection.runTransaction( + transaction -> { + if (preStmtAttempts.incrementAndGet() == 1) { + throw SpannerExceptionFactory.newSpannerException( + ErrorCode.ABORTED, "synthetic pre-statement abort"); + } + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + return "pre-stmt-ok"; + }); + assertEquals("pre-stmt-ok", preStmtResult); + assertEquals(2, preStmtAttempts.get()); + assertEquals(1, mockSpanner.countRequestsOfType(ExecuteSqlRequest.class)); + assertEquals(1, mockSpanner.countRequestsOfType(CommitRequest.class)); + assertEquals(0, mockSpanner.countRequestsOfType(RollbackRequest.class)); + mockSpanner.clearRequests(); + + // 2. Synthetic AbortedException thrown by callable after INSERT (rolls back STARTED + // txManager). + final AtomicInteger postInsertAttempts = new AtomicInteger(); + connection.runTransaction( + transaction -> { + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + if (postInsertAttempts.incrementAndGet() == 1) { + throw SpannerExceptionFactory.newSpannerException( + ErrorCode.ABORTED, "synthetic post-insert abort"); + } + return null; + }); + assertEquals(2, postInsertAttempts.get()); + assertEquals(2, mockSpanner.countRequestsOfType(ExecuteSqlRequest.class)); + assertEquals(1, mockSpanner.countRequestsOfType(RollbackRequest.class)); + assertEquals(1, mockSpanner.countRequestsOfType(CommitRequest.class)); + mockSpanner.clearRequests(); + + // 3. delayTransactionStartUntilFirstWrite=true read-then-abort (txManager remains null on + // Attempt 1). + connection.setDelayTransactionStartUntilFirstWrite(true); + try { + final AtomicInteger delayedAttempts = new AtomicInteger(); + connection.runTransaction( + transaction -> { + try (ResultSet rs = transaction.executeQuery(SELECT1_STATEMENT)) { + assertTrue(rs.next()); + } + if (delayedAttempts.incrementAndGet() == 1) { + throw SpannerExceptionFactory.newSpannerException( + ErrorCode.ABORTED, "synthetic delayed-start abort"); + } + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + return null; + }); + assertEquals(2, delayedAttempts.get()); + assertEquals(1, mockSpanner.countRequestsOfType(CommitRequest.class)); + assertEquals(0, mockSpanner.countRequestsOfType(RollbackRequest.class)); + } finally { + connection.setDelayTransactionStartUntilFirstWrite(false); + } + mockSpanner.clearRequests(); + + // 4a. Abort while DML batch is active resets batch state and succeeds on retry. + final AtomicInteger batchAbortAttempts = new AtomicInteger(); + connection.runTransaction( + transaction -> { + transaction.startBatchDml(); + transaction.executeUpdate(INSERT_STATEMENT); + if (batchAbortAttempts.incrementAndGet() == 1) { + throw SpannerExceptionFactory.newSpannerException( + ErrorCode.ABORTED, "abort during active batch"); + } + assertArrayEquals(new long[] {1L}, transaction.runBatch()); + return null; + }); + assertFalse(connection.isDmlBatchActive()); + assertEquals(2, batchAbortAttempts.get()); + assertEquals(1, mockSpanner.countRequestsOfType(ExecuteBatchDmlRequest.class)); + assertEquals(1, mockSpanner.countRequestsOfType(CommitRequest.class)); + mockSpanner.clearRequests(); + + // 4b. Non-abort failure while DML batch is active aborts batch and rolls back transaction. + IllegalStateException batchFailure = + assertThrows( + IllegalStateException.class, + () -> + connection.runTransaction( + transaction -> { + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + transaction.startBatchDml(); + transaction.executeUpdate(INSERT_STATEMENT); + throw new IllegalStateException("failure during active batch"); + })); + assertEquals("failure during active batch", batchFailure.getMessage()); + assertFalse(connection.isDmlBatchActive()); + assertFalse(connection.isInTransaction()); + assertEquals(1, mockSpanner.countRequestsOfType(RollbackRequest.class)); + } + } + + @Test + public void testApplicationCommitAbortClosesTransaction() { + SpannerOptionsTestHelper.resetActiveTracingFramework(); + SpannerOptions.enableOpenTelemetryTraces(); + InMemorySpanExporter spanExporter = InMemorySpanExporter.create(); + SdkTracerProvider tracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(spanExporter)) + .build(); + OpenTelemetrySdk openTelemetry = + OpenTelemetrySdk.builder() + .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance())) + .setTracerProvider(tracerProvider) + .build(); + + try { + try (Connection connection = createTracingConnection(openTelemetry, "")) { + connection.setRetryAbortsInternally(false); + assertFalse(connection.isRetryAbortsInternally()); + // 1. Direct application commit() outside runTransaction transitions to COMMIT_FAILED and + // closes txManager + unit-of-work span. + assertEquals(1L, connection.executeUpdate(INSERT_STATEMENT)); + UnitOfWork directUowBefore = + ((ConnectionImpl) connection).getCurrentUnitOfWorkOrStartNewUnitOfWork(); + mockSpanner.abortNextStatement(); + assertThrows(AbortedException.class, connection::commit); + assertEquals(UnitOfWorkState.COMMIT_FAILED, directUowBefore.getState()); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpannerJdbc.ReadWriteTransaction")); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpanner.ReadWriteTransaction")); + + // 2. Run runTransaction with commit abort on the same connection, then verify subsequent + // direct application commit() still closes transaction with COMMIT_FAILED. + spanExporter.reset(); + mockSpanner.clearRequests(); + connection.setAutocommit(true); + final AtomicInteger runnerAttempts = new AtomicInteger(); + connection.runTransaction( + transaction -> { + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + if (runnerAttempts.incrementAndGet() == 1) { + mockSpanner.abortNextStatement(); + } + return null; + }); + assertEquals(2, runnerAttempts.get()); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpannerJdbc.ReadWriteTransaction")); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpanner.ReadWriteTransaction")); + + spanExporter.reset(); + mockSpanner.clearRequests(); + connection.beginTransaction(); + assertFalse(connection.isRetryAbortsInternally()); + assertEquals(1L, connection.executeUpdate(INSERT_STATEMENT)); + UnitOfWork directUowAfter = + ((ConnectionImpl) connection).getCurrentUnitOfWorkOrStartNewUnitOfWork(); + mockSpanner.abortNextStatement(); + assertThrows(AbortedException.class, connection::commit); + assertEquals(UnitOfWorkState.COMMIT_FAILED, directUowAfter.getState()); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpannerJdbc.ReadWriteTransaction")); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpanner.ReadWriteTransaction")); + } + } finally { + SpannerPool.closeSpannerPool(); + openTelemetry.close(); + SpannerOptionsTestHelper.resetActiveTracingFramework(); + } + } + + @Test + public void testRunTransactionInterruptedDuringBackoffAfterCommitAbort() throws Exception { + Metadata trailers = new Metadata(); + trailers.put( + ProtoUtils.keyForProto(RetryInfo.getDefaultInstance()), + RetryInfo.newBuilder() + .setRetryDelay(com.google.protobuf.Duration.newBuilder().setSeconds(30L)) + .build()); + mockSpanner.setCommitExecutionTime( + SimulatedExecutionTime.ofException( + Status.ABORTED.withDescription("commit aborted").asRuntimeException(trailers))); + Thread runnerThread = Thread.currentThread(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30L); + // The 30s RetryInfo delay is the only timed wait the runner thread enters; interrupt it there. + Thread interrupter = + new Thread( + () -> { + while (runnerThread.getState() != Thread.State.TIMED_WAITING + && System.nanoTime() < deadline) { + Thread.yield(); + } + runnerThread.interrupt(); + }); + try (Connection connection = createConnection()) { + connection.setAutocommit(true); + interrupter.start(); + SpannerException exception = + assertThrows( + SpannerException.class, + () -> + connection.runTransaction( + transaction -> { + assertEquals(1L, transaction.executeUpdate(INSERT_STATEMENT)); + return null; + })); + assertEquals(ErrorCode.CANCELLED, exception.getErrorCode()); + assertTrue(Thread.interrupted()); + assertFalse(connection.isInTransaction()); + assertFalse(connection.isTransactionStarted()); + assertTrue(connection.isRetryAbortsInternally()); + } finally { + interrupter.join(); + Thread.interrupted(); + mockSpanner.removeAllExecutionTimes(); + } + assertEquals(1, mockSpanner.countRequestsOfType(CommitRequest.class)); + assertEquals(0, mockSpanner.countRequestsOfType(RollbackRequest.class)); + } + + @Test + public void testTimedOutAsyncStatementBeforeCommitEndsSpanAndClosesTransaction() { + SpannerOptionsTestHelper.resetActiveTracingFramework(); + SpannerOptions.enableOpenTelemetryTraces(); + InMemorySpanExporter spanExporter = InMemorySpanExporter.create(); + SdkTracerProvider tracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(spanExporter)) + .build(); + OpenTelemetrySdk openTelemetry = + OpenTelemetrySdk.builder() + .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance())) + .setTracerProvider(tracerProvider) + .build(); + + try { + try (Connection connection = createTracingConnection(openTelemetry, "")) { + connection.setRetryAbortsInternally(false); + assertEquals(1L, connection.executeUpdate(INSERT_STATEMENT)); + UnitOfWork uow = ((ConnectionImpl) connection).getCurrentUnitOfWorkOrStartNewUnitOfWork(); + + connection.setStatementTimeout(15L, TimeUnit.MILLISECONDS); + mockSpanner.freeze(); + ApiFuture timedOutDml; + ApiFuture queuedCommit; + try { + timedOutDml = connection.executeUpdateAsync(INSERT_STATEMENT); + queuedCommit = connection.commitAsync(); + SpannerException dmlTimeout = + assertThrows(SpannerException.class, () -> SpannerApiFutures.get(timedOutDml)); + assertEquals(ErrorCode.DEADLINE_EXCEEDED, dmlTimeout.getErrorCode()); + } finally { + mockSpanner.unfreeze(); + } + SpannerException commitFailure = + assertThrows(SpannerException.class, () -> SpannerApiFutures.get(queuedCommit)); + assertEquals(ErrorCode.FAILED_PRECONDITION, commitFailure.getErrorCode()); + assertEquals(UnitOfWorkState.COMMIT_FAILED, uow.getState()); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpannerJdbc.ReadWriteTransaction")); + assertEquals(1L, countFinishedSpans(spanExporter, "CloudSpanner.ReadWriteTransaction")); + } + } finally { + mockSpanner.unfreeze(); + SpannerPool.closeSpannerPool(); + openTelemetry.close(); + SpannerOptionsTestHelper.resetActiveTracingFramework(); + } + } + + @Test + public void testSyntheticAbortWithoutRetryInfoAppliesExponentialBackoff() { + try (Connection connection = createConnection()) { + connection.setAutocommit(true); + final AtomicInteger attempts = new AtomicInteger(); + long startNanos = System.nanoTime(); + String result = + connection.runTransaction( + transaction -> { + if (attempts.incrementAndGet() <= 6) { + throw SpannerExceptionFactory.newSpannerException( + ErrorCode.ABORTED, "synthetic abort without retry info"); + } + return "done"; + }); + long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos); + assertEquals("done", result); + assertEquals(7, attempts.get()); + assertTrue( + "Expected exponential fallback backoff (>= 50ms across 6 retries), got " + + elapsedMillis + + "ms", + elapsedMillis >= 50L); + } + } + + private Connection createTracingConnection( + OpenTelemetry openTelemetry, String additionalUrlOptions) { + return ConnectionOptions.newBuilder() + .setTracingPrefix("CloudSpannerJdbc") + .setOpenTelemetry(openTelemetry) + .setUri(getBaseUrl() + additionalUrlOptions) + .build() + .getConnection(); + } + + private static long countFinishedSpans(InMemorySpanExporter exporter, String spanName) { + assertEquals(CompletableResultCode.ofSuccess(), exporter.flush()); + return exporter.getFinishedSpanItems().stream() + .filter(span -> span.getName().equals(spanName)) + .count(); + } }