From e307dc5971e99d1a247724c1da5c12b694cc7b8f Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian <179120858+sakthivelmanii@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:27:43 +0000 Subject: [PATCH 1/2] chore(spanner): bound streaming server retry delays and fall back on non-positive values Sanitize server-supplied RetryInfo delays when resuming interrupted Cloud Spanner streaming reads and queries so zero, negative, or oversized retry trailers cannot trigger zero-delay retry spin loops or stall consumer threads for hours. - Fall back to exponential backoff (>= 1 ms) when a server retry delay is zero, sub-millisecond, or negative instead of retrying immediately without sleeping. - Cap positive server retry delays at the configured maximum retry delay on streaming retry settings (defaulting to 1000 ms when unset or non-positive) before evaluating the remaining total timeout budget. - Use the same 1000 ms ceiling for client-side exponential backoff when maximum retry delay is unset on streaming retry settings. --- .../spanner/ResumableStreamIterator.java | 79 +++- .../spanner/ResumableStreamIteratorTest.java | 386 +++++++++++++++++- 2 files changed, 424 insertions(+), 41 deletions(-) diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/ResumableStreamIterator.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/ResumableStreamIterator.java index e13245b1ef1d..c3c554668f11 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/ResumableStreamIterator.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/ResumableStreamIterator.java @@ -37,6 +37,7 @@ import io.grpc.Context; import io.opentelemetry.api.common.Attributes; import java.io.IOException; +import java.time.Duration; import java.util.ArrayDeque; import java.util.Set; import java.util.concurrent.CountDownLatch; @@ -62,7 +63,9 @@ * token that differs from the last seen resume token) resets both, so a long-running stream that * regularly makes progress is not terminated by an occasional transient error. The default settings * have no maximum number of attempts or total timeout. Setting maxAttempts to 1 disables streaming - * retries. + * retries. Positive server-supplied retry delays are capped at {@link + * RetrySettings#getMaxRetryDelayDuration()} (defaulting to 1000ms when unset or non-positive), and + * missing or non-positive server retry delays fall back to exponential backoff. * *

These limits bound the number of streams that this iterator starts. Each (re)started stream is * a call through the underlying GAX callable, and GAX applies the same {@link RetrySettings} to @@ -89,6 +92,7 @@ abstract class ResumableStreamIterator extends AbstractIterator retryableCodes; + private static final long DEFAULT_MAX_RETRY_DELAY_MILLIS = 1000L; private static final Logger logger = Logger.getLogger(ResumableStreamIterator.class.getName()); @Nullable private BackOff backOff; private final ArrayDeque buffer = new ArrayDeque<>(); @@ -200,8 +204,9 @@ private boolean maxAttemptsExhausted() { * zero means that no total timeout has been set, and that only maxAttempts (if set) limits the * retries. This mirrors the interpretation of these values in GAX. */ - private boolean totalTimeoutExceeded(long proposedDelayMillis) { - long totalTimeoutMillis = streamingRetrySettings.getTotalTimeout().toMillis(); + @VisibleForTesting + boolean totalTimeoutExceeded(long proposedDelayMillis) { + long totalTimeoutMillis = streamingRetrySettings.getTotalTimeoutDuration().toMillis(); if (totalTimeoutMillis <= 0L) { return false; } @@ -215,23 +220,48 @@ private boolean totalTimeoutExceeded(long proposedDelayMillis) { return Math.max(proposedDelayMillis, 0L) >= totalTimeoutMillis - elapsedMillis; } - private ExponentialBackOff newBackOff() { + /** + * Returns the effective maximum retry delay in milliseconds: the configured {@link + * RetrySettings#getMaxRetryDelayDuration()} when positive (floored at 1ms for sub-millisecond + * durations and saturated at {@link Long#MAX_VALUE}), or {@link #DEFAULT_MAX_RETRY_DELAY_MILLIS} + * when unset or non-positive. + */ + private long effectiveMaxRetryDelayMillis() { + Duration maxRetryDelay = streamingRetrySettings.getMaxRetryDelayDuration(); + if (maxRetryDelay.isZero() || maxRetryDelay.isNegative()) { + return DEFAULT_MAX_RETRY_DELAY_MILLIS; + } + try { + return Math.max(1L, maxRetryDelay.toMillis()); + } catch (ArithmeticException overflow) { + return Long.MAX_VALUE; + } + } + + private int effectiveInitialRetryDelayMillis() { + Duration initialRetryDelay = streamingRetrySettings.getInitialRetryDelayDuration(); + if (initialRetryDelay.isZero() || initialRetryDelay.isNegative()) { + return 1; + } + try { + return Math.max(1, (int) Math.min(initialRetryDelay.toMillis(), Integer.MAX_VALUE)); + } catch (ArithmeticException overflow) { + return Integer.MAX_VALUE; + } + } + + @VisibleForTesting + BackOff newBackOff() { + int initialIntervalMillis = effectiveInitialRetryDelayMillis(); + int maxIntervalMillis = + Math.max( + initialIntervalMillis, + (int) Math.min(effectiveMaxRetryDelayMillis(), Integer.MAX_VALUE)); return new ExponentialBackOff.Builder() .setMultiplier(streamingRetrySettings.getRetryDelayMultiplier()) // All of these values must be > 0. - .setInitialIntervalMillis( - Math.max( - 1, - (int) - Math.min( - streamingRetrySettings.getInitialRetryDelay().toMillis(), - Integer.MAX_VALUE))) - .setMaxIntervalMillis( - Math.max( - 1, - (int) - Math.min( - streamingRetrySettings.getMaxRetryDelay().toMillis(), Integer.MAX_VALUE))) + .setInitialIntervalMillis(initialIntervalMillis) + .setMaxIntervalMillis(maxIntervalMillis) // The total timeout is enforced explicitly in computeNext(), based on the elapsed time // since the first failure of the current retry sequence. Prevent the backoff from // returning BackOff.STOP, as that would misinterpret a total timeout of zero (that is, no @@ -261,7 +291,7 @@ private void backoffSleep(Context context, long backoffMillis) throws SpannerExc try { if (backoffMillis == BackOff.STOP) { // Highly unlikely but we handle it just in case. - backoffMillis = streamingRetrySettings.getMaxRetryDelay().toMillis(); + backoffMillis = effectiveMaxRetryDelayMillis(); } if (latch.await(backoffMillis, TimeUnit.MILLISECONDS)) { // Woken by context cancellation. @@ -478,14 +508,19 @@ long checkRetryBudgetAndGetDelay(SpannerException spannerException) { spannerException); throw spannerException; } - // Determine the retry delay: either the delay that the server included in the error, or - // otherwise a delay determined by the exponential backoff. + // Determine the retry delay: use a positive server-supplied retry delay (clamped to the + // effective max retry delay), or fall back to exponential backoff when absent or non-positive. long delayMillis = spannerException.getRetryDelayInMillis(); - if (delayMillis == -1L) { + long maxRetryDelayMillis = effectiveMaxRetryDelayMillis(); + if (delayMillis <= 0L) { if (this.backOff == null) { this.backOff = newBackOff(); } - delayMillis = nextBackOffMillis(this.backOff); + long backOffMillis = nextBackOffMillis(this.backOff); + delayMillis = + backOffMillis == BackOff.STOP ? maxRetryDelayMillis : Math.max(1L, backOffMillis); + } else { + delayMillis = Math.min(delayMillis, maxRetryDelayMillis); } // The total timeout budget applies regardless of whether the delay came from the server // or from the backoff. diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/ResumableStreamIteratorTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/ResumableStreamIteratorTest.java index 12a00a990c05..fb587768fe30 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/ResumableStreamIteratorTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/ResumableStreamIteratorTest.java @@ -26,6 +26,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.api.client.util.BackOff; import com.google.api.core.ApiFuture; import com.google.api.gax.core.ExecutorProvider; import com.google.api.gax.retrying.RetrySettings; @@ -105,19 +106,33 @@ public static List data() { } private static StatusRuntimeException statusWithRetryInfo(ErrorCode code) { + return statusWithRetryInfo(code, 0L, (int) TimeUnit.MILLISECONDS.toNanos(1L)); + } + + private static StatusRuntimeException statusWithRetryInfo( + ErrorCode code, long seconds, int nanos) { Metadata.Key key = ProtoUtils.keyForProto(RetryInfo.getDefaultInstance()); Metadata trailers = new Metadata(); RetryInfo retryInfo = RetryInfo.newBuilder() - .setRetryDelay( - Duration.newBuilder() - .setNanos((int) TimeUnit.MILLISECONDS.toNanos(1L)) - .setSeconds(0L)) + .setRetryDelay(Duration.newBuilder().setSeconds(seconds).setNanos(nanos)) .build(); trailers.put(key, retryInfo); return code.getGrpcStatus().asRuntimeException(trailers); } + private static BackOff fixedBackOff(long delayMillis) { + return new BackOff() { + @Override + public void reset() {} + + @Override + public long nextBackOffMillis() { + return delayMillis; + } + }; + } + static class RetryableException extends SpannerException { RetryableException(ErrorCode code, @Nullable String message) { // OK to instantiate SpannerException directly for this unit test. @@ -527,20 +542,11 @@ private boolean totalTimeoutExceeded(long timeoutMillis, long elapsedMillis, lon RetrySettings.newBuilder() .setTotalTimeoutDuration(java.time.Duration.ofMillis(timeoutMillis)) .build()); - SpannerException exception = - Mockito.spy(new RetryableException(errorCodeParameter, "failed by test")); - when(exception.getRetryDelayInMillis()).thenReturn(0L); + SpannerException exception = new RetryableException(errorCodeParameter, "failed by test"); nanoTime = () -> 0L; - assertEquals(0L, resumableStreamIterator.checkRetryBudgetAndGetDelay(exception)); + assertEquals(1L, resumableStreamIterator.checkRetryBudgetAndGetDelay(exception)); nanoTime = () -> TimeUnit.MILLISECONDS.toNanos(elapsedMillis); - when(exception.getRetryDelayInMillis()).thenReturn(delayMillis); - try { - assertEquals(delayMillis, resumableStreamIterator.checkRetryBudgetAndGetDelay(exception)); - return false; - } catch (SpannerException e) { - assertThat(e).isSameInstanceAs(exception); - return true; - } + return resumableStreamIterator.totalTimeoutExceeded(delayMillis); } @Test @@ -565,11 +571,17 @@ public void retryBudget_usesConfiguredBackoffWithoutRetryInfo() { public void rawGapicSettings_useConfiguredTimeoutWithoutSpecialCase() { RetrySettings settings = SpannerStubSettings.newBuilder().executeStreamingSqlSettings().getRetrySettings(); + long totalTimeoutMillis = settings.getTotalTimeoutDuration().toMillis(); initWithLimitAndRetrySettings(Integer.MAX_VALUE, settings); SpannerException exception = - Mockito.spy(new RetryableException(errorCodeParameter, "failed by test")); - when(exception.getRetryDelayInMillis()) - .thenReturn(settings.getTotalTimeoutDuration().toMillis()); + new RetryableException( + errorCodeParameter, + "failed by test", + statusWithRetryInfo( + errorCodeParameter, TimeUnit.MILLISECONDS.toSeconds(totalTimeoutMillis), 0)); + nanoTime = () -> 0L; + assertEquals(1000L, resumableStreamIterator.checkRetryBudgetAndGetDelay(exception)); + nanoTime = () -> TimeUnit.MILLISECONDS.toNanos(totalTimeoutMillis - 1000L); assertThat( assertThrows( SpannerException.class, @@ -577,6 +589,342 @@ public void rawGapicSettings_useConfiguredTimeoutWithoutSpecialCase() { .isSameInstanceAs(exception); } + @Test + public void retryInfoWithZeroDelay_fallsBackToExponentialBackoff() { + SpannerException zeroDelayException = + new RetryableException( + errorCodeParameter, "zero delay", statusWithRetryInfo(errorCodeParameter, 0L, 0)); + assertEquals(0L, zeroDelayException.getRetryDelayInMillis()); + long delayMillis = resumableStreamIterator.checkRetryBudgetAndGetDelay(zeroDelayException); + assertThat(delayMillis).isAtLeast(5L); + assertThat(delayMillis).isAtMost(15L); + + // Positive sub-millisecond RetryInfo truncates to 0L in Durations.toMillis and must also use + // exponential backoff rather than spinning at zero delay. + initWithLimit(Integer.MAX_VALUE); + SpannerException subMillisecondException = + new RetryableException( + errorCodeParameter, + "sub-millisecond delay", + statusWithRetryInfo(errorCodeParameter, 0L, 500_000)); + assertEquals(0L, subMillisecondException.getRetryDelayInMillis()); + long subMsBackoff = + resumableStreamIterator.checkRetryBudgetAndGetDelay(subMillisecondException); + assertThat(subMsBackoff).isAtLeast(5L); + assertThat(subMsBackoff).isAtMost(15L); + + initWithLimit(Integer.MAX_VALUE); + ResultSetStream s1 = Mockito.mock(ResultSetStream.class); + Mockito.when(starter.startStream(null, null)).thenReturn(new ResultSetIterator(s1)); + Mockito.when(s1.next()) + .thenReturn(resultSet(ByteString.copyFromUtf8("r1"), "a")) + .thenThrow(zeroDelayException); + + ResultSetStream s2 = Mockito.mock(ResultSetStream.class); + Mockito.when(starter.startStream(ByteString.copyFromUtf8("r1"), null)) + .thenReturn(new ResultSetIterator(s2)); + Mockito.when(s2.next()) + .thenReturn(resultSet(ByteString.copyFromUtf8("r2"), "b")) + .thenReturn(null); + assertThat(consume(resumableStreamIterator)).containsExactly("a", "b").inOrder(); + } + + @Test + public void retryInfoWithNegativeDelays_fallsBackToExponentialBackoff() { + SpannerException minusOneMsException = + new RetryableException( + errorCodeParameter, + "-1ms delay", + statusWithRetryInfo(errorCodeParameter, 0L, (int) -TimeUnit.MILLISECONDS.toNanos(1L))); + SpannerException minusTwoMsException = + new RetryableException( + errorCodeParameter, + "-2ms delay", + statusWithRetryInfo(errorCodeParameter, 0L, (int) -TimeUnit.MILLISECONDS.toNanos(2L))); + SpannerException minusFiveSecException = + new RetryableException( + errorCodeParameter, "-5s delay", statusWithRetryInfo(errorCodeParameter, -5L, 0)); + assertEquals(-1L, minusOneMsException.getRetryDelayInMillis()); + assertEquals(-2L, minusTwoMsException.getRetryDelayInMillis()); + assertEquals(-5000L, minusFiveSecException.getRetryDelayInMillis()); + + for (SpannerException exception : + ImmutableList.of(minusOneMsException, minusTwoMsException, minusFiveSecException)) { + initWithLimit(Integer.MAX_VALUE); + long delayMillis = resumableStreamIterator.checkRetryBudgetAndGetDelay(exception); + assertThat(delayMillis).isAtLeast(5L); + assertThat(delayMillis).isAtMost(15L); + } + + initWithLimit(Integer.MAX_VALUE); + ResultSetStream s1 = Mockito.mock(ResultSetStream.class); + Mockito.when(starter.startStream(null, null)).thenReturn(new ResultSetIterator(s1)); + Mockito.when(s1.next()) + .thenReturn(resultSet(ByteString.copyFromUtf8("r1"), "a")) + .thenThrow(minusFiveSecException); + + ResultSetStream s2 = Mockito.mock(ResultSetStream.class); + Mockito.when(starter.startStream(ByteString.copyFromUtf8("r1"), null)) + .thenReturn(new ResultSetIterator(s2)); + Mockito.when(s2.next()) + .thenReturn(resultSet(ByteString.copyFromUtf8("r2"), "b")) + .thenReturn(null); + assertThat(consume(resumableStreamIterator)).containsExactly("a", "b").inOrder(); + } + + @Test(timeout = 10_000L) + public void retryInfoWithMultiHourDelay_clampedToConfiguredMaxRetryDelay() { + RetrySettings settings = + RetrySettings.newBuilder() + .setInitialRetryDelayDuration(java.time.Duration.ofMillis(10L)) + .setMaxRetryDelayDuration(java.time.Duration.ofMillis(250L)) + .build(); + initWithLimitAndRetrySettings(Integer.MAX_VALUE, settings); + SpannerException threeHourException = + new RetryableException( + errorCodeParameter, + "3h delay", + statusWithRetryInfo(errorCodeParameter, TimeUnit.HOURS.toSeconds(3L), 0)); + assertEquals(TimeUnit.HOURS.toMillis(3L), threeHourException.getRetryDelayInMillis()); + assertEquals(250L, resumableStreamIterator.checkRetryBudgetAndGetDelay(threeHourException)); + + initWithLimitAndRetrySettings(Integer.MAX_VALUE, settings); + ResultSetStream s1 = Mockito.mock(ResultSetStream.class); + Mockito.when(starter.startStream(null, null)).thenReturn(new ResultSetIterator(s1)); + Mockito.when(s1.next()) + .thenReturn(resultSet(ByteString.copyFromUtf8("r1"), "a")) + .thenThrow(threeHourException); + + ResultSetStream s2 = Mockito.mock(ResultSetStream.class); + Mockito.when(starter.startStream(ByteString.copyFromUtf8("r1"), null)) + .thenReturn(new ResultSetIterator(s2)); + Mockito.when(s2.next()) + .thenReturn(resultSet(ByteString.copyFromUtf8("r2"), "b")) + .thenReturn(null); + + Stopwatch stopwatch = Stopwatch.createStarted(); + assertThat(consume(resumableStreamIterator)).containsExactly("a", "b").inOrder(); + assertThat(stopwatch.elapsed(TimeUnit.MILLISECONDS)).isLessThan(5000L); + } + + @Test(timeout = 10_000L) + public void retryInfoWithMultiHourDelayAndUnsetMaxRetryDelay_clampedToDefaultCap() { + RetrySettings unsetSettings = RetrySettings.newBuilder().build(); + assertTrue(unsetSettings.getMaxRetryDelayDuration().isZero()); + assertTrue(unsetSettings.getTotalTimeoutDuration().isZero()); + initWithLimitAndRetrySettings(Integer.MAX_VALUE, unsetSettings); + + SpannerException threeHourException = + new RetryableException( + errorCodeParameter, + "3h delay", + statusWithRetryInfo(errorCodeParameter, TimeUnit.HOURS.toSeconds(3L), 0)); + assertEquals(1000L, resumableStreamIterator.checkRetryBudgetAndGetDelay(threeHourException)); + + initWithLimitAndRetrySettings(Integer.MAX_VALUE, unsetSettings); + ResultSetStream s1 = Mockito.mock(ResultSetStream.class); + Mockito.when(starter.startStream(null, null)).thenReturn(new ResultSetIterator(s1)); + Mockito.when(s1.next()) + .thenReturn(resultSet(ByteString.copyFromUtf8("r1"), "a")) + .thenThrow(threeHourException); + + ResultSetStream s2 = Mockito.mock(ResultSetStream.class); + Mockito.when(starter.startStream(ByteString.copyFromUtf8("r1"), null)) + .thenReturn(new ResultSetIterator(s2)); + Mockito.when(s2.next()) + .thenReturn(resultSet(ByteString.copyFromUtf8("r2"), "b")) + .thenReturn(null); + + Stopwatch stopwatch = Stopwatch.createStarted(); + assertThat(consume(resumableStreamIterator)).containsExactly("a", "b").inOrder(); + assertThat(stopwatch.elapsed(TimeUnit.MILLISECONDS)).isLessThan(5000L); + } + + @Test + public void retryInfoWithinMaxRetryDelay_honorsServerDelayAndChecksTotalTimeout() { + SpannerException twoHundredMsException = + new RetryableException( + errorCodeParameter, + "200ms delay", + statusWithRetryInfo(errorCodeParameter, 0L, (int) TimeUnit.MILLISECONDS.toNanos(200L))); + assertEquals(200L, twoHundredMsException.getRetryDelayInMillis()); + assertEquals(200L, resumableStreamIterator.checkRetryBudgetAndGetDelay(twoHundredMsException)); + + // Clamping occurs before totalTimeoutExceeded: a 3-hour server delay clamped to 250ms fits in + // a 1000ms total timeout at t=0, then throws immediately once remaining budget drops to 250ms. + initWithLimitAndRetrySettings( + Integer.MAX_VALUE, + RetrySettings.newBuilder() + .setInitialRetryDelayDuration(java.time.Duration.ofMillis(10L)) + .setMaxRetryDelayDuration(java.time.Duration.ofMillis(250L)) + .setTotalTimeoutDuration(java.time.Duration.ofMillis(1000L)) + .build()); + SpannerException threeHourException = + new RetryableException( + errorCodeParameter, + "3h delay", + statusWithRetryInfo(errorCodeParameter, TimeUnit.HOURS.toSeconds(3L), 0)); + nanoTime = () -> 0L; + assertEquals(250L, resumableStreamIterator.checkRetryBudgetAndGetDelay(threeHourException)); + nanoTime = () -> TimeUnit.MILLISECONDS.toNanos(750L); + assertThat( + assertThrows( + SpannerException.class, + () -> resumableStreamIterator.checkRetryBudgetAndGetDelay(threeHourException))) + .isSameInstanceAs(threeHourException); + } + + @Test + public void nonPositiveRetryInfo_floorsZeroBackoffToOneMsAndMapsBackOffStopToEffectiveMax() { + SpannerException zeroDelayException = + new RetryableException( + errorCodeParameter, "zero delay", statusWithRetryInfo(errorCodeParameter, 0L, 0)); + SpannerException absentDelayException = + new RetryableException( + errorCodeParameter, + "absent delay", + errorCodeParameter.getGrpcStatus().asRuntimeException()); + + TraceWrapper noopTracer = + new TraceWrapper(Tracing.getTracer(), OpenTelemetry.noop().getTracer(""), false); + ResumableStreamIterator zeroBackoffIterator = + new TestResumableStreamIterator(noopTracer, RetrySettings.newBuilder().build()) { + @Override + BackOff newBackOff() { + return fixedBackOff(0L); + } + + @Override + CloseableIterator startStream( + @Nullable ByteString resumeToken, + @Nullable StreamMessageListener streamMessageListener, + XGoogSpannerRequestId requestId) { + throw new AssertionError("unexpected startStream"); + } + }; + assertEquals(1L, zeroBackoffIterator.checkRetryBudgetAndGetDelay(zeroDelayException)); + assertEquals(1L, zeroBackoffIterator.checkRetryBudgetAndGetDelay(absentDelayException)); + + ResumableStreamIterator stopBackoffUnsetMaxIterator = + new TestResumableStreamIterator(noopTracer, RetrySettings.newBuilder().build()) { + @Override + BackOff newBackOff() { + return fixedBackOff(BackOff.STOP); + } + + @Override + CloseableIterator startStream( + @Nullable ByteString resumeToken, + @Nullable StreamMessageListener streamMessageListener, + XGoogSpannerRequestId requestId) { + throw new AssertionError("unexpected startStream"); + } + }; + assertEquals( + 1000L, stopBackoffUnsetMaxIterator.checkRetryBudgetAndGetDelay(zeroDelayException)); + assertEquals( + 1000L, stopBackoffUnsetMaxIterator.checkRetryBudgetAndGetDelay(absentDelayException)); + + RetrySettings customMaxSettings = + RetrySettings.newBuilder() + .setInitialRetryDelayDuration(java.time.Duration.ofMillis(10L)) + .setMaxRetryDelayDuration(java.time.Duration.ofMillis(250L)) + .build(); + ResumableStreamIterator stopBackoffCustomMaxIterator = + new TestResumableStreamIterator(noopTracer, customMaxSettings) { + @Override + BackOff newBackOff() { + return fixedBackOff(BackOff.STOP); + } + + @Override + CloseableIterator startStream( + @Nullable ByteString resumeToken, + @Nullable StreamMessageListener streamMessageListener, + XGoogSpannerRequestId requestId) { + throw new AssertionError("unexpected startStream"); + } + }; + assertEquals( + 250L, stopBackoffCustomMaxIterator.checkRetryBudgetAndGetDelay(absentDelayException)); + + initWithLimitAndRetrySettings( + Integer.MAX_VALUE, + RetrySettings.newBuilder() + .setInitialRetryDelayDuration(java.time.Duration.ofMillis(1L)) + .setMaxRetryDelayDuration(java.time.Duration.ofMillis(1L)) + .build()); + for (int i = 0; i < 100; i++) { + assertThat(resumableStreamIterator.checkRetryBudgetAndGetDelay(zeroDelayException)) + .isAtLeast(1L); + } + } + + @Test + public void effectiveMaxRetryDelayAndNewBackOff_handleSubMillisecondAndOverflowingDurations() { + SpannerException twoHundredMsException = + new RetryableException( + errorCodeParameter, + "200ms delay", + statusWithRetryInfo(errorCodeParameter, 0L, (int) TimeUnit.MILLISECONDS.toNanos(200L))); + SpannerException zeroDelayException = + new RetryableException( + errorCodeParameter, "zero delay", statusWithRetryInfo(errorCodeParameter, 0L, 0)); + + RetrySettings subMsSettings = + RetrySettings.newBuilder() + .setInitialRetryDelayDuration(java.time.Duration.ofNanos(500_000L)) + .setMaxRetryDelayDuration(java.time.Duration.ofNanos(500_000L)) + .build(); + initWithLimitAndRetrySettings(Integer.MAX_VALUE, subMsSettings); + assertEquals(1L, resumableStreamIterator.checkRetryBudgetAndGetDelay(twoHundredMsException)); + assertThat(resumableStreamIterator.checkRetryBudgetAndGetDelay(zeroDelayException)) + .isAtLeast(1L); + + RetrySettings overflowMaxSettings = + RetrySettings.newBuilder() + .setInitialRetryDelayDuration(java.time.Duration.ofMillis(10L)) + .setMaxRetryDelayDuration(java.time.Duration.ofSeconds(Long.MAX_VALUE)) + .build(); + initWithLimitAndRetrySettings(Integer.MAX_VALUE, overflowMaxSettings); + assertEquals(200L, resumableStreamIterator.checkRetryBudgetAndGetDelay(twoHundredMsException)); + long backoffDelay = resumableStreamIterator.checkRetryBudgetAndGetDelay(zeroDelayException); + assertThat(backoffDelay).isAtLeast(5L); + assertThat(backoffDelay).isAtMost(15L); + + RetrySettings overflowInitialAndMaxSettings = mock(RetrySettings.class); + when(overflowInitialAndMaxSettings.getTotalTimeoutDuration()) + .thenReturn(java.time.Duration.ZERO); + when(overflowInitialAndMaxSettings.getRetryDelayMultiplier()).thenReturn(1.0); + when(overflowInitialAndMaxSettings.getInitialRetryDelayDuration()) + .thenReturn(java.time.Duration.ofSeconds(Long.MAX_VALUE)); + when(overflowInitialAndMaxSettings.getMaxRetryDelayDuration()) + .thenReturn(java.time.Duration.ofSeconds(Long.MAX_VALUE)); + initWithLimitAndRetrySettings(Integer.MAX_VALUE, overflowInitialAndMaxSettings); + assertThat(resumableStreamIterator.checkRetryBudgetAndGetDelay(zeroDelayException)) + .isAtLeast(1L); + } + + @Test + public void newBackOff_unsetMaxRetryDelayUsesDefaultCeilingForClientBackoff() { + initWithLimitAndRetrySettings( + Integer.MAX_VALUE, RetrySettings.newBuilder().setRetryDelayMultiplier(10.0).build()); + SpannerException absentDelayException = + new RetryableException( + errorCodeParameter, + "absent delay", + errorCodeParameter.getGrpcStatus().asRuntimeException()); + long maxObserved = 0L; + for (int i = 0; i < 6; i++) { + long delayMillis = resumableStreamIterator.checkRetryBudgetAndGetDelay(absentDelayException); + assertThat(delayMillis).isAtLeast(1L); + assertThat(delayMillis).isAtMost(1500L); + maxObserved = Math.max(maxObserved, delayMillis); + } + // Intervals 1, 10, 100, 1000, 1000, 1000 (+/-50% jitter): the 1000ms ceiling is reached. + assertThat(maxObserved).isAtLeast(500L); + } + @Test(timeout = 60000L) public void customTotalTimeoutSmallerThanRetryDelay_doesNotRetry() { // Retrying is only allowed if the retry delay still fits in the remaining total timeout From a7a37dd2804bd19b2bec42483a95c2c624285f04 Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian <179120858+sakthivelmanii@users.noreply.github.com> Date: Thu, 19 Nov 2026 03:02:11 +0000 Subject: [PATCH 2/2] Addressed comments --- .../cloud/spanner/ResumableStreamIteratorTest.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/ResumableStreamIteratorTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/ResumableStreamIteratorTest.java index fb587768fe30..e3aba0c783e3 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/ResumableStreamIteratorTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/ResumableStreamIteratorTest.java @@ -892,14 +892,11 @@ public void effectiveMaxRetryDelayAndNewBackOff_handleSubMillisecondAndOverflowi assertThat(backoffDelay).isAtLeast(5L); assertThat(backoffDelay).isAtMost(15L); - RetrySettings overflowInitialAndMaxSettings = mock(RetrySettings.class); - when(overflowInitialAndMaxSettings.getTotalTimeoutDuration()) - .thenReturn(java.time.Duration.ZERO); - when(overflowInitialAndMaxSettings.getRetryDelayMultiplier()).thenReturn(1.0); - when(overflowInitialAndMaxSettings.getInitialRetryDelayDuration()) - .thenReturn(java.time.Duration.ofSeconds(Long.MAX_VALUE)); - when(overflowInitialAndMaxSettings.getMaxRetryDelayDuration()) - .thenReturn(java.time.Duration.ofSeconds(Long.MAX_VALUE)); + RetrySettings overflowInitialAndMaxSettings = + RetrySettings.newBuilder() + .setInitialRetryDelayDuration(java.time.Duration.ofMillis(Long.MAX_VALUE)) + .setMaxRetryDelayDuration(java.time.Duration.ofSeconds(Long.MAX_VALUE)) + .build(); initWithLimitAndRetrySettings(Integer.MAX_VALUE, overflowInitialAndMaxSettings); assertThat(resumableStreamIterator.checkRetryBudgetAndGetDelay(zeroDelayException)) .isAtLeast(1L);