From e307dc5971e99d1a247724c1da5c12b694cc7b8f Mon Sep 17 00:00:00 2001
From: Sakthivel Subramanian
<179120858+sakthivelmanii@users.noreply.github.com>
Date: Wed, 18 Nov 2026 22:27:43 +0000
Subject: [PATCH 1/2] chore(spanner): bound streaming server retry delays and
fall back on non-positive values
Sanitize server-supplied RetryInfo delays when resuming interrupted Cloud Spanner streaming reads and queries so zero, negative, or oversized retry trailers cannot trigger zero-delay retry spin loops or stall consumer threads for hours.
- Fall back to exponential backoff (>= 1 ms) when a server retry delay is zero, sub-millisecond, or negative instead of retrying immediately without sleeping.
- Cap positive server retry delays at the configured maximum retry delay on streaming retry settings (defaulting to 1000 ms when unset or non-positive) before evaluating the remaining total timeout budget.
- Use the same 1000 ms ceiling for client-side exponential backoff when maximum retry delay is unset on streaming retry settings.
---
.../spanner/ResumableStreamIterator.java | 79 +++-
.../spanner/ResumableStreamIteratorTest.java | 386 +++++++++++++++++-
2 files changed, 424 insertions(+), 41 deletions(-)
diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/ResumableStreamIterator.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/ResumableStreamIterator.java
index e13245b1ef1d..c3c554668f11 100644
--- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/ResumableStreamIterator.java
+++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/ResumableStreamIterator.java
@@ -37,6 +37,7 @@
import io.grpc.Context;
import io.opentelemetry.api.common.Attributes;
import java.io.IOException;
+import java.time.Duration;
import java.util.ArrayDeque;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
@@ -62,7 +63,9 @@
* token that differs from the last seen resume token) resets both, so a long-running stream that
* regularly makes progress is not terminated by an occasional transient error. The default settings
* have no maximum number of attempts or total timeout. Setting maxAttempts to 1 disables streaming
- * retries.
+ * retries. Positive server-supplied retry delays are capped at {@link
+ * RetrySettings#getMaxRetryDelayDuration()} (defaulting to 1000ms when unset or non-positive), and
+ * missing or non-positive server retry delays fall back to exponential backoff.
*
*
These limits bound the number of streams that this iterator starts. Each (re)started stream is
* a call through the underlying GAX callable, and GAX applies the same {@link RetrySettings} to
@@ -89,6 +92,7 @@ abstract class ResumableStreamIterator extends AbstractIterator retryableCodes;
+ private static final long DEFAULT_MAX_RETRY_DELAY_MILLIS = 1000L;
private static final Logger logger = Logger.getLogger(ResumableStreamIterator.class.getName());
@Nullable private BackOff backOff;
private final ArrayDeque buffer = new ArrayDeque<>();
@@ -200,8 +204,9 @@ private boolean maxAttemptsExhausted() {
* zero means that no total timeout has been set, and that only maxAttempts (if set) limits the
* retries. This mirrors the interpretation of these values in GAX.
*/
- private boolean totalTimeoutExceeded(long proposedDelayMillis) {
- long totalTimeoutMillis = streamingRetrySettings.getTotalTimeout().toMillis();
+ @VisibleForTesting
+ boolean totalTimeoutExceeded(long proposedDelayMillis) {
+ long totalTimeoutMillis = streamingRetrySettings.getTotalTimeoutDuration().toMillis();
if (totalTimeoutMillis <= 0L) {
return false;
}
@@ -215,23 +220,48 @@ private boolean totalTimeoutExceeded(long proposedDelayMillis) {
return Math.max(proposedDelayMillis, 0L) >= totalTimeoutMillis - elapsedMillis;
}
- private ExponentialBackOff newBackOff() {
+ /**
+ * Returns the effective maximum retry delay in milliseconds: the configured {@link
+ * RetrySettings#getMaxRetryDelayDuration()} when positive (floored at 1ms for sub-millisecond
+ * durations and saturated at {@link Long#MAX_VALUE}), or {@link #DEFAULT_MAX_RETRY_DELAY_MILLIS}
+ * when unset or non-positive.
+ */
+ private long effectiveMaxRetryDelayMillis() {
+ Duration maxRetryDelay = streamingRetrySettings.getMaxRetryDelayDuration();
+ if (maxRetryDelay.isZero() || maxRetryDelay.isNegative()) {
+ return DEFAULT_MAX_RETRY_DELAY_MILLIS;
+ }
+ try {
+ return Math.max(1L, maxRetryDelay.toMillis());
+ } catch (ArithmeticException overflow) {
+ return Long.MAX_VALUE;
+ }
+ }
+
+ private int effectiveInitialRetryDelayMillis() {
+ Duration initialRetryDelay = streamingRetrySettings.getInitialRetryDelayDuration();
+ if (initialRetryDelay.isZero() || initialRetryDelay.isNegative()) {
+ return 1;
+ }
+ try {
+ return Math.max(1, (int) Math.min(initialRetryDelay.toMillis(), Integer.MAX_VALUE));
+ } catch (ArithmeticException overflow) {
+ return Integer.MAX_VALUE;
+ }
+ }
+
+ @VisibleForTesting
+ BackOff newBackOff() {
+ int initialIntervalMillis = effectiveInitialRetryDelayMillis();
+ int maxIntervalMillis =
+ Math.max(
+ initialIntervalMillis,
+ (int) Math.min(effectiveMaxRetryDelayMillis(), Integer.MAX_VALUE));
return new ExponentialBackOff.Builder()
.setMultiplier(streamingRetrySettings.getRetryDelayMultiplier())
// All of these values must be > 0.
- .setInitialIntervalMillis(
- Math.max(
- 1,
- (int)
- Math.min(
- streamingRetrySettings.getInitialRetryDelay().toMillis(),
- Integer.MAX_VALUE)))
- .setMaxIntervalMillis(
- Math.max(
- 1,
- (int)
- Math.min(
- streamingRetrySettings.getMaxRetryDelay().toMillis(), Integer.MAX_VALUE)))
+ .setInitialIntervalMillis(initialIntervalMillis)
+ .setMaxIntervalMillis(maxIntervalMillis)
// The total timeout is enforced explicitly in computeNext(), based on the elapsed time
// since the first failure of the current retry sequence. Prevent the backoff from
// returning BackOff.STOP, as that would misinterpret a total timeout of zero (that is, no
@@ -261,7 +291,7 @@ private void backoffSleep(Context context, long backoffMillis) throws SpannerExc
try {
if (backoffMillis == BackOff.STOP) {
// Highly unlikely but we handle it just in case.
- backoffMillis = streamingRetrySettings.getMaxRetryDelay().toMillis();
+ backoffMillis = effectiveMaxRetryDelayMillis();
}
if (latch.await(backoffMillis, TimeUnit.MILLISECONDS)) {
// Woken by context cancellation.
@@ -478,14 +508,19 @@ long checkRetryBudgetAndGetDelay(SpannerException spannerException) {
spannerException);
throw spannerException;
}
- // Determine the retry delay: either the delay that the server included in the error, or
- // otherwise a delay determined by the exponential backoff.
+ // Determine the retry delay: use a positive server-supplied retry delay (clamped to the
+ // effective max retry delay), or fall back to exponential backoff when absent or non-positive.
long delayMillis = spannerException.getRetryDelayInMillis();
- if (delayMillis == -1L) {
+ long maxRetryDelayMillis = effectiveMaxRetryDelayMillis();
+ if (delayMillis <= 0L) {
if (this.backOff == null) {
this.backOff = newBackOff();
}
- delayMillis = nextBackOffMillis(this.backOff);
+ long backOffMillis = nextBackOffMillis(this.backOff);
+ delayMillis =
+ backOffMillis == BackOff.STOP ? maxRetryDelayMillis : Math.max(1L, backOffMillis);
+ } else {
+ delayMillis = Math.min(delayMillis, maxRetryDelayMillis);
}
// The total timeout budget applies regardless of whether the delay came from the server
// or from the backoff.
diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/ResumableStreamIteratorTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/ResumableStreamIteratorTest.java
index 12a00a990c05..fb587768fe30 100644
--- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/ResumableStreamIteratorTest.java
+++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/ResumableStreamIteratorTest.java
@@ -26,6 +26,7 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import com.google.api.client.util.BackOff;
import com.google.api.core.ApiFuture;
import com.google.api.gax.core.ExecutorProvider;
import com.google.api.gax.retrying.RetrySettings;
@@ -105,19 +106,33 @@ public static List data() {
}
private static StatusRuntimeException statusWithRetryInfo(ErrorCode code) {
+ return statusWithRetryInfo(code, 0L, (int) TimeUnit.MILLISECONDS.toNanos(1L));
+ }
+
+ private static StatusRuntimeException statusWithRetryInfo(
+ ErrorCode code, long seconds, int nanos) {
Metadata.Key key = ProtoUtils.keyForProto(RetryInfo.getDefaultInstance());
Metadata trailers = new Metadata();
RetryInfo retryInfo =
RetryInfo.newBuilder()
- .setRetryDelay(
- Duration.newBuilder()
- .setNanos((int) TimeUnit.MILLISECONDS.toNanos(1L))
- .setSeconds(0L))
+ .setRetryDelay(Duration.newBuilder().setSeconds(seconds).setNanos(nanos))
.build();
trailers.put(key, retryInfo);
return code.getGrpcStatus().asRuntimeException(trailers);
}
+ private static BackOff fixedBackOff(long delayMillis) {
+ return new BackOff() {
+ @Override
+ public void reset() {}
+
+ @Override
+ public long nextBackOffMillis() {
+ return delayMillis;
+ }
+ };
+ }
+
static class RetryableException extends SpannerException {
RetryableException(ErrorCode code, @Nullable String message) {
// OK to instantiate SpannerException directly for this unit test.
@@ -527,20 +542,11 @@ private boolean totalTimeoutExceeded(long timeoutMillis, long elapsedMillis, lon
RetrySettings.newBuilder()
.setTotalTimeoutDuration(java.time.Duration.ofMillis(timeoutMillis))
.build());
- SpannerException exception =
- Mockito.spy(new RetryableException(errorCodeParameter, "failed by test"));
- when(exception.getRetryDelayInMillis()).thenReturn(0L);
+ SpannerException exception = new RetryableException(errorCodeParameter, "failed by test");
nanoTime = () -> 0L;
- assertEquals(0L, resumableStreamIterator.checkRetryBudgetAndGetDelay(exception));
+ assertEquals(1L, resumableStreamIterator.checkRetryBudgetAndGetDelay(exception));
nanoTime = () -> TimeUnit.MILLISECONDS.toNanos(elapsedMillis);
- when(exception.getRetryDelayInMillis()).thenReturn(delayMillis);
- try {
- assertEquals(delayMillis, resumableStreamIterator.checkRetryBudgetAndGetDelay(exception));
- return false;
- } catch (SpannerException e) {
- assertThat(e).isSameInstanceAs(exception);
- return true;
- }
+ return resumableStreamIterator.totalTimeoutExceeded(delayMillis);
}
@Test
@@ -565,11 +571,17 @@ public void retryBudget_usesConfiguredBackoffWithoutRetryInfo() {
public void rawGapicSettings_useConfiguredTimeoutWithoutSpecialCase() {
RetrySettings settings =
SpannerStubSettings.newBuilder().executeStreamingSqlSettings().getRetrySettings();
+ long totalTimeoutMillis = settings.getTotalTimeoutDuration().toMillis();
initWithLimitAndRetrySettings(Integer.MAX_VALUE, settings);
SpannerException exception =
- Mockito.spy(new RetryableException(errorCodeParameter, "failed by test"));
- when(exception.getRetryDelayInMillis())
- .thenReturn(settings.getTotalTimeoutDuration().toMillis());
+ new RetryableException(
+ errorCodeParameter,
+ "failed by test",
+ statusWithRetryInfo(
+ errorCodeParameter, TimeUnit.MILLISECONDS.toSeconds(totalTimeoutMillis), 0));
+ nanoTime = () -> 0L;
+ assertEquals(1000L, resumableStreamIterator.checkRetryBudgetAndGetDelay(exception));
+ nanoTime = () -> TimeUnit.MILLISECONDS.toNanos(totalTimeoutMillis - 1000L);
assertThat(
assertThrows(
SpannerException.class,
@@ -577,6 +589,342 @@ public void rawGapicSettings_useConfiguredTimeoutWithoutSpecialCase() {
.isSameInstanceAs(exception);
}
+ @Test
+ public void retryInfoWithZeroDelay_fallsBackToExponentialBackoff() {
+ SpannerException zeroDelayException =
+ new RetryableException(
+ errorCodeParameter, "zero delay", statusWithRetryInfo(errorCodeParameter, 0L, 0));
+ assertEquals(0L, zeroDelayException.getRetryDelayInMillis());
+ long delayMillis = resumableStreamIterator.checkRetryBudgetAndGetDelay(zeroDelayException);
+ assertThat(delayMillis).isAtLeast(5L);
+ assertThat(delayMillis).isAtMost(15L);
+
+ // Positive sub-millisecond RetryInfo truncates to 0L in Durations.toMillis and must also use
+ // exponential backoff rather than spinning at zero delay.
+ initWithLimit(Integer.MAX_VALUE);
+ SpannerException subMillisecondException =
+ new RetryableException(
+ errorCodeParameter,
+ "sub-millisecond delay",
+ statusWithRetryInfo(errorCodeParameter, 0L, 500_000));
+ assertEquals(0L, subMillisecondException.getRetryDelayInMillis());
+ long subMsBackoff =
+ resumableStreamIterator.checkRetryBudgetAndGetDelay(subMillisecondException);
+ assertThat(subMsBackoff).isAtLeast(5L);
+ assertThat(subMsBackoff).isAtMost(15L);
+
+ initWithLimit(Integer.MAX_VALUE);
+ ResultSetStream s1 = Mockito.mock(ResultSetStream.class);
+ Mockito.when(starter.startStream(null, null)).thenReturn(new ResultSetIterator(s1));
+ Mockito.when(s1.next())
+ .thenReturn(resultSet(ByteString.copyFromUtf8("r1"), "a"))
+ .thenThrow(zeroDelayException);
+
+ ResultSetStream s2 = Mockito.mock(ResultSetStream.class);
+ Mockito.when(starter.startStream(ByteString.copyFromUtf8("r1"), null))
+ .thenReturn(new ResultSetIterator(s2));
+ Mockito.when(s2.next())
+ .thenReturn(resultSet(ByteString.copyFromUtf8("r2"), "b"))
+ .thenReturn(null);
+ assertThat(consume(resumableStreamIterator)).containsExactly("a", "b").inOrder();
+ }
+
+ @Test
+ public void retryInfoWithNegativeDelays_fallsBackToExponentialBackoff() {
+ SpannerException minusOneMsException =
+ new RetryableException(
+ errorCodeParameter,
+ "-1ms delay",
+ statusWithRetryInfo(errorCodeParameter, 0L, (int) -TimeUnit.MILLISECONDS.toNanos(1L)));
+ SpannerException minusTwoMsException =
+ new RetryableException(
+ errorCodeParameter,
+ "-2ms delay",
+ statusWithRetryInfo(errorCodeParameter, 0L, (int) -TimeUnit.MILLISECONDS.toNanos(2L)));
+ SpannerException minusFiveSecException =
+ new RetryableException(
+ errorCodeParameter, "-5s delay", statusWithRetryInfo(errorCodeParameter, -5L, 0));
+ assertEquals(-1L, minusOneMsException.getRetryDelayInMillis());
+ assertEquals(-2L, minusTwoMsException.getRetryDelayInMillis());
+ assertEquals(-5000L, minusFiveSecException.getRetryDelayInMillis());
+
+ for (SpannerException exception :
+ ImmutableList.of(minusOneMsException, minusTwoMsException, minusFiveSecException)) {
+ initWithLimit(Integer.MAX_VALUE);
+ long delayMillis = resumableStreamIterator.checkRetryBudgetAndGetDelay(exception);
+ assertThat(delayMillis).isAtLeast(5L);
+ assertThat(delayMillis).isAtMost(15L);
+ }
+
+ initWithLimit(Integer.MAX_VALUE);
+ ResultSetStream s1 = Mockito.mock(ResultSetStream.class);
+ Mockito.when(starter.startStream(null, null)).thenReturn(new ResultSetIterator(s1));
+ Mockito.when(s1.next())
+ .thenReturn(resultSet(ByteString.copyFromUtf8("r1"), "a"))
+ .thenThrow(minusFiveSecException);
+
+ ResultSetStream s2 = Mockito.mock(ResultSetStream.class);
+ Mockito.when(starter.startStream(ByteString.copyFromUtf8("r1"), null))
+ .thenReturn(new ResultSetIterator(s2));
+ Mockito.when(s2.next())
+ .thenReturn(resultSet(ByteString.copyFromUtf8("r2"), "b"))
+ .thenReturn(null);
+ assertThat(consume(resumableStreamIterator)).containsExactly("a", "b").inOrder();
+ }
+
+ @Test(timeout = 10_000L)
+ public void retryInfoWithMultiHourDelay_clampedToConfiguredMaxRetryDelay() {
+ RetrySettings settings =
+ RetrySettings.newBuilder()
+ .setInitialRetryDelayDuration(java.time.Duration.ofMillis(10L))
+ .setMaxRetryDelayDuration(java.time.Duration.ofMillis(250L))
+ .build();
+ initWithLimitAndRetrySettings(Integer.MAX_VALUE, settings);
+ SpannerException threeHourException =
+ new RetryableException(
+ errorCodeParameter,
+ "3h delay",
+ statusWithRetryInfo(errorCodeParameter, TimeUnit.HOURS.toSeconds(3L), 0));
+ assertEquals(TimeUnit.HOURS.toMillis(3L), threeHourException.getRetryDelayInMillis());
+ assertEquals(250L, resumableStreamIterator.checkRetryBudgetAndGetDelay(threeHourException));
+
+ initWithLimitAndRetrySettings(Integer.MAX_VALUE, settings);
+ ResultSetStream s1 = Mockito.mock(ResultSetStream.class);
+ Mockito.when(starter.startStream(null, null)).thenReturn(new ResultSetIterator(s1));
+ Mockito.when(s1.next())
+ .thenReturn(resultSet(ByteString.copyFromUtf8("r1"), "a"))
+ .thenThrow(threeHourException);
+
+ ResultSetStream s2 = Mockito.mock(ResultSetStream.class);
+ Mockito.when(starter.startStream(ByteString.copyFromUtf8("r1"), null))
+ .thenReturn(new ResultSetIterator(s2));
+ Mockito.when(s2.next())
+ .thenReturn(resultSet(ByteString.copyFromUtf8("r2"), "b"))
+ .thenReturn(null);
+
+ Stopwatch stopwatch = Stopwatch.createStarted();
+ assertThat(consume(resumableStreamIterator)).containsExactly("a", "b").inOrder();
+ assertThat(stopwatch.elapsed(TimeUnit.MILLISECONDS)).isLessThan(5000L);
+ }
+
+ @Test(timeout = 10_000L)
+ public void retryInfoWithMultiHourDelayAndUnsetMaxRetryDelay_clampedToDefaultCap() {
+ RetrySettings unsetSettings = RetrySettings.newBuilder().build();
+ assertTrue(unsetSettings.getMaxRetryDelayDuration().isZero());
+ assertTrue(unsetSettings.getTotalTimeoutDuration().isZero());
+ initWithLimitAndRetrySettings(Integer.MAX_VALUE, unsetSettings);
+
+ SpannerException threeHourException =
+ new RetryableException(
+ errorCodeParameter,
+ "3h delay",
+ statusWithRetryInfo(errorCodeParameter, TimeUnit.HOURS.toSeconds(3L), 0));
+ assertEquals(1000L, resumableStreamIterator.checkRetryBudgetAndGetDelay(threeHourException));
+
+ initWithLimitAndRetrySettings(Integer.MAX_VALUE, unsetSettings);
+ ResultSetStream s1 = Mockito.mock(ResultSetStream.class);
+ Mockito.when(starter.startStream(null, null)).thenReturn(new ResultSetIterator(s1));
+ Mockito.when(s1.next())
+ .thenReturn(resultSet(ByteString.copyFromUtf8("r1"), "a"))
+ .thenThrow(threeHourException);
+
+ ResultSetStream s2 = Mockito.mock(ResultSetStream.class);
+ Mockito.when(starter.startStream(ByteString.copyFromUtf8("r1"), null))
+ .thenReturn(new ResultSetIterator(s2));
+ Mockito.when(s2.next())
+ .thenReturn(resultSet(ByteString.copyFromUtf8("r2"), "b"))
+ .thenReturn(null);
+
+ Stopwatch stopwatch = Stopwatch.createStarted();
+ assertThat(consume(resumableStreamIterator)).containsExactly("a", "b").inOrder();
+ assertThat(stopwatch.elapsed(TimeUnit.MILLISECONDS)).isLessThan(5000L);
+ }
+
+ @Test
+ public void retryInfoWithinMaxRetryDelay_honorsServerDelayAndChecksTotalTimeout() {
+ SpannerException twoHundredMsException =
+ new RetryableException(
+ errorCodeParameter,
+ "200ms delay",
+ statusWithRetryInfo(errorCodeParameter, 0L, (int) TimeUnit.MILLISECONDS.toNanos(200L)));
+ assertEquals(200L, twoHundredMsException.getRetryDelayInMillis());
+ assertEquals(200L, resumableStreamIterator.checkRetryBudgetAndGetDelay(twoHundredMsException));
+
+ // Clamping occurs before totalTimeoutExceeded: a 3-hour server delay clamped to 250ms fits in
+ // a 1000ms total timeout at t=0, then throws immediately once remaining budget drops to 250ms.
+ initWithLimitAndRetrySettings(
+ Integer.MAX_VALUE,
+ RetrySettings.newBuilder()
+ .setInitialRetryDelayDuration(java.time.Duration.ofMillis(10L))
+ .setMaxRetryDelayDuration(java.time.Duration.ofMillis(250L))
+ .setTotalTimeoutDuration(java.time.Duration.ofMillis(1000L))
+ .build());
+ SpannerException threeHourException =
+ new RetryableException(
+ errorCodeParameter,
+ "3h delay",
+ statusWithRetryInfo(errorCodeParameter, TimeUnit.HOURS.toSeconds(3L), 0));
+ nanoTime = () -> 0L;
+ assertEquals(250L, resumableStreamIterator.checkRetryBudgetAndGetDelay(threeHourException));
+ nanoTime = () -> TimeUnit.MILLISECONDS.toNanos(750L);
+ assertThat(
+ assertThrows(
+ SpannerException.class,
+ () -> resumableStreamIterator.checkRetryBudgetAndGetDelay(threeHourException)))
+ .isSameInstanceAs(threeHourException);
+ }
+
+ @Test
+ public void nonPositiveRetryInfo_floorsZeroBackoffToOneMsAndMapsBackOffStopToEffectiveMax() {
+ SpannerException zeroDelayException =
+ new RetryableException(
+ errorCodeParameter, "zero delay", statusWithRetryInfo(errorCodeParameter, 0L, 0));
+ SpannerException absentDelayException =
+ new RetryableException(
+ errorCodeParameter,
+ "absent delay",
+ errorCodeParameter.getGrpcStatus().asRuntimeException());
+
+ TraceWrapper noopTracer =
+ new TraceWrapper(Tracing.getTracer(), OpenTelemetry.noop().getTracer(""), false);
+ ResumableStreamIterator zeroBackoffIterator =
+ new TestResumableStreamIterator(noopTracer, RetrySettings.newBuilder().build()) {
+ @Override
+ BackOff newBackOff() {
+ return fixedBackOff(0L);
+ }
+
+ @Override
+ CloseableIterator startStream(
+ @Nullable ByteString resumeToken,
+ @Nullable StreamMessageListener streamMessageListener,
+ XGoogSpannerRequestId requestId) {
+ throw new AssertionError("unexpected startStream");
+ }
+ };
+ assertEquals(1L, zeroBackoffIterator.checkRetryBudgetAndGetDelay(zeroDelayException));
+ assertEquals(1L, zeroBackoffIterator.checkRetryBudgetAndGetDelay(absentDelayException));
+
+ ResumableStreamIterator stopBackoffUnsetMaxIterator =
+ new TestResumableStreamIterator(noopTracer, RetrySettings.newBuilder().build()) {
+ @Override
+ BackOff newBackOff() {
+ return fixedBackOff(BackOff.STOP);
+ }
+
+ @Override
+ CloseableIterator startStream(
+ @Nullable ByteString resumeToken,
+ @Nullable StreamMessageListener streamMessageListener,
+ XGoogSpannerRequestId requestId) {
+ throw new AssertionError("unexpected startStream");
+ }
+ };
+ assertEquals(
+ 1000L, stopBackoffUnsetMaxIterator.checkRetryBudgetAndGetDelay(zeroDelayException));
+ assertEquals(
+ 1000L, stopBackoffUnsetMaxIterator.checkRetryBudgetAndGetDelay(absentDelayException));
+
+ RetrySettings customMaxSettings =
+ RetrySettings.newBuilder()
+ .setInitialRetryDelayDuration(java.time.Duration.ofMillis(10L))
+ .setMaxRetryDelayDuration(java.time.Duration.ofMillis(250L))
+ .build();
+ ResumableStreamIterator stopBackoffCustomMaxIterator =
+ new TestResumableStreamIterator(noopTracer, customMaxSettings) {
+ @Override
+ BackOff newBackOff() {
+ return fixedBackOff(BackOff.STOP);
+ }
+
+ @Override
+ CloseableIterator startStream(
+ @Nullable ByteString resumeToken,
+ @Nullable StreamMessageListener streamMessageListener,
+ XGoogSpannerRequestId requestId) {
+ throw new AssertionError("unexpected startStream");
+ }
+ };
+ assertEquals(
+ 250L, stopBackoffCustomMaxIterator.checkRetryBudgetAndGetDelay(absentDelayException));
+
+ initWithLimitAndRetrySettings(
+ Integer.MAX_VALUE,
+ RetrySettings.newBuilder()
+ .setInitialRetryDelayDuration(java.time.Duration.ofMillis(1L))
+ .setMaxRetryDelayDuration(java.time.Duration.ofMillis(1L))
+ .build());
+ for (int i = 0; i < 100; i++) {
+ assertThat(resumableStreamIterator.checkRetryBudgetAndGetDelay(zeroDelayException))
+ .isAtLeast(1L);
+ }
+ }
+
+ @Test
+ public void effectiveMaxRetryDelayAndNewBackOff_handleSubMillisecondAndOverflowingDurations() {
+ SpannerException twoHundredMsException =
+ new RetryableException(
+ errorCodeParameter,
+ "200ms delay",
+ statusWithRetryInfo(errorCodeParameter, 0L, (int) TimeUnit.MILLISECONDS.toNanos(200L)));
+ SpannerException zeroDelayException =
+ new RetryableException(
+ errorCodeParameter, "zero delay", statusWithRetryInfo(errorCodeParameter, 0L, 0));
+
+ RetrySettings subMsSettings =
+ RetrySettings.newBuilder()
+ .setInitialRetryDelayDuration(java.time.Duration.ofNanos(500_000L))
+ .setMaxRetryDelayDuration(java.time.Duration.ofNanos(500_000L))
+ .build();
+ initWithLimitAndRetrySettings(Integer.MAX_VALUE, subMsSettings);
+ assertEquals(1L, resumableStreamIterator.checkRetryBudgetAndGetDelay(twoHundredMsException));
+ assertThat(resumableStreamIterator.checkRetryBudgetAndGetDelay(zeroDelayException))
+ .isAtLeast(1L);
+
+ RetrySettings overflowMaxSettings =
+ RetrySettings.newBuilder()
+ .setInitialRetryDelayDuration(java.time.Duration.ofMillis(10L))
+ .setMaxRetryDelayDuration(java.time.Duration.ofSeconds(Long.MAX_VALUE))
+ .build();
+ initWithLimitAndRetrySettings(Integer.MAX_VALUE, overflowMaxSettings);
+ assertEquals(200L, resumableStreamIterator.checkRetryBudgetAndGetDelay(twoHundredMsException));
+ long backoffDelay = resumableStreamIterator.checkRetryBudgetAndGetDelay(zeroDelayException);
+ assertThat(backoffDelay).isAtLeast(5L);
+ assertThat(backoffDelay).isAtMost(15L);
+
+ RetrySettings overflowInitialAndMaxSettings = mock(RetrySettings.class);
+ when(overflowInitialAndMaxSettings.getTotalTimeoutDuration())
+ .thenReturn(java.time.Duration.ZERO);
+ when(overflowInitialAndMaxSettings.getRetryDelayMultiplier()).thenReturn(1.0);
+ when(overflowInitialAndMaxSettings.getInitialRetryDelayDuration())
+ .thenReturn(java.time.Duration.ofSeconds(Long.MAX_VALUE));
+ when(overflowInitialAndMaxSettings.getMaxRetryDelayDuration())
+ .thenReturn(java.time.Duration.ofSeconds(Long.MAX_VALUE));
+ initWithLimitAndRetrySettings(Integer.MAX_VALUE, overflowInitialAndMaxSettings);
+ assertThat(resumableStreamIterator.checkRetryBudgetAndGetDelay(zeroDelayException))
+ .isAtLeast(1L);
+ }
+
+ @Test
+ public void newBackOff_unsetMaxRetryDelayUsesDefaultCeilingForClientBackoff() {
+ initWithLimitAndRetrySettings(
+ Integer.MAX_VALUE, RetrySettings.newBuilder().setRetryDelayMultiplier(10.0).build());
+ SpannerException absentDelayException =
+ new RetryableException(
+ errorCodeParameter,
+ "absent delay",
+ errorCodeParameter.getGrpcStatus().asRuntimeException());
+ long maxObserved = 0L;
+ for (int i = 0; i < 6; i++) {
+ long delayMillis = resumableStreamIterator.checkRetryBudgetAndGetDelay(absentDelayException);
+ assertThat(delayMillis).isAtLeast(1L);
+ assertThat(delayMillis).isAtMost(1500L);
+ maxObserved = Math.max(maxObserved, delayMillis);
+ }
+ // Intervals 1, 10, 100, 1000, 1000, 1000 (+/-50% jitter): the 1000ms ceiling is reached.
+ assertThat(maxObserved).isAtLeast(500L);
+ }
+
@Test(timeout = 60000L)
public void customTotalTimeoutSmallerThanRetryDelay_doesNotRetry() {
// Retrying is only allowed if the retry delay still fits in the remaining total timeout
From a7a37dd2804bd19b2bec42483a95c2c624285f04 Mon Sep 17 00:00:00 2001
From: Sakthivel Subramanian
<179120858+sakthivelmanii@users.noreply.github.com>
Date: Thu, 19 Nov 2026 03:02:11 +0000
Subject: [PATCH 2/2] Addressed comments
---
.../cloud/spanner/ResumableStreamIteratorTest.java | 13 +++++--------
1 file changed, 5 insertions(+), 8 deletions(-)
diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/ResumableStreamIteratorTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/ResumableStreamIteratorTest.java
index fb587768fe30..e3aba0c783e3 100644
--- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/ResumableStreamIteratorTest.java
+++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/ResumableStreamIteratorTest.java
@@ -892,14 +892,11 @@ public void effectiveMaxRetryDelayAndNewBackOff_handleSubMillisecondAndOverflowi
assertThat(backoffDelay).isAtLeast(5L);
assertThat(backoffDelay).isAtMost(15L);
- RetrySettings overflowInitialAndMaxSettings = mock(RetrySettings.class);
- when(overflowInitialAndMaxSettings.getTotalTimeoutDuration())
- .thenReturn(java.time.Duration.ZERO);
- when(overflowInitialAndMaxSettings.getRetryDelayMultiplier()).thenReturn(1.0);
- when(overflowInitialAndMaxSettings.getInitialRetryDelayDuration())
- .thenReturn(java.time.Duration.ofSeconds(Long.MAX_VALUE));
- when(overflowInitialAndMaxSettings.getMaxRetryDelayDuration())
- .thenReturn(java.time.Duration.ofSeconds(Long.MAX_VALUE));
+ RetrySettings overflowInitialAndMaxSettings =
+ RetrySettings.newBuilder()
+ .setInitialRetryDelayDuration(java.time.Duration.ofMillis(Long.MAX_VALUE))
+ .setMaxRetryDelayDuration(java.time.Duration.ofSeconds(Long.MAX_VALUE))
+ .build();
initWithLimitAndRetrySettings(Integer.MAX_VALUE, overflowInitialAndMaxSettings);
assertThat(resumableStreamIterator.checkRetryBudgetAndGetDelay(zeroDelayException))
.isAtLeast(1L);