From 7de55f8a4f18ccbf2929470214ca19c505b5ea46 Mon Sep 17 00:00:00 2001 From: Carlo Goetz Date: Fri, 23 Oct 2026 16:21:22 +0200 Subject: [PATCH 1/2] feat(dns): retry requests on 408, 502, 503, 504 --- stackit/internal/core/clientutils.go | 29 +++-- stackit/internal/core/clientutils_test.go | 19 ++- stackit/internal/core/retrytransport_test.go | 123 +++++++++++++++++++ 3 files changed, 157 insertions(+), 14 deletions(-) diff --git a/stackit/internal/core/clientutils.go b/stackit/internal/core/clientutils.go index 45ffa7990..eb08041b6 100644 --- a/stackit/internal/core/clientutils.go +++ b/stackit/internal/core/clientutils.go @@ -7,6 +7,7 @@ import ( "io" "math/rand/v2" "net/http" + "slices" "strconv" "time" @@ -215,13 +216,15 @@ type DefaultClientFactory struct { ProviderDefaultRegion string } -func (f *DefaultClientFactory) defaultConfigOptions(customEndpoint string) []config.ConfigurationOption { +func (f *DefaultClientFactory) defaultConfigOptions(customEndpoint string, additionalRetryStatusCodes ...int) []config.ConfigurationOption { // Retry HTTP 429 rate limits for every service, outside the authentication transport. + // Services can opt in to retrying additional status codes. retryRoundTripper := &RetryTransport{ - Base: f.RoundTripper, - MaxRetries: defaultMaxRetries, - BaseBackoff: defaultBaseBackoff, - MaxJitter: defaultMaxJitter, // Always added to wait time + Base: f.RoundTripper, + MaxRetries: defaultMaxRetries, + BaseBackoff: defaultBaseBackoff, + MaxJitter: defaultMaxJitter, // Always added to wait time + AdditionalRetryStatusCodes: additionalRetryStatusCodes, } apiClientConfigOptions := []config.ConfigurationOption{ @@ -545,7 +548,13 @@ func (f *DefaultClientFactory) newAuthorizationV2Client() (authorization.Default } func (f *DefaultClientFactory) newDnsV1Client() (dns.DefaultAPI, error) { - apiClientConfigOptions := f.defaultConfigOptions(f.CustomEndpoints.DnsCustomEndpoint) + apiClientConfigOptions := f.defaultConfigOptions( + f.CustomEndpoints.DnsCustomEndpoint, + http.StatusRequestTimeout, + http.StatusBadGateway, + http.StatusServiceUnavailable, + http.StatusGatewayTimeout, + ) apiClient, err := dns.NewAPIClient(apiClientConfigOptions...) if err != nil { @@ -699,12 +708,14 @@ func (f *DefaultClientFactory) newObservabilityV1Client() (observability.Default return apiClient.DefaultAPI, nil } -// RetryTransport wraps an underlying RoundTripper to retry on HTTP 429 rate limit errors with jitter. +// RetryTransport wraps an underlying RoundTripper to retry on HTTP 429 and configured additional status codes with jitter. type RetryTransport struct { Base http.RoundTripper MaxRetries int BaseBackoff time.Duration MaxJitter time.Duration + // AdditionalRetryStatusCodes are retried in addition to HTTP 429, which is always retried. + AdditionalRetryStatusCodes []int } func (t *RetryTransport) RoundTrip(req *http.Request) (*http.Response, error) { @@ -746,8 +757,8 @@ func (t *RetryTransport) RoundTrip(req *http.Request) (*http.Response, error) { resp, err = base.RoundTrip(reqClone) - // If success or non-429 error, return immediately - if err != nil || resp.StatusCode != http.StatusTooManyRequests { + // Return transport errors and responses with non-retryable status codes immediately. + if err != nil || (resp.StatusCode != http.StatusTooManyRequests && !slices.Contains(t.AdditionalRetryStatusCodes, resp.StatusCode)) { return resp, err } diff --git a/stackit/internal/core/clientutils_test.go b/stackit/internal/core/clientutils_test.go index 66be322fc..a36bfd2f7 100644 --- a/stackit/internal/core/clientutils_test.go +++ b/stackit/internal/core/clientutils_test.go @@ -211,9 +211,10 @@ func TestDefaultClientFactory_defaultConfigOptions(t *testing.T) { } type testHelper[T any] struct { - factoryMethod func(*DefaultClientFactory) (T, error) - clientInitFunc func(opts ...config.ConfigurationOption) T - customEndpointSetter func(endpointConfig *CustomEndpointConfig) + factoryMethod func(*DefaultClientFactory) (T, error) + clientInitFunc func(opts ...config.ConfigurationOption) T + customEndpointSetter func(endpointConfig *CustomEndpointConfig) + expectedAdditionalRetryStatusCodes []int } func (h *testHelper[T]) run(t *testing.T) { @@ -222,6 +223,8 @@ func (h *testHelper[T]) run(t *testing.T) { var testRoundTripper http.RoundTripper = &http.Transport{ TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS13}, } + retryTransport := expectedRetryTransport(testRoundTripper) + retryTransport.AdditionalRetryStatusCodes = h.expectedAdditionalRetryStatusCodes type fields struct { RoundTripper http.RoundTripper @@ -248,7 +251,7 @@ func (h *testHelper[T]) run(t *testing.T) { }, want: h.clientInitFunc( config.WithUserAgent("stackit-terraform-provider/1.2.3"), - config.WithCustomAuth(expectedRetryTransport(testRoundTripper)), + config.WithCustomAuth(retryTransport), ), }, { @@ -266,7 +269,7 @@ func (h *testHelper[T]) run(t *testing.T) { want: h.clientInitFunc( config.WithUserAgent("stackit-terraform-provider/1.2.3"), config.WithEndpoint(testCustomEndpoint), - config.WithCustomAuth(expectedRetryTransport(testRoundTripper)), + config.WithCustomAuth(retryTransport), ), }, } @@ -838,6 +841,12 @@ func TestDefaultClientFactory_newDnsV1Client(t *testing.T) { customEndpointSetter: func(cfg *CustomEndpointConfig) { cfg.DnsCustomEndpoint = testCustomEndpoint }, + expectedAdditionalRetryStatusCodes: []int{ + http.StatusRequestTimeout, + http.StatusBadGateway, + http.StatusServiceUnavailable, + http.StatusGatewayTimeout, + }, } test.run(t) diff --git a/stackit/internal/core/retrytransport_test.go b/stackit/internal/core/retrytransport_test.go index aedb41488..2c560cb84 100644 --- a/stackit/internal/core/retrytransport_test.go +++ b/stackit/internal/core/retrytransport_test.go @@ -148,6 +148,129 @@ func TestDefaultClientFactory_RetryRequests(t *testing.T) { } } +func TestRetryTransport_AdditionalStatusCodes(t *testing.T) { + tests := []struct { + name string + statusCode int + }{ + { + name: "request timeout", + statusCode: http.StatusRequestTimeout, + }, + { + name: "bad gateway", + statusCode: http.StatusBadGateway, + }, + { + name: "service unavailable", + statusCode: http.StatusServiceUnavailable, + }, + { + name: "gateway timeout", + statusCode: http.StatusGatewayTimeout, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + attempts := 0 + transport := expectedRetryTransport(retryTestRoundTripper(func(*http.Request) (*http.Response, error) { + attempts++ + if attempts > 2 { + t.Fatal("unexpected extra retry") + } + status := tt.statusCode + if attempts == 2 { + status = http.StatusOK + } + return &http.Response{ + StatusCode: status, + Body: http.NoBody, + }, nil + })) + transport.AdditionalRetryStatusCodes = []int{tt.statusCode} + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://example.com", http.NoBody) + if err != nil { + t.Fatal(err) + } + + resp, err := transport.RoundTrip(req) + + if err != nil { + t.Fatal(err) + } + if err := resp.Body.Close(); err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusOK { + t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusOK) + } + if attempts != 2 { + t.Errorf("attempts = %d, want 2", attempts) + } + }) + }) + } +} + +func TestRetryTransport_AdditionalStatusCodes_RetryAfter(t *testing.T) { + statusCodes := []int{ + http.StatusRequestTimeout, + http.StatusBadGateway, + http.StatusServiceUnavailable, + http.StatusGatewayTimeout, + } + + for _, statusCode := range statusCodes { + t.Run(http.StatusText(statusCode), func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + attempts := 0 + transport := expectedRetryTransport(retryTestRoundTripper(func(*http.Request) (*http.Response, error) { + attempts++ + if attempts > 2 { + t.Fatal("unexpected extra retry") + } + if attempts == 2 { + return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody}, nil + } + return &http.Response{ + StatusCode: statusCode, + Header: http.Header{"Retry-After": {"30"}}, + Body: http.NoBody, + }, nil + })) + transport.AdditionalRetryStatusCodes = []int{statusCode} + transport.MaxJitter = time.Nanosecond // use exact timing + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://example.com", http.NoBody) + if err != nil { + t.Fatal(err) + } + + start := time.Now() + + resp, err := transport.RoundTrip(req) + + if err != nil { + t.Fatal(err) + } + if err := resp.Body.Close(); err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusOK { + t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusOK) + } + if attempts != 2 { + t.Errorf("attempts = %d, want 2", attempts) + } + if elapsed := time.Since(start); elapsed != 30*time.Second { + t.Errorf("elapsed = %s, want Retry-After delay of 30s", elapsed) + } + }) + }) + } +} + func TestRetryTransport_ExponentialBackoffWithJitter(t *testing.T) { synctest.Test(t, func(t *testing.T) { attempts := 0 From 3ac6a18550006f81aee628a4ee4642f62c9e8991 Mon Sep 17 00:00:00 2001 From: Carlo Goetz Date: Mon, 26 Oct 2026 14:48:25 +0100 Subject: [PATCH 2/2] extend docs --- stackit/internal/core/clientutils.go | 1 + 1 file changed, 1 insertion(+) diff --git a/stackit/internal/core/clientutils.go b/stackit/internal/core/clientutils.go index eb08041b6..5b607e9c3 100644 --- a/stackit/internal/core/clientutils.go +++ b/stackit/internal/core/clientutils.go @@ -715,6 +715,7 @@ type RetryTransport struct { BaseBackoff time.Duration MaxJitter time.Duration // AdditionalRetryStatusCodes are retried in addition to HTTP 429, which is always retried. + // Retrying additional status codes is not yet generally available (GA) and is intended only for selected services. AdditionalRetryStatusCodes []int }