From 67a461477b1fbf30f7a37c04780ef83a3164006f Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Mon, 2 Nov 2026 06:39:39 +0000 Subject: [PATCH] Ruler: retry notifications the Alertmanager failed to accept The Prometheus notifier does not retry: it logs the error returned by the Do option and moves on to the next batch, so a notification the Alertmanager failed to accept is dropped and its alert is not delivered until the rule group is evaluated again. Two of those failures recover on a second attempt: the Alertmanager closing a keep-alive connection it considers idle just as the ruler reuses it, and an Alertmanager replying 503 while it shuts down with the connection still pinned to it. Retry a notification that failed with a transport error or a 5xx response, up to -ruler.notification.backoff-retries times, dropping the idle connections of the tenant's client first so that the next attempt opens a new connection. A 4xx response is not retried, the Alertmanager rejects the request however many times it is sent. The retries are disabled by default, so the behaviour does not change unless they are configured, and they are always bounded by the context of the notification, which carries -ruler.notification-timeout. Signed-off-by: Ben Ye --- CHANGELOG.md | 1 + docs/configuration/config-file-reference.md | 20 ++ pkg/ruler/manager.go | 135 ++++++++++++-- pkg/ruler/manager_test.go | 191 ++++++++++++++++++++ pkg/ruler/ruler.go | 14 ++ schemas/cortex-config-schema.json | 25 +++ 6 files changed, 367 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9daa87d2f2c..6f6a0ed1326 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ * [ENHANCEMENT] Ruler: Add rule information to query parameters sent to the Query Frontend when `-ruler.frontend-address` is configured, so the Query Frontend query stats logs (`-frontend.enabled-ruler-query-stats`) can identify the evaluated rule. In this mode, the Ruler query stats log (`-ruler.query-stats-enabled`) omits the fetched statistics fields, which are always zero. #6539 * [ENHANCEMENT] gRPC: When `-auth.enabled=true`, a gRPC request without the `X-Scope-OrgID` metadata is now rejected with the `UNAUTHENTICATED` status code instead of `UNKNOWN`. #7873 * [ENHANCEMENT] Upgrade promql-engine and Thanos to latest. #7889 +* [ENHANCEMENT] Ruler: Retry a notification the Alertmanager failed to accept with a transport error or a 5xx response, instead of dropping it and leaving its alert undelivered until the next evaluation of the rule group. Enable the retries with `-ruler.notification.backoff-retries` (`0`, no retry, by default), and tune the delay between them with `-ruler.notification.backoff-min-period` and `-ruler.notification.backoff-max-period`. The retries are bounded by `-ruler.notification-timeout` and counted by the new `cortex_ruler_notification_retries_total` metric. #7909 * [BUGFIX] Update prometheus alertmanager version to v0.34.1 to apply upstream inhibit rule bug fixes. #7877 * [BUGFIX] Tenant Federation: Fix regex tenant federation not working in single binary mode, and failing with `multiple org IDs present` when a matched tenant ID contains regex metacharacters and also matches other tenants as a regex. #7869 * [BUGFIX] Query Frontend: Fix the data selection time window computed for an instant query sent without a `time` parameter. The default evaluation time was passed to `util.ParseTimeParam` in milliseconds, but that helper formats the default and parses it as seconds, so the value overflowed `UnixNano()` and wrapped to a timestamp decades away from `now`. This made the `data_select_min_time` and `data_select_max_time` query stats wrong, and stopped the `time_window` query attribute of query priority and query rejection from matching such queries correctly. #7880 diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 628ee3f9027..001c6c0d3ed 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -6043,6 +6043,26 @@ ruler_client: # CLI flag: -ruler.notification-timeout [notification_timeout: | default = 10s] +notification_backoff: + # Minimum delay between notification retries. Only applies when + # -ruler.notification.backoff-retries is greater than 0. + # CLI flag: -ruler.notification.backoff-min-period + [min_period: | default = 100ms] + + # Maximum delay between notification retries. Only applies when + # -ruler.notification.backoff-retries is greater than 0. + # CLI flag: -ruler.notification.backoff-max-period + [max_period: | default = 1s] + + # Maximum number of times to retry a notification that the Alertmanager failed + # to accept with a retriable error, which is a transport error or a 5xx + # response. A notification that is not retried is dropped and the alert is not + # delivered until the next evaluation of its rule group. Retries happen within + # -ruler.notification-timeout and delay the other notifications of the same + # tenant, so keep this number small. Set to 0 to disable the retries. + # CLI flag: -ruler.notification.backoff-retries + [max_retries: | default = 0] + alertmanager_client: # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. diff --git a/pkg/ruler/manager.go b/pkg/ruler/manager.go index 924d461f9d7..9ae814e88a4 100644 --- a/pkg/ruler/manager.go +++ b/pkg/ruler/manager.go @@ -28,9 +28,14 @@ import ( "github.com/cortexproject/cortex/pkg/parser" "github.com/cortexproject/cortex/pkg/ring/client" "github.com/cortexproject/cortex/pkg/ruler/rulespb" + "github.com/cortexproject/cortex/pkg/util/backoff" "github.com/cortexproject/cortex/pkg/util/users" ) +// maxNotificationErrorMessageLength is the number of characters of the Alertmanager +// response body kept in the error logged for a notification it did not accept. +const maxNotificationErrorMessageLength = 150 + type DefaultMultiTenantManager struct { cfg Config notifierCfg *config.Config @@ -64,6 +69,7 @@ type DefaultMultiTenantManager struct { lastReloadSuccessful *prometheus.GaugeVec lastReloadSuccessfulTimestamp *prometheus.GaugeVec configUpdatesTotal *prometheus.CounterVec + notificationRetriesTotal *prometheus.CounterVec registry prometheus.Registerer logger log.Logger @@ -137,6 +143,11 @@ func NewDefaultMultiTenantManager(cfg Config, limits RulesLimits, managerFactory Name: "ruler_config_updates_total", Help: "Total number of config updates triggered by a user", }, []string{"user"}), + notificationRetriesTotal: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ + Namespace: "cortex", + Name: "ruler_notification_retries_total", + Help: "Total number of notifications to the Alertmanager retried after a retriable failure.", + }, []string{"user"}), registry: reg, logger: logger, ruleGroupIterationFunc: defaultRuleGroupIterationFunc, @@ -184,6 +195,7 @@ func (r *DefaultMultiTenantManager) SyncRuleGroups(ctx context.Context, ruleGrou r.lastReloadSuccessful.DeleteLabelValues(userID) r.lastReloadSuccessfulTimestamp.DeleteLabelValues(userID) r.configUpdatesTotal.DeleteLabelValues(userID) + r.notificationRetriesTotal.DeleteLabelValues(userID) r.userManagerMetrics.RemoveUserRegistry(userID) if r.ruleEvalMetrics != nil { r.ruleEvalMetrics.deletePerUserMetrics(userID) @@ -420,6 +432,7 @@ func (r *DefaultMultiTenantManager) getOrCreateNotifier(userID string, userManag } logger := log.With(r.logger, "user", userID) + notificationRetries := r.notificationRetriesTotal.WithLabelValues(userID) n = newRulerNotifier(¬ifier.Options{ QueueCapacity: r.cfg.NotificationQueueCapacity, @@ -437,25 +450,7 @@ func (r *DefaultMultiTenantManager) getOrCreateNotifier(userID string, userManag defer sp.Finish() ctx = ot.ContextWithSpan(ctx, sp) _ = ot.GlobalTracer().Inject(sp.Context(), ot.HTTPHeaders, ot.HTTPHeadersCarrier(req.Header)) - resp, err := ctxhttp.Do(ctx, client, req) - if err != nil { - level.Error(logger).Log("msg", "error occurred while sending alerts", "error", err) - return resp, err - } - defer resp.Body.Close() - if resp.StatusCode/100 != 2 { - bodyBytes, err := io.ReadAll(resp.Body) - if err != nil { - level.Error(logger).Log("msg", "error reading response body", "error", err, "response code", resp.StatusCode) - return resp, err - } - customErrorMessage := string(bodyBytes) - if len(customErrorMessage) >= 150 { - customErrorMessage = customErrorMessage[:150] - } - level.Error(logger).Log("msg", "error occurred sending notification", "error", customErrorMessage, "response code", resp.StatusCode) - } - return resp, err + return sendNotification(ctx, client, req, r.cfg.NotificationBackoff, notificationRetries, logger) }, }, r.cfg.NameValidationScheme, logger, userManagerRegistry, r.notifiersDiscoveryMetrics) @@ -470,6 +465,108 @@ func (r *DefaultMultiTenantManager) getOrCreateNotifier(userID string, userManag return n.notifier, nil } +// sendNotification sends the alerts of a tenant to one Alertmanager, retrying +// the failures that another attempt may recover from. +// +// The Prometheus notifier does not retry: it logs the error returned here and +// moves on to the next batch, so a notification failing once is never delivered +// again until the rule group is evaluated again. Two failures recover on a +// second attempt, and both are routine when the ruler keeps connections alive: +// the Alertmanager closing an idle connection just as it is reused, and an +// Alertmanager replying 503 because it is shutting down while the connection is +// still pinned to it. +// +// Retries are disabled unless backoffCfg.MaxRetries is greater than 0, and are +// always bounded by the deadline of ctx, which carries the notification timeout. +func sendNotification(ctx context.Context, client *http.Client, req *http.Request, backoffCfg backoff.Config, retries prometheus.Counter, logger log.Logger) (*http.Response, error) { + // A MaxRetries of 0 means infinite retries to backoff.Backoff, so the retries + // have to be skipped here instead of relying on Ongoing() to stop them. + var retry *backoff.Backoff + if backoffCfg.MaxRetries > 0 { + retry = backoff.New(ctx, backoffCfg) + } + + for { + resp, retriable, err := doNotification(ctx, client, req, logger) + if !retriable || retry == nil || !retry.Ongoing() { + return resp, err + } + + retryReq, rewindErr := rewindNotificationRequest(req) + if rewindErr != nil { + level.Error(logger).Log("msg", "notification cannot be retried", "error", rewindErr) + return resp, err + } + req = retryReq + + // The connection the failed attempt used may be half-closed by the + // Alertmanager, or pinned to an instance that is refusing the alerts while it + // shuts down. Dropping the idle connections of this tenant's client makes the + // next attempt open a new one, which is also how it reaches another instance + // when the Alertmanagers are behind a load balancer. + client.CloseIdleConnections() + + retries.Inc() + retry.Wait() + } +} + +// doNotification sends the alerts once and reports whether the failure, if any, +// is worth another attempt. The response body is always closed, so the returned +// response carries the status and nothing else. +func doNotification(ctx context.Context, client *http.Client, req *http.Request, logger log.Logger) (*http.Response, bool, error) { + resp, err := ctxhttp.Do(ctx, client, req) + if err != nil { + level.Error(logger).Log("msg", "error occurred while sending alerts", "error", err) + return resp, isRetriableNotificationError(err), err + } + defer resp.Body.Close() + + if resp.StatusCode/100 != 2 { + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + level.Error(logger).Log("msg", "error reading response body", "error", err, "response code", resp.StatusCode) + return resp, false, err + } + customErrorMessage := string(bodyBytes) + if len(customErrorMessage) >= maxNotificationErrorMessageLength { + customErrorMessage = customErrorMessage[:maxNotificationErrorMessageLength] + } + level.Error(logger).Log("msg", "error occurred sending notification", "error", customErrorMessage, "response code", resp.StatusCode) + } + + // Only a server error can succeed on another attempt. A request the + // Alertmanager rejected is rejected again, however many times it is sent. + return resp, resp.StatusCode/100 == 5, nil +} + +// isRetriableNotificationError returns whether the transport error may be +// recovered from by sending the alerts again. An error caused by the context +// being cancelled or its deadline being exceeded is not retriable, there is no +// time left to retry. +func isRetriableNotificationError(err error) bool { + return !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) +} + +// rewindNotificationRequest returns a copy of req that can be sent again. The +// body of a request that has already been sent is consumed and closed. +func rewindNotificationRequest(req *http.Request) (*http.Request, error) { + // GetBody is always set by the Prometheus notifier, which builds the request + // over a bytes.Reader, but Do is a public option of the notifier. + if req.GetBody == nil { + return nil, errors.New("the request body cannot be replayed") + } + + body, err := req.GetBody() + if err != nil { + return nil, err + } + + rewound := req.Clone(req.Context()) + rewound.Body = body + return rewound, nil +} + func (r *DefaultMultiTenantManager) notifierApplyExternalLabels(userID string, externalLabels labels.Labels) error { r.notifiersMtx.Lock() defer r.notifiersMtx.Unlock() diff --git a/pkg/ruler/manager_test.go b/pkg/ruler/manager_test.go index c22fe857039..9e44cb7c4cf 100644 --- a/pkg/ruler/manager_test.go +++ b/pkg/ruler/manager_test.go @@ -1,13 +1,19 @@ package ruler import ( + "bytes" "context" + "io" + "net" + "net/http" + "net/http/httptest" "sync" "testing" "time" "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/common/promslog" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/rulefmt" @@ -21,6 +27,7 @@ import ( "github.com/cortexproject/cortex/pkg/ring/client" "github.com/cortexproject/cortex/pkg/ruler/rulespb" "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/backoff" "github.com/cortexproject/cortex/pkg/util/test" "github.com/cortexproject/cortex/pkg/util/users" ) @@ -601,3 +608,187 @@ func TestSyncRuleGroups_FederatedRuleGroups(t *testing.T) { require.Empty(t, m.federatedGroups) }) } + +const testAlertsPayload = `[{"labels":{"alertname":"TestAlert"}}]` + +func newTestNotificationRequest(t *testing.T, url string) *http.Request { + t.Helper() + + req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader([]byte(testAlertsPayload))) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + return req +} + +func newTestNotificationRetriesCounter() prometheus.Counter { + return prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "cortex", + Name: "ruler_notification_retries_total", + Help: "Total number of notifications to the Alertmanager retried after a retriable failure.", + }) +} + +func TestSendNotification(t *testing.T) { + for name, tc := range map[string]struct { + failures int + failureStatus int + maxRetries int + expectedCalls int64 + expectedStatus int + expectedRetries float64 + }{ + "the retries are disabled by default": { + failures: 1, + failureStatus: http.StatusServiceUnavailable, + maxRetries: 0, + expectedCalls: 1, + expectedStatus: http.StatusServiceUnavailable, + expectedRetries: 0, + }, + "a server error is retried until the Alertmanager accepts the alerts": { + failures: 1, + failureStatus: http.StatusServiceUnavailable, + maxRetries: 2, + expectedCalls: 2, + expectedStatus: http.StatusOK, + expectedRetries: 1, + }, + "a server error is retried no more than the configured number of times": { + failures: 10, + failureStatus: http.StatusServiceUnavailable, + maxRetries: 2, + expectedCalls: 3, + expectedStatus: http.StatusServiceUnavailable, + expectedRetries: 2, + }, + "a client error is not retried": { + failures: 1, + failureStatus: http.StatusBadRequest, + maxRetries: 2, + expectedCalls: 1, + expectedStatus: http.StatusBadRequest, + expectedRetries: 0, + }, + } { + t.Run(name, func(t *testing.T) { + calls := atomic.NewInt64(0) + callsWithTheAlerts := atomic.NewInt64(0) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Every attempt, and not only the first one, must carry the alerts. + if body, err := io.ReadAll(r.Body); err == nil && string(body) == testAlertsPayload { + callsWithTheAlerts.Inc() + } + if calls.Inc() <= int64(tc.failures) { + w.WriteHeader(tc.failureStatus) + return + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + retries := newTestNotificationRetriesCounter() + resp, err := sendNotification(context.Background(), server.Client(), newTestNotificationRequest(t, server.URL), backoff.Config{ + MinBackoff: time.Millisecond, + MaxBackoff: 5 * time.Millisecond, + MaxRetries: tc.maxRetries, + }, retries, log.NewNopLogger()) + + require.NoError(t, err) + require.Equal(t, tc.expectedStatus, resp.StatusCode) + require.Equal(t, tc.expectedCalls, calls.Load()) + require.Equal(t, tc.expectedCalls, callsWithTheAlerts.Load()) + require.Equal(t, tc.expectedRetries, testutil.ToFloat64(retries)) + }) + } +} + +func TestSendNotificationRetriesAConnectionClosedByTheAlertmanager(t *testing.T) { + calls := atomic.NewInt64(0) + connections := atomic.NewInt64(0) + + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if calls.Inc() > 1 { + w.WriteHeader(http.StatusOK) + return + } + // Close the connection without replying, which is what the ruler gets as an + // EOF when the Alertmanager closes a connection it considers idle. + conn, _, err := w.(http.Hijacker).Hijack() + if err == nil { + _ = conn.Close() + } + })) + server.Config.ConnState = func(_ net.Conn, state http.ConnState) { + if state == http.StateNew { + connections.Inc() + } + } + server.Start() + defer server.Close() + + retries := newTestNotificationRetriesCounter() + resp, err := sendNotification(context.Background(), server.Client(), newTestNotificationRequest(t, server.URL), backoff.Config{ + MinBackoff: time.Millisecond, + MaxBackoff: 5 * time.Millisecond, + MaxRetries: 1, + }, retries, log.NewNopLogger()) + + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode) + require.Equal(t, float64(1), testutil.ToFloat64(retries)) + // The retry cannot reuse the connection the Alertmanager closed. + require.GreaterOrEqual(t, connections.Load(), int64(2)) +} + +func TestSendNotificationStopsWhenTheNotificationTimeoutIsReached(t *testing.T) { + calls := atomic.NewInt64(0) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls.Inc() + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer server.Close() + + // The notification timeout, which the ruler puts on the context of the + // notification, leaves time for one retry out of the hundred configured. + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + retries := newTestNotificationRetriesCounter() + start := time.Now() + _, err := sendNotification(ctx, server.Client(), newTestNotificationRequest(t, server.URL), backoff.Config{ + MinBackoff: 60 * time.Millisecond, + MaxBackoff: 60 * time.Millisecond, + MaxRetries: 100, + }, retries, log.NewNopLogger()) + + require.ErrorIs(t, err, context.DeadlineExceeded) + require.Less(t, time.Since(start), time.Second) + require.Less(t, calls.Load(), int64(100)) +} + +func TestSendNotificationDoesNotRetryARequestThatCannotBeReplayed(t *testing.T) { + calls := atomic.NewInt64(0) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls.Inc() + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer server.Close() + + req := newTestNotificationRequest(t, server.URL) + req.GetBody = nil + + retries := newTestNotificationRetriesCounter() + resp, err := sendNotification(context.Background(), server.Client(), req, backoff.Config{ + MinBackoff: time.Millisecond, + MaxBackoff: 5 * time.Millisecond, + MaxRetries: 2, + }, retries, log.NewNopLogger()) + + require.NoError(t, err) + require.Equal(t, http.StatusServiceUnavailable, resp.StatusCode) + require.Equal(t, int64(1), calls.Load()) + require.Equal(t, float64(0), testutil.ToFloat64(retries)) +} diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index cf1d089154a..b60e6fc6f3d 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -41,6 +41,7 @@ import ( "github.com/cortexproject/cortex/pkg/ruler/rulestore" "github.com/cortexproject/cortex/pkg/util" util_api "github.com/cortexproject/cortex/pkg/util/api" + "github.com/cortexproject/cortex/pkg/util/backoff" "github.com/cortexproject/cortex/pkg/util/concurrency" "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/cortexproject/cortex/pkg/util/grpcclient" @@ -60,6 +61,7 @@ var ( errInvalidTenantShardSize = errors.New("invalid tenant shard size, the value must be greater than 0") errInvalidMaxConcurrentEvals = errors.New("invalid max concurrent evals, the value must be greater than 0") errInvalidQueryResponseFormat = errors.New("invalid query response format") + errInvalidNotificationRetries = errors.New("invalid notification backoff retries, the value must not be negative") ) const ( @@ -142,6 +144,11 @@ type Config struct { NotificationQueueCapacity int `yaml:"notification_queue_capacity"` // HTTP timeout duration when sending notifications to the Alertmanager. NotificationTimeout time.Duration `yaml:"notification_timeout"` + // Retries of the notifications that failed with a retriable error. Unlike + // everywhere else in Cortex, MaxRetries set to 0 disables the retries instead + // of making them infinite, so that the default does not change the behaviour + // of the ruler. + NotificationBackoff backoff.Config `yaml:"notification_backoff"` // Client configs for interacting with the Alertmanager Notifier NotifierConfig `yaml:"alertmanager_client"` @@ -222,6 +229,10 @@ func (cfg *Config) Validate(limits validation.Limits, log log.Logger) error { return errInvalidQueryResponseFormat } + if cfg.NotificationBackoff.MaxRetries < 0 { + return errInvalidNotificationRetries + } + if err := cfg.ThanosEngine.Validate(); err != nil { return err } @@ -260,6 +271,9 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.DurationVar(&cfg.AlertmanagerRefreshInterval, "ruler.alertmanager-refresh-interval", 1*time.Minute, "How long to wait between refreshing DNS resolutions of Alertmanager hosts.") f.IntVar(&cfg.NotificationQueueCapacity, "ruler.notification-queue-capacity", 10000, "Capacity of the queue for notifications to be sent to the Alertmanager.") f.DurationVar(&cfg.NotificationTimeout, "ruler.notification-timeout", 10*time.Second, "HTTP timeout duration when sending notifications to the Alertmanager.") + f.IntVar(&cfg.NotificationBackoff.MaxRetries, "ruler.notification.backoff-retries", 0, "Maximum number of times to retry a notification that the Alertmanager failed to accept with a retriable error, which is a transport error or a 5xx response. A notification that is not retried is dropped and the alert is not delivered until the next evaluation of its rule group. Retries happen within -ruler.notification-timeout and delay the other notifications of the same tenant, so keep this number small. Set to 0 to disable the retries.") + f.DurationVar(&cfg.NotificationBackoff.MinBackoff, "ruler.notification.backoff-min-period", 100*time.Millisecond, "Minimum delay between notification retries. Only applies when -ruler.notification.backoff-retries is greater than 0.") + f.DurationVar(&cfg.NotificationBackoff.MaxBackoff, "ruler.notification.backoff-max-period", 1*time.Second, "Maximum delay between notification retries. Only applies when -ruler.notification.backoff-retries is greater than 0.") f.DurationVar(&cfg.SearchPendingFor, "ruler.search-pending-for", 5*time.Minute, "Time to spend searching for a pending ruler when shutting down.") f.BoolVar(&cfg.EnableSharding, "ruler.enable-sharding", false, "Distribute rule evaluation using ring backend") diff --git a/schemas/cortex-config-schema.json b/schemas/cortex-config-schema.json index b48a37271d3..4c946f76577 100644 --- a/schemas/cortex-config-schema.json +++ b/schemas/cortex-config-schema.json @@ -7586,6 +7586,31 @@ "type": "number", "x-cli-flag": "ruler.max-concurrent-evals" }, + "notification_backoff": { + "properties": { + "max_period": { + "default": "1s", + "description": "Maximum delay between notification retries. Only applies when -ruler.notification.backoff-retries is greater than 0.", + "type": "string", + "x-cli-flag": "ruler.notification.backoff-max-period", + "x-format": "duration" + }, + "max_retries": { + "default": 0, + "description": "Maximum number of times to retry a notification that the Alertmanager failed to accept with a retriable error, which is a transport error or a 5xx response. A notification that is not retried is dropped and the alert is not delivered until the next evaluation of its rule group. Retries happen within -ruler.notification-timeout and delay the other notifications of the same tenant, so keep this number small. Set to 0 to disable the retries.", + "type": "number", + "x-cli-flag": "ruler.notification.backoff-retries" + }, + "min_period": { + "default": "100ms", + "description": "Minimum delay between notification retries. Only applies when -ruler.notification.backoff-retries is greater than 0.", + "type": "string", + "x-cli-flag": "ruler.notification.backoff-min-period", + "x-format": "duration" + } + }, + "type": "object" + }, "notification_queue_capacity": { "default": 10000, "description": "Capacity of the queue for notifications to be sent to the Alertmanager.",