diff --git a/CHANGELOG.md b/CHANGELOG.md index 9daa87d2f2c..6f6a0ed1326 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ * [ENHANCEMENT] Ruler: Add rule information to query parameters sent to the Query Frontend when `-ruler.frontend-address` is configured, so the Query Frontend query stats logs (`-frontend.enabled-ruler-query-stats`) can identify the evaluated rule. In this mode, the Ruler query stats log (`-ruler.query-stats-enabled`) omits the fetched statistics fields, which are always zero. #6539 * [ENHANCEMENT] gRPC: When `-auth.enabled=true`, a gRPC request without the `X-Scope-OrgID` metadata is now rejected with the `UNAUTHENTICATED` status code instead of `UNKNOWN`. #7873 * [ENHANCEMENT] Upgrade promql-engine and Thanos to latest. #7889 +* [ENHANCEMENT] Ruler: Retry a notification the Alertmanager failed to accept with a transport error or a 5xx response, instead of dropping it and leaving its alert undelivered until the next evaluation of the rule group. Enable the retries with `-ruler.notification.backoff-retries` (`0`, no retry, by default), and tune the delay between them with `-ruler.notification.backoff-min-period` and `-ruler.notification.backoff-max-period`. The retries are bounded by `-ruler.notification-timeout` and counted by the new `cortex_ruler_notification_retries_total` metric. #7909 * [BUGFIX] Update prometheus alertmanager version to v0.34.1 to apply upstream inhibit rule bug fixes. #7877 * [BUGFIX] Tenant Federation: Fix regex tenant federation not working in single binary mode, and failing with `multiple org IDs present` when a matched tenant ID contains regex metacharacters and also matches other tenants as a regex. #7869 * [BUGFIX] Query Frontend: Fix the data selection time window computed for an instant query sent without a `time` parameter. The default evaluation time was passed to `util.ParseTimeParam` in milliseconds, but that helper formats the default and parses it as seconds, so the value overflowed `UnixNano()` and wrapped to a timestamp decades away from `now`. This made the `data_select_min_time` and `data_select_max_time` query stats wrong, and stopped the `time_window` query attribute of query priority and query rejection from matching such queries correctly. #7880 diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 628ee3f9027..001c6c0d3ed 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -6043,6 +6043,26 @@ ruler_client: # CLI flag: -ruler.notification-timeout [notification_timeout: | default = 10s] +notification_backoff: + # Minimum delay between notification retries. Only applies when + # -ruler.notification.backoff-retries is greater than 0. + # CLI flag: -ruler.notification.backoff-min-period + [min_period: | default = 100ms] + + # Maximum delay between notification retries. Only applies when + # -ruler.notification.backoff-retries is greater than 0. + # CLI flag: -ruler.notification.backoff-max-period + [max_period: | default = 1s] + + # Maximum number of times to retry a notification that the Alertmanager failed + # to accept with a retriable error, which is a transport error or a 5xx + # response. A notification that is not retried is dropped and the alert is not + # delivered until the next evaluation of its rule group. Retries happen within + # -ruler.notification-timeout and delay the other notifications of the same + # tenant, so keep this number small. Set to 0 to disable the retries. + # CLI flag: -ruler.notification.backoff-retries + [max_retries: | default = 0] + alertmanager_client: # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. diff --git a/pkg/ruler/manager.go b/pkg/ruler/manager.go index 924d461f9d7..9ae814e88a4 100644 --- a/pkg/ruler/manager.go +++ b/pkg/ruler/manager.go @@ -28,9 +28,14 @@ import ( "github.com/cortexproject/cortex/pkg/parser" "github.com/cortexproject/cortex/pkg/ring/client" "github.com/cortexproject/cortex/pkg/ruler/rulespb" + "github.com/cortexproject/cortex/pkg/util/backoff" "github.com/cortexproject/cortex/pkg/util/users" ) +// maxNotificationErrorMessageLength is the number of characters of the Alertmanager +// response body kept in the error logged for a notification it did not accept. +const maxNotificationErrorMessageLength = 150 + type DefaultMultiTenantManager struct { cfg Config notifierCfg *config.Config @@ -64,6 +69,7 @@ type DefaultMultiTenantManager struct { lastReloadSuccessful *prometheus.GaugeVec lastReloadSuccessfulTimestamp *prometheus.GaugeVec configUpdatesTotal *prometheus.CounterVec + notificationRetriesTotal *prometheus.CounterVec registry prometheus.Registerer logger log.Logger @@ -137,6 +143,11 @@ func NewDefaultMultiTenantManager(cfg Config, limits RulesLimits, managerFactory Name: "ruler_config_updates_total", Help: "Total number of config updates triggered by a user", }, []string{"user"}), + notificationRetriesTotal: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ + Namespace: "cortex", + Name: "ruler_notification_retries_total", + Help: "Total number of notifications to the Alertmanager retried after a retriable failure.", + }, []string{"user"}), registry: reg, logger: logger, ruleGroupIterationFunc: defaultRuleGroupIterationFunc, @@ -184,6 +195,7 @@ func (r *DefaultMultiTenantManager) SyncRuleGroups(ctx context.Context, ruleGrou r.lastReloadSuccessful.DeleteLabelValues(userID) r.lastReloadSuccessfulTimestamp.DeleteLabelValues(userID) r.configUpdatesTotal.DeleteLabelValues(userID) + r.notificationRetriesTotal.DeleteLabelValues(userID) r.userManagerMetrics.RemoveUserRegistry(userID) if r.ruleEvalMetrics != nil { r.ruleEvalMetrics.deletePerUserMetrics(userID) @@ -420,6 +432,7 @@ func (r *DefaultMultiTenantManager) getOrCreateNotifier(userID string, userManag } logger := log.With(r.logger, "user", userID) + notificationRetries := r.notificationRetriesTotal.WithLabelValues(userID) n = newRulerNotifier(¬ifier.Options{ QueueCapacity: r.cfg.NotificationQueueCapacity, @@ -437,25 +450,7 @@ func (r *DefaultMultiTenantManager) getOrCreateNotifier(userID string, userManag defer sp.Finish() ctx = ot.ContextWithSpan(ctx, sp) _ = ot.GlobalTracer().Inject(sp.Context(), ot.HTTPHeaders, ot.HTTPHeadersCarrier(req.Header)) - resp, err := ctxhttp.Do(ctx, client, req) - if err != nil { - level.Error(logger).Log("msg", "error occurred while sending alerts", "error", err) - return resp, err - } - defer resp.Body.Close() - if resp.StatusCode/100 != 2 { - bodyBytes, err := io.ReadAll(resp.Body) - if err != nil { - level.Error(logger).Log("msg", "error reading response body", "error", err, "response code", resp.StatusCode) - return resp, err - } - customErrorMessage := string(bodyBytes) - if len(customErrorMessage) >= 150 { - customErrorMessage = customErrorMessage[:150] - } - level.Error(logger).Log("msg", "error occurred sending notification", "error", customErrorMessage, "response code", resp.StatusCode) - } - return resp, err + return sendNotification(ctx, client, req, r.cfg.NotificationBackoff, notificationRetries, logger) }, }, r.cfg.NameValidationScheme, logger, userManagerRegistry, r.notifiersDiscoveryMetrics) @@ -470,6 +465,108 @@ func (r *DefaultMultiTenantManager) getOrCreateNotifier(userID string, userManag return n.notifier, nil } +// sendNotification sends the alerts of a tenant to one Alertmanager, retrying +// the failures that another attempt may recover from. +// +// The Prometheus notifier does not retry: it logs the error returned here and +// moves on to the next batch, so a notification failing once is never delivered +// again until the rule group is evaluated again. Two failures recover on a +// second attempt, and both are routine when the ruler keeps connections alive: +// the Alertmanager closing an idle connection just as it is reused, and an +// Alertmanager replying 503 because it is shutting down while the connection is +// still pinned to it. +// +// Retries are disabled unless backoffCfg.MaxRetries is greater than 0, and are +// always bounded by the deadline of ctx, which carries the notification timeout. +func sendNotification(ctx context.Context, client *http.Client, req *http.Request, backoffCfg backoff.Config, retries prometheus.Counter, logger log.Logger) (*http.Response, error) { + // A MaxRetries of 0 means infinite retries to backoff.Backoff, so the retries + // have to be skipped here instead of relying on Ongoing() to stop them. + var retry *backoff.Backoff + if backoffCfg.MaxRetries > 0 { + retry = backoff.New(ctx, backoffCfg) + } + + for { + resp, retriable, err := doNotification(ctx, client, req, logger) + if !retriable || retry == nil || !retry.Ongoing() { + return resp, err + } + + retryReq, rewindErr := rewindNotificationRequest(req) + if rewindErr != nil { + level.Error(logger).Log("msg", "notification cannot be retried", "error", rewindErr) + return resp, err + } + req = retryReq + + // The connection the failed attempt used may be half-closed by the + // Alertmanager, or pinned to an instance that is refusing the alerts while it + // shuts down. Dropping the idle connections of this tenant's client makes the + // next attempt open a new one, which is also how it reaches another instance + // when the Alertmanagers are behind a load balancer. + client.CloseIdleConnections() + + retries.Inc() + retry.Wait() + } +} + +// doNotification sends the alerts once and reports whether the failure, if any, +// is worth another attempt. The response body is always closed, so the returned +// response carries the status and nothing else. +func doNotification(ctx context.Context, client *http.Client, req *http.Request, logger log.Logger) (*http.Response, bool, error) { + resp, err := ctxhttp.Do(ctx, client, req) + if err != nil { + level.Error(logger).Log("msg", "error occurred while sending alerts", "error", err) + return resp, isRetriableNotificationError(err), err + } + defer resp.Body.Close() + + if resp.StatusCode/100 != 2 { + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + level.Error(logger).Log("msg", "error reading response body", "error", err, "response code", resp.StatusCode) + return resp, false, err + } + customErrorMessage := string(bodyBytes) + if len(customErrorMessage) >= maxNotificationErrorMessageLength { + customErrorMessage = customErrorMessage[:maxNotificationErrorMessageLength] + } + level.Error(logger).Log("msg", "error occurred sending notification", "error", customErrorMessage, "response code", resp.StatusCode) + } + + // Only a server error can succeed on another attempt. A request the + // Alertmanager rejected is rejected again, however many times it is sent. + return resp, resp.StatusCode/100 == 5, nil +} + +// isRetriableNotificationError returns whether the transport error may be +// recovered from by sending the alerts again. An error caused by the context +// being cancelled or its deadline being exceeded is not retriable, there is no +// time left to retry. +func isRetriableNotificationError(err error) bool { + return !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) +} + +// rewindNotificationRequest returns a copy of req that can be sent again. The +// body of a request that has already been sent is consumed and closed. +func rewindNotificationRequest(req *http.Request) (*http.Request, error) { + // GetBody is always set by the Prometheus notifier, which builds the request + // over a bytes.Reader, but Do is a public option of the notifier. + if req.GetBody == nil { + return nil, errors.New("the request body cannot be replayed") + } + + body, err := req.GetBody() + if err != nil { + return nil, err + } + + rewound := req.Clone(req.Context()) + rewound.Body = body + return rewound, nil +} + func (r *DefaultMultiTenantManager) notifierApplyExternalLabels(userID string, externalLabels labels.Labels) error { r.notifiersMtx.Lock() defer r.notifiersMtx.Unlock() diff --git a/pkg/ruler/manager_test.go b/pkg/ruler/manager_test.go index c22fe857039..9e44cb7c4cf 100644 --- a/pkg/ruler/manager_test.go +++ b/pkg/ruler/manager_test.go @@ -1,13 +1,19 @@ package ruler import ( + "bytes" "context" + "io" + "net" + "net/http" + "net/http/httptest" "sync" "testing" "time" "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/common/promslog" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/rulefmt" @@ -21,6 +27,7 @@ import ( "github.com/cortexproject/cortex/pkg/ring/client" "github.com/cortexproject/cortex/pkg/ruler/rulespb" "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/backoff" "github.com/cortexproject/cortex/pkg/util/test" "github.com/cortexproject/cortex/pkg/util/users" ) @@ -601,3 +608,187 @@ func TestSyncRuleGroups_FederatedRuleGroups(t *testing.T) { require.Empty(t, m.federatedGroups) }) } + +const testAlertsPayload = `[{"labels":{"alertname":"TestAlert"}}]` + +func newTestNotificationRequest(t *testing.T, url string) *http.Request { + t.Helper() + + req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader([]byte(testAlertsPayload))) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + return req +} + +func newTestNotificationRetriesCounter() prometheus.Counter { + return prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "cortex", + Name: "ruler_notification_retries_total", + Help: "Total number of notifications to the Alertmanager retried after a retriable failure.", + }) +} + +func TestSendNotification(t *testing.T) { + for name, tc := range map[string]struct { + failures int + failureStatus int + maxRetries int + expectedCalls int64 + expectedStatus int + expectedRetries float64 + }{ + "the retries are disabled by default": { + failures: 1, + failureStatus: http.StatusServiceUnavailable, + maxRetries: 0, + expectedCalls: 1, + expectedStatus: http.StatusServiceUnavailable, + expectedRetries: 0, + }, + "a server error is retried until the Alertmanager accepts the alerts": { + failures: 1, + failureStatus: http.StatusServiceUnavailable, + maxRetries: 2, + expectedCalls: 2, + expectedStatus: http.StatusOK, + expectedRetries: 1, + }, + "a server error is retried no more than the configured number of times": { + failures: 10, + failureStatus: http.StatusServiceUnavailable, + maxRetries: 2, + expectedCalls: 3, + expectedStatus: http.StatusServiceUnavailable, + expectedRetries: 2, + }, + "a client error is not retried": { + failures: 1, + failureStatus: http.StatusBadRequest, + maxRetries: 2, + expectedCalls: 1, + expectedStatus: http.StatusBadRequest, + expectedRetries: 0, + }, + } { + t.Run(name, func(t *testing.T) { + calls := atomic.NewInt64(0) + callsWithTheAlerts := atomic.NewInt64(0) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Every attempt, and not only the first one, must carry the alerts. + if body, err := io.ReadAll(r.Body); err == nil && string(body) == testAlertsPayload { + callsWithTheAlerts.Inc() + } + if calls.Inc() <= int64(tc.failures) { + w.WriteHeader(tc.failureStatus) + return + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + retries := newTestNotificationRetriesCounter() + resp, err := sendNotification(context.Background(), server.Client(), newTestNotificationRequest(t, server.URL), backoff.Config{ + MinBackoff: time.Millisecond, + MaxBackoff: 5 * time.Millisecond, + MaxRetries: tc.maxRetries, + }, retries, log.NewNopLogger()) + + require.NoError(t, err) + require.Equal(t, tc.expectedStatus, resp.StatusCode) + require.Equal(t, tc.expectedCalls, calls.Load()) + require.Equal(t, tc.expectedCalls, callsWithTheAlerts.Load()) + require.Equal(t, tc.expectedRetries, testutil.ToFloat64(retries)) + }) + } +} + +func TestSendNotificationRetriesAConnectionClosedByTheAlertmanager(t *testing.T) { + calls := atomic.NewInt64(0) + connections := atomic.NewInt64(0) + + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if calls.Inc() > 1 { + w.WriteHeader(http.StatusOK) + return + } + // Close the connection without replying, which is what the ruler gets as an + // EOF when the Alertmanager closes a connection it considers idle. + conn, _, err := w.(http.Hijacker).Hijack() + if err == nil { + _ = conn.Close() + } + })) + server.Config.ConnState = func(_ net.Conn, state http.ConnState) { + if state == http.StateNew { + connections.Inc() + } + } + server.Start() + defer server.Close() + + retries := newTestNotificationRetriesCounter() + resp, err := sendNotification(context.Background(), server.Client(), newTestNotificationRequest(t, server.URL), backoff.Config{ + MinBackoff: time.Millisecond, + MaxBackoff: 5 * time.Millisecond, + MaxRetries: 1, + }, retries, log.NewNopLogger()) + + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode) + require.Equal(t, float64(1), testutil.ToFloat64(retries)) + // The retry cannot reuse the connection the Alertmanager closed. + require.GreaterOrEqual(t, connections.Load(), int64(2)) +} + +func TestSendNotificationStopsWhenTheNotificationTimeoutIsReached(t *testing.T) { + calls := atomic.NewInt64(0) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls.Inc() + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer server.Close() + + // The notification timeout, which the ruler puts on the context of the + // notification, leaves time for one retry out of the hundred configured. + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + retries := newTestNotificationRetriesCounter() + start := time.Now() + _, err := sendNotification(ctx, server.Client(), newTestNotificationRequest(t, server.URL), backoff.Config{ + MinBackoff: 60 * time.Millisecond, + MaxBackoff: 60 * time.Millisecond, + MaxRetries: 100, + }, retries, log.NewNopLogger()) + + require.ErrorIs(t, err, context.DeadlineExceeded) + require.Less(t, time.Since(start), time.Second) + require.Less(t, calls.Load(), int64(100)) +} + +func TestSendNotificationDoesNotRetryARequestThatCannotBeReplayed(t *testing.T) { + calls := atomic.NewInt64(0) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls.Inc() + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer server.Close() + + req := newTestNotificationRequest(t, server.URL) + req.GetBody = nil + + retries := newTestNotificationRetriesCounter() + resp, err := sendNotification(context.Background(), server.Client(), req, backoff.Config{ + MinBackoff: time.Millisecond, + MaxBackoff: 5 * time.Millisecond, + MaxRetries: 2, + }, retries, log.NewNopLogger()) + + require.NoError(t, err) + require.Equal(t, http.StatusServiceUnavailable, resp.StatusCode) + require.Equal(t, int64(1), calls.Load()) + require.Equal(t, float64(0), testutil.ToFloat64(retries)) +} diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index cf1d089154a..b60e6fc6f3d 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -41,6 +41,7 @@ import ( "github.com/cortexproject/cortex/pkg/ruler/rulestore" "github.com/cortexproject/cortex/pkg/util" util_api "github.com/cortexproject/cortex/pkg/util/api" + "github.com/cortexproject/cortex/pkg/util/backoff" "github.com/cortexproject/cortex/pkg/util/concurrency" "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/cortexproject/cortex/pkg/util/grpcclient" @@ -60,6 +61,7 @@ var ( errInvalidTenantShardSize = errors.New("invalid tenant shard size, the value must be greater than 0") errInvalidMaxConcurrentEvals = errors.New("invalid max concurrent evals, the value must be greater than 0") errInvalidQueryResponseFormat = errors.New("invalid query response format") + errInvalidNotificationRetries = errors.New("invalid notification backoff retries, the value must not be negative") ) const ( @@ -142,6 +144,11 @@ type Config struct { NotificationQueueCapacity int `yaml:"notification_queue_capacity"` // HTTP timeout duration when sending notifications to the Alertmanager. NotificationTimeout time.Duration `yaml:"notification_timeout"` + // Retries of the notifications that failed with a retriable error. Unlike + // everywhere else in Cortex, MaxRetries set to 0 disables the retries instead + // of making them infinite, so that the default does not change the behaviour + // of the ruler. + NotificationBackoff backoff.Config `yaml:"notification_backoff"` // Client configs for interacting with the Alertmanager Notifier NotifierConfig `yaml:"alertmanager_client"` @@ -222,6 +229,10 @@ func (cfg *Config) Validate(limits validation.Limits, log log.Logger) error { return errInvalidQueryResponseFormat } + if cfg.NotificationBackoff.MaxRetries < 0 { + return errInvalidNotificationRetries + } + if err := cfg.ThanosEngine.Validate(); err != nil { return err } @@ -260,6 +271,9 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.DurationVar(&cfg.AlertmanagerRefreshInterval, "ruler.alertmanager-refresh-interval", 1*time.Minute, "How long to wait between refreshing DNS resolutions of Alertmanager hosts.") f.IntVar(&cfg.NotificationQueueCapacity, "ruler.notification-queue-capacity", 10000, "Capacity of the queue for notifications to be sent to the Alertmanager.") f.DurationVar(&cfg.NotificationTimeout, "ruler.notification-timeout", 10*time.Second, "HTTP timeout duration when sending notifications to the Alertmanager.") + f.IntVar(&cfg.NotificationBackoff.MaxRetries, "ruler.notification.backoff-retries", 0, "Maximum number of times to retry a notification that the Alertmanager failed to accept with a retriable error, which is a transport error or a 5xx response. A notification that is not retried is dropped and the alert is not delivered until the next evaluation of its rule group. Retries happen within -ruler.notification-timeout and delay the other notifications of the same tenant, so keep this number small. Set to 0 to disable the retries.") + f.DurationVar(&cfg.NotificationBackoff.MinBackoff, "ruler.notification.backoff-min-period", 100*time.Millisecond, "Minimum delay between notification retries. Only applies when -ruler.notification.backoff-retries is greater than 0.") + f.DurationVar(&cfg.NotificationBackoff.MaxBackoff, "ruler.notification.backoff-max-period", 1*time.Second, "Maximum delay between notification retries. Only applies when -ruler.notification.backoff-retries is greater than 0.") f.DurationVar(&cfg.SearchPendingFor, "ruler.search-pending-for", 5*time.Minute, "Time to spend searching for a pending ruler when shutting down.") f.BoolVar(&cfg.EnableSharding, "ruler.enable-sharding", false, "Distribute rule evaluation using ring backend") diff --git a/schemas/cortex-config-schema.json b/schemas/cortex-config-schema.json index b48a37271d3..4c946f76577 100644 --- a/schemas/cortex-config-schema.json +++ b/schemas/cortex-config-schema.json @@ -7586,6 +7586,31 @@ "type": "number", "x-cli-flag": "ruler.max-concurrent-evals" }, + "notification_backoff": { + "properties": { + "max_period": { + "default": "1s", + "description": "Maximum delay between notification retries. Only applies when -ruler.notification.backoff-retries is greater than 0.", + "type": "string", + "x-cli-flag": "ruler.notification.backoff-max-period", + "x-format": "duration" + }, + "max_retries": { + "default": 0, + "description": "Maximum number of times to retry a notification that the Alertmanager failed to accept with a retriable error, which is a transport error or a 5xx response. A notification that is not retried is dropped and the alert is not delivered until the next evaluation of its rule group. Retries happen within -ruler.notification-timeout and delay the other notifications of the same tenant, so keep this number small. Set to 0 to disable the retries.", + "type": "number", + "x-cli-flag": "ruler.notification.backoff-retries" + }, + "min_period": { + "default": "100ms", + "description": "Minimum delay between notification retries. Only applies when -ruler.notification.backoff-retries is greater than 0.", + "type": "string", + "x-cli-flag": "ruler.notification.backoff-min-period", + "x-format": "duration" + } + }, + "type": "object" + }, "notification_queue_capacity": { "default": 10000, "description": "Capacity of the queue for notifications to be sent to the Alertmanager.",