From b4397343444c6e22401dcce4647e594bf18be4f5 Mon Sep 17 00:00:00 2001 From: Kyle Stang Date: Wed, 19 Aug 2026 23:32:24 +0000 Subject: [PATCH] feat(ruler): add limit to list-rules API This commit adds a limit to the number of rules returned by the ListRules prometheus API, after which the API will return 400 and an error message encourages the caller to use pagination. This is being implemented after we found that serializing very large numbers of rules massively spikes the ruler CPU, which causes issues with rule evaluations and request timeouts. Signed-off-by: Kyle Stang --- CHANGELOG.md | 1 + docs/configuration/config-file-reference.md | 5 + pkg/ruler/api.go | 7 ++ pkg/ruler/api_test.go | 129 ++++++++++++++++++++ pkg/ruler/ruler.go | 4 + schemas/cortex-config-schema.json | 6 + 6 files changed, 152 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16148eeae63..84b083dc73b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,7 @@ * [ENHANCEMENT] Upgrade Thanos and promql-engine to latest. #7740 * [ENHANCEMENT] Ruler: Adjust ruler frontend decoder to not wrap query error messages with execution prefix, this makes error responses consistent between internal and external ruler paths. #7741 * [ENHANCEMENT] Distributor: Deduplicate metric metadata when converting PRW 2.0 requests. PRW 2.0 attaches metadata to every series, so a metric family was previously expanded into one `MetricMetadata` per series. #7760 +* [ENHANCEMENT] Ruler: Add new limit `-ruler.list-rules-max-rules` on the total number of rules returned by the Prometheus ListRules API. Requests exceeding the limit are rejected with HTTP 400. Defaults to 0, which is unlimited. #7785 * [BUGFIX] Querier: Fix queryWithRetry and labelsWithRetry returning (nil, nil) on cancelled context by propagating ctx.Err(). #7370 * [BUGFIX] Metrics Helper: Fix non-deterministic bucket order in merged histograms by sorting buckets after map iteration, matching Prometheus client library behavior. #7380 * [BUGFIX] Distributor: Return HTTP 401 Unauthorized when tenant ID resolution fails in the Prometheus Remote Write 2.0 path. #7389 diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index b7922a94376..de69617339f 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -6241,6 +6241,11 @@ ring: # CLI flag: -ruler.disabled-tenants [disabled_tenants: | default = ""] +# Maximum number of rules returned by the Prometheus ListRules API. Defaults to +# 0, which is unlimited +# CLI flag: -ruler.list-rules-max-rules +[list_rules_max_rules: | default = 0] + # Report query statistics for ruler queries to complete as a per user metric and # as an info level log message. # CLI flag: -ruler.query-stats-enabled diff --git a/pkg/ruler/api.go b/pkg/ruler/api.go index 177782e607b..9411c818667 100644 --- a/pkg/ruler/api.go +++ b/pkg/ruler/api.go @@ -199,6 +199,7 @@ func (a *API) PrometheusRules(w http.ResponseWriter, req *http.Request) { groups := make([]*RuleGroup, 0, len(response.Groups)) + rulesCount := 0 for _, g := range response.Groups { grp := RuleGroup{ Name: g.Group.Name, @@ -254,9 +255,15 @@ func (a *API) PrometheusRules(w http.ResponseWriter, req *http.Request) { } } } + rulesCount += len(g.ActiveRules) groups = append(groups, &grp) } + if a.ruler.cfg.ListRulesMaxRules > 0 && uint(rulesCount) > a.ruler.cfg.ListRulesMaxRules { + util_api.RespondError(logger, w, v1.ErrBadData, fmt.Sprintf("request returned %d rules, more than the limit of %d. Consider listing a smaller page size", rulesCount, a.ruler.cfg.ListRulesMaxRules), http.StatusBadRequest) + return + } + sort.Slice(groups, func(i, j int) bool { if groups[i].File == groups[j].File { return groups[i].Name < groups[j].Name diff --git a/pkg/ruler/api_test.go b/pkg/ruler/api_test.go index f55b1e0e314..5e351c734bb 100644 --- a/pkg/ruler/api_test.go +++ b/pkg/ruler/api_test.go @@ -14,6 +14,7 @@ import ( "github.com/go-kit/log" "github.com/gorilla/mux" + v1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/stretchr/testify/require" "github.com/weaveworks/common/user" @@ -426,6 +427,134 @@ func TestRuler_rules_limit(t *testing.T) { require.JSONEq(t, string(expectedResponse), string(actual)) } +func TestRuler_rules_max_rules(t *testing.T) { + // mockRulesNamespaces gives user1 two groups of two rules each, so filters and + // pagination can be used to vary how many rules a single request returns. + for _, tc := range []struct { + name string + rules map[string]rulespb.RuleGroupList + userID string + maxRules uint + queryParams string + expectedGroups int + expectedRules int + expectedError string + }{ + { + name: "limit disabled returns every rule", + rules: mockRulesNamespaces, + userID: "user1", + maxRules: 0, + expectedGroups: 2, + expectedRules: 4, + }, + { + name: "count below the limit", + rules: mockRulesNamespaces, + userID: "user1", + maxRules: 10, + expectedGroups: 2, + expectedRules: 4, + }, + { + name: "count exactly at the limit", + rules: mockRulesNamespaces, + userID: "user1", + maxRules: 4, + expectedGroups: 2, + expectedRules: 4, + }, + { + name: "count over the limit is rejected, summed across groups", + rules: mockRulesNamespaces, + userID: "user1", + maxRules: 3, + expectedError: "request returned 4 rules, more than the limit of 3", + }, + { + name: "only the rules matching the filters are counted", + rules: mockRulesNamespaces, + userID: "user1", + maxRules: 3, + queryParams: "?type=alert", + expectedGroups: 2, + expectedRules: 2, + }, + { + name: "paginating below the limit succeeds", + rules: mockRulesNamespaces, + userID: "user1", + maxRules: 3, + queryParams: "?group_limit=1", + expectedGroups: 1, + expectedRules: 2, + }, + { + name: "limit applies per request: tenant over the limit", + rules: mockRules, + userID: "user1", + maxRules: 1, + expectedError: "request returned 2 rules, more than the limit of 1", + }, + { + name: "limit applies per request: tenant under the limit", + rules: mockRules, + userID: "user2", + maxRules: 1, + expectedGroups: 1, + expectedRules: 1, + }, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := defaultRulerConfig(t) + cfg.ListRulesMaxRules = tc.maxRules + + r := newTestRuler(t, cfg, newMockRuleStore(tc.rules, nil), nil) + defer services.StopAndAwaitTerminated(context.Background(), r) //nolint:errcheck + + a := NewAPI(r, r.store, log.NewNopLogger()) + + req := requestFor(t, http.MethodGet, "https://localhost:8080/api/prom/api/v1/rules"+tc.queryParams, nil, tc.userID) + w := httptest.NewRecorder() + a.PrometheusRules(w, req) + + resp := w.Result() + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + // util_api.Response holds Data as an any, so decode into a shape that + // exposes the rule groups directly. + parsed := struct { + Status string `json:"status"` + Data RuleDiscovery `json:"data"` + ErrorType v1.ErrorType `json:"errorType"` + Error string `json:"error"` + }{} + require.NoError(t, json.Unmarshal(body, &parsed)) + + if tc.expectedError != "" { + require.Equal(t, http.StatusBadRequest, resp.StatusCode) + require.Equal(t, "error", parsed.Status) + require.Equal(t, v1.ErrBadData, parsed.ErrorType) + require.Contains(t, parsed.Error, tc.expectedError) + require.Empty(t, parsed.Data.RuleGroups) + return + } + + require.Equal(t, http.StatusOK, resp.StatusCode) + require.Equal(t, "success", parsed.Status) + require.Empty(t, parsed.Error) + require.Len(t, parsed.Data.RuleGroups, tc.expectedGroups) + + rulesCount := 0 + for _, g := range parsed.Data.RuleGroups { + rulesCount += len(g.Rules) + } + require.Equal(t, tc.expectedRules, rulesCount) + }) + } +} + func TestRuler_alerts(t *testing.T) { store := newMockRuleStore(mockRules, nil) cfg := defaultRulerConfig(t) diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index 82f7c57fb0d..1a633987257 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -168,6 +168,8 @@ type Config struct { EnabledTenants flagext.StringSliceCSV `yaml:"enabled_tenants"` DisabledTenants flagext.StringSliceCSV `yaml:"disabled_tenants"` + ListRulesMaxRules uint `yaml:"list_rules_max_rules"` + RingCheckPeriod time.Duration `yaml:"-"` // Field will be populated during runtime. @@ -268,6 +270,8 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.Var(&cfg.EnabledTenants, "ruler.enabled-tenants", "Comma separated list of tenants whose rules this ruler can evaluate. If specified, only these tenants will be handled by ruler, otherwise this ruler can process rules from all tenants. Subject to sharding.") f.Var(&cfg.DisabledTenants, "ruler.disabled-tenants", "Comma separated list of tenants whose rules this ruler cannot evaluate. If specified, a ruler that would normally pick the specified tenant(s) for processing will ignore them instead. Subject to sharding.") + f.UintVar(&cfg.ListRulesMaxRules, "ruler.list-rules-max-rules", 0, "Maximum number of rules returned by the Prometheus ListRules API. Defaults to 0, which is unlimited") + f.BoolVar(&cfg.EnableQueryStats, "ruler.query-stats-enabled", false, "Report query statistics for ruler queries to complete as a per user metric and as an info level log message.") f.BoolVar(&cfg.DisableRuleGroupLabel, "ruler.disable-rule-group-label", false, "Disable the rule_group label on exported metrics") diff --git a/schemas/cortex-config-schema.json b/schemas/cortex-config-schema.json index aed2998e063..28fde8fb55c 100644 --- a/schemas/cortex-config-schema.json +++ b/schemas/cortex-config-schema.json @@ -7552,6 +7552,12 @@ }, "type": "object" }, + "list_rules_max_rules": { + "default": 0, + "description": "Maximum number of rules returned by the Prometheus ListRules API. Defaults to 0, which is unlimited", + "type": "number", + "x-cli-flag": "ruler.list-rules-max-rules" + }, "liveness_check_timeout": { "default": "1s", "description": "Timeout duration for non-primary rulers during liveness checks. If the check times out, the non-primary ruler will evaluate the rule group. Applicable when ruler.enable-ha-evaluation is true.",