diff --git a/CHANGELOG.md b/CHANGELOG.md index 16148eeae6..84b083dc73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,7 @@ * [ENHANCEMENT] Upgrade Thanos and promql-engine to latest. #7740 * [ENHANCEMENT] Ruler: Adjust ruler frontend decoder to not wrap query error messages with execution prefix, this makes error responses consistent between internal and external ruler paths. #7741 * [ENHANCEMENT] Distributor: Deduplicate metric metadata when converting PRW 2.0 requests. PRW 2.0 attaches metadata to every series, so a metric family was previously expanded into one `MetricMetadata` per series. #7760 +* [ENHANCEMENT] Ruler: Add new limit `-ruler.list-rules-max-rules` on the total number of rules returned by the Prometheus ListRules API. Requests exceeding the limit are rejected with HTTP 400. Defaults to 0, which is unlimited. #7785 * [BUGFIX] Querier: Fix queryWithRetry and labelsWithRetry returning (nil, nil) on cancelled context by propagating ctx.Err(). #7370 * [BUGFIX] Metrics Helper: Fix non-deterministic bucket order in merged histograms by sorting buckets after map iteration, matching Prometheus client library behavior. #7380 * [BUGFIX] Distributor: Return HTTP 401 Unauthorized when tenant ID resolution fails in the Prometheus Remote Write 2.0 path. #7389 diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index b7922a9437..de69617339 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -6241,6 +6241,11 @@ ring: # CLI flag: -ruler.disabled-tenants [disabled_tenants: | default = ""] +# Maximum number of rules returned by the Prometheus ListRules API. Defaults to +# 0, which is unlimited +# CLI flag: -ruler.list-rules-max-rules +[list_rules_max_rules: | default = 0] + # Report query statistics for ruler queries to complete as a per user metric and # as an info level log message. # CLI flag: -ruler.query-stats-enabled diff --git a/pkg/ruler/api.go b/pkg/ruler/api.go index 177782e607..9411c81866 100644 --- a/pkg/ruler/api.go +++ b/pkg/ruler/api.go @@ -199,6 +199,7 @@ func (a *API) PrometheusRules(w http.ResponseWriter, req *http.Request) { groups := make([]*RuleGroup, 0, len(response.Groups)) + rulesCount := 0 for _, g := range response.Groups { grp := RuleGroup{ Name: g.Group.Name, @@ -254,9 +255,15 @@ func (a *API) PrometheusRules(w http.ResponseWriter, req *http.Request) { } } } + rulesCount += len(g.ActiveRules) groups = append(groups, &grp) } + if a.ruler.cfg.ListRulesMaxRules > 0 && uint(rulesCount) > a.ruler.cfg.ListRulesMaxRules { + util_api.RespondError(logger, w, v1.ErrBadData, fmt.Sprintf("request returned %d rules, more than the limit of %d. Consider listing a smaller page size", rulesCount, a.ruler.cfg.ListRulesMaxRules), http.StatusBadRequest) + return + } + sort.Slice(groups, func(i, j int) bool { if groups[i].File == groups[j].File { return groups[i].Name < groups[j].Name diff --git a/pkg/ruler/api_test.go b/pkg/ruler/api_test.go index f55b1e0e31..5e351c734b 100644 --- a/pkg/ruler/api_test.go +++ b/pkg/ruler/api_test.go @@ -14,6 +14,7 @@ import ( "github.com/go-kit/log" "github.com/gorilla/mux" + v1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/stretchr/testify/require" "github.com/weaveworks/common/user" @@ -426,6 +427,134 @@ func TestRuler_rules_limit(t *testing.T) { require.JSONEq(t, string(expectedResponse), string(actual)) } +func TestRuler_rules_max_rules(t *testing.T) { + // mockRulesNamespaces gives user1 two groups of two rules each, so filters and + // pagination can be used to vary how many rules a single request returns. + for _, tc := range []struct { + name string + rules map[string]rulespb.RuleGroupList + userID string + maxRules uint + queryParams string + expectedGroups int + expectedRules int + expectedError string + }{ + { + name: "limit disabled returns every rule", + rules: mockRulesNamespaces, + userID: "user1", + maxRules: 0, + expectedGroups: 2, + expectedRules: 4, + }, + { + name: "count below the limit", + rules: mockRulesNamespaces, + userID: "user1", + maxRules: 10, + expectedGroups: 2, + expectedRules: 4, + }, + { + name: "count exactly at the limit", + rules: mockRulesNamespaces, + userID: "user1", + maxRules: 4, + expectedGroups: 2, + expectedRules: 4, + }, + { + name: "count over the limit is rejected, summed across groups", + rules: mockRulesNamespaces, + userID: "user1", + maxRules: 3, + expectedError: "request returned 4 rules, more than the limit of 3", + }, + { + name: "only the rules matching the filters are counted", + rules: mockRulesNamespaces, + userID: "user1", + maxRules: 3, + queryParams: "?type=alert", + expectedGroups: 2, + expectedRules: 2, + }, + { + name: "paginating below the limit succeeds", + rules: mockRulesNamespaces, + userID: "user1", + maxRules: 3, + queryParams: "?group_limit=1", + expectedGroups: 1, + expectedRules: 2, + }, + { + name: "limit applies per request: tenant over the limit", + rules: mockRules, + userID: "user1", + maxRules: 1, + expectedError: "request returned 2 rules, more than the limit of 1", + }, + { + name: "limit applies per request: tenant under the limit", + rules: mockRules, + userID: "user2", + maxRules: 1, + expectedGroups: 1, + expectedRules: 1, + }, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := defaultRulerConfig(t) + cfg.ListRulesMaxRules = tc.maxRules + + r := newTestRuler(t, cfg, newMockRuleStore(tc.rules, nil), nil) + defer services.StopAndAwaitTerminated(context.Background(), r) //nolint:errcheck + + a := NewAPI(r, r.store, log.NewNopLogger()) + + req := requestFor(t, http.MethodGet, "https://localhost:8080/api/prom/api/v1/rules"+tc.queryParams, nil, tc.userID) + w := httptest.NewRecorder() + a.PrometheusRules(w, req) + + resp := w.Result() + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + // util_api.Response holds Data as an any, so decode into a shape that + // exposes the rule groups directly. + parsed := struct { + Status string `json:"status"` + Data RuleDiscovery `json:"data"` + ErrorType v1.ErrorType `json:"errorType"` + Error string `json:"error"` + }{} + require.NoError(t, json.Unmarshal(body, &parsed)) + + if tc.expectedError != "" { + require.Equal(t, http.StatusBadRequest, resp.StatusCode) + require.Equal(t, "error", parsed.Status) + require.Equal(t, v1.ErrBadData, parsed.ErrorType) + require.Contains(t, parsed.Error, tc.expectedError) + require.Empty(t, parsed.Data.RuleGroups) + return + } + + require.Equal(t, http.StatusOK, resp.StatusCode) + require.Equal(t, "success", parsed.Status) + require.Empty(t, parsed.Error) + require.Len(t, parsed.Data.RuleGroups, tc.expectedGroups) + + rulesCount := 0 + for _, g := range parsed.Data.RuleGroups { + rulesCount += len(g.Rules) + } + require.Equal(t, tc.expectedRules, rulesCount) + }) + } +} + func TestRuler_alerts(t *testing.T) { store := newMockRuleStore(mockRules, nil) cfg := defaultRulerConfig(t) diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index 82f7c57fb0..1a63398725 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -168,6 +168,8 @@ type Config struct { EnabledTenants flagext.StringSliceCSV `yaml:"enabled_tenants"` DisabledTenants flagext.StringSliceCSV `yaml:"disabled_tenants"` + ListRulesMaxRules uint `yaml:"list_rules_max_rules"` + RingCheckPeriod time.Duration `yaml:"-"` // Field will be populated during runtime. @@ -268,6 +270,8 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.Var(&cfg.EnabledTenants, "ruler.enabled-tenants", "Comma separated list of tenants whose rules this ruler can evaluate. If specified, only these tenants will be handled by ruler, otherwise this ruler can process rules from all tenants. Subject to sharding.") f.Var(&cfg.DisabledTenants, "ruler.disabled-tenants", "Comma separated list of tenants whose rules this ruler cannot evaluate. If specified, a ruler that would normally pick the specified tenant(s) for processing will ignore them instead. Subject to sharding.") + f.UintVar(&cfg.ListRulesMaxRules, "ruler.list-rules-max-rules", 0, "Maximum number of rules returned by the Prometheus ListRules API. Defaults to 0, which is unlimited") + f.BoolVar(&cfg.EnableQueryStats, "ruler.query-stats-enabled", false, "Report query statistics for ruler queries to complete as a per user metric and as an info level log message.") f.BoolVar(&cfg.DisableRuleGroupLabel, "ruler.disable-rule-group-label", false, "Disable the rule_group label on exported metrics") diff --git a/schemas/cortex-config-schema.json b/schemas/cortex-config-schema.json index aed2998e06..28fde8fb55 100644 --- a/schemas/cortex-config-schema.json +++ b/schemas/cortex-config-schema.json @@ -7552,6 +7552,12 @@ }, "type": "object" }, + "list_rules_max_rules": { + "default": 0, + "description": "Maximum number of rules returned by the Prometheus ListRules API. Defaults to 0, which is unlimited", + "type": "number", + "x-cli-flag": "ruler.list-rules-max-rules" + }, "liveness_check_timeout": { "default": "1s", "description": "Timeout duration for non-primary rulers during liveness checks. If the check times out, the non-primary ruler will evaluate the rule group. Applicable when ruler.enable-ha-evaluation is true.",