From a29e5db30a6f9030aca2c32260ef7c225dca6129 Mon Sep 17 00:00:00 2001 From: Charlie Le Date: Wed, 19 Aug 2026 15:11:49 -0700 Subject: [PATCH 1/2] Formally deprecate -blocks-storage.tsdb.max-exemplars The flag's help text has called itself deprecated for a long time, but the deprecation was never announced in CHANGELOG.md, so the two-minor removal window in v1-guarantees.md never started. It is also still live code: getMaxExemplars uses it as the global fallback whenever a tenant's max_exemplars limit is 0, so removing it now would silently disable exemplars for those tenants. Start the clock properly instead. The flag keeps working, but setting it logs a warning and increments deprecated_flags_inuse_total so operators can find and migrate the usages, and the help text names v1.24.0 as the removal target. Signed-off-by: Charlie Le --- CHANGELOG.md | 1 + docs/blocks-storage/querier.md | 8 ++++---- docs/blocks-storage/store-gateway.md | 8 ++++---- docs/configuration/config-file-reference.md | 8 ++++---- pkg/ingester/ingester.go | 5 +++++ pkg/storage/tsdb/config.go | 2 +- schemas/cortex-config-schema.json | 2 +- 7 files changed, 20 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16148eeae6..95fd37e173 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## master / unreleased * [FEATURE] Engine: Add `-querier.selector-batch-size` and `-ruler.selector-batch-size` flags to configure series batching in the Thanos promQL engine. 0 disables batching. #7763 +* [CHANGE] Ingester: Formally deprecate `-blocks-storage.tsdb.max-exemplars`, scheduled for removal in v1.24.0. Use the per-tenant `max_exemplars` limit instead. The flag still works as the global fallback when `max_exemplars` is 0, but setting it now logs a warning and increments `deprecated_flags_inuse_total`. #7793 * [CHANGE] Querier: Make query time range configurations per-tenant: `query_ingesters_within`, `query_store_after`, and `shuffle_sharding_ingesters_lookback_period`. Uses `model.Duration` instead of `time.Duration` to support serialization but has minimum unit of 1ms (nanoseconds/microseconds not supported). #7160 * [CHANGE] Cache: Setting `-blocks-storage.bucket-store.metadata-cache.bucket-index-content-ttl` to 0 will disable the bucket-index cache. #7446 * [CHANGE] HA Tracker: Move `-distributor.ha-tracker.failover-timeout` from a global config to a per-tenant runtime config. The flag name and default value (30s) remain the same. #7481 diff --git a/docs/blocks-storage/querier.md b/docs/blocks-storage/querier.md index 91c39e2d74..3dc3c704b2 100644 --- a/docs/blocks-storage/querier.md +++ b/docs/blocks-storage/querier.md @@ -2259,10 +2259,10 @@ blocks_storage: # CLI flag: -blocks-storage.tsdb.max-tsdb-opening-concurrency-on-startup [max_tsdb_opening_concurrency_on_startup: | default = 10] - # Deprecated, use maxExemplars in limits instead. If the MaxExemplars value - # in limits is set to zero, cortex will fallback on this value. This setting - # enables support for exemplars in TSDB and sets the maximum number that - # will be stored. 0 or less means disabled. + # Deprecated (use the per-tenant max_exemplars limit instead) and will be + # removed in v1.24.0: the global fallback for the maximum number of + # exemplars stored in TSDB, used only when the per-tenant max_exemplars + # limit is 0. 0 or less means exemplars are disabled. # CLI flag: -blocks-storage.tsdb.max-exemplars [max_exemplars: | default = 0] diff --git a/docs/blocks-storage/store-gateway.md b/docs/blocks-storage/store-gateway.md index 8e9e7d9e44..8c43bcb465 100644 --- a/docs/blocks-storage/store-gateway.md +++ b/docs/blocks-storage/store-gateway.md @@ -2312,10 +2312,10 @@ blocks_storage: # CLI flag: -blocks-storage.tsdb.max-tsdb-opening-concurrency-on-startup [max_tsdb_opening_concurrency_on_startup: | default = 10] - # Deprecated, use maxExemplars in limits instead. If the MaxExemplars value - # in limits is set to zero, cortex will fallback on this value. This setting - # enables support for exemplars in TSDB and sets the maximum number that - # will be stored. 0 or less means disabled. + # Deprecated (use the per-tenant max_exemplars limit instead) and will be + # removed in v1.24.0: the global fallback for the maximum number of + # exemplars stored in TSDB, used only when the per-tenant max_exemplars + # limit is 0. 0 or less means exemplars are disabled. # CLI flag: -blocks-storage.tsdb.max-exemplars [max_exemplars: | default = 0] diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index b7922a9437..39737b78d4 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -2955,10 +2955,10 @@ tsdb: # CLI flag: -blocks-storage.tsdb.max-tsdb-opening-concurrency-on-startup [max_tsdb_opening_concurrency_on_startup: | default = 10] - # Deprecated, use maxExemplars in limits instead. If the MaxExemplars value in - # limits is set to zero, cortex will fallback on this value. This setting - # enables support for exemplars in TSDB and sets the maximum number that will - # be stored. 0 or less means disabled. + # Deprecated (use the per-tenant max_exemplars limit instead) and will be + # removed in v1.24.0: the global fallback for the maximum number of exemplars + # stored in TSDB, used only when the per-tenant max_exemplars limit is 0. 0 or + # less means exemplars are disabled. # CLI flag: -blocks-storage.tsdb.max-exemplars [max_exemplars: | default = 0] diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 3a6c03449c..933a2e3183 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -832,6 +832,11 @@ func New(cfg Config, limits *validation.Overrides, registerer prometheus.Registe matchersCache: storecache.NoopMatchersCache, } + if cfg.BlocksStorageConfig.TSDB.MaxExemplars != 0 { + flagext.DeprecatedFlagsUsed.Inc() + level.Warn(logger).Log("msg", "running with DEPRECATED flag blocks-storage.tsdb.max-exemplars, use the per-tenant max_exemplars limit instead") + } + if cfg.ActiveQueriedSeriesMetricsEnabled || cfg.HeadQueriedSeriesMetricsEnabled { i.activeQueriedSeriesService = NewActiveQueriedSeriesService(logger, registerer) } diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go index f3817fe32c..3ae454d19b 100644 --- a/pkg/storage/tsdb/config.go +++ b/pkg/storage/tsdb/config.go @@ -209,7 +209,7 @@ func (cfg *TSDBConfig) RegisterFlags(f *flag.FlagSet) { f.BoolVar(&cfg.FlushBlocksOnShutdown, "blocks-storage.tsdb.flush-blocks-on-shutdown", false, "True to flush blocks to storage on shutdown. If false, incomplete blocks will be reused after restart.") f.DurationVar(&cfg.CloseIdleTSDBTimeout, "blocks-storage.tsdb.close-idle-tsdb-timeout", 0, "If TSDB has not received any data for this duration, and all blocks from TSDB have been shipped, TSDB is closed and deleted from local disk. If set to positive value, this value must be greater than -limits.query-ingesters-within flag to make sure that TSDB is not closed prematurely, which could cause partial query results. 0 or negative value disables closing of idle TSDB.") f.IntVar(&cfg.HeadChunksWriteQueueSize, "blocks-storage.tsdb.head-chunks-write-queue-size", chunks.DefaultWriteQueueSize, "The size of the in-memory queue used before flushing chunks to the disk.") - f.IntVar(&cfg.MaxExemplars, "blocks-storage.tsdb.max-exemplars", 0, "Deprecated, use maxExemplars in limits instead. If the MaxExemplars value in limits is set to zero, cortex will fallback on this value. This setting enables support for exemplars in TSDB and sets the maximum number that will be stored. 0 or less means disabled.") + f.IntVar(&cfg.MaxExemplars, "blocks-storage.tsdb.max-exemplars", 0, "Deprecated (use the per-tenant max_exemplars limit instead) and will be removed in v1.24.0: the global fallback for the maximum number of exemplars stored in TSDB, used only when the per-tenant max_exemplars limit is 0. 0 or less means exemplars are disabled.") f.BoolVar(&cfg.MemorySnapshotOnShutdown, "blocks-storage.tsdb.memory-snapshot-on-shutdown", false, "True to enable snapshotting of in-memory TSDB data on disk when shutting down.") f.Int64Var(&cfg.OutOfOrderCapMax, "blocks-storage.tsdb.out-of-order-cap-max", tsdb.DefaultOutOfOrderCapMax, "[EXPERIMENTAL] Configures the maximum number of samples per chunk that can be out-of-order.") diff --git a/schemas/cortex-config-schema.json b/schemas/cortex-config-schema.json index aed2998e06..dffd947606 100644 --- a/schemas/cortex-config-schema.json +++ b/schemas/cortex-config-schema.json @@ -3579,7 +3579,7 @@ }, "max_exemplars": { "default": 0, - "description": "Deprecated, use maxExemplars in limits instead. If the MaxExemplars value in limits is set to zero, cortex will fallback on this value. This setting enables support for exemplars in TSDB and sets the maximum number that will be stored. 0 or less means disabled.", + "description": "Deprecated (use the per-tenant max_exemplars limit instead) and will be removed in v1.24.0: the global fallback for the maximum number of exemplars stored in TSDB, used only when the per-tenant max_exemplars limit is 0. 0 or less means exemplars are disabled.", "type": "number", "x-cli-flag": "blocks-storage.tsdb.max-exemplars" }, From 6b29a0d4769028c2d9e2db4b47bdf618c6473ace Mon Sep 17 00:00:00 2001 From: Charlie Le Date: Fri, 21 Aug 2026 10:53:21 -0700 Subject: [PATCH 2/2] Note the deprecated fallback in -ingester.max-exemplars help text The per-tenant max_exemplars limit pointed operators at blocks-storage.tsdb.max-exemplars without saying that flag is deprecated or that the fallback disappears in v1.24.0. An operator reading only the limits reference would follow the pointer to a flag on its way out. Both flags now tell the same story. Help text only; the fallback behavior is unchanged. Signed-off-by: Charlie Le --- docs/configuration/config-file-reference.md | 3 ++- pkg/util/validation/limits.go | 2 +- schemas/cortex-config-schema.json | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 39737b78d4..2b11f47f82 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -4671,7 +4671,8 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s # Enables support for exemplars in TSDB and sets the maximum number that will be # stored. less than zero means disabled. If the value is set to zero, cortex -# will fallback to blocks-storage.tsdb.max-exemplars value. +# will fallback to the deprecated blocks-storage.tsdb.max-exemplars value; that +# fallback is removed in v1.24.0. # CLI flag: -ingester.max-exemplars [max_exemplars: | default = 0] diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 019a5adc3e..98b8175f0a 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -316,7 +316,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.IntVar(&l.MaxLocalNativeHistogramSeriesPerUser, "ingester.max-native-histogram-series-per-user", 0, "The maximum number of active native histogram series per user, per ingester. 0 to disable. Supported only if ingester.active-series-metrics-enabled is true.") f.IntVar(&l.MaxGlobalNativeHistogramSeriesPerUser, "ingester.max-global-native-histogram-series-per-user", 0, "The maximum number of active native histogram series per user, across the cluster before replication. 0 to disable. Supported only if -distributor.shard-by-all-labels and ingester.active-series-metrics-enabled is true.") f.BoolVar(&l.EnableNativeHistograms, "blocks-storage.tsdb.enable-native-histograms", false, "[EXPERIMENTAL] True to enable native histogram.") - f.IntVar(&l.MaxExemplars, "ingester.max-exemplars", 0, "Enables support for exemplars in TSDB and sets the maximum number that will be stored. less than zero means disabled. If the value is set to zero, cortex will fallback to blocks-storage.tsdb.max-exemplars value.") + f.IntVar(&l.MaxExemplars, "ingester.max-exemplars", 0, "Enables support for exemplars in TSDB and sets the maximum number that will be stored. less than zero means disabled. If the value is set to zero, cortex will fallback to the deprecated blocks-storage.tsdb.max-exemplars value; that fallback is removed in v1.24.0.") f.Var(&l.OutOfOrderTimeWindow, "ingester.out-of-order-time-window", "[Experimental] Configures the allowed time window for ingestion of out-of-order samples. Disabled (0s) by default.") f.IntVar(&l.MaxLocalMetricsWithMetadataPerUser, "ingester.max-metadata-per-user", 8000, "The maximum number of active metrics with metadata per user, per ingester. 0 to disable.") diff --git a/schemas/cortex-config-schema.json b/schemas/cortex-config-schema.json index dffd947606..939e9134db 100644 --- a/schemas/cortex-config-schema.json +++ b/schemas/cortex-config-schema.json @@ -5770,7 +5770,7 @@ }, "max_exemplars": { "default": 0, - "description": "Enables support for exemplars in TSDB and sets the maximum number that will be stored. less than zero means disabled. If the value is set to zero, cortex will fallback to blocks-storage.tsdb.max-exemplars value.", + "description": "Enables support for exemplars in TSDB and sets the maximum number that will be stored. less than zero means disabled. If the value is set to zero, cortex will fallback to the deprecated blocks-storage.tsdb.max-exemplars value; that fallback is removed in v1.24.0.", "type": "number", "x-cli-flag": "ingester.max-exemplars" },