diff --git a/CHANGELOG.md b/CHANGELOG.md index 9daa87d2f2c..0452e6ab5cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ * [FEATURE] Ruler: Add experimental support for federated rule groups. A rule group listing tenants in its `source_tenants` field is evaluated against those tenants while the resulting series and alerts are written to the tenant owning the rule group. Enabled with `-ruler.enable-federated-rules` (requires `-tenant-federation.enabled`), and restricted to selected tenants with `-ruler.allowed-federated-tenants` and `-ruler.disallowed-federated-tenants`. #7828 * [FEATURE] Distributor: Add experimental OTLP metrics ingestion over gRPC (`opentelemetry.proto.collector.metrics.v1.MetricsService/Export`) on the gRPC server port. Enable it with `-distributor.otlp.grpc-enabled`. The tenant is read from the `X-Scope-OrgID` gRPC metadata, and the request size is limited by `-server.grpc-max-recv-msg-size-bytes`. #7873 +* [ENHANCEMENT] Ring: Add the `cortex_ring_member_zonal_ownership_percent{name, member, zone}` metric, reporting the fraction of its own zone's token space owned by each ring member. With zone-aware replication a write is routed within a single zone, so the existing `cortex_ring_member_ownership_percent`, which is computed over the merged token list of all zones, does not reflect the hash range a member actually serves. Reported when ring detailed metrics and zone awareness are both enabled. #7910 * [ENHANCEMENT] Ruler: Add rule information to query parameters sent to the Query Frontend when `-ruler.frontend-address` is configured, so the Query Frontend query stats logs (`-frontend.enabled-ruler-query-stats`) can identify the evaluated rule. In this mode, the Ruler query stats log (`-ruler.query-stats-enabled`) omits the fetched statistics fields, which are always zero. #6539 * [ENHANCEMENT] gRPC: When `-auth.enabled=true`, a gRPC request without the `X-Scope-OrgID` metadata is now rejected with the `UNAUTHENTICATED` status code instead of `UNKNOWN`. #7873 * [ENHANCEMENT] Upgrade promql-engine and Thanos to latest. #7889 diff --git a/pkg/ring/ring.go b/pkg/ring/ring.go index b510a907625..c0cfbb3b0bf 100644 --- a/pkg/ring/ring.go +++ b/pkg/ring/ring.go @@ -208,17 +208,26 @@ type Ring struct { // If set to nil, no caching is done (used by tests, and subrings). shuffledSubringCache map[subringCacheKey]*cachedSubring - memberOwnershipGaugeVec *prometheus.GaugeVec - numMembersGaugeVec *prometheus.GaugeVec - totalTokensGauge prometheus.Gauge - numTokensGaugeVec *prometheus.GaugeVec - oldestTimestampGaugeVec *prometheus.GaugeVec - duplicateTokensGauge prometheus.Gauge - reportedOwners map[string]struct{} + memberOwnershipGaugeVec *prometheus.GaugeVec + memberZonalOwnershipGaugeVec *prometheus.GaugeVec + numMembersGaugeVec *prometheus.GaugeVec + totalTokensGauge prometheus.Gauge + numTokensGaugeVec *prometheus.GaugeVec + oldestTimestampGaugeVec *prometheus.GaugeVec + duplicateTokensGauge prometheus.Gauge + reportedOwners map[string]struct{} + reportedZonalOwners map[zonalOwner]struct{} logger log.Logger } +// zonalOwner identifies a single series of the per-zone ownership metric, so that stale +// label values can be deleted when a member leaves the ring or moves to another zone. +type zonalOwner struct { + member string + zone string +} + type subringCacheKey struct { identifier string shardSize int @@ -274,6 +283,11 @@ func NewWithStoreClientAndStrategy(cfg Config, name, key string, store kv.Client Help: "The percent ownership of the ring by member", ConstLabels: map[string]string{"name": name}}, []string{"member"}), + memberZonalOwnershipGaugeVec: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ + Name: "ring_member_zonal_ownership_percent", + Help: "The fraction of its own zone's token space owned by the member. Values within a zone sum to 1. Only reported when zone awareness is enabled.", + ConstLabels: map[string]string{"name": name}}, + []string{"member", "zone"}), numMembersGaugeVec: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ Name: "ring_members", Help: "Number of members in the ring", @@ -626,11 +640,18 @@ func (r *Ring) GetReplicationSetForOperation(op Operation) (ReplicationSet, erro }, nil } +// countTokensByAz returns the number of tokens per zone and member, and the token range +// owned by each member within its own zone. With zone-aware replication each zone's tokens +// span the whole hash space independently, so the ranges owned by the members of a zone sum +// to the full space (math.MaxUint32+1). +// The ring read lock must be already taken when calling this function. func (r *Ring) countTokensByAz() (map[string]map[string]uint32, map[string]int64) { numTokens := map[string]map[string]uint32{} owned := map[string]int64{} - for zone, zonalTokens := range r.ringDesc.getTokensByZone() { + // r.ringTokensByZone is already merged and sorted per zone by updateRingState, so reuse it + // rather than recomputing it here. + for zone, zonalTokens := range r.ringTokensByZone { numTokens[zone] = map[string]uint32{} for i := 1; i <= len(zonalTokens); i++ { index := i % len(zonalTokens) @@ -645,6 +666,9 @@ func (r *Ring) countTokensByAz() (map[string]map[string]uint32, map[string]int64 for id, info := range r.ringDesc.Ingesters { if _, ok := owned[id]; !ok { owned[id] = 0 + if _, ok := numTokens[info.Zone]; !ok { + numTokens[info.Zone] = map[string]uint32{} + } numTokens[info.Zone][id] = 0 } } @@ -741,6 +765,28 @@ func (r *Ring) updateRingMetrics(compareResult CompareResult) { r.memberOwnershipGaugeVec.DeleteLabelValues(k) r.numTokensGaugeVec.DeleteLabelValues(k) } + + // Zone-aware replication routes each write within a single zone, so the ownership + // computed above over the merged token list of all zones is not the hash range a + // member actually serves: it also depends on where the other zones' tokens fall. + // Report the per-zone ownership too, which is what the write path uses. + if r.cfg.ZoneAwarenessEnabled { + prevZonalOwners := r.reportedZonalOwners + r.reportedZonalOwners = make(map[zonalOwner]struct{}, len(prevZonalOwners)) + numTokensByZone, ownedRangeByZone := r.countTokensByAz() + for zone, members := range numTokensByZone { + for id := range members { + r.memberZonalOwnershipGaugeVec.WithLabelValues(id, zone).Set(float64(ownedRangeByZone[id]) / float64(math.MaxUint32+1)) + owner := zonalOwner{member: id, zone: zone} + delete(prevZonalOwners, owner) + r.reportedZonalOwners[owner] = struct{}{} + } + } + + for owner := range prevZonalOwners { + r.memberZonalOwnershipGaugeVec.DeleteLabelValues(owner.member, owner.zone) + } + } } r.totalTokensGauge.Set(float64(len(r.ringTokens))) diff --git a/pkg/ring/ring_test.go b/pkg/ring/ring_test.go index 4e59db38179..c6e252453d5 100644 --- a/pkg/ring/ring_test.go +++ b/pkg/ring/ring_test.go @@ -3306,6 +3306,10 @@ func TestUpdateMetrics(t *testing.T) { # TYPE ring_member_ownership_percent gauge ring_member_ownership_percent{member="A",name="test"} 0.49999999976716936 ring_member_ownership_percent{member="B",name="test"} 0.5000000002328306 + # HELP ring_member_zonal_ownership_percent The fraction of its own zone's token space owned by the member. Values within a zone sum to 1. Only reported when zone awareness is enabled. + # TYPE ring_member_zonal_ownership_percent gauge + ring_member_zonal_ownership_percent{member="A",name="test",zone=""} 0.49999999976716936 + ring_member_zonal_ownership_percent{member="B",name="test",zone=""} 0.5000000002328306 # HELP ring_members Number of members in the ring # TYPE ring_members gauge ring_members{name="test",state="ACTIVE",zone=""} 2 @@ -3420,6 +3424,10 @@ func TestUpdateMetricsWithRemoval(t *testing.T) { # TYPE ring_member_ownership_percent gauge ring_member_ownership_percent{member="A",name="test"} 0.49999999976716936 ring_member_ownership_percent{member="B",name="test"} 0.5000000002328306 + # HELP ring_member_zonal_ownership_percent The fraction of its own zone's token space owned by the member. Values within a zone sum to 1. Only reported when zone awareness is enabled. + # TYPE ring_member_zonal_ownership_percent gauge + ring_member_zonal_ownership_percent{member="A",name="test",zone=""} 0.49999999976716936 + ring_member_zonal_ownership_percent{member="B",name="test",zone=""} 0.5000000002328306 # HELP ring_members Number of members in the ring # TYPE ring_members gauge ring_members{name="test",state="ACTIVE",zone=""} 2 @@ -3460,6 +3468,9 @@ func TestUpdateMetricsWithRemoval(t *testing.T) { # HELP ring_member_ownership_percent The percent ownership of the ring by member # TYPE ring_member_ownership_percent gauge ring_member_ownership_percent{member="A",name="test"} 1 + # HELP ring_member_zonal_ownership_percent The fraction of its own zone's token space owned by the member. Values within a zone sum to 1. Only reported when zone awareness is enabled. + # TYPE ring_member_zonal_ownership_percent gauge + ring_member_zonal_ownership_percent{member="A",name="test",zone=""} 1 # HELP ring_members Number of members in the ring # TYPE ring_members gauge ring_members{name="test",state="ACTIVE",zone=""} 1 @@ -3519,6 +3530,11 @@ func TestUpdateMetricsWithZone(t *testing.T) { ring_member_ownership_percent{member="A",name="test"} 0.3333333332557231 ring_member_ownership_percent{member="B",name="test"} 0.3333333330228925 ring_member_ownership_percent{member="C",name="test"} 0.3333333337213844 + # HELP ring_member_zonal_ownership_percent The fraction of its own zone's token space owned by the member. Values within a zone sum to 1. Only reported when zone awareness is enabled. + # TYPE ring_member_zonal_ownership_percent gauge + ring_member_zonal_ownership_percent{member="A",name="test",zone="zone1"} 1 + ring_member_zonal_ownership_percent{member="B",name="test",zone="zone2"} 1 + ring_member_zonal_ownership_percent{member="C",name="test",zone="zone3"} 1 # HELP ring_members Number of members in the ring # TYPE ring_members gauge ring_members{name="test",state="ACTIVE",zone="zone1"} 1 @@ -3572,6 +3588,9 @@ func TestUpdateMetricsWithZone(t *testing.T) { # HELP ring_member_ownership_percent The percent ownership of the ring by member # TYPE ring_member_ownership_percent gauge ring_member_ownership_percent{member="A",name="test"} 1 + # HELP ring_member_zonal_ownership_percent The fraction of its own zone's token space owned by the member. Values within a zone sum to 1. Only reported when zone awareness is enabled. + # TYPE ring_member_zonal_ownership_percent gauge + ring_member_zonal_ownership_percent{member="A",name="test",zone="zone1"} 1 # HELP ring_members Number of members in the ring # TYPE ring_members gauge ring_members{name="test",state="ACTIVE",zone="zone1"} 1 @@ -3610,6 +3629,181 @@ func TestUpdateMetricsWithZone(t *testing.T) { assert.NoError(t, err) } +const ( + zonalOwnershipMetricName = "ring_member_zonal_ownership_percent" + + // fullTokenSpace is the size of the whole token space. Each zone's tokens span it + // independently, so it is the denominator of the per-zone ownership metric. + fullTokenSpace = float64(math.MaxUint32) + 1 + + // Tokens used by the per-zone ownership tests. They are globally unique so that no + // token ends up owned by more than one instance. + zonalTokenA = uint32(1 << 30) + zonalTokenB = uint32(math.MaxUint32 - 3) + zonalTokenC = uint32(1 << 31) + zonalTokenD = uint32(math.MaxUint32 - 2) +) + +// gatherZonalOwnership returns the current values of the per-zone ownership metric keyed by +// member and zone, so tests can assert both on values and on series having been removed. +func gatherZonalOwnership(t *testing.T, reg *prometheus.Registry) map[zonalOwner]float64 { + t.Helper() + + families, err := reg.Gather() + require.NoError(t, err) + + owned := map[zonalOwner]float64{} + for _, family := range families { + if family.GetName() != zonalOwnershipMetricName { + continue + } + for _, metric := range family.GetMetric() { + owner := zonalOwner{} + for _, label := range metric.GetLabel() { + switch label.GetName() { + case "member": + owner.member = label.GetValue() + case "zone": + owner.zone = label.GetValue() + } + } + owned[owner] = metric.GetGauge().GetValue() + } + } + + return owned +} + +// zonalOwnershipTestRing returns a ring with two zones holding two members each. Zone "zone-a" +// is deliberately skewed, with B owning three quarters of the zone's token space, while zone +// "zone-b" is balanced. +func zonalOwnershipTestRing(t *testing.T, cfg Config) (*Ring, *prometheus.Registry) { + t.Helper() + + registry := prometheus.NewRegistry() + ring, err := NewWithStoreClientAndStrategy(cfg, testRingName, testRingKey, &MockClient{}, NewDefaultReplicationStrategy(), registry, log.NewNopLogger()) + require.NoError(t, err) + + ring.updateRingState(&Desc{ + Ingesters: map[string]InstanceDesc{ + "A": {Addr: "127.0.0.1", Timestamp: 11, Zone: "zone-a", Tokens: []uint32{zonalTokenA}}, + "B": {Addr: "127.0.0.2", Timestamp: 11, Zone: "zone-a", Tokens: []uint32{zonalTokenB}}, + "C": {Addr: "127.0.0.3", Timestamp: 11, Zone: "zone-b", Tokens: []uint32{zonalTokenC}}, + "D": {Addr: "127.0.0.4", Timestamp: 11, Zone: "zone-b", Tokens: []uint32{zonalTokenD}}, + }, + }) + + return ring, registry +} + +func TestUpdateMetricsZonalOwnershipSkew(t *testing.T) { + cfg := Config{ + KVStore: kv.Config{}, + HeartbeatTimeout: 0, // get healthy stats + ReplicationFactor: 3, + ZoneAwarenessEnabled: true, + DetailedMetricsEnabled: true, + } + + ring, registry := zonalOwnershipTestRing(t, cfg) + owned := gatherZonalOwnership(t, registry) + + // Within a zone, a member owns the distance from the previous token of the same zone, + // wrapping around the end of the token space. + require.Len(t, owned, 4) + require.Equal(t, 1073741828/fullTokenSpace, owned[zonalOwner{member: "A", zone: "zone-a"}]) + require.Equal(t, 3221225468/fullTokenSpace, owned[zonalOwner{member: "B", zone: "zone-a"}]) + require.Equal(t, 2147483651/fullTokenSpace, owned[zonalOwner{member: "C", zone: "zone-b"}]) + require.Equal(t, 2147483645/fullTokenSpace, owned[zonalOwner{member: "D", zone: "zone-b"}]) + + // Each zone's tokens span the whole token space, so the members of a zone own all of it. + zoneTotals := map[string]float64{} + for owner, value := range owned { + zoneTotals[owner.zone] += value + } + require.Len(t, zoneTotals, 2) + for zone, total := range zoneTotals { + require.Equal(t, 1.0, total, "ownership of zone %s should sum to 1", zone) + } + + // The skew the metric is meant to expose: zone-a is 1.5x its average, zone-b is balanced. + require.InDelta(t, 1.5, owned[zonalOwner{member: "B", zone: "zone-a"}]/0.5, 0.0001) + require.InDelta(t, 1.0, owned[zonalOwner{member: "D", zone: "zone-b"}]/0.5, 0.0001) + + // The pre-existing global metric merges the tokens of all zones into one list, so it does + // not report the per-zone skew above. Here it claims B owns half the ring while it actually + // serves three quarters of zone-a, and that D owns almost nothing while it actually serves + // half of zone-b. This is the gap the per-zone metric closes. + require.Equal(t, 2147483644/fullTokenSpace, testutil.ToFloat64(ring.memberOwnershipGaugeVec.WithLabelValues("B"))) + require.Equal(t, 1/fullTokenSpace, testutil.ToFloat64(ring.memberOwnershipGaugeVec.WithLabelValues("D"))) +} + +func TestUpdateMetricsZonalOwnershipWithRemoval(t *testing.T) { + cfg := Config{ + KVStore: kv.Config{}, + HeartbeatTimeout: 0, // get healthy stats + ReplicationFactor: 3, + ZoneAwarenessEnabled: true, + DetailedMetricsEnabled: true, + } + + ring, registry := zonalOwnershipTestRing(t, cfg) + require.Len(t, gatherZonalOwnership(t, registry), 4) + + // B leaves the ring, and C moves from zone-b to zone-a. + ring.updateRingState(&Desc{ + Ingesters: map[string]InstanceDesc{ + "A": {Addr: "127.0.0.1", Timestamp: 11, Zone: "zone-a", Tokens: []uint32{zonalTokenA}}, + "C": {Addr: "127.0.0.3", Timestamp: 11, Zone: "zone-a", Tokens: []uint32{zonalTokenC}}, + "D": {Addr: "127.0.0.4", Timestamp: 11, Zone: "zone-b", Tokens: []uint32{zonalTokenD}}, + }, + }) + + owned := gatherZonalOwnership(t, registry) + + // The series of the member that left is gone, and so is the stale series of the member + // that changed zone. + require.Len(t, owned, 3) + require.NotContains(t, owned, zonalOwner{member: "B", zone: "zone-a"}) + require.NotContains(t, owned, zonalOwner{member: "C", zone: "zone-b"}) + + // D is now alone in zone-b and owns all of it. + require.Equal(t, 1.0, owned[zonalOwner{member: "D", zone: "zone-b"}]) + require.Equal(t, 1.0, owned[zonalOwner{member: "A", zone: "zone-a"}]+owned[zonalOwner{member: "C", zone: "zone-a"}]) +} + +func TestUpdateMetricsZonalOwnershipGating(t *testing.T) { + for _, tc := range []struct { + detailedMetricsEnabled bool + zoneAwarenessEnabled bool + expectReported bool + }{ + {detailedMetricsEnabled: true, zoneAwarenessEnabled: true, expectReported: true}, + {detailedMetricsEnabled: true, zoneAwarenessEnabled: false, expectReported: false}, + {detailedMetricsEnabled: false, zoneAwarenessEnabled: true, expectReported: false}, + {detailedMetricsEnabled: false, zoneAwarenessEnabled: false, expectReported: false}, + } { + t.Run(fmt.Sprintf("detailedMetricsEnabled=%v/zoneAwarenessEnabled=%v", tc.detailedMetricsEnabled, tc.zoneAwarenessEnabled), func(t *testing.T) { + cfg := Config{ + KVStore: kv.Config{}, + HeartbeatTimeout: 0, // get healthy stats + ReplicationFactor: 3, + ZoneAwarenessEnabled: tc.zoneAwarenessEnabled, + DetailedMetricsEnabled: tc.detailedMetricsEnabled, + } + + _, registry := zonalOwnershipTestRing(t, cfg) + + owned := gatherZonalOwnership(t, registry) + if tc.expectReported { + require.Len(t, owned, 4) + } else { + require.Empty(t, owned) + } + }) + } +} + func TestUpdateMetricsDuplicateTokens(t *testing.T) { cfg := Config{ KVStore: kv.Config{},