Skip to content

Ring: add per-zone member ownership metric #7910

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
yeya24 wants to merge 1 commit into
base: master
Choose a base branch
from
+249 −8
Open

Ring: add per-zone member ownership metric #7910

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

* [FEATURE] Ruler: Add experimental support for federated rule groups. A rule group listing tenants in its `source_tenants` field is evaluated against those tenants while the resulting series and alerts are written to the tenant owning the rule group. Enabled with `-ruler.enable-federated-rules` (requires `-tenant-federation.enabled`), and restricted to selected tenants with `-ruler.allowed-federated-tenants` and `-ruler.disallowed-federated-tenants`. #7828
* [FEATURE] Distributor: Add experimental OTLP metrics ingestion over gRPC (`opentelemetry.proto.collector.metrics.v1.MetricsService/Export`) on the gRPC server port. Enable it with `-distributor.otlp.grpc-enabled`. The tenant is read from the `X-Scope-OrgID` gRPC metadata, and the request size is limited by `-server.grpc-max-recv-msg-size-bytes`. #7873
* [ENHANCEMENT] Ring: Add the `cortex_ring_member_zonal_ownership_percent{name, member, zone}` metric, reporting the fraction of its own zone's token space owned by each ring member. With zone-aware replication a write is routed within a single zone, so the existing `cortex_ring_member_ownership_percent`, which is computed over the merged token list of all zones, does not reflect the hash range a member actually serves. Reported when ring detailed metrics and zone awareness are both enabled. #7910
* [ENHANCEMENT] Ruler: Add rule information to query parameters sent to the Query Frontend when `-ruler.frontend-address` is configured, so the Query Frontend query stats logs (`-frontend.enabled-ruler-query-stats`) can identify the evaluated rule. In this mode, the Ruler query stats log (`-ruler.query-stats-enabled`) omits the fetched statistics fields, which are always zero. #6539
* [ENHANCEMENT] gRPC: When `-auth.enabled=true`, a gRPC request without the `X-Scope-OrgID` metadata is now rejected with the `UNAUTHENTICATED` status code instead of `UNKNOWN`. #7873
* [ENHANCEMENT] Upgrade promql-engine and Thanos to latest. #7889
Expand Down
62 changes: 54 additions & 8 deletions pkg/ring/ring.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,17 +208,26 @@ type Ring struct {
// If set to nil, no caching is done (used by tests, and subrings).
shuffledSubringCache map[subringCacheKey]*cachedSubring

memberOwnershipGaugeVec *prometheus.GaugeVec
numMembersGaugeVec *prometheus.GaugeVec
totalTokensGauge prometheus.Gauge
numTokensGaugeVec *prometheus.GaugeVec
oldestTimestampGaugeVec *prometheus.GaugeVec
duplicateTokensGauge prometheus.Gauge
reportedOwners map[string]struct{}
memberOwnershipGaugeVec *prometheus.GaugeVec
memberZonalOwnershipGaugeVec *prometheus.GaugeVec
numMembersGaugeVec *prometheus.GaugeVec
totalTokensGauge prometheus.Gauge
numTokensGaugeVec *prometheus.GaugeVec
oldestTimestampGaugeVec *prometheus.GaugeVec
duplicateTokensGauge prometheus.Gauge
reportedOwners map[string]struct{}
reportedZonalOwners map[zonalOwner]struct{}

logger log.Logger
}

// zonalOwner identifies a single series of the per-zone ownership metric, so that stale
// label values can be deleted when a member leaves the ring or moves to another zone.
type zonalOwner struct {
member string
zone string
}

type subringCacheKey struct {
identifier string
shardSize int
Expand Down Expand Up @@ -274,6 +283,11 @@ func NewWithStoreClientAndStrategy(cfg Config, name, key string, store kv.Client
Help: "The percent ownership of the ring by member",
ConstLabels: map[string]string{"name": name}},
[]string{"member"}),
memberZonalOwnershipGaugeVec: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{
Name: "ring_member_zonal_ownership_percent",
Help: "The fraction of its own zone's token space owned by the member. Values within a zone sum to 1. Only reported when zone awareness is enabled.",
ConstLabels: map[string]string{"name": name}},
[]string{"member", "zone"}),
numMembersGaugeVec: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{
Name: "ring_members",
Help: "Number of members in the ring",
Expand Down Expand Up @@ -626,11 +640,18 @@ func (r *Ring) GetReplicationSetForOperation(op Operation) (ReplicationSet, erro
}, nil
}

// countTokensByAz returns the number of tokens per zone and member, and the token range
// owned by each member within its own zone. With zone-aware replication each zone's tokens
// span the whole hash space independently, so the ranges owned by the members of a zone sum
// to the full space (math.MaxUint32+1).
// The ring read lock must be already taken when calling this function.
func (r *Ring) countTokensByAz() (map[string]map[string]uint32, map[string]int64) {
numTokens := map[string]map[string]uint32{}
owned := map[string]int64{}

for zone, zonalTokens := range r.ringDesc.getTokensByZone() {
// r.ringTokensByZone is already merged and sorted per zone by updateRingState, so reuse it
// rather than recomputing it here.
for zone, zonalTokens := range r.ringTokensByZone {
numTokens[zone] = map[string]uint32{}
for i := 1; i <= len(zonalTokens); i++ {
index := i % len(zonalTokens)
Expand All @@ -645,6 +666,9 @@ func (r *Ring) countTokensByAz() (map[string]map[string]uint32, map[string]int64
for id, info := range r.ringDesc.Ingesters {
if _, ok := owned[id]; !ok {
owned[id] = 0
if _, ok := numTokens[info.Zone]; !ok {
numTokens[info.Zone] = map[string]uint32{}
}
numTokens[info.Zone][id] = 0
}
}
Expand Down Expand Up @@ -741,6 +765,28 @@ func (r *Ring) updateRingMetrics(compareResult CompareResult) {
r.memberOwnershipGaugeVec.DeleteLabelValues(k)
r.numTokensGaugeVec.DeleteLabelValues(k)
}

// Zone-aware replication routes each write within a single zone, so the ownership
// computed above over the merged token list of all zones is not the hash range a
// member actually serves: it also depends on where the other zones' tokens fall.
// Report the per-zone ownership too, which is what the write path uses.
if r.cfg.ZoneAwarenessEnabled {
prevZonalOwners := r.reportedZonalOwners
r.reportedZonalOwners = make(map[zonalOwner]struct{}, len(prevZonalOwners))
numTokensByZone, ownedRangeByZone := r.countTokensByAz()
for zone, members := range numTokensByZone {
for id := range members {
r.memberZonalOwnershipGaugeVec.WithLabelValues(id, zone).Set(float64(ownedRangeByZone[id]) / float64(math.MaxUint32+1))
owner := zonalOwner{member: id, zone: zone}
delete(prevZonalOwners, owner)
r.reportedZonalOwners[owner] = struct{}{}
}
}

for owner := range prevZonalOwners {
r.memberZonalOwnershipGaugeVec.DeleteLabelValues(owner.member, owner.zone)
}
}
}

r.totalTokensGauge.Set(float64(len(r.ringTokens)))
Expand Down
194 changes: 194 additions & 0 deletions pkg/ring/ring_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3306,6 +3306,10 @@ func TestUpdateMetrics(t *testing.T) {
# TYPE ring_member_ownership_percent gauge
ring_member_ownership_percent{member="A",name="test"} 0.49999999976716936
ring_member_ownership_percent{member="B",name="test"} 0.5000000002328306
# HELP ring_member_zonal_ownership_percent The fraction of its own zone's token space owned by the member. Values within a zone sum to 1. Only reported when zone awareness is enabled.
# TYPE ring_member_zonal_ownership_percent gauge
ring_member_zonal_ownership_percent{member="A",name="test",zone=""} 0.49999999976716936
ring_member_zonal_ownership_percent{member="B",name="test",zone=""} 0.5000000002328306
# HELP ring_members Number of members in the ring
# TYPE ring_members gauge
ring_members{name="test",state="ACTIVE",zone=""} 2
Expand Down Expand Up @@ -3420,6 +3424,10 @@ func TestUpdateMetricsWithRemoval(t *testing.T) {
# TYPE ring_member_ownership_percent gauge
ring_member_ownership_percent{member="A",name="test"} 0.49999999976716936
ring_member_ownership_percent{member="B",name="test"} 0.5000000002328306
# HELP ring_member_zonal_ownership_percent The fraction of its own zone's token space owned by the member. Values within a zone sum to 1. Only reported when zone awareness is enabled.
# TYPE ring_member_zonal_ownership_percent gauge
ring_member_zonal_ownership_percent{member="A",name="test",zone=""} 0.49999999976716936
ring_member_zonal_ownership_percent{member="B",name="test",zone=""} 0.5000000002328306
# HELP ring_members Number of members in the ring
# TYPE ring_members gauge
ring_members{name="test",state="ACTIVE",zone=""} 2
Expand Down Expand Up @@ -3460,6 +3468,9 @@ func TestUpdateMetricsWithRemoval(t *testing.T) {
# HELP ring_member_ownership_percent The percent ownership of the ring by member
# TYPE ring_member_ownership_percent gauge
ring_member_ownership_percent{member="A",name="test"} 1
# HELP ring_member_zonal_ownership_percent The fraction of its own zone's token space owned by the member. Values within a zone sum to 1. Only reported when zone awareness is enabled.
# TYPE ring_member_zonal_ownership_percent gauge
ring_member_zonal_ownership_percent{member="A",name="test",zone=""} 1
# HELP ring_members Number of members in the ring
# TYPE ring_members gauge
ring_members{name="test",state="ACTIVE",zone=""} 1
Expand Down Expand Up @@ -3519,6 +3530,11 @@ func TestUpdateMetricsWithZone(t *testing.T) {
ring_member_ownership_percent{member="A",name="test"} 0.3333333332557231
ring_member_ownership_percent{member="B",name="test"} 0.3333333330228925
ring_member_ownership_percent{member="C",name="test"} 0.3333333337213844
# HELP ring_member_zonal_ownership_percent The fraction of its own zone's token space owned by the member. Values within a zone sum to 1. Only reported when zone awareness is enabled.
# TYPE ring_member_zonal_ownership_percent gauge
ring_member_zonal_ownership_percent{member="A",name="test",zone="zone1"} 1
ring_member_zonal_ownership_percent{member="B",name="test",zone="zone2"} 1
ring_member_zonal_ownership_percent{member="C",name="test",zone="zone3"} 1
# HELP ring_members Number of members in the ring
# TYPE ring_members gauge
ring_members{name="test",state="ACTIVE",zone="zone1"} 1
Expand Down Expand Up @@ -3572,6 +3588,9 @@ func TestUpdateMetricsWithZone(t *testing.T) {
# HELP ring_member_ownership_percent The percent ownership of the ring by member
# TYPE ring_member_ownership_percent gauge
ring_member_ownership_percent{member="A",name="test"} 1
# HELP ring_member_zonal_ownership_percent The fraction of its own zone's token space owned by the member. Values within a zone sum to 1. Only reported when zone awareness is enabled.
# TYPE ring_member_zonal_ownership_percent gauge
ring_member_zonal_ownership_percent{member="A",name="test",zone="zone1"} 1
# HELP ring_members Number of members in the ring
# TYPE ring_members gauge
ring_members{name="test",state="ACTIVE",zone="zone1"} 1
Expand Down Expand Up @@ -3610,6 +3629,181 @@ func TestUpdateMetricsWithZone(t *testing.T) {
assert.NoError(t, err)
}

const (
zonalOwnershipMetricName = "ring_member_zonal_ownership_percent"

// fullTokenSpace is the size of the whole token space. Each zone's tokens span it
// independently, so it is the denominator of the per-zone ownership metric.
fullTokenSpace = float64(math.MaxUint32) + 1

// Tokens used by the per-zone ownership tests. They are globally unique so that no
// token ends up owned by more than one instance.
zonalTokenA = uint32(1 << 30)
zonalTokenB = uint32(math.MaxUint32 - 3)
zonalTokenC = uint32(1 << 31)
zonalTokenD = uint32(math.MaxUint32 - 2)
)

// gatherZonalOwnership returns the current values of the per-zone ownership metric keyed by
// member and zone, so tests can assert both on values and on series having been removed.
func gatherZonalOwnership(t *testing.T, reg *prometheus.Registry) map[zonalOwner]float64 {
t.Helper()

families, err := reg.Gather()
require.NoError(t, err)

owned := map[zonalOwner]float64{}
for _, family := range families {
if family.GetName() != zonalOwnershipMetricName {
continue
}
for _, metric := range family.GetMetric() {
owner := zonalOwner{}
for _, label := range metric.GetLabel() {
switch label.GetName() {
case "member":
owner.member = label.GetValue()
case "zone":
owner.zone = label.GetValue()
}
}
owned[owner] = metric.GetGauge().GetValue()
}
}

return owned
}

// zonalOwnershipTestRing returns a ring with two zones holding two members each. Zone "zone-a"
// is deliberately skewed, with B owning three quarters of the zone's token space, while zone
// "zone-b" is balanced.
func zonalOwnershipTestRing(t *testing.T, cfg Config) (*Ring, *prometheus.Registry) {
t.Helper()

registry := prometheus.NewRegistry()
ring, err := NewWithStoreClientAndStrategy(cfg, testRingName, testRingKey, &MockClient{}, NewDefaultReplicationStrategy(), registry, log.NewNopLogger())
require.NoError(t, err)

ring.updateRingState(&Desc{
Ingesters: map[string]InstanceDesc{
"A": {Addr: "127.0.0.1", Timestamp: 11, Zone: "zone-a", Tokens: []uint32{zonalTokenA}},
"B": {Addr: "127.0.0.2", Timestamp: 11, Zone: "zone-a", Tokens: []uint32{zonalTokenB}},
"C": {Addr: "127.0.0.3", Timestamp: 11, Zone: "zone-b", Tokens: []uint32{zonalTokenC}},
"D": {Addr: "127.0.0.4", Timestamp: 11, Zone: "zone-b", Tokens: []uint32{zonalTokenD}},
},
})

return ring, registry
}

func TestUpdateMetricsZonalOwnershipSkew(t *testing.T) {
cfg := Config{
KVStore: kv.Config{},
HeartbeatTimeout: 0, // get healthy stats
ReplicationFactor: 3,
ZoneAwarenessEnabled: true,
DetailedMetricsEnabled: true,
}

ring, registry := zonalOwnershipTestRing(t, cfg)
owned := gatherZonalOwnership(t, registry)

// Within a zone, a member owns the distance from the previous token of the same zone,
// wrapping around the end of the token space.
require.Len(t, owned, 4)
require.Equal(t, 1073741828/fullTokenSpace, owned[zonalOwner{member: "A", zone: "zone-a"}])
require.Equal(t, 3221225468/fullTokenSpace, owned[zonalOwner{member: "B", zone: "zone-a"}])
require.Equal(t, 2147483651/fullTokenSpace, owned[zonalOwner{member: "C", zone: "zone-b"}])
require.Equal(t, 2147483645/fullTokenSpace, owned[zonalOwner{member: "D", zone: "zone-b"}])

// Each zone's tokens span the whole token space, so the members of a zone own all of it.
zoneTotals := map[string]float64{}
for owner, value := range owned {
zoneTotals[owner.zone] += value
}
require.Len(t, zoneTotals, 2)
for zone, total := range zoneTotals {
require.Equal(t, 1.0, total, "ownership of zone %s should sum to 1", zone)
}

// The skew the metric is meant to expose: zone-a is 1.5x its average, zone-b is balanced.
require.InDelta(t, 1.5, owned[zonalOwner{member: "B", zone: "zone-a"}]/0.5, 0.0001)
require.InDelta(t, 1.0, owned[zonalOwner{member: "D", zone: "zone-b"}]/0.5, 0.0001)

// The pre-existing global metric merges the tokens of all zones into one list, so it does
// not report the per-zone skew above. Here it claims B owns half the ring while it actually
// serves three quarters of zone-a, and that D owns almost nothing while it actually serves
// half of zone-b. This is the gap the per-zone metric closes.
require.Equal(t, 2147483644/fullTokenSpace, testutil.ToFloat64(ring.memberOwnershipGaugeVec.WithLabelValues("B")))
require.Equal(t, 1/fullTokenSpace, testutil.ToFloat64(ring.memberOwnershipGaugeVec.WithLabelValues("D")))
}

func TestUpdateMetricsZonalOwnershipWithRemoval(t *testing.T) {
cfg := Config{
KVStore: kv.Config{},
HeartbeatTimeout: 0, // get healthy stats
ReplicationFactor: 3,
ZoneAwarenessEnabled: true,
DetailedMetricsEnabled: true,
}

ring, registry := zonalOwnershipTestRing(t, cfg)
require.Len(t, gatherZonalOwnership(t, registry), 4)

// B leaves the ring, and C moves from zone-b to zone-a.
ring.updateRingState(&Desc{
Ingesters: map[string]InstanceDesc{
"A": {Addr: "127.0.0.1", Timestamp: 11, Zone: "zone-a", Tokens: []uint32{zonalTokenA}},
"C": {Addr: "127.0.0.3", Timestamp: 11, Zone: "zone-a", Tokens: []uint32{zonalTokenC}},
"D": {Addr: "127.0.0.4", Timestamp: 11, Zone: "zone-b", Tokens: []uint32{zonalTokenD}},
},
})

owned := gatherZonalOwnership(t, registry)

// The series of the member that left is gone, and so is the stale series of the member
// that changed zone.
require.Len(t, owned, 3)
require.NotContains(t, owned, zonalOwner{member: "B", zone: "zone-a"})
require.NotContains(t, owned, zonalOwner{member: "C", zone: "zone-b"})

// D is now alone in zone-b and owns all of it.
require.Equal(t, 1.0, owned[zonalOwner{member: "D", zone: "zone-b"}])
require.Equal(t, 1.0, owned[zonalOwner{member: "A", zone: "zone-a"}]+owned[zonalOwner{member: "C", zone: "zone-a"}])
}

func TestUpdateMetricsZonalOwnershipGating(t *testing.T) {
for _, tc := range []struct {
detailedMetricsEnabled bool
zoneAwarenessEnabled bool
expectReported bool
}{
{detailedMetricsEnabled: true, zoneAwarenessEnabled: true, expectReported: true},
{detailedMetricsEnabled: true, zoneAwarenessEnabled: false, expectReported: false},
{detailedMetricsEnabled: false, zoneAwarenessEnabled: true, expectReported: false},
{detailedMetricsEnabled: false, zoneAwarenessEnabled: false, expectReported: false},
} {
t.Run(fmt.Sprintf("detailedMetricsEnabled=%v/zoneAwarenessEnabled=%v", tc.detailedMetricsEnabled, tc.zoneAwarenessEnabled), func(t *testing.T) {
cfg := Config{
KVStore: kv.Config{},
HeartbeatTimeout: 0, // get healthy stats
ReplicationFactor: 3,
ZoneAwarenessEnabled: tc.zoneAwarenessEnabled,
DetailedMetricsEnabled: tc.detailedMetricsEnabled,
}

_, registry := zonalOwnershipTestRing(t, cfg)

owned := gatherZonalOwnership(t, registry)
if tc.expectReported {
require.Len(t, owned, 4)
} else {
require.Empty(t, owned)
}
})
}
}

func TestUpdateMetricsDuplicateTokens(t *testing.T) {
cfg := Config{
KVStore: kv.Config{},
Expand Down