From 167071461e9809f43839314aa1c8b312abf38e1a Mon Sep 17 00:00:00 2001 From: Timofei Larkin Date: Fri, 6 Nov 2026 13:12:15 +0300 Subject: [PATCH 1/2] Report the ledger as metrics, with an optional ServiceMonitor and alerts Two gauges count IPAddress objects by class and phase and claims by namespace, resolved class and phase, read from the cache at scrape time. The chart's metrics.enabled serves metrics over authenticated HTTPS on 8443 behind a Service, with the TokenReview and SubjectAccessReview RBAC this needs and a reader ClusterRole for scrapers; a ServiceMonitor and a PrometheusRule alerting on Conflict, Lost and long-Pending claims are optional. Assisted-By: LLM Signed-off-by: Timofei Larkin --- README.md | 18 +++ .../templates/controller.yaml | 7 +- .../address-controller/templates/metrics.yaml | 118 ++++++++++++++++++ chart/address-controller/values.yaml | 17 +++ cmd/main.go | 3 + go.mod | 2 +- internal/controller/metrics.go | 83 ++++++++++++ internal/controller/metrics_test.go | 49 ++++++++ 8 files changed, 295 insertions(+), 2 deletions(-) create mode 100644 chart/address-controller/templates/metrics.yaml create mode 100644 internal/controller/metrics.go create mode 100644 internal/controller/metrics_test.go diff --git a/README.md b/README.md index cfa3555..82116f8 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,24 @@ The controller image is published as `ghcr.io/cozystack/address-controller:main` (plus `main-` and semver tags) by the release workflow on every push to main. + +## Metrics + +With `metrics.enabled`, the controller serves metrics over HTTPS on port +8443 behind the Service `address-controller-metrics`. Scrapes are +authenticated and authorized against the API server, so the scraper's +identity needs the `address-controller-metrics-reader` ClusterRole. Besides +the controller-runtime metrics, two gauges count the ledger: + +| metric | labels | +|---|---| +| `address_controller_ipaddresses` | `class`, `phase` | +| `address_controller_ipaddressclaims` | `namespace`, `class` (resolved), `phase` | + +`metrics.serviceMonitor.enabled` and `metrics.prometheusRule.enabled` add a +`ServiceMonitor` and alerts where the `monitoring.coreos.com` API exists: +addresses in `Conflict` or `Lost` for 5 minutes, and claims `Pending` for +15 minutes. ## Development ```sh diff --git a/chart/address-controller/templates/controller.yaml b/chart/address-controller/templates/controller.yaml index 4a15854..3322c59 100644 --- a/chart/address-controller/templates/controller.yaml +++ b/chart/address-controller/templates/controller.yaml @@ -58,8 +58,13 @@ spec: imagePullPolicy: {{ .Values.imagePullPolicy }} args: - --leader-elect={{ .Values.controller.leaderElect }} - - --metrics-bind-address={{ .Values.controller.metricsBindAddress }} + - --metrics-bind-address={{ if .Values.metrics.enabled }}:8443{{ else }}{{ .Values.controller.metricsBindAddress }}{{ end }} - --health-probe-bind-address=:8081 + {{- if .Values.metrics.enabled }} + ports: + - name: metrics + containerPort: 8443 + {{- end }} securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true diff --git a/chart/address-controller/templates/metrics.yaml b/chart/address-controller/templates/metrics.yaml new file mode 100644 index 0000000..42fed7e --- /dev/null +++ b/chart/address-controller/templates/metrics.yaml @@ -0,0 +1,118 @@ +{{- if .Values.metrics.enabled }} +--- +apiVersion: v1 +kind: Service +metadata: + name: address-controller-metrics + namespace: {{ .Release.Namespace }} + labels: + app: address-controller-metrics + {{- include "address-controller.labels" . | nindent 4 }} +spec: + selector: + app: address-controller + ports: + - name: metrics + port: 8443 + targetPort: metrics +--- +# Lets the controller authenticate and authorize scrape requests. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: address-controller-metrics-auth + labels: + {{- include "address-controller.labels" . | nindent 4 }} +rules: + - apiGroups: ["authentication.k8s.io"] + resources: ["tokenreviews"] + verbs: ["create"] + - apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: address-controller-metrics-auth + labels: + {{- include "address-controller.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: address-controller-metrics-auth +subjects: + - kind: ServiceAccount + name: address-controller + namespace: {{ .Release.Namespace }} +--- +# Bind a scraper's identity to this role to let it read the metrics. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: address-controller-metrics-reader + labels: + {{- include "address-controller.labels" . | nindent 4 }} +rules: + - nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if and .Values.metrics.serviceMonitor.enabled (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/ServiceMonitor") }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: address-controller + namespace: {{ .Release.Namespace }} + labels: + {{- include "address-controller.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: address-controller-metrics + endpoints: + - port: metrics + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + # The controller serves a self-signed certificate. + insecureSkipVerify: true +{{- end }} +{{- if and .Values.metrics.prometheusRule.enabled (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/PrometheusRule") }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: address-controller + namespace: {{ .Release.Namespace }} + labels: + {{- include "address-controller.labels" . | nindent 4 }} +spec: + groups: + - name: address-controller + rules: + - alert: IPAddressConflict + expr: sum by (class) (address_controller_ipaddresses{phase="Conflict"}) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "{{ "{{ $value }}" }} IPAddress(es) of class {{ "{{ $labels.class }}" }} in Conflict" + description: A Service holds an address that its binding does not authorize. kubectl get ipaddresses names the offender. + - alert: IPAddressLost + expr: sum by (class) (address_controller_ipaddresses{phase="Lost"}) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "{{ "{{ $value }}" }} IPAddress(es) of class {{ "{{ $labels.class }}" }} Lost" + description: The backend no longer holds a reserved address. Its claim reports Lost until the address is restored or deleted. + - alert: IPAddressClaimPending + expr: sum by (namespace, class) (address_controller_ipaddressclaims{phase="Pending"}) > 0 + for: 15m + labels: + severity: warning + annotations: + summary: "{{ "{{ $value }}" }} IPAddressClaim(s) in {{ "{{ $labels.namespace }}" }} Pending for 15m" + description: The claim's class may be exhausted, missing or refused by its driver. kubectl describe on the claim shows why. +{{- end }} +{{- end }} diff --git a/chart/address-controller/values.yaml b/chart/address-controller/values.yaml index 08b9f72..4ed8e53 100644 --- a/chart/address-controller/values.yaml +++ b/chart/address-controller/values.yaml @@ -18,3 +18,20 @@ controller: memory: 64Mi limits: memory: 128Mi + +metrics: + # Serve controller-runtime and inventory metrics (address_controller_*) + # over HTTPS on 8443, behind the Service address-controller-metrics. + # Requests are authenticated and authorized against the API server, so a + # scraper needs GET on the /metrics non-resource URL: bind its identity + # to the address-controller-metrics-reader ClusterRole. Overrides + # controller.metricsBindAddress. + enabled: false + serviceMonitor: + # A monitoring.coreos.com ServiceMonitor for the Service; rendered only + # where that API exists. + enabled: false + prometheusRule: + # Alerts on Conflict and Lost addresses and on long-Pending claims; + # rendered only where monitoring.coreos.com exists. + enabled: false diff --git a/cmd/main.go b/cmd/main.go index 7ab2de7..c697324 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -34,6 +34,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -207,6 +208,8 @@ func main() { os.Exit(1) } + metrics.Registry.MustRegister(&controller.InventoryCollector{Reader: mgr.GetClient()}) + if err = controller.SetupIndexes(context.Background(), mgr); err != nil { setupLog.Error(err, "unable to set up field indexes") os.Exit(1) diff --git a/go.mod b/go.mod index a3293d2..7be6d91 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.26 godebug default=go1.23 require ( + github.com/prometheus/client_golang v1.19.1 k8s.io/api v0.32.0 k8s.io/apimachinery v0.32.0 k8s.io/client-go v0.32.0 @@ -48,7 +49,6 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.19.1 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect diff --git a/internal/controller/metrics.go b/internal/controller/metrics.go new file mode 100644 index 0000000..856ba58 --- /dev/null +++ b/internal/controller/metrics.go @@ -0,0 +1,83 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "time" + + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/client" + + localv1alpha1 "github.com/cozystack/address-controller/api/v1alpha1" +) + +var ( + addressesDesc = prometheus.NewDesc( + "address_controller_ipaddresses", + "IPAddress objects by class and phase.", + []string{"class", "phase"}, nil) + claimsDesc = prometheus.NewDesc( + "address_controller_ipaddressclaims", + "IPAddressClaim objects by namespace, resolved class and phase.", + []string{"namespace", "class", "phase"}, nil) +) + +// InventoryCollector reports the ledger as gauges, counted from the cache +// at scrape time, so the counts hold no state of their own. +type InventoryCollector struct { + Reader client.Reader +} + +// Describe implements prometheus.Collector. +func (c *InventoryCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- addressesDesc + ch <- claimsDesc +} + +// Collect implements prometheus.Collector. A failed list reports an +// invalid metric, which fails the scrape visibly instead of reporting zero. +func (c *InventoryCollector) Collect(ch chan<- prometheus.Metric) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + addrs := &localv1alpha1.IPAddressList{} + if err := c.Reader.List(ctx, addrs); err != nil { + ch <- prometheus.NewInvalidMetric(addressesDesc, err) + } else { + counts := map[[2]string]int{} + for _, a := range addrs.Items { + counts[[2]string{a.Spec.ClassName, string(a.Status.Phase)}]++ + } + for k, n := range counts { + ch <- prometheus.MustNewConstMetric(addressesDesc, prometheus.GaugeValue, float64(n), k[0], k[1]) + } + } + + claims := &localv1alpha1.IPAddressClaimList{} + if err := c.Reader.List(ctx, claims); err != nil { + ch <- prometheus.NewInvalidMetric(claimsDesc, err) + return + } + counts := map[[3]string]int{} + for _, cl := range claims.Items { + counts[[3]string{cl.Namespace, cl.Status.ClassName, string(cl.Status.Phase)}]++ + } + for k, n := range counts { + ch <- prometheus.MustNewConstMetric(claimsDesc, prometheus.GaugeValue, float64(n), k[0], k[1], k[2]) + } +} diff --git a/internal/controller/metrics_test.go b/internal/controller/metrics_test.go new file mode 100644 index 0000000..d528bd2 --- /dev/null +++ b/internal/controller/metrics_test.go @@ -0,0 +1,49 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" + + localv1alpha1 "github.com/cozystack/address-controller/api/v1alpha1" +) + +func TestInventoryCollectorCountsByPhase(t *testing.T) { + bound := boundAddress("ip-a", "203.0.113.50", localv1alpha1.IPAddressBound) + conflict := boundAddress("ip-b", "203.0.113.51", localv1alpha1.IPAddressConflict) + conflict2 := boundAddress("ip-c", "203.0.113.52", localv1alpha1.IPAddressConflict) + claim := pendingClaim(testClassName) + claim.Status.Phase = localv1alpha1.ClaimBound + claim.Status.ClassName = testClassName + c := testClient(t, bound, conflict, conflict2, claim) + + want := ` +# HELP address_controller_ipaddresses IPAddress objects by class and phase. +# TYPE address_controller_ipaddresses gauge +address_controller_ipaddresses{class="public",phase="Bound"} 1 +address_controller_ipaddresses{class="public",phase="Conflict"} 2 +# HELP address_controller_ipaddressclaims IPAddressClaim objects by namespace, resolved class and phase. +# TYPE address_controller_ipaddressclaims gauge +address_controller_ipaddressclaims{class="public",namespace="tenant-a",phase="Bound"} 1 +` + if err := testutil.CollectAndCompare(&InventoryCollector{Reader: c}, strings.NewReader(want)); err != nil { + t.Error(err) + } +} From c22125bce863facc6de89038c5566e4e6855eab7 Mon Sep 17 00:00:00 2001 From: Timofei Larkin Date: Tue, 10 Nov 2026 18:40:36 +0300 Subject: [PATCH 2/2] Document the scraper binding and let the ServiceMonitor carry labels metrics.serviceMonitor.labels adds labels to the ServiceMonitor for a Prometheus serviceMonitorSelector. The values and the README spell out the ClusterRoleBinding a scraper's ServiceAccount needs, which the chart cannot create, and the README names claims as the billable unit. Assisted-By: LLM Signed-off-by: Timofei Larkin --- README.md | 21 ++++++++++++++++--- .../address-controller/templates/metrics.yaml | 3 +++ chart/address-controller/values.yaml | 7 ++++++- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 82116f8..4eee31c 100644 --- a/README.md +++ b/README.md @@ -130,18 +130,33 @@ by the release workflow on every push to main. With `metrics.enabled`, the controller serves metrics over HTTPS on port 8443 behind the Service `address-controller-metrics`. Scrapes are authenticated and authorized against the API server, so the scraper's -identity needs the `address-controller-metrics-reader` ClusterRole. Besides -the controller-runtime metrics, two gauges count the ledger: +identity needs the `address-controller-metrics-reader` ClusterRole, for +example: + +```sh +kubectl create clusterrolebinding prometheus-address-controller-metrics \ + --clusterrole=address-controller-metrics-reader \ + --serviceaccount=: +``` + +Besides the controller-runtime metrics, two gauges count the ledger: | metric | labels | |---|---| | `address_controller_ipaddresses` | `class`, `phase` | | `address_controller_ipaddressclaims` | `namespace`, `class` (resolved), `phase` | +Claims are the billable unit, as `PersistentVolumeClaim`s are for storage: a +claim holds its addresses whether or not a Service uses them, so +`address_controller_ipaddressclaims{phase="Bound"}` counts what each +namespace holds. + `metrics.serviceMonitor.enabled` and `metrics.prometheusRule.enabled` add a `ServiceMonitor` and alerts where the `monitoring.coreos.com` API exists: addresses in `Conflict` or `Lost` for 5 minutes, and claims `Pending` for -15 minutes. +15 minutes. `metrics.serviceMonitor.labels` adds labels to the +`ServiceMonitor`, for a Prometheus `serviceMonitorSelector`. + ## Development ```sh diff --git a/chart/address-controller/templates/metrics.yaml b/chart/address-controller/templates/metrics.yaml index 42fed7e..225483c 100644 --- a/chart/address-controller/templates/metrics.yaml +++ b/chart/address-controller/templates/metrics.yaml @@ -65,6 +65,9 @@ metadata: namespace: {{ .Release.Namespace }} labels: {{- include "address-controller.labels" . | nindent 4 }} + {{- with .Values.metrics.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} spec: selector: matchLabels: diff --git a/chart/address-controller/values.yaml b/chart/address-controller/values.yaml index 4ed8e53..8bf619b 100644 --- a/chart/address-controller/values.yaml +++ b/chart/address-controller/values.yaml @@ -29,8 +29,13 @@ metrics: enabled: false serviceMonitor: # A monitoring.coreos.com ServiceMonitor for the Service; rendered only - # where that API exists. + # where that API exists. The scraping Prometheus's ServiceAccount still + # needs a ClusterRoleBinding to address-controller-metrics-reader, which + # the chart cannot create without knowing that ServiceAccount. enabled: false + # Extra labels on the ServiceMonitor, e.g. to match a Prometheus + # serviceMonitorSelector. + labels: {} prometheusRule: # Alerts on Conflict and Lost addresses and on long-Pending claims; # rendered only where monitoring.coreos.com exists.