diff --git a/README.md b/README.md index cfa3555..4eee31c 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,39 @@ The controller image is published as `ghcr.io/cozystack/address-controller:main` (plus `main-` and semver tags) by the release workflow on every push to main. + +## Metrics + +With `metrics.enabled`, the controller serves metrics over HTTPS on port +8443 behind the Service `address-controller-metrics`. Scrapes are +authenticated and authorized against the API server, so the scraper's +identity needs the `address-controller-metrics-reader` ClusterRole, for +example: + +```sh +kubectl create clusterrolebinding prometheus-address-controller-metrics \ + --clusterrole=address-controller-metrics-reader \ + --serviceaccount=: +``` + +Besides the controller-runtime metrics, two gauges count the ledger: + +| metric | labels | +|---|---| +| `address_controller_ipaddresses` | `class`, `phase` | +| `address_controller_ipaddressclaims` | `namespace`, `class` (resolved), `phase` | + +Claims are the billable unit, as `PersistentVolumeClaim`s are for storage: a +claim holds its addresses whether or not a Service uses them, so +`address_controller_ipaddressclaims{phase="Bound"}` counts what each +namespace holds. + +`metrics.serviceMonitor.enabled` and `metrics.prometheusRule.enabled` add a +`ServiceMonitor` and alerts where the `monitoring.coreos.com` API exists: +addresses in `Conflict` or `Lost` for 5 minutes, and claims `Pending` for +15 minutes. `metrics.serviceMonitor.labels` adds labels to the +`ServiceMonitor`, for a Prometheus `serviceMonitorSelector`. + ## Development ```sh diff --git a/chart/address-controller/templates/controller.yaml b/chart/address-controller/templates/controller.yaml index 4a15854..3322c59 100644 --- a/chart/address-controller/templates/controller.yaml +++ b/chart/address-controller/templates/controller.yaml @@ -58,8 +58,13 @@ spec: imagePullPolicy: {{ .Values.imagePullPolicy }} args: - --leader-elect={{ .Values.controller.leaderElect }} - - --metrics-bind-address={{ .Values.controller.metricsBindAddress }} + - --metrics-bind-address={{ if .Values.metrics.enabled }}:8443{{ else }}{{ .Values.controller.metricsBindAddress }}{{ end }} - --health-probe-bind-address=:8081 + {{- if .Values.metrics.enabled }} + ports: + - name: metrics + containerPort: 8443 + {{- end }} securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true diff --git a/chart/address-controller/templates/metrics.yaml b/chart/address-controller/templates/metrics.yaml new file mode 100644 index 0000000..225483c --- /dev/null +++ b/chart/address-controller/templates/metrics.yaml @@ -0,0 +1,121 @@ +{{- if .Values.metrics.enabled }} +--- +apiVersion: v1 +kind: Service +metadata: + name: address-controller-metrics + namespace: {{ .Release.Namespace }} + labels: + app: address-controller-metrics + {{- include "address-controller.labels" . | nindent 4 }} +spec: + selector: + app: address-controller + ports: + - name: metrics + port: 8443 + targetPort: metrics +--- +# Lets the controller authenticate and authorize scrape requests. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: address-controller-metrics-auth + labels: + {{- include "address-controller.labels" . | nindent 4 }} +rules: + - apiGroups: ["authentication.k8s.io"] + resources: ["tokenreviews"] + verbs: ["create"] + - apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: address-controller-metrics-auth + labels: + {{- include "address-controller.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: address-controller-metrics-auth +subjects: + - kind: ServiceAccount + name: address-controller + namespace: {{ .Release.Namespace }} +--- +# Bind a scraper's identity to this role to let it read the metrics. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: address-controller-metrics-reader + labels: + {{- include "address-controller.labels" . | nindent 4 }} +rules: + - nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if and .Values.metrics.serviceMonitor.enabled (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/ServiceMonitor") }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: address-controller + namespace: {{ .Release.Namespace }} + labels: + {{- include "address-controller.labels" . | nindent 4 }} + {{- with .Values.metrics.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + app: address-controller-metrics + endpoints: + - port: metrics + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + # The controller serves a self-signed certificate. + insecureSkipVerify: true +{{- end }} +{{- if and .Values.metrics.prometheusRule.enabled (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/PrometheusRule") }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: address-controller + namespace: {{ .Release.Namespace }} + labels: + {{- include "address-controller.labels" . | nindent 4 }} +spec: + groups: + - name: address-controller + rules: + - alert: IPAddressConflict + expr: sum by (class) (address_controller_ipaddresses{phase="Conflict"}) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "{{ "{{ $value }}" }} IPAddress(es) of class {{ "{{ $labels.class }}" }} in Conflict" + description: A Service holds an address that its binding does not authorize. kubectl get ipaddresses names the offender. + - alert: IPAddressLost + expr: sum by (class) (address_controller_ipaddresses{phase="Lost"}) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "{{ "{{ $value }}" }} IPAddress(es) of class {{ "{{ $labels.class }}" }} Lost" + description: The backend no longer holds a reserved address. Its claim reports Lost until the address is restored or deleted. + - alert: IPAddressClaimPending + expr: sum by (namespace, class) (address_controller_ipaddressclaims{phase="Pending"}) > 0 + for: 15m + labels: + severity: warning + annotations: + summary: "{{ "{{ $value }}" }} IPAddressClaim(s) in {{ "{{ $labels.namespace }}" }} Pending for 15m" + description: The claim's class may be exhausted, missing or refused by its driver. kubectl describe on the claim shows why. +{{- end }} +{{- end }} diff --git a/chart/address-controller/values.yaml b/chart/address-controller/values.yaml index 08b9f72..8bf619b 100644 --- a/chart/address-controller/values.yaml +++ b/chart/address-controller/values.yaml @@ -18,3 +18,25 @@ controller: memory: 64Mi limits: memory: 128Mi + +metrics: + # Serve controller-runtime and inventory metrics (address_controller_*) + # over HTTPS on 8443, behind the Service address-controller-metrics. + # Requests are authenticated and authorized against the API server, so a + # scraper needs GET on the /metrics non-resource URL: bind its identity + # to the address-controller-metrics-reader ClusterRole. Overrides + # controller.metricsBindAddress. + enabled: false + serviceMonitor: + # A monitoring.coreos.com ServiceMonitor for the Service; rendered only + # where that API exists. The scraping Prometheus's ServiceAccount still + # needs a ClusterRoleBinding to address-controller-metrics-reader, which + # the chart cannot create without knowing that ServiceAccount. + enabled: false + # Extra labels on the ServiceMonitor, e.g. to match a Prometheus + # serviceMonitorSelector. + labels: {} + prometheusRule: + # Alerts on Conflict and Lost addresses and on long-Pending claims; + # rendered only where monitoring.coreos.com exists. + enabled: false diff --git a/cmd/main.go b/cmd/main.go index 7ab2de7..c697324 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -34,6 +34,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -207,6 +208,8 @@ func main() { os.Exit(1) } + metrics.Registry.MustRegister(&controller.InventoryCollector{Reader: mgr.GetClient()}) + if err = controller.SetupIndexes(context.Background(), mgr); err != nil { setupLog.Error(err, "unable to set up field indexes") os.Exit(1) diff --git a/go.mod b/go.mod index a3293d2..7be6d91 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.26 godebug default=go1.23 require ( + github.com/prometheus/client_golang v1.19.1 k8s.io/api v0.32.0 k8s.io/apimachinery v0.32.0 k8s.io/client-go v0.32.0 @@ -48,7 +49,6 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.19.1 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect diff --git a/internal/controller/metrics.go b/internal/controller/metrics.go new file mode 100644 index 0000000..856ba58 --- /dev/null +++ b/internal/controller/metrics.go @@ -0,0 +1,83 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "time" + + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/client" + + localv1alpha1 "github.com/cozystack/address-controller/api/v1alpha1" +) + +var ( + addressesDesc = prometheus.NewDesc( + "address_controller_ipaddresses", + "IPAddress objects by class and phase.", + []string{"class", "phase"}, nil) + claimsDesc = prometheus.NewDesc( + "address_controller_ipaddressclaims", + "IPAddressClaim objects by namespace, resolved class and phase.", + []string{"namespace", "class", "phase"}, nil) +) + +// InventoryCollector reports the ledger as gauges, counted from the cache +// at scrape time, so the counts hold no state of their own. +type InventoryCollector struct { + Reader client.Reader +} + +// Describe implements prometheus.Collector. +func (c *InventoryCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- addressesDesc + ch <- claimsDesc +} + +// Collect implements prometheus.Collector. A failed list reports an +// invalid metric, which fails the scrape visibly instead of reporting zero. +func (c *InventoryCollector) Collect(ch chan<- prometheus.Metric) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + addrs := &localv1alpha1.IPAddressList{} + if err := c.Reader.List(ctx, addrs); err != nil { + ch <- prometheus.NewInvalidMetric(addressesDesc, err) + } else { + counts := map[[2]string]int{} + for _, a := range addrs.Items { + counts[[2]string{a.Spec.ClassName, string(a.Status.Phase)}]++ + } + for k, n := range counts { + ch <- prometheus.MustNewConstMetric(addressesDesc, prometheus.GaugeValue, float64(n), k[0], k[1]) + } + } + + claims := &localv1alpha1.IPAddressClaimList{} + if err := c.Reader.List(ctx, claims); err != nil { + ch <- prometheus.NewInvalidMetric(claimsDesc, err) + return + } + counts := map[[3]string]int{} + for _, cl := range claims.Items { + counts[[3]string{cl.Namespace, cl.Status.ClassName, string(cl.Status.Phase)}]++ + } + for k, n := range counts { + ch <- prometheus.MustNewConstMetric(claimsDesc, prometheus.GaugeValue, float64(n), k[0], k[1], k[2]) + } +} diff --git a/internal/controller/metrics_test.go b/internal/controller/metrics_test.go new file mode 100644 index 0000000..d528bd2 --- /dev/null +++ b/internal/controller/metrics_test.go @@ -0,0 +1,49 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" + + localv1alpha1 "github.com/cozystack/address-controller/api/v1alpha1" +) + +func TestInventoryCollectorCountsByPhase(t *testing.T) { + bound := boundAddress("ip-a", "203.0.113.50", localv1alpha1.IPAddressBound) + conflict := boundAddress("ip-b", "203.0.113.51", localv1alpha1.IPAddressConflict) + conflict2 := boundAddress("ip-c", "203.0.113.52", localv1alpha1.IPAddressConflict) + claim := pendingClaim(testClassName) + claim.Status.Phase = localv1alpha1.ClaimBound + claim.Status.ClassName = testClassName + c := testClient(t, bound, conflict, conflict2, claim) + + want := ` +# HELP address_controller_ipaddresses IPAddress objects by class and phase. +# TYPE address_controller_ipaddresses gauge +address_controller_ipaddresses{class="public",phase="Bound"} 1 +address_controller_ipaddresses{class="public",phase="Conflict"} 2 +# HELP address_controller_ipaddressclaims IPAddressClaim objects by namespace, resolved class and phase. +# TYPE address_controller_ipaddressclaims gauge +address_controller_ipaddressclaims{class="public",namespace="tenant-a",phase="Bound"} 1 +` + if err := testutil.CollectAndCompare(&InventoryCollector{Reader: c}, strings.NewReader(want)); err != nil { + t.Error(err) + } +}