From 1e6313565fe846eb47c871cd62be3c7147c52de5 Mon Sep 17 00:00:00 2001 From: morazow Date: Mon, 14 Sep 2026 15:41:05 +0200 Subject: [PATCH 1/2] [helm] Add existingSecret form for the readiness probe credential The cluster-health readiness probe authenticates as a Fluss client, so a SASL-enforced cluster needs a credential for it. tablet.readinessProbe.healthCheckAuth takes it as a literal, which the chart inlines into the StatefulSet's exec command, leaving it in plain text in the rendered manifest. Keeping it out is possible today only through the generic secrets.env list, where the credential sits far from the probe settings it belongs to and the link between the two is the naming convention on READINESS_HEALTH_CHECK_AUTH rather than something the values schema expresses. Accept the chart's existing existingSecret {name, key} idiom for it: security: readinessProbe: existingSecret: name: fluss-readiness-probe-auth key: auth The chart renders READINESS_HEALTH_CHECK_AUTH as a secretKeyRef on every server container; the exec probe inherits the container environment, so readiness-check.sh and ClusterHealthReadinessCheck are unchanged. Two placement decisions. The key lives under security rather than under tablet, because every component's probe authenticates with the same credential - the coordinator gains a health-based probe in #4115 and needs no second copy. And it is a new key rather than a map accepted on tablet.readinessProbe.healthCheckAuth, so that key keeps its string type and existing values files keep rendering the inline form unchanged. The credential now has three possible sources, and any two of them conflict: the probe command exports the literal over the Secret-sourced env var, and two env entries of the same name make the StatefulSet invalid. Reject every combination at render time. Closes #4330. Supersedes #4329, which documents the secrets.env route for the same credential. --- helm/templates/_security.tpl | 46 +++++++ helm/templates/sts-coordinator.yaml | 3 + helm/templates/sts-tablet.yaml | 3 + helm/tests/readiness_probe_auth_test.yaml | 114 ++++++++++++++++++ helm/values.yaml | 19 +++ .../install-deploy/deploying-with-helm.md | 42 +++++++ 6 files changed, 227 insertions(+) create mode 100644 helm/tests/readiness_probe_auth_test.yaml diff --git a/helm/templates/_security.tpl b/helm/templates/_security.tpl index b6a52d4060f..bd2ffe0ab7e 100644 --- a/helm/templates/_security.tpl +++ b/helm/templates/_security.tpl @@ -286,6 +286,51 @@ Usage: {{- include "fluss.security.sasl.warnInternalUser" . -}} {{- end -}} +{{/* +secretKeyRef env entry for the readiness probe credential, or empty when no +Secret is referenced. Rendered on every server container, so the exec probe +inherits it whichever component runs the cluster-health check. +Usage: + include "fluss.security.readinessProbe.env" . +*/}} +{{- define "fluss.security.readinessProbe.env" -}} +{{- $ref := .Values.security.readinessProbe.existingSecret | default (dict) -}} +{{- if $ref.name }} +- name: READINESS_HEALTH_CHECK_AUTH + valueFrom: + secretKeyRef: + name: {{ $ref.name }} + key: {{ $ref.key | default "auth" }} +{{- end }} +{{- end -}} + +{{/* +Validates the readiness probe credential. It has three possible sources, and +setting more than one is an error: the probe command would export the literal +over the Secret-sourced env var, and two env entries of the same name make the +StatefulSet invalid. +Returns an error message if invalid, empty string otherwise. +Usage: + include "fluss.security.readinessProbe.validateAuth" . +*/}} +{{- define "fluss.security.readinessProbe.validateAuth" -}} +{{- $sources := list -}} +{{- if .Values.tablet.readinessProbe.healthCheckAuth -}} + {{- $sources = append $sources "tablet.readinessProbe.healthCheckAuth" -}} +{{- end -}} +{{- if (.Values.security.readinessProbe.existingSecret | default (dict)).name -}} + {{- $sources = append $sources "security.readinessProbe.existingSecret" -}} +{{- end -}} +{{- range .Values.secrets.env -}} + {{- if eq .name "READINESS_HEALTH_CHECK_AUTH" -}} + {{- $sources = append $sources "the secrets.env entry named READINESS_HEALTH_CHECK_AUTH" -}} + {{- end -}} +{{- end -}} +{{- if gt (len $sources) 1 -}} + {{- printf "the readiness probe credential is set more than once (%s); set only one" (join ", " $sources) -}} +{{- end -}} +{{- end -}} + {{/* Collects security error messages. Usage: @@ -299,6 +344,7 @@ Usage: {{- $errMessages = append $errMessages (include "fluss.security.zookeeper.sasl.validateLoginModuleClass" .) -}} {{- $errMessages = append $errMessages (include "fluss.security.zookeeper.sasl.validateUsername" .) -}} {{- $errMessages = append $errMessages (include "fluss.security.zookeeper.sasl.validatePassword" .) -}} +{{- $errMessages = append $errMessages (include "fluss.security.readinessProbe.validateAuth" .) -}} {{- $errMessages = without $errMessages "" -}} {{- join "\n" $errMessages -}} {{- end -}} diff --git a/helm/templates/sts-coordinator.yaml b/helm/templates/sts-coordinator.yaml index 583cfcd9cbd..c097f0be03b 100644 --- a/helm/templates/sts-coordinator.yaml +++ b/helm/templates/sts-coordinator.yaml @@ -105,6 +105,9 @@ spec: {{- if (include "fluss.secrets.enabled" .) }} {{- include "fluss.secrets.env" . | trim | nindent 12 }} {{- end }} + {{- with (include "fluss.security.readinessProbe.env" .) }} + {{- . | trim | nindent 12 }} + {{- end }} {{- with .Values.coordinator.extraEnv }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/helm/templates/sts-tablet.yaml b/helm/templates/sts-tablet.yaml index 5407b2a6a95..4f0526cfd10 100644 --- a/helm/templates/sts-tablet.yaml +++ b/helm/templates/sts-tablet.yaml @@ -101,6 +101,9 @@ spec: {{- if (include "fluss.secrets.enabled" .) }} {{- include "fluss.secrets.env" . | trim | nindent 12 }} {{- end }} + {{- with (include "fluss.security.readinessProbe.env" .) }} + {{- . | trim | nindent 12 }} + {{- end }} {{- with .Values.tablet.extraEnv }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/helm/tests/readiness_probe_auth_test.yaml b/helm/tests/readiness_probe_auth_test.yaml new file mode 100644 index 00000000000..b1b10aa1a9c --- /dev/null +++ b/helm/tests/readiness_probe_auth_test.yaml @@ -0,0 +1,114 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: readiness-probe-auth +templates: + - templates/sts-coordinator.yaml + - templates/sts-tablet.yaml + - templates/NOTES.txt +tests: + - it: renders no probe credential by default + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: READINESS_HEALTH_CHECK_AUTH + any: true + template: templates/sts-tablet.yaml + - notMatchRegex: + path: spec.template.spec.containers[0].readinessProbe.exec.command[2] + pattern: 'READINESS_HEALTH_CHECK_AUTH' + template: templates/sts-tablet.yaml + + - it: sources the probe credential from a Secret on every server container + set: + security.readinessProbe.existingSecret: + name: fluss-readiness-probe-auth + key: auth + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: READINESS_HEALTH_CHECK_AUTH + valueFrom: + secretKeyRef: + name: fluss-readiness-probe-auth + key: auth + template: templates/sts-tablet.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: READINESS_HEALTH_CHECK_AUTH + valueFrom: + secretKeyRef: + name: fluss-readiness-probe-auth + key: auth + template: templates/sts-coordinator.yaml + # The probe command must not export the variable, otherwise it would + # override the Secret-sourced value inherited from the container. + - notMatchRegex: + path: spec.template.spec.containers[0].readinessProbe.exec.command[2] + pattern: 'export READINESS_HEALTH_CHECK_AUTH' + template: templates/sts-tablet.yaml + + - it: defaults the Secret key to auth + set: + security.readinessProbe.existingSecret.name: fluss-readiness-probe-auth + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: READINESS_HEALTH_CHECK_AUTH + valueFrom: + secretKeyRef: + name: fluss-readiness-probe-auth + key: auth + template: templates/sts-tablet.yaml + + - it: fails when the credential is set both inline and via existingSecret + set: + tablet.readinessProbe.healthCheckAuth: "client.security.protocol:SASL" + security.readinessProbe.existingSecret.name: fluss-readiness-probe-auth + asserts: + - failedTemplate: + errorMessage: "VALUES VALIDATION:\nthe readiness probe credential is set more than once (tablet.readinessProbe.healthCheckAuth, security.readinessProbe.existingSecret); set only one" + template: templates/NOTES.txt + + - it: fails when the credential is set both via existingSecret and secrets.env + set: + security.readinessProbe.existingSecret.name: fluss-readiness-probe-auth + secrets.env: + - name: READINESS_HEALTH_CHECK_AUTH + secretName: fluss-readiness-probe-auth + key: auth + asserts: + - failedTemplate: + errorMessage: "VALUES VALIDATION:\nthe readiness probe credential is set more than once (security.readinessProbe.existingSecret, the secrets.env entry named READINESS_HEALTH_CHECK_AUTH); set only one" + template: templates/NOTES.txt + + - it: fails when the credential is set both inline and via secrets.env + set: + tablet.readinessProbe.healthCheckAuth: "client.security.protocol:SASL" + secrets.env: + - name: READINESS_HEALTH_CHECK_AUTH + secretName: fluss-readiness-probe-auth + key: auth + asserts: + - failedTemplate: + errorMessage: "VALUES VALIDATION:\nthe readiness probe credential is set more than once (tablet.readinessProbe.healthCheckAuth, the secrets.env entry named READINESS_HEALTH_CHECK_AUTH); set only one" + template: templates/NOTES.txt diff --git a/helm/values.yaml b/helm/values.yaml index de9c8eb42e0..3f46fac24ef 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -79,6 +79,12 @@ tablet: # or escaping needed). Required only when the local client listener # enforces SASL. Example: # healthCheckAuth: "client.security.protocol:SASL;client.sasl.mechanism:PLAIN;client.security.sasl.username:admin;client.security.sasl.password:admin-pass" + # + # This value is inlined into the StatefulSet's probe command, so it is + # readable by anyone who can read the rendered manifest. To keep the + # credential in a Secret instead, leave this empty and set + # `security.readinessProbe.existingSecret`. Setting both is rejected at + # render time, because the probe command would silently override the Secret. healthCheckAuth: "" extraVolumes: [] extraVolumeMounts: [] @@ -197,6 +203,19 @@ security: # usernameKey: username # passwordKey: password + # Credential for the cluster-health readiness probes, sourced from a Secret. + # The referenced key holds the same single-line string that + # `.readinessProbe.healthCheckAuth` takes as a literal; the chart + # renders it as the READINESS_HEALTH_CHECK_AUTH environment variable on every + # server container, so the exec probe picks it up unchanged and the credential + # never appears in the rendered manifest. It lives here rather than under a + # component because every component's probe authenticates with it. + readinessProbe: + existingSecret: {} + # existingSecret: + # name: fluss-readiness-probe-auth + # key: auth + zookeeper: sasl: # "" | plain diff --git a/website/docs/install-deploy/deploying-with-helm.md b/website/docs/install-deploy/deploying-with-helm.md index 4a483e14a74..fa6809ad04a 100644 --- a/website/docs/install-deploy/deploying-with-helm.md +++ b/website/docs/install-deploy/deploying-with-helm.md @@ -193,6 +193,7 @@ The following table lists the configurable parameters of the Fluss chart, and th | `security.internal.sasl.plain.username` | Internal listener PLAIN username | `""` | | `security.internal.sasl.plain.password` | Internal listener PLAIN password | `""` | | `security.internal.sasl.plain.existingSecret` | Reference to a pre-existing Secret for internal SASL credentials | `{}` | +| `security.readinessProbe.existingSecret` | Reference to a pre-existing Secret holding the [readiness probe credential](#probe-credentials-on-a-sasl-cluster), `{name, key}` | `{}` | Only `plain` mechanism is supported for now. An empty string disables the SASL authentication, and maps to the `PLAINTEXT` protocol. @@ -873,6 +874,47 @@ tablet: | `failureThreshold` | `200` | Max consecutive probe failures before marking the pod as unready. With `periodSeconds=5`, this allows up to ~16 minutes for recovery. | | `periodSeconds` | `5` | How often the probe runs. | +##### Probe credentials on a SASL cluster + +The probe connects to the pod's own client listener as a regular Fluss client. When that +listener enforces SASL, the probe needs credentials or no TabletServer ever becomes Ready. + +The credential is a single-line string of semicolon-separated `key:value` pairs. Supply it in +one of two ways. + +Inline, which is simplest but puts the credential in plain text in the rendered StatefulSet: + +```yaml +tablet: + readinessProbe: + healthCheckAuth: "client.security.protocol:SASL;client.security.sasl.mechanism:PLAIN;client.security.sasl.username:probe;client.security.sasl.password:probe-pass" +``` + +Or from a Secret, which keeps it out of the manifest. Leave `healthCheckAuth` empty and +reference the Secret instead: + +```bash +kubectl create secret generic fluss-readiness-probe-auth \ + --from-literal=auth="client.security.protocol:SASL;client.security.sasl.mechanism:PLAIN;client.security.sasl.username:probe;client.security.sasl.password:probe-pass" +``` + +```yaml +security: + readinessProbe: + existingSecret: + name: fluss-readiness-probe-auth + key: auth # optional, defaults to `auth` +``` + +The chart renders it as the `READINESS_HEALTH_CHECK_AUTH` environment variable on every server +container, and the exec probe inherits the container environment, so the value reaches the +check unchanged. The setting sits under `security` rather than under a component because every +component's probe authenticates with the same credential. + +Setting the credential more than once fails the render — the inline form would silently +override the Secret, and a second `READINESS_HEALTH_CHECK_AUTH` entry under +[`secrets.env`](#secrets-in-configuration-overrides) would make the StatefulSet invalid. + :::note The CoordinatorServer does not need the Cluster Health API probe — it does not host data replicas, so a simple TCP check is sufficient. The Coordinator should be upgraded **after** all TabletServers are fully upgraded and recovered. From 8b5c295bcb6d1a7a0a62c4268facd1b4370a6a14 Mon Sep 17 00:00:00 2001 From: morazow Date: Mon, 14 Sep 2026 16:27:28 +0200 Subject: [PATCH 2/2] refac --- helm/templates/_security.tpl | 4 +--- helm/values.yaml | 5 +++-- website/docs/install-deploy/deploying-with-helm.md | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/helm/templates/_security.tpl b/helm/templates/_security.tpl index bd2ffe0ab7e..2b09c100c84 100644 --- a/helm/templates/_security.tpl +++ b/helm/templates/_security.tpl @@ -306,9 +306,7 @@ Usage: {{/* Validates the readiness probe credential. It has three possible sources, and -setting more than one is an error: the probe command would export the literal -over the Secret-sourced env var, and two env entries of the same name make the -StatefulSet invalid. +setting more than one is an error. Returns an error message if invalid, empty string otherwise. Usage: include "fluss.security.readinessProbe.validateAuth" . diff --git a/helm/values.yaml b/helm/values.yaml index 3f46fac24ef..e125a229f66 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -83,8 +83,9 @@ tablet: # This value is inlined into the StatefulSet's probe command, so it is # readable by anyone who can read the rendered manifest. To keep the # credential in a Secret instead, leave this empty and set - # `security.readinessProbe.existingSecret`. Setting both is rejected at - # render time, because the probe command would silently override the Secret. + # `security.readinessProbe.existingSecret`. + # Setting both is rejected at render time, because the probe command + # would silently override the Secret. healthCheckAuth: "" extraVolumes: [] extraVolumeMounts: [] diff --git a/website/docs/install-deploy/deploying-with-helm.md b/website/docs/install-deploy/deploying-with-helm.md index fa6809ad04a..1cb5c45c51b 100644 --- a/website/docs/install-deploy/deploying-with-helm.md +++ b/website/docs/install-deploy/deploying-with-helm.md @@ -911,7 +911,7 @@ container, and the exec probe inherits the container environment, so the value r check unchanged. The setting sits under `security` rather than under a component because every component's probe authenticates with the same credential. -Setting the credential more than once fails the render — the inline form would silently +Setting the credential more than once fails the render. The inline form would silently override the Secret, and a second `READINESS_HEALTH_CHECK_AUTH` entry under [`secrets.env`](#secrets-in-configuration-overrides) would make the StatefulSet invalid.