From 85af30478803d27cfee4c89d05eb4d00924e3560 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:16:06 -0600 Subject: [PATCH] sourcegraph: standardize health probes with shared helpers Add sourcegraph.probes, sourcegraph.httpProbes, sourcegraph.postgresProbes, and sourcegraph.redisProbes under templates/_helpers, and use them for every probed container in the chart. Each container gets a startup probe that holds liveness and readiness off until it passes, a liveness probe that restarts after 30s of failures, and a readiness probe every 5s. Add sourcegraph.startupDeadlineSeconds (default 300) with a per-service .startupDeadlineSeconds override; the Postgres databases and prometheus default to 3600 for crash recovery, upgrades, and WAL replay. Add the missing probes to codeinsights-db, a liveness probe to searcher, cadvisor, prometheus, jaeger, and zoekt-indexserver, and liveness and readiness probes to grafana. Probe blobstore on its debug port instead of the S3 API root. Fixes #296 Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a1572b-fce8-7159-abd5-f4142c8f68a2 --- charts/sourcegraph/CHANGELOG.md | 3 + charts/sourcegraph/README.md | 5 + .../templates/_helpers/_postgresProbes.tpl | 30 +++++ .../templates/_helpers/_probes.tpl | 63 +++++++++ .../templates/_helpers/_redisProbes.tpl | 40 ++++++ charts/sourcegraph/templates/_worker.tpl | 15 +-- .../blobstore/blobstore.Deployment.yaml | 17 +-- .../cadvisor/cadvisor.DaemonSet.yaml | 1 + .../codeinsights-db.StatefulSet.yaml | 1 + .../codeintel-db.StatefulSet.yaml | 16 +-- .../sourcegraph-frontend.Deployment.yaml | 15 +-- .../gitserver/gitserver.StatefulSet.yaml | 25 +--- .../grafana/grafana.StatefulSet.yaml | 1 + .../indexed-search.StatefulSet.yaml | 1 + .../templates/jaeger/jaeger.Deployment.yaml | 6 +- .../node-exporter.DaemonSet.yaml | 19 +-- .../otel-collector/otel-agent.DaemonSet.yaml | 9 +- .../otel-collector.Deployment.yaml | 9 +- .../templates/pgsql/pgsql.StatefulSet.yaml | 16 +-- .../precise-code-intel/worker.Deployment.yaml | 15 +-- .../prometheus/prometheus.Deployment.yaml | 8 +- .../redis/redis-cache.Deployment.yaml | 29 +--- .../redis/redis-store.Deployment.yaml | 28 +--- .../searcher/searcher.StatefulSet.yaml | 9 +- .../worker.Deployment.yaml | 15 +-- .../syntect-server.Deployment.yaml | 11 +- charts/sourcegraph/tests/probes_test.yaml | 127 ++++++++++++++++++ charts/sourcegraph/values.yaml | 13 ++ 28 files changed, 304 insertions(+), 243 deletions(-) create mode 100644 charts/sourcegraph/templates/_helpers/_postgresProbes.tpl create mode 100644 charts/sourcegraph/templates/_helpers/_probes.tpl create mode 100644 charts/sourcegraph/templates/_helpers/_redisProbes.tpl create mode 100644 charts/sourcegraph/tests/probes_test.yaml diff --git a/charts/sourcegraph/CHANGELOG.md b/charts/sourcegraph/CHANGELOG.md index c3a3baccf..13ee320f8 100644 --- a/charts/sourcegraph/CHANGELOG.md +++ b/charts/sourcegraph/CHANGELOG.md @@ -8,6 +8,9 @@ Use `**BREAKING**:` to denote a breaking change ## Unreleased +- Standardized health probes across the chart with shared helpers, and added `sourcegraph.startupDeadlineSeconds` (default `300`) with a per-service `.startupDeadlineSeconds` override. Every probed container now has a startup probe that holds liveness and readiness off until it passes, so slow or resource-starved starts are no longer killed and no probe fails before the container is up. Liveness probes restart a container after 30 seconds of failures; readiness probes run every 5 seconds. Pods affected: frontend, worker, precise-code-intel-worker, syntactic-code-intel-worker, gitserver, searcher, blobstore, pgsql, codeintel-db, codeinsights-db, redis-cache, redis-store, grafana, cadvisor, prometheus, jaeger, otel-collector, otel-agent, node-exporter, syntect-server, and the zoekt-indexserver container of indexed-search. Fixes [#296](https://github.com/sourcegraph/deploy-sourcegraph-helm/issues/296). +- Added the previously missing probes to codeinsights-db, matching pgsql and codeintel-db. Added a liveness probe to searcher, cadvisor, prometheus, jaeger, and zoekt-indexserver, and liveness and readiness probes to grafana. The Postgres databases and prometheus default to a one-hour startup deadline to cover crash recovery, major-version upgrades, and write-ahead log replay. +- blobstore probes now use the `/healthz` and `/ready` endpoints on its debug port `6060` instead of the S3 API root. - Added opt-in Redis config management through `redisCache.config.enabled` and `redisStore.config.enabled` (both default to `false`). Default deployments retain their image configs, custom mounts, authentication, and memory settings. When enabled, the chart mounts `/etc/redis/redis.conf` and supports `existingConfig`, `maxmemory`, `maxmemoryRatio`, `maxmemoryPolicy`, and `additionalConfig`. See [Enabling Redis config management](README.md#enabling-redis-config-management) before opting in, especially when using custom images or Secret-mounted configs. - With Redis config management enabled, the chart sizes `maxmemory` at 75% of `resources.limits.memory` to leave room for overhead; this reduces, but does not eliminate, OOM risk. Ratios must be greater than 0 and less than 1. Auto-sizing is skipped in local development mode or when the limit is absent or unrecognised. At the default 7Gi limit, opting in lowers the effective cap from 6GiB to 5.25GiB: `redis-cache` evicts earlier, but `redis-store` uses `noeviction`, so its write-error ceiling drops too. Set `redisStore.config.maxmemory: 6gb` to keep the old ceiling if the pod has sufficient memory. - `searcher.storageSize` now accepts any Kubernetes quantity (e.g. `150G`, `1Ti`, `1.5Ti`, `500000M`) when deriving `SEARCHER_CACHE_SIZE_MB` and `SYMBOLS_CACHE_SIZE_MB`. Previously anything other than `Gi` silently rendered both as `0`, which disables cache eviction and lets the cache volume fill up. Unparseable values, or values below `3M`, now fail the render unless `searcher.autoCacheSize` is enabled. The derived size now matches searcher's own auto-sizing arithmetic (45% of the volume in decimal megabytes), so existing `Gi` deployments see a slightly higher value on upgrade, e.g. the default `26Gi` renders `12562` instead of `11700`. diff --git a/charts/sourcegraph/README.md b/charts/sourcegraph/README.md index 8c19366ce..b94052e8f 100644 --- a/charts/sourcegraph/README.md +++ b/charts/sourcegraph/README.md @@ -134,6 +134,7 @@ In addition to the documented values, all services also support the following va | codeInsightsDB.resources | object | `{"limits":{"cpu":"4","memory":"2Gi"},"requests":{"cpu":"4","memory":"2Gi"}}` | Resource requests & limits for the `codeinsights-db` container, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) | | codeInsightsDB.serviceAccount.create | bool | `false` | Enable creation of ServiceAccount for `codeinsights-db` | | codeInsightsDB.serviceAccount.name | string | `""` | Name of the ServiceAccount to be created or an existing ServiceAccount | +| codeInsightsDB.startupDeadlineSeconds | int | `3600` | Time `codeinsights-db` has to recover from a crash or finish a major-version upgrade before Kubernetes restarts it | | codeInsightsDB.storageAnnotations | object | `{}` | Optional annotations to add to the `codeinsights-db` PVC | | codeInsightsDB.storageSize | string | `"200Gi"` | PVC Storage Request for `codeinsights-db` data volume | | codeInsightsDB.storageSubPath | string | `""` | Optional subPath for the `codeinsights-db` primary data volume mount | @@ -156,6 +157,7 @@ In addition to the documented values, all services also support the following va | codeIntelDB.resources | object | `{"limits":{"cpu":"4","memory":"4Gi"},"requests":{"cpu":"4","memory":"4Gi"}}` | Resource requests & limits for the `codeintel-db` container, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) | | codeIntelDB.serviceAccount.create | bool | `false` | Enable creation of ServiceAccount for `codeintel-db` | | codeIntelDB.serviceAccount.name | string | `""` | Name of the ServiceAccount to be created or an existing ServiceAccount | +| codeIntelDB.startupDeadlineSeconds | int | `3600` | Time `codeintel-db` has to recover from a crash or finish a major-version upgrade before Kubernetes restarts it | | codeIntelDB.storageAnnotations | object | `{}` | Optional annotations to add to the `codeintel-db` PVC | | codeIntelDB.storageSize | string | `"200Gi"` | PVC Storage Request for `codeintel-db` data volume | | codeIntelDB.storageSubPath | string | `""` | Optional subPath for the `codeintel-db` primary data volume mount | @@ -324,6 +326,7 @@ In addition to the documented values, all services also support the following va | pgsql.resources | object | `{"limits":{"cpu":"4","memory":"4Gi"},"requests":{"cpu":"4","memory":"4Gi"}}` | Resource requests & limits for the `pgsql` container, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) | | pgsql.serviceAccount.create | bool | `false` | Enable creation of ServiceAccount for `pgsql` | | pgsql.serviceAccount.name | string | `""` | Name of the ServiceAccount to be created or an existing ServiceAccount | +| pgsql.startupDeadlineSeconds | int | `3600` | Time `pgsql` has to recover from a crash or finish a major-version upgrade before Kubernetes restarts it | | pgsql.storageAnnotations | object | `{}` | Optional annotations to add to the `pgsql` PVC | | pgsql.storageSize | string | `"200Gi"` | PVC Storage Request for `pgsql` data volume | | pgsql.storageSubPath | string | `""` | Optional subPath for the `pgsql` primary data volume mount | @@ -357,6 +360,7 @@ In addition to the documented values, all services also support the following va | prometheus.resources | object | `{"limits":{"cpu":"2","memory":"6G"},"requests":{"cpu":"500m","memory":"6G"}}` | Resource requests & limits for the `prometheus` container, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) Prometheus is relied upon to monitor services for sending alerts to site admins when something is wrong with Sourcegraph, thus its memory requests and limits are the same to guarantee it has enough memory to perform its job reliably and prevent conflicts with other pods on the same host node. The limit chosen here is based on what works reliably on Sourcegraph.com with lots of traffic. | | prometheus.serviceAccount.create | bool | `true` | Enable creation of ServiceAccount | | prometheus.serviceAccount.name | string | `"prometheus"` | Name of the ServiceAccount to be created or an existing ServiceAccount | +| prometheus.startupDeadlineSeconds | int | `3600` | Time `prometheus` has to replay its write-ahead log before Kubernetes restarts it | | prometheus.storageAnnotations | object | `{}` | Optional annotations to add to the `prometheus` PVC | | prometheus.storageSize | string | `"200Gi"` | PVC Storage Request for `prometheus` data volume | | prometheus.storageSubPath | string | `""` | Optional subPath for the `prometheus` primary data volume mount | @@ -440,6 +444,7 @@ In addition to the documented values, all services also support the following va | sourcegraph.priorityClassName | string | `""` | Assign a priorityClass to all pods (daemonSets, deployments, and statefulSets) | | sourcegraph.revisionHistoryLimit | int | `10` | Global deployment clean up policy, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#clean-up-policy) | | sourcegraph.serviceLabels | object | `{}` | Add extra labels to all services | +| sourcegraph.startupDeadlineSeconds | int | `300` | Time a container has to pass its startup probe before Kubernetes restarts it. Liveness and readiness probes do not run until the startup probe passes | | sourcegraph.terminationGracePeriodSeconds | int | `30` | Time Kubernetes allows Sourcegraph services to shut down before forced termination | | sourcegraph.tolerations | list | `[]` | Global Tolerations, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) | | storageClass.allowedTopologies | list | `[]` | Persistent volumes topology configuration, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/storage/storage-classes/#allowed-topologies) | diff --git a/charts/sourcegraph/templates/_helpers/_postgresProbes.tpl b/charts/sourcegraph/templates/_helpers/_postgresProbes.tpl new file mode 100644 index 000000000..0d271def4 --- /dev/null +++ b/charts/sourcegraph/templates/_helpers/_postgresProbes.tpl @@ -0,0 +1,30 @@ +{{/* +Render the standard probes for a Sourcegraph PostgreSQL container (pgsql, codeintel-db, +codeinsights-db). The images ship /liveness.sh, which passes while Postgres is re-indexing +after a major-version upgrade, and /ready.sh, which runs a query over TCP. +The startup deadline is long because crash recovery and upgrades can take a while. +*/}} +{{- define "sourcegraph.postgresProbes" -}} +{{- $top := index . 0 -}} +{{- $service := index $top.Values (index . 1) -}} +startupProbe: + exec: + command: + - /liveness.sh + periodSeconds: 1 + timeoutSeconds: 5 + failureThreshold: {{ include "sourcegraph.startupDeadlineSeconds" (list $top $service) }} +livenessProbe: + exec: + command: + - /liveness.sh + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 +readinessProbe: + exec: + command: + - /ready.sh + periodSeconds: 5 + timeoutSeconds: 5 +{{- end }} diff --git a/charts/sourcegraph/templates/_helpers/_probes.tpl b/charts/sourcegraph/templates/_helpers/_probes.tpl new file mode 100644 index 000000000..2fa56d4ed --- /dev/null +++ b/charts/sourcegraph/templates/_helpers/_probes.tpl @@ -0,0 +1,63 @@ +{{/* +Startup deadline for a service: .startupDeadlineSeconds, falling back to +sourcegraph.startupDeadlineSeconds. Startup probes run once a second, so this is also +their failureThreshold. +*/}} +{{- define "sourcegraph.startupDeadlineSeconds" -}} +{{- $top := index . 0 -}} +{{- $service := index . 1 -}} +{{- default $top.Values.sourcegraph.startupDeadlineSeconds $service.startupDeadlineSeconds -}} +{{- end }} + +{{/* +Render startup, liveness, and readiness probes for a container that serves HTTP health endpoints. + +Liveness and readiness stay disabled until the startup probe passes, so a slow or +resource-starved start is never killed and no probe fires before the container is up. + +Parameters (dict): + top chart root context + service the service's values map, e.g. .Values.grafana + port container port name or number + livenessPath path that returns 200 whenever the process is serving + readinessPath path that returns 200 only when the container can take traffic (optional); + the startup probe also uses it when set +*/}} +{{- define "sourcegraph.httpProbes" -}} +startupProbe: + httpGet: + path: {{ default .livenessPath .readinessPath }} + port: {{ .port }} + scheme: HTTP + periodSeconds: 1 + timeoutSeconds: 5 + failureThreshold: {{ include "sourcegraph.startupDeadlineSeconds" (list .top .service) }} +livenessProbe: + httpGet: + path: {{ .livenessPath }} + port: {{ .port }} + scheme: HTTP + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 +{{- if .readinessPath }} +readinessProbe: + httpGet: + path: {{ .readinessPath }} + port: {{ .port }} + scheme: HTTP + periodSeconds: 5 + timeoutSeconds: 5 +{{- end }} +{{- end }} + +{{/* +Render the standard probes for a Sourcegraph service. Every service built from +sourcegraph/sourcegraph serves /healthz (200 once listening) and /ready (200 once +startup has finished) on its http-debug port. +*/}} +{{- define "sourcegraph.probes" -}} +{{- $top := index . 0 -}} +{{- $service := index $top.Values (index . 1) -}} +{{- include "sourcegraph.httpProbes" (dict "top" $top "service" $service "port" "http-debug" "livenessPath" "/healthz" "readinessPath" "/ready") -}} +{{- end }} diff --git a/charts/sourcegraph/templates/_helpers/_redisProbes.tpl b/charts/sourcegraph/templates/_helpers/_redisProbes.tpl new file mode 100644 index 000000000..dc887dc14 --- /dev/null +++ b/charts/sourcegraph/templates/_helpers/_redisProbes.tpl @@ -0,0 +1,40 @@ +{{/* +Render the standard probes for a Redis container (redis-cache, redis-store). +Redis accepts TCP connections while loading its dataset but answers commands with +LOADING, so liveness checks the socket and readiness checks for PONG. +*/}} +{{- define "sourcegraph.redisProbes" -}} +{{- $top := index . 0 -}} +{{- $service := index $top.Values (index . 1) -}} +startupProbe: + tcpSocket: + port: redis + periodSeconds: 1 + timeoutSeconds: 5 + failureThreshold: {{ include "sourcegraph.startupDeadlineSeconds" (list $top $service) }} +livenessProbe: + tcpSocket: + port: redis + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 +readinessProbe: + exec: + command: + - /bin/sh + - -c + - | + PASS_CHECK=$(grep -h "requirepass" /etc/redis/redis.conf | cut -d ' ' -f 2) + if [ ! -z "$PASS_CHECK" ]; then + export REDISCLI_AUTH="$PASS_CHECK" + fi + response=$( + redis-cli ping + ) + if [ "$response" != "PONG" ]; then + echo "$response" + exit 1 + fi + periodSeconds: 5 + timeoutSeconds: 5 +{{- end }} diff --git a/charts/sourcegraph/templates/_worker.tpl b/charts/sourcegraph/templates/_worker.tpl index 2b087e7f2..fb9ab6953 100644 --- a/charts/sourcegraph/templates/_worker.tpl +++ b/charts/sourcegraph/templates/_worker.tpl @@ -100,20 +100,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} terminationMessagePolicy: FallbackToLogsOnError - livenessProbe: - httpGet: - path: /healthz - port: http-debug - scheme: HTTP - initialDelaySeconds: 60 - timeoutSeconds: 5 - readinessProbe: - httpGet: - path: /ready - port: http-debug - scheme: HTTP - periodSeconds: 5 - timeoutSeconds: 5 + {{- include "sourcegraph.probes" (list $top "worker") | nindent 8 }} ports: - name: http containerPort: 3189 diff --git a/charts/sourcegraph/templates/blobstore/blobstore.Deployment.yaml b/charts/sourcegraph/templates/blobstore/blobstore.Deployment.yaml index 2a330d383..1dd106454 100644 --- a/charts/sourcegraph/templates/blobstore/blobstore.Deployment.yaml +++ b/charts/sourcegraph/templates/blobstore/blobstore.Deployment.yaml @@ -57,20 +57,9 @@ spec: ports: - name: http containerPort: 9000 - livenessProbe: - httpGet: - path: / - port: http - scheme: HTTP - initialDelaySeconds: 60 - timeoutSeconds: 5 - readinessProbe: - httpGet: - path: / - port: http - scheme: HTTP - periodSeconds: 5 - timeoutSeconds: 5 + - name: http-debug + containerPort: 6060 + {{- include "sourcegraph.probes" (list . "blobstore") | nindent 8 }} {{- if not .Values.sourcegraph.localDevMode }} resources: {{- toYaml .Values.blobstore.resources | nindent 10 }} diff --git a/charts/sourcegraph/templates/cadvisor/cadvisor.DaemonSet.yaml b/charts/sourcegraph/templates/cadvisor/cadvisor.DaemonSet.yaml index e9814a4e7..bb1d61f04 100644 --- a/charts/sourcegraph/templates/cadvisor/cadvisor.DaemonSet.yaml +++ b/charts/sourcegraph/templates/cadvisor/cadvisor.DaemonSet.yaml @@ -94,6 +94,7 @@ spec: - name: http containerPort: 48080 protocol: TCP + {{- include "sourcegraph.httpProbes" (dict "top" . "service" .Values.cadvisor "port" "http" "livenessPath" "/healthz") | nindent 8 }} {{- if .Values.cadvisor.extraContainers }} {{- toYaml .Values.cadvisor.extraContainers | nindent 6 }} {{- end }} diff --git a/charts/sourcegraph/templates/codeinsights-db/codeinsights-db.StatefulSet.yaml b/charts/sourcegraph/templates/codeinsights-db/codeinsights-db.StatefulSet.yaml index d0585e3fa..7434f23d1 100644 --- a/charts/sourcegraph/templates/codeinsights-db/codeinsights-db.StatefulSet.yaml +++ b/charts/sourcegraph/templates/codeinsights-db/codeinsights-db.StatefulSet.yaml @@ -81,6 +81,7 @@ spec: - name: POSTGRESQL_CONF_DIR value: "/conf" terminationMessagePolicy: FallbackToLogsOnError + {{- include "sourcegraph.postgresProbes" (list . "codeInsightsDB") | nindent 8 }} ports: - name: codeinsights-db containerPort: 5432 diff --git a/charts/sourcegraph/templates/codeintel-db/codeintel-db.StatefulSet.yaml b/charts/sourcegraph/templates/codeintel-db/codeintel-db.StatefulSet.yaml index eff5c7275..1c4b0fff1 100644 --- a/charts/sourcegraph/templates/codeintel-db/codeintel-db.StatefulSet.yaml +++ b/charts/sourcegraph/templates/codeintel-db/codeintel-db.StatefulSet.yaml @@ -77,21 +77,7 @@ spec: - name: {{ $name }} {{- $item | toYaml | nindent 10 }} {{- end }} - readinessProbe: - exec: - command: - - /ready.sh - livenessProbe: - initialDelaySeconds: 15 - exec: - command: - - /liveness.sh - startupProbe: - exec: - command: - - /liveness.sh - failureThreshold: 360 - periodSeconds: 10 + {{- include "sourcegraph.postgresProbes" (list . "codeIntelDB") | nindent 8 }} ports: - name: pgsql containerPort: 5432 diff --git a/charts/sourcegraph/templates/frontend/sourcegraph-frontend.Deployment.yaml b/charts/sourcegraph/templates/frontend/sourcegraph-frontend.Deployment.yaml index 41575281d..ac7cca4d7 100644 --- a/charts/sourcegraph/templates/frontend/sourcegraph-frontend.Deployment.yaml +++ b/charts/sourcegraph/templates/frontend/sourcegraph-frontend.Deployment.yaml @@ -106,20 +106,7 @@ spec: value: http://blobstore:9000 {{- end }} terminationMessagePolicy: FallbackToLogsOnError - livenessProbe: - httpGet: - path: /healthz - port: http-debug - scheme: HTTP - initialDelaySeconds: 300 - timeoutSeconds: 5 - readinessProbe: - httpGet: - path: /ready - port: http-debug - scheme: HTTP - periodSeconds: 5 - timeoutSeconds: 5 + {{- include "sourcegraph.probes" (list . "frontend") | nindent 8 }} ports: - name: grpc-internal containerPort: 3090 diff --git a/charts/sourcegraph/templates/gitserver/gitserver.StatefulSet.yaml b/charts/sourcegraph/templates/gitserver/gitserver.StatefulSet.yaml index 4a4f7ccae..e5176944f 100644 --- a/charts/sourcegraph/templates/gitserver/gitserver.StatefulSet.yaml +++ b/charts/sourcegraph/templates/gitserver/gitserver.StatefulSet.yaml @@ -58,30 +58,7 @@ spec: {{- end }} {{- include "sourcegraph.openTelemetryEnv" . | nindent 8 }} terminationMessagePolicy: FallbackToLogsOnError - # Temporary: when migrating from repo names to repo IDs on disk, - # gitserver can take a little while to start up. To avoid killing the - # pod because of a failed liveness probe, we give it 2 minutes to start up. - startupProbe: - httpGet: - path: /ready - port: http-debug - scheme: HTTP - failureThreshold: 120 - periodSeconds: 1 - livenessProbe: - httpGet: - path: /healthz - port: http-debug - scheme: HTTP - initialDelaySeconds: 5 - timeoutSeconds: 5 - readinessProbe: - httpGet: - path: /ready - port: http-debug - scheme: HTTP - periodSeconds: 5 - timeoutSeconds: 5 + {{- include "sourcegraph.probes" (list . "gitserver") | nindent 8 }} ports: - name: grpc containerPort: 3178 diff --git a/charts/sourcegraph/templates/grafana/grafana.StatefulSet.yaml b/charts/sourcegraph/templates/grafana/grafana.StatefulSet.yaml index 1dfc52b51..d66b504bc 100644 --- a/charts/sourcegraph/templates/grafana/grafana.StatefulSet.yaml +++ b/charts/sourcegraph/templates/grafana/grafana.StatefulSet.yaml @@ -66,6 +66,7 @@ spec: ports: - name: http containerPort: 3370 + {{- include "sourcegraph.httpProbes" (dict "top" . "service" .Values.grafana "port" "http" "livenessPath" "/healthz" "readinessPath" "/api/health") | nindent 8 }} volumeMounts: - mountPath: /var/lib/grafana name: grafana-data diff --git a/charts/sourcegraph/templates/indexed-search/indexed-search.StatefulSet.yaml b/charts/sourcegraph/templates/indexed-search/indexed-search.StatefulSet.yaml index 561afe8c5..bef048206 100644 --- a/charts/sourcegraph/templates/indexed-search/indexed-search.StatefulSet.yaml +++ b/charts/sourcegraph/templates/indexed-search/indexed-search.StatefulSet.yaml @@ -112,6 +112,7 @@ spec: ports: - name: grpc-indexer containerPort: 6072 + {{- include "sourcegraph.httpProbes" (dict "top" . "service" .Values.indexedSearchIndexer "port" "grpc-indexer" "livenessPath" "/debug") | nindent 8 }} {{- if not .Values.sourcegraph.localDevMode }} resources: {{- toYaml .Values.indexedSearchIndexer.resources | nindent 10 }} diff --git a/charts/sourcegraph/templates/jaeger/jaeger.Deployment.yaml b/charts/sourcegraph/templates/jaeger/jaeger.Deployment.yaml index 75c8961e8..a01f459b7 100644 --- a/charts/sourcegraph/templates/jaeger/jaeger.Deployment.yaml +++ b/charts/sourcegraph/templates/jaeger/jaeger.Deployment.yaml @@ -77,11 +77,7 @@ spec: - containerPort: 13133 name: health protocol: TCP - readinessProbe: - httpGet: - path: "/status" - port: 13133 - initialDelaySeconds: 5 + {{- include "sourcegraph.httpProbes" (dict "top" . "service" .Values.jaeger "port" "health" "livenessPath" "/status" "readinessPath" "/status") | nindent 8 }} {{- if not .Values.sourcegraph.localDevMode }} resources: {{- toYaml .Values.jaeger.resources | nindent 10 }} diff --git a/charts/sourcegraph/templates/node-exporter/node-exporter.DaemonSet.yaml b/charts/sourcegraph/templates/node-exporter/node-exporter.DaemonSet.yaml index db0bc0c3e..8e6272553 100644 --- a/charts/sourcegraph/templates/node-exporter/node-exporter.DaemonSet.yaml +++ b/charts/sourcegraph/templates/node-exporter/node-exporter.DaemonSet.yaml @@ -92,24 +92,7 @@ spec: - name: metrics containerPort: 9100 protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - scheme: HTTP - port: metrics - initialDelaySeconds: 0 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - livenessProbe: - failureThreshold: 3 - httpGet: - scheme: HTTP - port: metrics - initialDelaySeconds: 0 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 + {{- include "sourcegraph.httpProbes" (dict "top" . "service" .Values.nodeExporter "port" "metrics" "livenessPath" "/" "readinessPath" "/") | nindent 8 }} terminationMessagePolicy: FallbackToLogsOnError {{- if .Values.nodeExporter.extraContainers }} {{- toYaml .Values.nodeExporter.extraContainers | nindent 6 }} diff --git a/charts/sourcegraph/templates/otel-collector/otel-agent.DaemonSet.yaml b/charts/sourcegraph/templates/otel-collector/otel-agent.DaemonSet.yaml index d93c64b62..20ccdf555 100644 --- a/charts/sourcegraph/templates/otel-collector/otel-agent.DaemonSet.yaml +++ b/charts/sourcegraph/templates/otel-collector/otel-agent.DaemonSet.yaml @@ -60,14 +60,7 @@ spec: {{- toYaml .Values.openTelemetry.agent.resources | nindent 10 }} securityContext: {{- toYaml .Values.openTelemetry.agent.containerSecurityContext | nindent 10 }} - readinessProbe: - httpGet: - path: / - port: 13133 - livenessProbe: - httpGet: - path: / - port: 13133 + {{- include "sourcegraph.httpProbes" (dict "top" . "service" .Values.openTelemetry.agent "port" 13133 "livenessPath" "/" "readinessPath" "/") | nindent 8 }} ports: - name: grpc-otlp containerPort: 4317 diff --git a/charts/sourcegraph/templates/otel-collector/otel-collector.Deployment.yaml b/charts/sourcegraph/templates/otel-collector/otel-collector.Deployment.yaml index 38d6d7c2f..42900f43d 100644 --- a/charts/sourcegraph/templates/otel-collector/otel-collector.Deployment.yaml +++ b/charts/sourcegraph/templates/otel-collector/otel-collector.Deployment.yaml @@ -76,14 +76,7 @@ spec: {{- toYaml .Values.openTelemetry.gateway.resources | nindent 10 }} securityContext: {{- toYaml .Values.openTelemetry.gateway.containerSecurityContext | nindent 10 }} - readinessProbe: - httpGet: - path: / - port: 13133 - livenessProbe: - httpGet: - path: / - port: 13133 + {{- include "sourcegraph.httpProbes" (dict "top" . "service" .Values.openTelemetry.gateway "port" 13133 "livenessPath" "/" "readinessPath" "/") | nindent 8 }} ports: - name: grpc-otlp containerPort: 4317 diff --git a/charts/sourcegraph/templates/pgsql/pgsql.StatefulSet.yaml b/charts/sourcegraph/templates/pgsql/pgsql.StatefulSet.yaml index adccd43fb..7f7a98c3c 100644 --- a/charts/sourcegraph/templates/pgsql/pgsql.StatefulSet.yaml +++ b/charts/sourcegraph/templates/pgsql/pgsql.StatefulSet.yaml @@ -69,21 +69,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} terminationMessagePolicy: FallbackToLogsOnError - readinessProbe: - exec: - command: - - /ready.sh - livenessProbe: - initialDelaySeconds: 15 - exec: - command: - - /liveness.sh - startupProbe: - exec: - command: - - /liveness.sh - failureThreshold: 360 - periodSeconds: 10 + {{- include "sourcegraph.postgresProbes" (list . "pgsql") | nindent 8 }} env: {{- include "sourcegraph.databaseAuth" (list . "pgsql" "POSTGRES_") | nindent 8 }} - name: POSTGRES_DB diff --git a/charts/sourcegraph/templates/precise-code-intel/worker.Deployment.yaml b/charts/sourcegraph/templates/precise-code-intel/worker.Deployment.yaml index 74e59a483..2640ab6b7 100644 --- a/charts/sourcegraph/templates/precise-code-intel/worker.Deployment.yaml +++ b/charts/sourcegraph/templates/precise-code-intel/worker.Deployment.yaml @@ -76,20 +76,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} terminationMessagePolicy: FallbackToLogsOnError - livenessProbe: - httpGet: - path: /healthz - port: http-debug - scheme: HTTP - initialDelaySeconds: 60 - timeoutSeconds: 5 - readinessProbe: - httpGet: - path: /ready - port: http-debug - scheme: HTTP - periodSeconds: 5 - timeoutSeconds: 5 + {{- include "sourcegraph.probes" (list . "preciseCodeIntel") | nindent 8 }} ports: - name: http-debug containerPort: 6060 diff --git a/charts/sourcegraph/templates/prometheus/prometheus.Deployment.yaml b/charts/sourcegraph/templates/prometheus/prometheus.Deployment.yaml index 608379c47..20f26f23a 100644 --- a/charts/sourcegraph/templates/prometheus/prometheus.Deployment.yaml +++ b/charts/sourcegraph/templates/prometheus/prometheus.Deployment.yaml @@ -57,13 +57,7 @@ spec: - name: {{ $name }} {{- $item | toYaml | nindent 10 }} {{- end }} - readinessProbe: - httpGet: - path: /-/ready - port: http - timeoutSeconds: 3 - failureThreshold: 120 - periodSeconds: 5 + {{- include "sourcegraph.httpProbes" (dict "top" . "service" .Values.prometheus "port" "http" "livenessPath" "/-/healthy" "readinessPath" "/-/ready") | nindent 8 }} ports: - name: http containerPort: 9090 diff --git a/charts/sourcegraph/templates/redis/redis-cache.Deployment.yaml b/charts/sourcegraph/templates/redis/redis-cache.Deployment.yaml index 1ea57e114..c928f4221 100644 --- a/charts/sourcegraph/templates/redis/redis-cache.Deployment.yaml +++ b/charts/sourcegraph/templates/redis/redis-cache.Deployment.yaml @@ -58,34 +58,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} terminationMessagePolicy: FallbackToLogsOnError - livenessProbe: - initialDelaySeconds: 60 - periodSeconds: 30 - failureThreshold: 5 - timeoutSeconds: 5 - tcpSocket: - port: redis - readinessProbe: - initialDelaySeconds: 10 - failureThreshold: 5 - timeoutSeconds: 5 - exec: - command: - - /bin/sh - - -c - - | - #!/bin/bash - PASS_CHECK=$(grep -h "requirepass" /etc/redis/redis.conf | cut -d ' ' -f 2) - if [ ! -z "$PASS_CHECK" ]; then - export REDISCLI_AUTH="$PASS_CHECK" - fi - response=$( - redis-cli ping - ) - if [ "$response" != "PONG" ]; then - echo "$response" - exit 1 - fi + {{- include "sourcegraph.redisProbes" (list . "redisCache") | nindent 8 }} env: {{- range $name, $item := .Values.redisCache.env}} - name: {{ $name }} diff --git a/charts/sourcegraph/templates/redis/redis-store.Deployment.yaml b/charts/sourcegraph/templates/redis/redis-store.Deployment.yaml index e557c4ebb..742c9d54b 100644 --- a/charts/sourcegraph/templates/redis/redis-store.Deployment.yaml +++ b/charts/sourcegraph/templates/redis/redis-store.Deployment.yaml @@ -63,33 +63,7 @@ spec: - name: {{ $name }} {{- $item | toYaml | nindent 10 }} {{- end }} - livenessProbe: - initialDelaySeconds: 60 - periodSeconds: 30 - failureThreshold: 2 - timeoutSeconds: 5 - tcpSocket: - port: redis - readinessProbe: - initialDelaySeconds: 10 - timeoutSeconds: 5 - exec: - command: - - /bin/sh - - -c - - | - #!/bin/bash - PASS_CHECK=$(grep -h "requirepass" /etc/redis/redis.conf | cut -d ' ' -f 2) - if [ ! -z "$PASS_CHECK" ]; then - export REDISCLI_AUTH="$PASS_CHECK" - fi - response=$( - redis-cli ping - ) - if [ "$response" != "PONG" ]; then - echo "$response" - exit 1 - fi + {{- include "sourcegraph.redisProbes" (list . "redisStore") | nindent 8 }} ports: - name: redis containerPort: 6379 diff --git a/charts/sourcegraph/templates/searcher/searcher.StatefulSet.yaml b/charts/sourcegraph/templates/searcher/searcher.StatefulSet.yaml index 76c71d47a..890fa176d 100644 --- a/charts/sourcegraph/templates/searcher/searcher.StatefulSet.yaml +++ b/charts/sourcegraph/templates/searcher/searcher.StatefulSet.yaml @@ -83,14 +83,7 @@ spec: containerPort: 3181 - name: http-debug containerPort: 6060 - readinessProbe: - failureThreshold: 3 - httpGet: - path: /ready - port: http-debug - scheme: HTTP - periodSeconds: 5 - timeoutSeconds: 5 + {{- include "sourcegraph.probes" (list . "searcher") | nindent 8 }} volumeMounts: - mountPath: /mnt/cache name: cache diff --git a/charts/sourcegraph/templates/syntactic-code-intel/worker.Deployment.yaml b/charts/sourcegraph/templates/syntactic-code-intel/worker.Deployment.yaml index b5ace0f20..9110f51c5 100644 --- a/charts/sourcegraph/templates/syntactic-code-intel/worker.Deployment.yaml +++ b/charts/sourcegraph/templates/syntactic-code-intel/worker.Deployment.yaml @@ -73,20 +73,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} terminationMessagePolicy: FallbackToLogsOnError - livenessProbe: - httpGet: - path: /healthz - port: http-debug - scheme: HTTP - initialDelaySeconds: 60 - timeoutSeconds: 5 - readinessProbe: - httpGet: - path: /ready - port: http-debug - scheme: HTTP - periodSeconds: 5 - timeoutSeconds: 5 + {{- include "sourcegraph.probes" (list . "syntacticCodeIntel") | nindent 8 }} ports: - name: http-debug containerPort: 6060 diff --git a/charts/sourcegraph/templates/syntect-server/syntect-server.Deployment.yaml b/charts/sourcegraph/templates/syntect-server/syntect-server.Deployment.yaml index 1195f6ad9..b36b7997d 100644 --- a/charts/sourcegraph/templates/syntect-server/syntect-server.Deployment.yaml +++ b/charts/sourcegraph/templates/syntect-server/syntect-server.Deployment.yaml @@ -61,21 +61,12 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} terminationMessagePolicy: FallbackToLogsOnError - livenessProbe: - httpGet: - path: /health - port: http - scheme: HTTP - initialDelaySeconds: 5 - timeoutSeconds: 5 + {{- include "sourcegraph.httpProbes" (dict "top" . "service" .Values.syntectServer "port" "http" "livenessPath" "/health" "readinessPath" "/health") | nindent 8 }} ports: - name: http containerPort: 9238 - name: http-debug containerPort: 6060 - readinessProbe: - tcpSocket: - port: http {{- if not .Values.sourcegraph.localDevMode }} resources: {{- toYaml .Values.syntectServer.resources | nindent 10 }} diff --git a/charts/sourcegraph/tests/probes_test.yaml b/charts/sourcegraph/tests/probes_test.yaml new file mode 100644 index 000000000..81003fbf7 --- /dev/null +++ b/charts/sourcegraph/tests/probes_test.yaml @@ -0,0 +1,127 @@ +suite: probes +release: + name: sourcegraph + namespace: sourcegraph +tests: +- it: should render the standard probes for a Sourcegraph service + template: frontend/sourcegraph-frontend.Deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].startupProbe + value: + httpGet: + path: /ready + port: http-debug + scheme: HTTP + periodSeconds: 1 + timeoutSeconds: 5 + failureThreshold: 300 + - equal: + path: spec.template.spec.containers[0].livenessProbe + value: + httpGet: + path: /healthz + port: http-debug + scheme: HTTP + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + - equal: + path: spec.template.spec.containers[0].readinessProbe + value: + httpGet: + path: /ready + port: http-debug + scheme: HTTP + periodSeconds: 5 + timeoutSeconds: 5 + - notExists: + path: spec.template.spec.containers[0].livenessProbe.initialDelaySeconds +- it: should apply the global startup deadline to every helper + set: + sourcegraph.startupDeadlineSeconds: 42 + pgsql.startupDeadlineSeconds: null + templates: + - gitserver/gitserver.StatefulSet.yaml + - blobstore/blobstore.Deployment.yaml + - pgsql/pgsql.StatefulSet.yaml + - redis/redis-store.Deployment.yaml + - grafana/grafana.StatefulSet.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].startupProbe.failureThreshold + value: 42 +- it: should let a service override the global startup deadline + set: + worker.startupDeadlineSeconds: 900 + templates: + - worker/worker.Deployment.yaml + - searcher/searcher.StatefulSet.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].startupProbe.failureThreshold + value: 900 + template: worker/worker.Deployment.yaml + - equal: + path: spec.template.spec.containers[0].startupProbe.failureThreshold + value: 300 + template: searcher/searcher.StatefulSet.yaml +- it: should give the databases and prometheus a one-hour startup deadline by default + templates: + - pgsql/pgsql.StatefulSet.yaml + - codeintel-db/codeintel-db.StatefulSet.yaml + - codeinsights-db/codeinsights-db.StatefulSet.yaml + - prometheus/prometheus.Deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].startupProbe.failureThreshold + value: 3600 +- it: should probe codeinsights-db like the other databases + template: codeinsights-db/codeinsights-db.StatefulSet.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].livenessProbe.exec.command + value: [/liveness.sh] + - equal: + path: spec.template.spec.containers[0].readinessProbe.exec.command + value: [/ready.sh] +- it: should keep redis readiness on PONG and liveness on the socket + template: redis/redis-cache.Deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].livenessProbe.tcpSocket.port + value: redis + - matchRegex: + path: spec.template.spec.containers[0].readinessProbe.exec.command[2] + pattern: redis-cli ping +- it: should probe blobstore on its debug port + template: blobstore/blobstore.Deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].ports + content: + name: http-debug + containerPort: 6060 + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.port + value: http-debug +- it: should give grafana root-level health paths + template: grafana/grafana.StatefulSet.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet.path + value: /healthz + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.path + value: /api/health +- it: should not add a readiness probe to zoekt-indexserver + template: indexed-search/indexed-search.StatefulSet.yaml + asserts: + - equal: + path: spec.template.spec.containers[1].livenessProbe.httpGet.path + value: /debug + - notExists: + path: spec.template.spec.containers[1].readinessProbe + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.path + value: /healthz diff --git a/charts/sourcegraph/values.yaml b/charts/sourcegraph/values.yaml index 74d4b7a3f..86e9c8b4a 100644 --- a/charts/sourcegraph/values.yaml +++ b/charts/sourcegraph/values.yaml @@ -44,6 +44,9 @@ sourcegraph: gracefulShutdownTimeout: 15s # -- Time Kubernetes allows Sourcegraph services to shut down before forced termination terminationGracePeriodSeconds: 30 + # -- Time a container has to pass its startup probe before Kubernetes restarts it. + # Liveness and readiness probes do not run until the startup probe passes + startupDeadlineSeconds: 300 # -- Add extra labels to all services serviceLabels: {} # -- Disable the creation of Kubernetes secrets objects @@ -75,6 +78,8 @@ sourcegraph: # preShutdownPause: "" # gracefulShutdownTimeout: "" # terminationGracePeriodSeconds: null +# # Override the global sourcegraph.startupDeadlineSeconds +# startupDeadlineSeconds: null # # Set resource requests / limits # resources: {} # containerSecurityContext: {} @@ -235,6 +240,8 @@ codeInsightsDB: create: false # -- Name of the ServiceAccount to be created or an existing ServiceAccount name: "" + # -- Time `codeinsights-db` has to recover from a crash or finish a major-version upgrade before Kubernetes restarts it + startupDeadlineSeconds: 3600 # -- PVC Storage Request for `codeinsights-db` data volume storageSize: 200Gi # -- Optional subPath for the `codeinsights-db` primary data volume mount @@ -304,6 +311,8 @@ codeIntelDB: create: false # -- Name of the ServiceAccount to be created or an existing ServiceAccount name: "" + # -- Time `codeintel-db` has to recover from a crash or finish a major-version upgrade before Kubernetes restarts it + startupDeadlineSeconds: 3600 # -- PVC Storage Request for `codeintel-db` data volume storageSize: 200Gi # -- Optional subPath for the `codeintel-db` primary data volume mount @@ -870,6 +879,8 @@ pgsql: create: false # -- Name of the ServiceAccount to be created or an existing ServiceAccount name: "" + # -- Time `pgsql` has to recover from a crash or finish a major-version upgrade before Kubernetes restarts it + startupDeadlineSeconds: 3600 # -- PVC Storage Request for `pgsql` data volume storageSize: 200Gi # -- Optional subPath for the `pgsql` primary data volume mount @@ -1047,6 +1058,8 @@ prometheus: create: true # -- Name of the ServiceAccount to be created or an existing ServiceAccount name: prometheus + # -- Time `prometheus` has to replay its write-ahead log before Kubernetes restarts it + startupDeadlineSeconds: 3600 # -- PVC Storage Request for `prometheus` data volume storageSize: 200Gi # -- Optional subPath for the `prometheus` primary data volume mount