From 318ac451f7e358de36c7f38d4adc91096a67a603 Mon Sep 17 00:00:00 2001 From: slayerjain Date: Fri, 4 Sep 2026 14:51:42 +0530 Subject: [PATCH] fix(postgres): probe over TCP so dependents aren't released early postgres' docker-entrypoint runs a temporary server reachable only over the Unix socket (listen_addresses='') once initdb has finished, so it can create the database and run docker-entrypoint-initdb.d. A pg_isready probe without -h talks to that socket, so it answers "accepting connections" while no TCP listener exists at all. compose then marks the service healthy and depends_on: service_healthy releases the dependent straight into ECONNREFUSED. Measured at ~235ms with no initdb.d scripts, and at 1.2s of false-healthy followed by a 2.7s shutdown checkpoint in the CI failure this was diagnosed from. Reproduced on postgres 13.3, 15 and 16. Only bites on a fresh volume, since a populated one skips the temporary server entirely -- which is why it surfaced as a rare flake rather than a consistent failure. Passing -h 127.0.0.1 forces a TCP probe, which tracks the real listener in every phase: the stock images ship listen_addresses = '*', and during startup or recovery pg_isready reports "rejecting connections", so the probe correctly keeps failing until the server can actually serve. The k8s manifest gets the same change on its readinessProbe only. The livenessProbe deliberately keeps the socket check: liveness asks whether the process is alive and a failure restarts the pod, so a TCP probe there would restart-loop a pod whose initdb outruns initialDelaySeconds + failureThreshold * periodSeconds. Signed-off-by: slayerjain --- ps-cache-kotlin/docker-compose.yml | 16 +++++++++++++++- sap-demo-java/docker-compose.yml | 16 +++++++++++++++- sap-demo-java/k8s/postgres.yaml | 11 ++++++++++- 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/ps-cache-kotlin/docker-compose.yml b/ps-cache-kotlin/docker-compose.yml index ca799f2e..fc2ff39b 100644 --- a/ps-cache-kotlin/docker-compose.yml +++ b/ps-cache-kotlin/docker-compose.yml @@ -11,7 +11,21 @@ services: - pgdata:/var/lib/postgresql/data - ./init.sql:/docker-entrypoint-initdb.d/init.sql healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres"] + # -h 127.0.0.1 is load-bearing: it forces a TCP probe. + # + # Without it pg_isready talks to the Unix socket, and postgres' + # docker-entrypoint runs a *temporary* server reachable only over that + # socket (listen_addresses='') once initdb has finished, to create the + # database and run docker-entrypoint-initdb.d. The socket therefore + # answers "accepting connections" while no TCP listener exists at all, + # so compose marks this service healthy and depends_on: + # service_healthy releases the dependent straight into ECONNREFUSED. + # + # Short but real: ~235ms with no initdb.d scripts, and 1.2s of + # false-healthy plus a 2.7s shutdown checkpoint in the CI failure this + # was diagnosed from. Only bites on a fresh volume -- a populated one + # skips the temporary server entirely, which is why it failed rarely. + test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U postgres"] interval: 2s timeout: 5s retries: 5 diff --git a/sap-demo-java/docker-compose.yml b/sap-demo-java/docker-compose.yml index 93faf890..bfce0100 100644 --- a/sap-demo-java/docker-compose.yml +++ b/sap-demo-java/docker-compose.yml @@ -17,7 +17,21 @@ services: volumes: - pgdata:/var/lib/postgresql/data healthcheck: - test: ["CMD", "pg_isready", "-U", "customer360", "-d", "customer360"] + # -h 127.0.0.1 is load-bearing: it forces a TCP probe. + # + # Without it pg_isready talks to the Unix socket, and postgres' + # docker-entrypoint runs a *temporary* server reachable only over that + # socket (listen_addresses='') once initdb has finished, to create the + # database and run docker-entrypoint-initdb.d. The socket therefore + # answers "accepting connections" while no TCP listener exists at all, + # so compose marks this service healthy and depends_on: + # service_healthy releases customer360 straight into ECONNREFUSED. + # + # Short but real: ~235ms with no initdb.d scripts, and 1.2s of + # false-healthy plus a 2.7s shutdown checkpoint in the CI failure this + # was diagnosed from. Only bites on a fresh volume -- a populated one + # skips the temporary server entirely, which is why it failed rarely. + test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "customer360", "-d", "customer360"] interval: 5s timeout: 3s retries: 10 diff --git a/sap-demo-java/k8s/postgres.yaml b/sap-demo-java/k8s/postgres.yaml index ed6dddb9..ca69cf66 100644 --- a/sap-demo-java/k8s/postgres.yaml +++ b/sap-demo-java/k8s/postgres.yaml @@ -88,12 +88,21 @@ spec: cpu: "500m" memory: "256Mi" readinessProbe: + # Readiness means "can serve clients", so probe over TCP. + # Without -h, pg_isready uses the Unix socket, which the + # temporary post-initdb server (listen_addresses='') already + # answers -- marking the pod Ready while no TCP listener exists + # and letting dependents connect to a refused port. exec: - command: ["pg_isready", "-U", "customer360", "-d", "customer360"] + command: ["pg_isready", "-h", "127.0.0.1", "-U", "customer360", "-d", "customer360"] periodSeconds: 5 timeoutSeconds: 3 failureThreshold: 5 livenessProbe: + # Deliberately the socket probe, unlike readiness above: liveness + # asks "is the process alive", and a failure restarts the pod. A + # TCP probe here would restart-loop a pod whose initdb outruns + # initialDelaySeconds + failureThreshold * periodSeconds. exec: command: ["pg_isready", "-U", "customer360", "-d", "customer360"] initialDelaySeconds: 15