diff --git a/.changeset/haproxy-blue-green.md b/.changeset/haproxy-blue-green.md new file mode 100644 index 00000000..38137823 --- /dev/null +++ b/.changeset/haproxy-blue-green.md @@ -0,0 +1,7 @@ +--- +"nostream": minor +--- + +feat(deploy): add HAProxy blue/green compose stack + +Two relays behind HAProxy with `/readyz` health checks and `option redispatch`, plus a rolling recreate script that replaces one relay at a time for zero-downtime image updates. diff --git a/deploy/README.md b/deploy/README.md index 937878db..fad7d2e3 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -152,3 +152,31 @@ the new checkout (or copy the updated files). Automated sync is planned separate ``` Existing `.env` and `.nostr/settings.yaml` are preserved. + +## Zero-downtime updates (HAProxy blue/green) + +`deploy/docker-compose.haproxy.yml` replaces the single-relay stack with two +relays (`nostream-blue`, `nostream-green`) behind HAProxy on `127.0.0.1:8008`. +Postgres, Redis, and migrations are unchanged. + +Install alongside `.env` and `postgresql.conf`, then start: + +```bash +cp deploy/docker-compose.haproxy.yml deploy/rolling-relay-recreate.sh /opt/nostream/ +cp -r deploy/haproxy /opt/nostream/ +cd /opt/nostream +docker compose -f docker-compose.haproxy.yml up -d +curl -s http://127.0.0.1:8008/readyz +``` + +To update, load the new image, then replace relays one at a time: + +```bash +./rolling-relay-recreate.sh +``` + +The script stops a relay, waits for the replacement to report healthy, and only +then moves to the second one, so a ready backend is always serving. HAProxy +health-checks `/readyz` every 2s and retries failed requests on the other +backend (`option redispatch`). Relays get `stop_grace_period: 45s` so the +`WS_DRAIN_TIMEOUT_MS` drain (default 30s) finishes before Docker sends SIGKILL. diff --git a/deploy/docker-compose.haproxy.yml b/deploy/docker-compose.haproxy.yml new file mode 100644 index 00000000..fb8e5984 --- /dev/null +++ b/deploy/docker-compose.haproxy.yml @@ -0,0 +1,126 @@ +# Blue/green alternative to docker-compose.prod.yml: two relays behind HAProxy. +# Keep the shared services below in sync with docker-compose.prod.yml. +# +# docker compose -f docker-compose.haproxy.yml up -d + +x-nostream-relay: &nostream-relay + image: ghcr.io/cameri/nostream:main + pull_policy: never + env_file: .env + environment: + RELAY_PORT: 8008 + NOSTR_CONFIG_DIR: /home/node/.nostr + DB_HOST: nostream-db + DB_PORT: 5432 + DB_USER: ${DB_USER} + DB_PASSWORD: ${DB_PASSWORD} + DB_NAME: ${DB_NAME} + DB_MIN_POOL_SIZE: ${DB_MIN_POOL_SIZE:-16} + DB_MAX_POOL_SIZE: ${DB_MAX_POOL_SIZE:-64} + DB_ACQUIRE_CONNECTION_TIMEOUT: ${DB_ACQUIRE_CONNECTION_TIMEOUT:-60000} + REDIS_HOST: nostream-cache + REDIS_PORT: 6379 + REDIS_USER: default + REDIS_PASSWORD: ${REDIS_PASSWORD} + READ_REPLICA_ENABLED: 'false' + WORKER_COUNT: ${WORKER_COUNT:-2} + WS_DRAIN_TIMEOUT_MS: ${WS_DRAIN_TIMEOUT_MS:-30000} + user: node:node + volumes: + - ${PWD}/.nostr:/home/node/.nostr + depends_on: + nostream-cache: + condition: service_healthy + nostream-db: + condition: service_healthy + nostream-migrate: + condition: service_completed_successfully + restart: on-failure + stop_grace_period: 45s + # Gates `up --wait` during rolling recreate, so the next relay is only + # replaced once this one serves traffic. + healthcheck: + test: + [ + 'CMD-SHELL', + "node -e \"fetch('http://127.0.0.1:8008/readyz').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))\"", + ] + interval: 5s + timeout: 5s + retries: 6 + start_period: 60s + +services: + haproxy: + image: haproxy:3.0-alpine + container_name: nostream-haproxy + volumes: + - ./haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - 127.0.0.1:8008:8008 + depends_on: + - nostream-blue + - nostream-green + restart: on-failure + + nostream-blue: + <<: *nostream-relay + container_name: nostream-blue + + nostream-green: + <<: *nostream-relay + container_name: nostream-green + + nostream-db: + image: postgres:15 + container_name: nostream-db + environment: + POSTGRES_DB: ${DB_NAME} + POSTGRES_USER: ${DB_USER} + POSTGRES_PASSWORD: ${DB_PASSWORD} + volumes: + - ${PWD}/.nostr/data:/var/lib/postgresql/data + - ${PWD}/.nostr/db-logs:/var/log/postgresql + - ${PWD}/postgresql.conf:/postgresql.conf + command: postgres -c 'config_file=/postgresql.conf' + restart: always + healthcheck: + test: ['CMD-SHELL', 'pg_isready -U ${DB_USER}'] + interval: 5s + timeout: 5s + retries: 5 + start_period: 360s + + nostream-cache: + image: redis:7.0.5-alpine3.16 + container_name: nostream-cache + environment: + REDIS_PASSWORD: ${REDIS_PASSWORD} + volumes: + - cache:/data + command: sh -c 'redis-server --loglevel warning --requirepass "$$REDIS_PASSWORD"' + restart: always + healthcheck: + test: ['CMD-SHELL', 'redis-cli -a "$$REDIS_PASSWORD" ping | grep PONG'] + interval: 2s + timeout: 5s + retries: 10 + + nostream-migrate: + image: ghcr.io/cameri/nostream:main + pull_policy: never + container_name: nostream-migrate + user: node:node + command: ['node_modules/.bin/knex', 'migrate:latest'] + environment: + DB_HOST: nostream-db + DB_PORT: 5432 + DB_USER: ${DB_USER} + DB_PASSWORD: ${DB_PASSWORD} + DB_NAME: ${DB_NAME} + depends_on: + nostream-db: + condition: service_healthy + +volumes: + cache: diff --git a/deploy/haproxy/haproxy.cfg b/deploy/haproxy/haproxy.cfg new file mode 100644 index 00000000..c699bb9d --- /dev/null +++ b/deploy/haproxy/haproxy.cfg @@ -0,0 +1,39 @@ +global + log stdout format raw local0 info + maxconn 4096 + +defaults + mode http + log global + option httplog + timeout connect 5s + timeout client 30s + timeout server 30s + timeout tunnel 1h + timeout check 5s + retries 3 + option redispatch + retry-on conn-failure empty-response response-timeout 502 503 504 + +# Docker's embedded DNS, so recreated relay containers are picked up by IP change. +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s + +frontend nostream_in + bind *:8008 + default_backend nostream_relays + +backend nostream_relays + balance roundrobin + option httpchk + http-check send meth GET uri /readyz + http-check expect status 200 + + default-server check inter 2s fall 2 rise 1 resolvers docker resolve-prefer ipv4 init-addr libc,none + + server blue nostream-blue:8008 + server green nostream-green:8008 diff --git a/deploy/rolling-relay-recreate.sh b/deploy/rolling-relay-recreate.sh new file mode 100755 index 00000000..62fba0bd --- /dev/null +++ b/deploy/rolling-relay-recreate.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Rolling relay recreate for the HAProxy blue/green stack. Replaces one relay +# at a time so the other keeps serving traffic. +# +# Usage: +# ./rolling-relay-recreate.sh [/opt/nostream] +# +# Load the new image and run migrations before calling this. + +TARGET="${1:-.}" +COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.haproxy.yml}" + +cd "$TARGET" + +if [[ ! -f "$COMPOSE_FILE" ]]; then + echo "error: compose file not found: $TARGET/$COMPOSE_FILE" >&2 + exit 1 +fi + +compose() { + docker compose -f "$COMPOSE_FILE" "$@" +} + +for service in nostream-blue nostream-green; do + if [[ -z "$(compose ps -q "$service")" ]]; then + echo "Skipping $service (not running)" + continue + fi + + echo "Replacing $service..." + compose stop "$service" + compose rm -f "$service" + compose up -d --no-deps --wait "$service" +done + +echo "Rolling recreate complete"