diff --git a/.env.sample.hoodi b/.env.sample.hoodi index 9e934bc6..f7a15328 100644 --- a/.env.sample.hoodi +++ b/.env.sample.hoodi @@ -190,7 +190,9 @@ MEV_RELAYS=https://0x98f0ef62f00780cf8eb06701a7d22725b9437d4768bb19b363e882ae871 # See available tags https://github.com/grafana/grafana/releases. #GRAFANA_VERSION= -# Grafana host exposed IP and port. +# Grafana host exposed IP and port. Defaults to 127.0.0.1 so Grafana is not reachable from the internet. +# Docker-published ports bypass host firewalls such as ufw, so only bind to another IP if a provider or network +# firewall restricts access to it. To view Grafana remotely, prefer an SSH tunnel: ssh -L 3000:127.0.0.1:3000 @ #MONITORING_IP_GRAFANA= #MONITORING_PORT_GRAFANA= diff --git a/.env.sample.mainnet b/.env.sample.mainnet index a2e2543c..f6073a86 100644 --- a/.env.sample.mainnet +++ b/.env.sample.mainnet @@ -190,7 +190,9 @@ MEV_RELAYS=https://0xa15b52576bcbf1072f4a011c0f99f9fb6c66f3e1ff321f11f461d15e31b # See available tags https://github.com/grafana/grafana/releases. #GRAFANA_VERSION= -# Grafana host exposed IP and port. +# Grafana host exposed IP and port. Defaults to 127.0.0.1 so Grafana is not reachable from the internet. +# Docker-published ports bypass host firewalls such as ufw, so only bind to another IP if a provider or network +# firewall restricts access to it. To view Grafana remotely, prefer an SSH tunnel: ssh -L 3000:127.0.0.1:3000 @ #MONITORING_IP_GRAFANA= #MONITORING_PORT_GRAFANA= diff --git a/.github/scripts/check_exposure.py b/.github/scripts/check_exposure.py new file mode 100644 index 00000000..d8ed4496 --- /dev/null +++ b/.github/scripts/check_exposure.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Fails if the default configuration exposes more than the P2P ports to the internet. + +- Every port published by a top-level compose file must, with its default values, either bind to + a loopback address or be a P2P port. +- Anonymous Grafana users must not get more than the Viewer role. +""" + +import configparser +import glob +import re +import sys + +import yaml + +# Container ports that must be reachable by peers: EL, CL (incl. Prysm's 13000/12000) and charon. +P2P_PORTS = {"30303", "9000", "12000", "13000", "3610"} +LOOPBACK = {"127.0.0.1", "::1", "[::1]"} + + +def expand_defaults(value): + """Replaces ${VAR:-default} and ${VAR-default} with the default, and ${VAR} with an empty string.""" + return re.sub(r"\$\{\w+(?::?-([^}]*))?\}", lambda m: m.group(1) or "", value) + + +def check_port(where, port): + """Returns an error if the port is published on all interfaces by default and is not a P2P port.""" + if not isinstance(port, str): + return f"{where}: use the short port syntax so this check can read it: {port}" + parts = expand_defaults(port).split("/")[0].rsplit(":", 2) + host_ip = parts[0] if len(parts) == 3 else "" + if host_ip in LOOPBACK or parts[-1] in P2P_PORTS: + return None + return f"{where}: '{port}' is published on all interfaces by default; bind it to 127.0.0.1" + + +def check_ports(): + errors = [] + for path in sorted(glob.glob("*.yml")): + with open(path) as f: + compose = yaml.safe_load(f) or {} + for name, service in (compose.get("services") or {}).items(): + for port in (service or {}).get("ports") or []: + errors.append(check_port(f"{path}: {name}", port)) + return [error for error in errors if error] + + +def check_grafana(): + ini = configparser.ConfigParser(interpolation=None) + ini.read("grafana/grafana.ini") + anonymous = ini["auth.anonymous"] if ini.has_section("auth.anonymous") else {} + if anonymous.get("enabled", "false").lower() == "true" and anonymous.get("org_role", "Viewer") != "Viewer": + return [f"grafana/grafana.ini: anonymous users get the {anonymous['org_role']} role; use Viewer"] + return [] + + +def main(): + errors = check_ports() + check_grafana() + for error in errors: + print(f"::error::{error}") + if not errors: + print("Only P2P ports are exposed by default and anonymous Grafana access is read-only.") + return 1 if errors else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4e85bfdd..acb28ff5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,6 +4,8 @@ on: push: branches: - main +permissions: + contents: read jobs: create_containers: # Ensures default fresh checkout can create containers. @@ -12,3 +14,25 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - run: docker compose up --no-start - run: docker compose -f docker-compose.yml -f compose-debug.yml up --no-start + + check_exposure: + # Ensures the defaults only expose P2P ports and keep anonymous Grafana access read-only. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - run: 'pip install --only-binary :all: pyyaml==6.0.2' + - run: python3 .github/scripts/check_exposure.py + + render_configs: + # Ensures every sample .env and the logging overlay produce a valid merged compose config. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - run: | + for env in .env.sample.*; do + echo "Rendering $env" + cp "$env" .env + docker compose --profile '*' config --quiet + done + rm .env + docker compose -f docker-compose.yml -f logging.yml config --quiet diff --git a/README.md b/README.md index 2566b15a..241941c0 100644 --- a/README.md +++ b/README.md @@ -206,6 +206,25 @@ docker compose down cl-grandine docker compose up cl-grandine -d ``` +# Security + +- **Grafana is only published on `127.0.0.1`.** To view it from another machine, use an SSH tunnel: `ssh -L 3000:127.0.0.1:3000 @`, then open `http://localhost:3000`. Anonymous visitors get read-only access, and password logins are disabled because dashboards and datasources are provisioned from this repo. +- **Docker-published ports bypass host firewalls such as `ufw`.** Only bind a port to a non-loopback address (e.g. `MONITORING_IP_GRAFANA`, `EL_IP_HTTP`) if a provider or network firewall restricts who can reach it. By default only the P2P ports are published on all interfaces. +- **Avoid descriptive hostnames.** DNS names and TLS certificates naming the network or what the node runs (e.g. `mainnet`, `validator`) make a node easy to find with internet-wide scans. +- **If you ran a release before the Grafana hardening,** Grafana now starts from a fresh Docker volume and no longer uses `data/grafana`. After upgrading, run `./grafana/security-check.sh` (it may need `sudo`). It reports whether Grafana is published on a public address and lists any service accounts, tokens, users, datasources and plugins in the old `data/grafana` that this stack does not create. Report findings to security@obol.tech, then delete `data/grafana`. +- **Upgrading leaves the old Grafana database behind.** Anything created in the Grafana UI, such as custom dashboards, alert rules, contact points or users, stays in the unused `data/grafana` and does not carry over to the fresh Grafana. Before upgrading, export any custom dashboards (Share > Export) and note your alert rules. Do not copy the old database across, as it may contain unauthorized changes. To set customizations up again, add dashboards as JSON files under `grafana/dashboards/`, or re-enable logins with your own admin password in a `docker-compose.override.yml`, adding `:docker-compose.override.yml` to the end of `COMPOSE_FILE` in `.env` so it is loaded: + + ```yaml + services: + grafana: + environment: + GF_AUTH_DISABLE_LOGIN_FORM: "false" + GF_AUTH_BASIC_ENABLED: "true" + GF_SECURITY_ADMIN_PASSWORD: "" + ``` + + `GF_SECURITY_ADMIN_PASSWORD` only applies when Grafana creates its database. If Grafana has already started, set the password with `docker compose exec grafana grafana cli admin reset-admin-password ''`. Keep Grafana bound to `127.0.0.1` either way. + # FAQs Check the Obol docs for frequent [errors and resolutions](https://docs.obol.org/docs/faq/errors) diff --git a/alloy/config.alloy.example b/alloy/config.alloy.example index 91577b4e..32dade6d 100644 --- a/alloy/config.alloy.example +++ b/alloy/config.alloy.example @@ -1,5 +1,5 @@ discovery.docker "docker" { - host = "unix:///var/run/docker.sock" + host = "tcp://docker-socket-proxy:2375" } loki.process "docker" { @@ -70,7 +70,7 @@ discovery.relabel "docker" { } loki.source.docker "docker" { - host = "unix:///var/run/docker.sock" + host = "tcp://docker-socket-proxy:2375" targets = discovery.docker.docker.targets forward_to = [loki.process.docker.receiver] relabel_rules = discovery.relabel.docker.rules diff --git a/compose-debug.yml b/compose-debug.yml index 8e6429fe..2644f176 100644 --- a/compose-debug.yml +++ b/compose-debug.yml @@ -28,6 +28,8 @@ services: volumes: - ./tempo:/etc/tempo - ./data/tempo:/opt/tempo + # Grafana reaches Tempo and Loki over the monitoring network defined in docker-compose.yml. + networks: [default, monitoring] restart: unless-stopped loki: @@ -37,6 +39,7 @@ services: volumes: - ./loki/loki.yml:/etc/loki/loki.yml - ./data/loki:/loki + networks: [default, monitoring] restart: unless-stopped charon: diff --git a/docker-compose.yml b/docker-compose.yml index cb5436a7..fdfd580b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,7 +52,7 @@ services: profiles: [""] ports: - ${LIGHTHOUSE_PORT_P2P:-9000}:9000/tcp # P2P TCP - - 5054:5054/tcp # P2P TCP + - ${LIGHTHOUSE_IP_METRICS:-127.0.0.1}:5054:5054/tcp # Metrics - ${LIGHTHOUSE_PORT_P2P:-9000}:9000/udp # P2P UDP - ${LIGHTHOUSE_IP_HTTP:-127.0.0.1}:${LIGHTHOUSE_PORT_HTTP:-5052}:5052 # BN TCP labels: @@ -174,7 +174,7 @@ services: prometheus: image: prom/prometheus:${PROMETHEUS_VERSION:-v3.15.1} user: ":" - networks: [dvnode] + networks: [dvnode, monitoring] environment: PROM_REMOTE_WRITE_TOKEN: ${PROM_REMOTE_WRITE_TOKEN} SERVICE_OWNER: ${SERVICE_OWNER:-obol-cdvn} @@ -188,17 +188,24 @@ services: grafana: image: grafana/grafana:${GRAFANA_VERSION:-13.2.4} - user: ":" ports: - - ${MONITORING_IP_GRAFANA:-0.0.0.0}:${MONITORING_PORT_GRAFANA:-3000}:3000 - networks: [dvnode] + - ${MONITORING_IP_GRAFANA:-127.0.0.1}:${MONITORING_PORT_GRAFANA:-3000}:3000 + networks: [monitoring] + security_opt: [no-new-privileges:true] + cap_drop: [ALL] volumes: - ./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml - ./grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/datasource.yml - ./grafana/grafana.ini:/etc/grafana/grafana.ini:ro - ./grafana/dashboards:/etc/dashboards - - ./data/grafana:/var/lib/grafana + # A named volume lets Grafana run as its own non-root user. Nothing in it needs keeping: dashboards and datasources are provisioned. + - grafana-data:/var/lib/grafana restart: unless-stopped networks: dvnode: + # Grafana only joins this network, so it can reach the monitoring backends but not charon or the clients. + monitoring: + +volumes: + grafana-data: diff --git a/grafana/datasource.yml b/grafana/datasource.yml index ace88848..3fae8d0d 100644 --- a/grafana/datasource.yml +++ b/grafana/datasource.yml @@ -20,7 +20,7 @@ datasources: tlsAuth: false tlsAuthWithCACert: false version: 1 - editable: true + editable: false - name: Loki type: loki @@ -30,7 +30,7 @@ datasources: basicAuth: false isDefault: false version: 1 - editable: true + editable: false - name: Tempo type: tempo @@ -40,4 +40,4 @@ datasources: basicAuth: false isDefault: false version: 1 - editable: true \ No newline at end of file + editable: false \ No newline at end of file diff --git a/grafana/grafana.ini b/grafana/grafana.ini index d087b643..95eac884 100644 --- a/grafana/grafana.ini +++ b/grafana/grafana.ini @@ -1,3 +1,35 @@ +# Grafana is published on 127.0.0.1 by default (see MONITORING_IP_GRAFANA). To view it from +# another machine use an SSH tunnel: ssh -L 3000:127.0.0.1:3000 @ + +# Anonymous users can view the provisioned dashboards but cannot change anything. [auth.anonymous] enabled = true -org_role = Admin \ No newline at end of file +org_role = Viewer + +# No password logins: dashboards and datasources are provisioned from files in this repo. +# This also disables the factory admin:admin account, including on existing data/grafana databases. +[auth] +disable_login_form = true + +[auth.basic] +enabled = false + +[users] +allow_sign_up = false +allow_org_create = false + +[security] +disable_gravatar = true +cookie_samesite = strict +# Only allow the datasource proxy to reach the provisioned monitoring backends. +data_source_proxy_whitelist = prometheus:9090 loki:3100 tempo:3200 + +[plugins] +plugin_admin_enabled = false +preinstall_disabled = true + +[public_dashboards] +enabled = false + +[snapshots] +enabled = false diff --git a/grafana/security-check.sh b/grafana/security-check.sh new file mode 100755 index 00000000..0e2f6e51 --- /dev/null +++ b/grafana/security-check.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# Checks this node's Grafana for internet exposure and for signs of unauthorized changes. +# Releases before the Grafana hardening granted anonymous users the Admin role and published +# Grafana on all interfaces, so anyone who could reach port 3000 could change Grafana. +# +# Grafana used to keep its data in data/grafana. It now uses a fresh Docker volume, so this script +# inspects the old data/grafana for signs of tampering, after which data/grafana can be deleted. +# +# Usage, from the repository root, after upgrading: ./grafana/security-check.sh +# Reading data/grafana may need sudo, as it was written by Grafana's container. + +set -uo pipefail + +DB="${GRAFANA_DB:-data/grafana/grafana.db}" +PLUGINS_DIR="${GRAFANA_PLUGINS_DIR:-data/grafana/plugins}" +EXPECTED_DATASOURCES="Prometheus Loki Tempo" +# Grafana Labs apps that Grafana preinstalls itself. +EXPECTED_PLUGINS="grafana-lokiexplore-app grafana-exploretraces-app grafana-metricsdrilldown-app grafana-pyroscope-app" +findings=0 + +warn() { + echo " [!] $*" + findings=$((findings + 1)) + return 0 +} + +echo "== Grafana port binding" +if binding=$(docker compose port grafana 3000 2>/dev/null) && [[ -n "$binding" ]]; then + echo " grafana:3000 is published on $binding" + case "$binding" in + 127.0.0.1:* | "[::1]:"*) echo " OK: only reachable from this host" ;; + *) warn "Grafana is published on a non-loopback address. Docker-published ports bypass ufw, so it is reachable unless a provider or network firewall blocks it. Set MONITORING_IP_GRAFANA=127.0.0.1 in .env and run: docker compose up -d grafana" ;; + esac +else + echo " grafana is not running or not published" +fi + +echo +echo "== Grafana database ($DB)" +if [[ ! -e "$DB" ]]; then + echo " not found, nothing to check" +elif [[ ! -r "$DB" ]]; then + echo " cannot read it, re-run with sudo" + exit 2 +elif ! command -v python3 >/dev/null; then + echo " python3 is required to inspect the database" + exit 2 +else + db_findings=$( + python3 -I - "$DB" "$EXPECTED_DATASOURCES" <<'EOF' +import sqlite3 +import sys + +db, expected = sys.argv[1], sys.argv[2].split() +conn = sqlite3.connect(f"file:{db}?mode=ro&immutable=1", uri=True) + + +def rows(query): + try: + return conn.execute(query).fetchall() + except sqlite3.Error: + return [] + + +for login, created in rows("SELECT login, created FROM user WHERE is_service_account = 1"): + print(f"service account: {login} (created {created})") +for name, created in rows("SELECT name, created FROM api_key"): + print(f"API or service account token: {name} (created {created})") +for login, created in rows("SELECT login, created FROM user WHERE is_service_account = 0 AND login != 'admin'"): + print(f"user: {login} (created {created})") +for name, kind, url, created in rows("SELECT name, type, url, created FROM data_source"): + if name not in expected: + print(f"datasource: {name} type={kind} url={url} (created {created})") +EOF + ) + if [[ -n "$db_findings" ]]; then + while IFS= read -r line; do warn "$line"; done <<<"$db_findings" + else + echo " OK: no service accounts, tokens, extra users or unexpected datasources" + fi +fi + +echo +echo "== Grafana plugins ($PLUGINS_DIR)" +unexpected_plugins=0 +if [[ -d "$PLUGINS_DIR" ]]; then + for plugin in "$PLUGINS_DIR"/*; do + [[ -e "$plugin" ]] || continue + name=$(basename "$plugin") + case " $EXPECTED_PLUGINS " in + *" $name "*) echo " preinstalled by Grafana: $name" ;; + *) + warn "installed plugin: $name" + unexpected_plugins=1 + ;; + esac + done +fi +[[ "$unexpected_plugins" -eq 0 ]] && echo " OK: no unexpected plugins installed" + +echo +if [[ "$findings" -eq 0 ]]; then + echo "No issues found." +else + echo "Found $findings issue(s). This stack provisions everything Grafana needs from files, so service accounts," + echo "tokens, extra users, extra datasources or plugins were most likely created by someone else." + echo "Please report it to security@obol.tech." +fi + +if [[ -d "$(dirname "$DB")" ]]; then + echo + echo "Grafana no longer uses $(dirname "$DB"). Delete it with: sudo rm -rf $(dirname "$DB")" +fi + +[[ "$findings" -eq 0 ]] diff --git a/logging.yml b/logging.yml index 3f0f8763..6206bd3b 100644 --- a/logging.yml +++ b/logging.yml @@ -7,11 +7,26 @@ services: CLUSTER_PEER: ${CLUSTER_PEER} volumes: - ./alloy:/etc/alloy - - /var/run/docker.sock:/var/run/docker.sock - networks: [dvnode] + networks: [dvnode, docker-api] + depends_on: [docker-socket-proxy] entrypoint: /etc/alloy/run.sh command: ["/bin/alloy", "run", "/etc/alloy/config.alloy", "--storage.path=/var/lib/alloy/data"] restart: unless-stopped + # Gives Alloy read-only access to container metadata and logs instead of the full Docker API. + docker-socket-proxy: + image: tecnativa/docker-socket-proxy:${DOCKER_SOCKET_PROXY_VERSION:-v0.5.0} + environment: + CONTAINERS: 1 + # Alloy's container discovery also lists networks. + NETWORKS: 1 + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + networks: [docker-api] + restart: unless-stopped + networks: dvnode: + # Only Alloy and the Docker socket proxy join this network, and it has no external access. + docker-api: + internal: true