monitoring: harden Grafana and keep non-P2P ports local

.env.sample.hoodi

@@ -194,7 +194,9 @@ CHARON_BUILDER_URLS=https://builder-hoodi.nflaig.dev,https://hoodi.titanrelay.xy
 # See available tags https://github.com/grafana/grafana/releases.
 #GRAFANA_VERSION=
 
-# Grafana host exposed IP and port.
+# Grafana host exposed IP and port. Defaults to 127.0.0.1 so Grafana is not reachable from the internet.
+# Docker-published ports bypass host firewalls such as ufw, so only bind to another IP if a provider or network
+# firewall restricts access to it. To view Grafana remotely, prefer an SSH tunnel: ssh -L 3000:127.0.0.1:3000 <user>@<node>
 #MONITORING_IP_GRAFANA=
 #MONITORING_PORT_GRAFANA=
 

.env.sample.mainnet

@@ -194,7 +194,9 @@ MEV_RELAYS=https://0xa15b52576bcbf1072f4a011c0f99f9fb6c66f3e1ff321f11f461d15e31b
 # See available tags https://github.com/grafana/grafana/releases.
 #GRAFANA_VERSION=
 
-# Grafana host exposed IP and port.
+# Grafana host exposed IP and port. Defaults to 127.0.0.1 so Grafana is not reachable from the internet.
+# Docker-published ports bypass host firewalls such as ufw, so only bind to another IP if a provider or network
+# firewall restricts access to it. To view Grafana remotely, prefer an SSH tunnel: ssh -L 3000:127.0.0.1:3000 <user>@<node>
 #MONITORING_IP_GRAFANA=
 #MONITORING_PORT_GRAFANA=
 

.env.sample.sepolia

@@ -194,7 +194,9 @@ CHARON_BUILDER_URLS=https://builder-sepolia.nflaig.dev,https://sepolia.titanrela
 # See available tags https://github.com/grafana/grafana/releases.
 #GRAFANA_VERSION=
 
-# Grafana host exposed IP and port.
+# Grafana host exposed IP and port. Defaults to 127.0.0.1 so Grafana is not reachable from the internet.
+# Docker-published ports bypass host firewalls such as ufw, so only bind to another IP if a provider or network
+# firewall restricts access to it. To view Grafana remotely, prefer an SSH tunnel: ssh -L 3000:127.0.0.1:3000 <user>@<node>
 #MONITORING_IP_GRAFANA=
 #MONITORING_PORT_GRAFANA=
 

.github/scripts/check_exposure.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""Fails if the default configuration exposes more than the P2P ports to the internet.
+
+- Every port published by a top-level compose file must, with its default values, either bind to
+  a loopback address or be a P2P port.
+- Anonymous Grafana users must not get more than the Viewer role.
+"""
+
+import configparser
+import glob
+import re
+import sys
+
+import yaml
+
+# Container ports that must be reachable by peers: EL, CL (incl. Prysm's 13000/12000) and charon.
+P2P_PORTS = {"30303", "9000", "12000", "13000", "3610"}
+LOOPBACK = {"127.0.0.1", "::1", "[::1]"}
+
+
+def expand_defaults(value):
+    """Replaces ${VAR:-default} and ${VAR-default} with the default, and ${VAR} with an empty string."""
+    return re.sub(r"\$\{\w+(?::?-([^}]*))?\}", lambda m: m.group(1) or "", value)
+
+
+def check_port(where, port):
+    """Returns an error if the port is published on all interfaces by default and is not a P2P port."""
+    if not isinstance(port, str):
+        return f"{where}: use the short port syntax so this check can read it: {port}"
+    parts = expand_defaults(port).split("/")[0].rsplit(":", 2)
+    host_ip = parts[0] if len(parts) == 3 else ""
+    if host_ip in LOOPBACK or parts[-1] in P2P_PORTS:
+        return None
+    return f"{where}: '{port}' is published on all interfaces by default; bind it to 127.0.0.1"
+
+
+def check_ports():
+    errors = []
+    for path in sorted(glob.glob("*.yml")):
+        with open(path) as f:
+            compose = yaml.safe_load(f) or {}
+        for name, service in (compose.get("services") or {}).items():
+            for port in (service or {}).get("ports") or []:
+                errors.append(check_port(f"{path}: {name}", port))
+    return [error for error in errors if error]
+
+
+def check_grafana():
+    ini = configparser.ConfigParser(interpolation=None)
+    ini.read("grafana/grafana.ini")
+    anonymous = ini["auth.anonymous"] if ini.has_section("auth.anonymous") else {}
+    if anonymous.get("enabled", "false").lower() == "true" and anonymous.get("org_role", "Viewer") != "Viewer":
+        return [f"grafana/grafana.ini: anonymous users get the {anonymous['org_role']} role; use Viewer"]
+    return []
+
+
+def main():
+    errors = check_ports() + check_grafana()
+    for error in errors:
+        print(f"::error::{error}")
+    if not errors:
+        print("Only P2P ports are exposed by default and anonymous Grafana access is read-only.")
+    return 1 if errors else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/workflows/test.yml

@@ -4,6 +4,8 @@ on:
   push:
     branches:
       - main
+permissions:
+  contents: read
 jobs:
   create_containers:
     # Ensures default fresh checkout can create containers.
@@ -12,3 +14,25 @@ jobs:
     - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
     - run: docker compose up --no-start
     - run: docker compose -f docker-compose.yml -f compose-debug.yml up --no-start
+
+  check_exposure:
+    # Ensures the defaults only expose P2P ports and keep anonymous Grafana access read-only.
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+    - run: 'pip install --only-binary :all: pyyaml==6.0.2'
+    - run: python3 .github/scripts/check_exposure.py
+
+  render_configs:
+    # Ensures every sample .env and the logging overlay produce a valid merged compose config.
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+    - run: |
+        for env in .env.sample.*; do
+          echo "Rendering $env"
+          cp "$env" .env
+          docker compose --profile '*' config --quiet
+        done
+        rm .env
+        docker compose -f docker-compose.yml -f logging.yml config --quiet

README.md

@@ -206,6 +206,25 @@ docker compose down cl-grandine
 docker compose up cl-grandine -d
 ```
 
+# Security
+
+- **Grafana is only published on `127.0.0.1`.** To view it from another machine, use an SSH tunnel: `ssh -L 3000:127.0.0.1:3000 <user>@<node>`, then open `http://localhost:3000`. Anonymous visitors get read-only access, and password logins are disabled because dashboards and datasources are provisioned from this repo.
+- **Docker-published ports bypass host firewalls such as `ufw`.** Only bind a port to a non-loopback address (e.g. `MONITORING_IP_GRAFANA`, `EL_IP_HTTP`) if a provider or network firewall restricts who can reach it. By default only the P2P ports are published on all interfaces.
+- **Avoid descriptive hostnames.** DNS names and TLS certificates naming the network or what the node runs (e.g. `mainnet`, `validator`) make a node easy to find with internet-wide scans.
+- **If you ran a release before the Grafana hardening,** Grafana now starts from a fresh Docker volume and no longer uses `data/grafana`. After upgrading, run `./grafana/security-check.sh` (it may need `sudo`). It reports whether Grafana is published on a public address and lists any service accounts, tokens, users, datasources and plugins in the old `data/grafana` that this stack does not create. Report findings to security@obol.tech, then delete `data/grafana`.
+- **Upgrading leaves the old Grafana database behind.** Anything created in the Grafana UI, such as custom dashboards, alert rules, contact points or users, stays in the unused `data/grafana` and does not carry over to the fresh Grafana. Before upgrading, export any custom dashboards (Share > Export) and note your alert rules. Do not copy the old database across, as it may contain unauthorized changes. To set customizations up again, add dashboards as JSON files under `grafana/dashboards/`, or re-enable logins with your own admin password in a `docker-compose.override.yml`, adding `:docker-compose.override.yml` to the end of `COMPOSE_FILE` in `.env` so it is loaded:
+
+  ```yaml
+  services:
+    grafana:
+      environment:
+        GF_AUTH_DISABLE_LOGIN_FORM: "false"
+        GF_AUTH_BASIC_ENABLED: "true"
+        GF_SECURITY_ADMIN_PASSWORD: "<a strong password>"
+  ```
+
+  `GF_SECURITY_ADMIN_PASSWORD` only applies when Grafana creates its database. If Grafana has already started, set the password with `docker compose exec grafana grafana cli admin reset-admin-password '<a strong password>'`. Keep Grafana bound to `127.0.0.1` either way.
+
 # FAQs
 
 Check the Obol docs for frequent [errors and resolutions](https://docs.obol.org/docs/faq/errors)

alloy/config.alloy.example

@@ -1,5 +1,5 @@
 discovery.docker "docker" {
-	host = "unix:///var/run/docker.sock"
+	host = "tcp://docker-socket-proxy:2375"
 }
 
 loki.process "docker" {
@@ -70,7 +70,7 @@ discovery.relabel "docker" {
 }
 
 loki.source.docker "docker" {
-	host          = "unix:///var/run/docker.sock"
+	host          = "tcp://docker-socket-proxy:2375"
 	targets       = discovery.docker.docker.targets
 	forward_to    = [loki.process.docker.receiver]
 	relabel_rules = discovery.relabel.docker.rules

compose-debug.yml

@@ -28,6 +28,8 @@ services:
     volumes:
       - ./tempo:/etc/tempo
       - ./data/tempo:/opt/tempo
+    # Grafana reaches Tempo and Loki over the monitoring network defined in docker-compose.yml.
+    networks: [default, monitoring]
     restart: unless-stopped
 
   loki:
@@ -37,6 +39,7 @@ services:
     volumes:
       - ./loki/loki.yml:/etc/loki/loki.yml
       - ./data/loki:/loki
+    networks: [default, monitoring]
     restart: unless-stopped
 
   charon:

docker-compose.yml

@@ -52,7 +52,7 @@ services:
     profiles: [""]
     ports:
       - ${LIGHTHOUSE_PORT_P2P:-9000}:9000/tcp # P2P TCP
-      - 5054:5054/tcp # P2P TCP
+      - ${LIGHTHOUSE_IP_METRICS:-127.0.0.1}:5054:5054/tcp # Metrics
       - ${LIGHTHOUSE_PORT_P2P:-9000}:9000/udp # P2P UDP
       - ${LIGHTHOUSE_IP_HTTP:-127.0.0.1}:${LIGHTHOUSE_PORT_HTTP:-5052}:5052 # BN TCP
     labels:
@@ -175,7 +175,7 @@ services:
   prometheus:
     image: prom/prometheus:${PROMETHEUS_VERSION:-v3.15.1}
     user: ":"
-    networks: [dvnode]
+    networks: [dvnode, monitoring]
     environment:
       PROM_REMOTE_WRITE_TOKEN: ${PROM_REMOTE_WRITE_TOKEN}
       SERVICE_OWNER: ${SERVICE_OWNER:-obol-cdvn}
@@ -189,17 +189,24 @@ services:
 
   grafana:
     image: grafana/grafana:${GRAFANA_VERSION:-13.2.4}
-    user: ":"
     ports:
-      - ${MONITORING_IP_GRAFANA:-0.0.0.0}:${MONITORING_PORT_GRAFANA:-3000}:3000
-    networks: [dvnode]
+      - ${MONITORING_IP_GRAFANA:-127.0.0.1}:${MONITORING_PORT_GRAFANA:-3000}:3000
+    networks: [monitoring]
+    security_opt: [no-new-privileges:true]
+    cap_drop: [ALL]
     volumes:
       - ./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml
       - ./grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/datasource.yml
       - ./grafana/grafana.ini:/etc/grafana/grafana.ini:ro
       - ./grafana/dashboards:/etc/dashboards
-      - ./data/grafana:/var/lib/grafana
+      # A named volume lets Grafana run as its own non-root user. Nothing in it needs keeping: dashboards and datasources are provisioned.
+      - grafana-data:/var/lib/grafana
     restart: unless-stopped
 
 networks:
   dvnode:
+  # Grafana only joins this network, so it can reach the monitoring backends but not charon or the clients.
+  monitoring:
+
+volumes:
+  grafana-data:

grafana/datasource.yml

@@ -20,7 +20,7 @@ datasources:
       tlsAuth: false
       tlsAuthWithCACert: false
     version: 1
-    editable: true
+    editable: false
 
   - name: Loki
     type: loki
@@ -30,7 +30,7 @@ datasources:
     basicAuth: false
     isDefault: false
     version: 1
-    editable: true
+    editable: false
 
   - name: Tempo
     type: tempo
@@ -40,4 +40,4 @@ datasources:
     basicAuth: false
     isDefault: false
     version: 1
-    editable: true
+    editable: false

grafana/grafana.ini

@@ -1,3 +1,35 @@
+# Grafana is published on 127.0.0.1 by default (see MONITORING_IP_GRAFANA). To view it from
+# another machine use an SSH tunnel: ssh -L 3000:127.0.0.1:3000 <user>@<node>
+
+# Anonymous users can view the provisioned dashboards but cannot change anything.
 [auth.anonymous]
 enabled = true
-org_role = Admin
+org_role = Viewer
+
+# No password logins: dashboards and datasources are provisioned from files in this repo.
+# This also disables the factory admin:admin account, including on existing data/grafana databases.
+[auth]
+disable_login_form = true
+
+[auth.basic]
+enabled = false
+
+[users]
+allow_sign_up = false
+allow_org_create = false
+
+[security]
+disable_gravatar = true
+cookie_samesite = strict
+# Only allow the datasource proxy to reach the provisioned monitoring backends.
+data_source_proxy_whitelist = prometheus:9090 loki:3100 tempo:3200
+
+[plugins]
+plugin_admin_enabled = false
+preinstall_disabled = true
+
+[public_dashboards]
+enabled = false
+
+[snapshots]
+enabled = false