feat: v1.11 hardening follow up

.github/scripts/check_exposure.py

@@ -2,8 +2,15 @@
 """Fails if the default configuration exposes more than the P2P ports to the internet.
 
 - Every port published by a top-level compose file must, with its default values, either bind to
-  a loopback address or be a P2P port.
+  a loopback address or be a P2P port. Example ports in *.sample files must bind to 127.0.0.1 too.
 - Anonymous Grafana users must not get more than the Viewer role.
+- Grafana must not accept passwords, neither through the login form and POST /login nor through
+  HTTP basic auth on its API. Compose files must not configure Grafana with GF_* variables or an
+  env_file, since those override grafana.ini.
+- Grafana must only join the monitoring network, so it cannot reach charon, the clients or the ejector.
+- Grafana must mount files from this repository read-only, so it cannot change its own provisioning.
+- Only the Docker socket proxy may mount the Docker socket.
+- Outside compose-debug.yml, no service may run privileged or share the host's network or PID namespace.
 """
 
 import configparser
@@ -16,17 +23,30 @@
 # Container ports that must be reachable by peers: EL, CL (incl. Prysm's 13000/12000) and charon.
 P2P_PORTS = {"30303", "9000", "12000", "13000", "3610"}
 LOOPBACK = {"127.0.0.1", "::1", "[::1]"}
+SOCKET_PROXY = "docker-socket-proxy"
+DEBUG_FILES = {"compose-debug.yml"}
+
+
+def compose_files():
+    """Yields (path, services) for every top-level compose file."""
+    for path in sorted(glob.glob("*.yml") + glob.glob("*.yaml")):
+        with open(path) as f:
+            compose = yaml.safe_load(f) or {}
+        yield path, {name: service or {} for name, service in (compose.get("services") or {}).items()}
 
 
 def expand_defaults(value):
-    """Replaces ${VAR:-default} and ${VAR-default} with the default, and ${VAR} with an empty string."""
-    return re.sub(r"\$\{\w+(?::?-([^}]*))?\}", lambda m: m.group(1) or "", value)
+    """Replaces ${VAR:-default} with the default, and ${VAR} with an empty string."""
+    return re.sub(r"\$\{\w+(?::-([^}]*))?\}", lambda m: m.group(1) or "", value)
 
 
 def check_port(where, port):
     """Returns an error if the port is published on all interfaces by default and is not a P2P port."""
     if not isinstance(port, str):
         return f"{where}: use the short port syntax so this check can read it: {port}"
+    if re.search(r"\$\{\w+-", port):
+        # ${VAR-default} keeps an empty VAR= from .env, which publishes on all interfaces.
+        return f"{where}: '{port}' uses ${{VAR-default}}; use ${{VAR:-default}} so an empty value keeps the default"
     parts = expand_defaults(port).split("/")[0].rsplit(":", 2)
     host_ip = parts[0] if len(parts) == 3 else ""
     if host_ip in LOOPBACK or parts[-1] in P2P_PORTS:
@@ -36,30 +56,95 @@
 
 def check_ports():
     errors = []
-    for path in sorted(glob.glob("*.yml")):
-        with open(path) as f:
-            compose = yaml.safe_load(f) or {}
-        for name, service in (compose.get("services") or {}).items():
-            for port in (service or {}).get("ports") or []:
+    for path, services in compose_files():
+        for name, service in services.items():
+            for port in service.get("ports") or []:
                 errors.append(check_port(f"{path}: {name}", port))
     return [error for error in errors if error]
 
 
+def check_sample_ports():
+    """Returns errors for commented-out example ports in *.sample files that would publish on all interfaces."""
+    errors = []
+    for path in sorted(glob.glob("*.sample")):
+        with open(path) as f:
+            for number, line in enumerate(f, 1):
+                match = re.match(r"\s*#?\s*-\s*['\"]?([^\s'\"#]+:\d+(?:/\w+)?)", line)
+                if match:
+                    errors.append(check_port(f"{path}:{number}", match.group(1)))
+    return [error for error in errors if error]
+
+
+# Grafana settings that, with their GF_* environment variable, must keep password logins off.
+GRAFANA_REQUIRED = {
+    ("auth", "disable_login_form"): "true",
+    ("auth.basic", "enabled"): "false",
+}
+
+
 def check_grafana():
     ini = configparser.ConfigParser(interpolation=None)
     ini.read("grafana/grafana.ini")
+    errors = []
     anonymous = ini["auth.anonymous"] if ini.has_section("auth.anonymous") else {}
     if anonymous.get("enabled", "false").lower() == "true" and anonymous.get("org_role", "Viewer") != "Viewer":
-        return [f"grafana/grafana.ini: anonymous users get the {anonymous['org_role']} role; use Viewer"]
-    return []
+        errors.append(f"grafana/grafana.ini: anonymous users get the {anonymous['org_role']} role; use Viewer")
+    for (section, key), want in GRAFANA_REQUIRED.items():
+        if ini.get(section, key, fallback="").lower() != want:
+            errors.append(f"grafana/grafana.ini: set [{section}] {key} = {want} so Grafana does not accept passwords")
+    return errors
+
+
+def check_grafana_service():
+    """Returns errors for compose files that configure Grafana around grafana.ini, join it to other networks or mount it writable."""
+    errors = []
+    for path, services in compose_files():
+        grafana = services.get("grafana")
+        if grafana is None:
+            continue
+        env = grafana.get("environment") or {}
+        keys = env.keys() if isinstance(env, dict) else [item.split("=")[0] for item in env]
+        errors += [f"{path}: grafana sets {key}; configure Grafana in grafana/grafana.ini" for key in keys if key.startswith("GF_")]
+        if grafana.get("env_file"):
+            errors.append(f"{path}: grafana has an env_file, which can override grafana.ini; remove it")
+        networks = grafana.get("networks")
+        if networks is not None and list(networks) != ["monitoring"]:
+            errors.append(f"{path}: grafana joins {list(networks)}; it must only join the monitoring network")
+        for volume in grafana.get("volumes") or []:
+            if isinstance(volume, dict):
+                source, read_only = str(volume.get("source", "")), volume.get("read_only", False)
+            else:
+                source, read_only = volume, volume.endswith(":ro")
+            if source.startswith(".") and not read_only:
+                errors.append(f"{path}: grafana mounts '{source}' writable; make it read-only")
+    return errors
+
+
+def check_privileges():
+    """Returns errors for Docker socket mounts outside the socket proxy, and for privileged or host-namespace services."""
+    errors = []
+    for path, services in compose_files():
+        for name, service in services.items():
+            for volume in service.get("volumes") or []:
+                source = str(volume.get("source", "")) if isinstance(volume, dict) else volume.split(":")[0]
+                if source.endswith("docker.sock") and name != SOCKET_PROXY:
+                    errors.append(f"{path}: {name} mounts the Docker socket; go through {SOCKET_PROXY}")
+            if path in DEBUG_FILES:
+                continue
+            if service.get("privileged"):
+                errors.append(f"{path}: {name} runs privileged")
+            for key in ("network_mode", "pid", "ipc"):
+                if service.get(key) == "host":
+                    errors.append(f"{path}: {name} sets {key}: host")
+    return errors
 
 
 def main():
-    errors = check_ports() + check_grafana()
+    errors = check_ports() + check_sample_ports() + check_grafana() + check_grafana_service() + check_privileges()
     for error in errors:
         print(f"::error::{error}")
     if not errors:
-        print("Only P2P ports are exposed by default and anonymous Grafana access is read-only.")
+        print("Only P2P ports are exposed by default and anonymous Grafana access is read-only without password logins.")
     return 1 if errors else 0
 
 

README.md

@@ -211,19 +211,26 @@ docker compose up cl-grandine -d
 - **Grafana is only published on `127.0.0.1`.** To view it from another machine, use an SSH tunnel: `ssh -L 3000:127.0.0.1:3000 <user>@<node>`, then open `http://localhost:3000`. Anonymous visitors get read-only access, and password logins are disabled because dashboards and datasources are provisioned from this repo.
 - **Docker-published ports bypass host firewalls such as `ufw`.** Only bind a port to a non-loopback address (e.g. `MONITORING_IP_GRAFANA`, `EL_IP_HTTP`) if a provider or network firewall restricts who can reach it. By default only the P2P ports are published on all interfaces.
 - **Avoid descriptive hostnames.** DNS names and TLS certificates naming the network or what the node runs (e.g. `mainnet`, `validator`) make a node easy to find with internet-wide scans.
-- **If you ran a release before the Grafana hardening,** Grafana now starts from a fresh Docker volume and no longer uses `data/grafana`. After upgrading, run `./grafana/security-check.sh` (it may need `sudo`). It reports whether Grafana is published on a public address and lists any service accounts, tokens, users, datasources and plugins in the old `data/grafana` that this stack does not create. Report findings to security@obol.tech, then delete `data/grafana`.
+- **If you ran a release before the Grafana hardening,** Grafana now starts from a fresh Docker volume and no longer uses `data/grafana`. After upgrading, run `./grafana/security-check.sh` (it may need `sudo`). It reports whether Grafana is published on a public address. It lists anything in the old `data/grafana` that this stack does not create, such as service accounts, tokens, users, datasources, plugins, organizations, dashboards, alert rules and contact points. It also lists changed or added files under `grafana/`, which older releases mounted writable. Report findings to security@obol.tech, then delete `data/grafana`. Grafana stores secrets with a default encryption key, so if your Grafana was reachable, rotate any credentials you stored in it, such as webhook URLs or tokens used by contact points.
 - **Upgrading leaves the old Grafana database behind.** Anything created in the Grafana UI, such as custom dashboards, alert rules, contact points or users, stays in the unused `data/grafana` and does not carry over to the fresh Grafana. Before upgrading, export any custom dashboards (Share > Export) and note your alert rules. Do not copy the old database across, as it may contain unauthorized changes. To set customizations up again, add dashboards as JSON files under `grafana/dashboards/`, or re-enable logins with your own admin password in a `docker-compose.override.yml`, adding `:docker-compose.override.yml` to the end of `COMPOSE_FILE` in `.env` so it is loaded:
 
   ```yaml
   services:
     grafana:
       environment:
         GF_AUTH_DISABLE_LOGIN_FORM: "false"
-        GF_AUTH_BASIC_ENABLED: "true"
+        # Visitors must log in to see anything.
+        GF_AUTH_ANONYMOUS_ENABLED: "false"
         GF_SECURITY_ADMIN_PASSWORD: "<a strong password>"
+        # Generate with: openssl rand -hex 32
+        GF_SECURITY_SECRET_KEY: "<your own random key>"
   ```
 
-  `GF_SECURITY_ADMIN_PASSWORD` only applies when Grafana creates its database. If Grafana has already started, set the password with `docker compose exec grafana grafana cli admin reset-admin-password '<a strong password>'`. Keep Grafana bound to `127.0.0.1` either way.
+  **Set the admin password before you restart with this override.** `GF_SECURITY_ADMIN_PASSWORD` only applies when Grafana creates its database, and your Grafana has most likely already created it with the default `admin` / `admin`. Run `docker compose exec grafana grafana cli admin reset-admin-password '<a strong password>'` first, then `docker compose up -d`.
+
+  `GF_SECURITY_SECRET_KEY` replaces Grafana's publicly known default key for encrypting stored secrets, such as the webhook URLs and tokens of contact points. Set it before adding any. Changing it later makes the secrets already stored unreadable, and you'll have to enter them again.
+
+  Keep Grafana bound to `127.0.0.1` and reach it through an SSH tunnel. Logins go over plain HTTP, so only bind it elsewhere behind a provider or network firewall.
 
 # FAQs
 

compose-cl.yml

@@ -12,7 +12,7 @@ services:
 
   cl-grandine:
     profiles: [ cl-grandine ]
-    image: sifrai/grandine:${GRANDINE_VERSION:-2.0.6}
+    image: sifrai/grandine:${CL_GRANDINE_VERSION:-${GRANDINE_VERSION:-2.0.6}}
     restart: unless-stopped
     labels:
       - "alloy-monitored=${CL_GRANDINE_ALLOY_MONITORED:-true}"
@@ -47,7 +47,7 @@ services:
 
   cl-lighthouse:
     profiles: [ cl-lighthouse ]
-    image: sigp/lighthouse:${LIGHTHOUSE_VERSION:-v8.3.1}
+    image: sigp/lighthouse:${CL_LIGHTHOUSE_VERSION:-${LIGHTHOUSE_VERSION:-v8.3.1}}
     restart: unless-stopped
     labels:
       - "alloy-monitored=${CL_LIGHTHOUSE_ALLOY_MONITORED:-true}"

compose-el.yml

@@ -11,7 +11,7 @@ services:
 
   el-nethermind:
     profiles: [el-nethermind]
-    image: nethermind/nethermind:${EL_NETHERMIND_VERSION:-2.2.0}
+    image: nethermind/nethermind:${NETHERMIND_VERSION:-${EL_NETHERMIND_VERSION:-2.2.0}}
     restart: unless-stopped
     ports:
       - ${EL_PORT_P2P:-30303}:30303/tcp # P2P TCP
@@ -30,6 +30,7 @@ services:
       --JsonRpc.EnginePort=8551
       --JsonRpc.Host=0.0.0.0
       --JsonRpc.Port=8545
+      --JsonRpc.EnabledModules=[Eth,Subscribe,TxPool,Web3,Proof,Net,Health,Rpc]
       --Metrics.Enabled=true
       --Metrics.ExposePort=8008
       --Sync.SnapSync=true
@@ -48,7 +49,7 @@ services:
 
   el-reth:
     profiles: [el-reth]
-    image: ghcr.io/paradigmxyz/reth:${EL_RETH_VERSION:-v2.8.0}
+    image: ghcr.io/paradigmxyz/reth:${RETH_VERSION:-${EL_RETH_VERSION:-v2.8.0}}
     restart: unless-stopped
     ports:
       - ${EL_PORT_P2P:-30303}:30303/tcp # P2P TCP

docker-compose.override.yml.sample

@@ -18,48 +18,48 @@
   #nethermind:
     # Disable nethermind
     #profiles: [disable]
-    # Bind nethermind internal ports to host ports
+    # Bind nethermind internal ports to host ports. Keep 127.0.0.1: Docker-published ports bypass ufw and other host firewalls.
     #ports:
-      #- 8545:8545 # JSON-RPC
-      #- 8551:8551 # AUTH-RPC
-      #- 8008:8008 # Metrics
+      #- 127.0.0.1:8545:8545 # JSON-RPC
+      #- 127.0.0.1:8551:8551 # AUTH-RPC
+      #- 127.0.0.1:8008:8008 # Metrics
 
   #lighthouse:
     # Disable lighthouse
     #profiles: [disable]
-    # Bind lighthouse internal ports to host ports
+    # Bind lighthouse internal ports to host ports. Keep 127.0.0.1: Docker-published ports bypass ufw and other host firewalls.
     #ports:
-      #- 5052:5052 # HTTP
-      #- 5054:5054 # Metrics
+      #- 127.0.0.1:5052:5052 # HTTP
+      #- 127.0.0.1:5054:5054 # Metrics
 
   #charon:
     # Configure any additional env var flags in .env.charon.more
     #env_file: [.env.charon.more]
     # Uncomment the extra_hosts section if you are trying to communicate with a CL running in a different docker network on the same machine 
     #extra_hosts:
       #- "host.docker.internal:host-gateway"
-    # Bind charon internal ports to host ports
+    # Bind charon internal ports to host ports. Keep 127.0.0.1: Docker-published ports bypass ufw and other host firewalls.
     #ports:
-      #- 3600:3600/tcp # Validator API
-      #- 3620:3620/tcp # Monitoring
+      #- 127.0.0.1:3600:3600/tcp # Validator API
+      #- 127.0.0.1:3620:3620/tcp # Monitoring
 
   #lodestar:
     # Disable lodestar
     #profiles: [disable]
-    # Bind lodestar internal ports to host ports
+    # Bind lodestar internal ports to host ports. Keep 127.0.0.1: Docker-published ports bypass ufw and other host firewalls.
     #ports:
-      #- 5064:5064 # Metrics
+      #- 127.0.0.1:5064:5064 # Metrics
 
   #prometheus:
     # Disable prometheus
     #profiles: [disable]
-    # Bind prometheus internal ports to host ports
+    # Bind prometheus internal ports to host ports. Keep 127.0.0.1: Docker-published ports bypass ufw and other host firewalls.
     #ports:
-      #- 9090:9090 # Metrics
+      #- 127.0.0.1:9090:9090 # Metrics
 
   #mev-boost:
     # Disable mev-boost
     #profiles: [disable]
-    # Bind mev-boost internal ports to host ports
+    # Bind mev-boost internal ports to host ports. Keep 127.0.0.1: Docker-published ports bypass ufw and other host firewalls.
     #ports:
-      #- 18550:18550 # Metrics
+      #- 127.0.0.1:18550:18550 # Metrics

docker-compose.yml

@@ -30,6 +30,7 @@ services:
       --JsonRpc.EnginePort=8551
       --JsonRpc.Host=0.0.0.0
       --JsonRpc.Port=8545
+      --JsonRpc.EnabledModules=[Eth,Subscribe,TxPool,Web3,Proof,Net,Health,Rpc]
       --Metrics.Enabled=true
       --Metrics.ExposePort=8008
       --Sync.SnapSync=true
@@ -86,7 +87,7 @@ services:
   #  \___|_| |_|\__,_|_|  \___/|_| |_|
 
   charon:
-    image: obolnetwork/charon:${CHARON_VERSION:-v1.11.2}
+    image: obolnetwork/charon:${CHARON_VERSION:-v1.11.3}
     environment:
       - CHARON_BEACON_NODE_ENDPOINTS=${CHARON_BEACON_NODE_ENDPOINTS:-http://${CL:-lighthouse}:5052}
       - CHARON_BEACON_NODE_TIMEOUT=${CHARON_BEACON_NODE_TIMEOUT:-3s}
@@ -194,10 +195,10 @@ services:
     security_opt: [no-new-privileges:true]
     cap_drop: [ALL]
     volumes:
-      - ./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml
-      - ./grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/datasource.yml
+      - ./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml:ro
+      - ./grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/datasource.yml:ro
       - ./grafana/grafana.ini:/etc/grafana/grafana.ini:ro
-      - ./grafana/dashboards:/etc/dashboards
+      - ./grafana/dashboards:/etc/dashboards:ro
       # A named volume lets Grafana run as its own non-root user. Nothing in it needs keeping: dashboards and datasources are provisioned.
       - grafana-data:/var/lib/grafana
     restart: unless-stopped