From 2d7a1ac8c429e1d44ba93814e658c00602082912 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ois=C3=ADn=20Kyne?= Date: Wed, 28 Oct 2026 11:14:55 +0000 Subject: [PATCH 1/2] security: close the remaining exposure gaps after the Grafana hardening - Bind the example ports in docker-compose.override.yml.sample to 127.0.0.1, since Docker-published ports bypass ufw. - Extend check_exposure.py to example ports, GF_* variables, env_file, networks and writable mounts on Grafana, Docker socket mounts and privileged or host-namespace services. - Report every non-P2P port published on a public address in security-check.sh, separately from signs of tampering. - Have the README's re-enable-logins snippet turn anonymous access off, set a secret key, and reset the admin password first. - Read the client version variables that .env.sample documents. - Limit Nethermind's JSON-RPC modules to the ones the stack needs. --- .github/scripts/check_exposure.py | 109 +++++++++++++++++++++++++---- README.md | 13 +++- compose-cl.yml | 4 +- compose-el.yml | 5 +- docker-compose.override.yml.sample | 32 ++++----- docker-compose.yml | 7 +- grafana/security-check.sh | 97 ++++++++++++++++++++++--- 7 files changed, 221 insertions(+), 46 deletions(-) diff --git a/.github/scripts/check_exposure.py b/.github/scripts/check_exposure.py index d8ed4496..0d7a277c 100644 --- a/.github/scripts/check_exposure.py +++ b/.github/scripts/check_exposure.py @@ -2,8 +2,15 @@ """Fails if the default configuration exposes more than the P2P ports to the internet. - Every port published by a top-level compose file must, with its default values, either bind to - a loopback address or be a P2P port. + a loopback address or be a P2P port. Example ports in *.sample files must bind to 127.0.0.1 too. - Anonymous Grafana users must not get more than the Viewer role. +- Grafana must not accept passwords, neither through the login form and POST /login nor through + HTTP basic auth on its API. Compose files must not configure Grafana with GF_* variables or an + env_file, since those override grafana.ini. +- Grafana must only join the monitoring network, so it cannot reach charon, the clients or the ejector. +- Grafana must mount files from this repository read-only, so it cannot change its own provisioning. +- Only the Docker socket proxy may mount the Docker socket. +- Outside compose-debug.yml, no service may run privileged or share the host's network or PID namespace. """ import configparser @@ -16,17 +23,30 @@ # Container ports that must be reachable by peers: EL, CL (incl. Prysm's 13000/12000) and charon. P2P_PORTS = {"30303", "9000", "12000", "13000", "3610"} LOOPBACK = {"127.0.0.1", "::1", "[::1]"} +SOCKET_PROXY = "docker-socket-proxy" +DEBUG_FILES = {"compose-debug.yml"} + + +def compose_files(): + """Yields (path, services) for every top-level compose file.""" + for path in sorted(glob.glob("*.yml") + glob.glob("*.yaml")): + with open(path) as f: + compose = yaml.safe_load(f) or {} + yield path, {name: service or {} for name, service in (compose.get("services") or {}).items()} def expand_defaults(value): - """Replaces ${VAR:-default} and ${VAR-default} with the default, and ${VAR} with an empty string.""" - return re.sub(r"\$\{\w+(?::?-([^}]*))?\}", lambda m: m.group(1) or "", value) + """Replaces ${VAR:-default} with the default, and ${VAR} with an empty string.""" + return re.sub(r"\$\{\w+(?::-([^}]*))?\}", lambda m: m.group(1) or "", value) def check_port(where, port): """Returns an error if the port is published on all interfaces by default and is not a P2P port.""" if not isinstance(port, str): return f"{where}: use the short port syntax so this check can read it: {port}" + if re.search(r"\$\{\w+-", port): + # ${VAR-default} keeps an empty VAR= from .env, which publishes on all interfaces. + return f"{where}: '{port}' uses ${{VAR-default}}; use ${{VAR:-default}} so an empty value keeps the default" parts = expand_defaults(port).split("/")[0].rsplit(":", 2) host_ip = parts[0] if len(parts) == 3 else "" if host_ip in LOOPBACK or parts[-1] in P2P_PORTS: @@ -36,30 +56,95 @@ def check_port(where, port): def check_ports(): errors = [] - for path in sorted(glob.glob("*.yml")): - with open(path) as f: - compose = yaml.safe_load(f) or {} - for name, service in (compose.get("services") or {}).items(): - for port in (service or {}).get("ports") or []: + for path, services in compose_files(): + for name, service in services.items(): + for port in service.get("ports") or []: errors.append(check_port(f"{path}: {name}", port)) return [error for error in errors if error] +def check_sample_ports(): + """Returns errors for commented-out example ports in *.sample files that would publish on all interfaces.""" + errors = [] + for path in sorted(glob.glob("*.sample")): + with open(path) as f: + for number, line in enumerate(f, 1): + match = re.match(r"\s*#?\s*-\s*['\"]?([^\s'\"#]+:\d+(?:/\w+)?)", line) + if match: + errors.append(check_port(f"{path}:{number}", match.group(1))) + return [error for error in errors if error] + + +# Grafana settings that, with their GF_* environment variable, must keep password logins off. +GRAFANA_REQUIRED = { + ("auth", "disable_login_form"): "true", + ("auth.basic", "enabled"): "false", +} + + def check_grafana(): ini = configparser.ConfigParser(interpolation=None) ini.read("grafana/grafana.ini") + errors = [] anonymous = ini["auth.anonymous"] if ini.has_section("auth.anonymous") else {} if anonymous.get("enabled", "false").lower() == "true" and anonymous.get("org_role", "Viewer") != "Viewer": - return [f"grafana/grafana.ini: anonymous users get the {anonymous['org_role']} role; use Viewer"] - return [] + errors.append(f"grafana/grafana.ini: anonymous users get the {anonymous['org_role']} role; use Viewer") + for (section, key), want in GRAFANA_REQUIRED.items(): + if ini.get(section, key, fallback="").lower() != want: + errors.append(f"grafana/grafana.ini: set [{section}] {key} = {want} so Grafana does not accept passwords") + return errors + + +def check_grafana_service(): + """Returns errors for compose files that configure Grafana around grafana.ini, join it to other networks or mount it writable.""" + errors = [] + for path, services in compose_files(): + grafana = services.get("grafana") + if grafana is None: + continue + env = grafana.get("environment") or {} + keys = env.keys() if isinstance(env, dict) else [item.split("=")[0] for item in env] + errors += [f"{path}: grafana sets {key}; configure Grafana in grafana/grafana.ini" for key in keys if key.startswith("GF_")] + if grafana.get("env_file"): + errors.append(f"{path}: grafana has an env_file, which can override grafana.ini; remove it") + networks = grafana.get("networks") + if networks is not None and list(networks) != ["monitoring"]: + errors.append(f"{path}: grafana joins {list(networks)}; it must only join the monitoring network") + for volume in grafana.get("volumes") or []: + if isinstance(volume, dict): + source, read_only = str(volume.get("source", "")), volume.get("read_only", False) + else: + source, read_only = volume, volume.endswith(":ro") + if source.startswith(".") and not read_only: + errors.append(f"{path}: grafana mounts '{source}' writable; make it read-only") + return errors + + +def check_privileges(): + """Returns errors for Docker socket mounts outside the socket proxy, and for privileged or host-namespace services.""" + errors = [] + for path, services in compose_files(): + for name, service in services.items(): + for volume in service.get("volumes") or []: + source = str(volume.get("source", "")) if isinstance(volume, dict) else volume.split(":")[0] + if source.endswith("docker.sock") and name != SOCKET_PROXY: + errors.append(f"{path}: {name} mounts the Docker socket; go through {SOCKET_PROXY}") + if path in DEBUG_FILES: + continue + if service.get("privileged"): + errors.append(f"{path}: {name} runs privileged") + for key in ("network_mode", "pid", "ipc"): + if service.get(key) == "host": + errors.append(f"{path}: {name} sets {key}: host") + return errors def main(): - errors = check_ports() + check_grafana() + errors = check_ports() + check_sample_ports() + check_grafana() + check_grafana_service() + check_privileges() for error in errors: print(f"::error::{error}") if not errors: - print("Only P2P ports are exposed by default and anonymous Grafana access is read-only.") + print("Only P2P ports are exposed by default and anonymous Grafana access is read-only without password logins.") return 1 if errors else 0 diff --git a/README.md b/README.md index 241941c0..127662f6 100644 --- a/README.md +++ b/README.md @@ -211,7 +211,7 @@ docker compose up cl-grandine -d - **Grafana is only published on `127.0.0.1`.** To view it from another machine, use an SSH tunnel: `ssh -L 3000:127.0.0.1:3000 @`, then open `http://localhost:3000`. Anonymous visitors get read-only access, and password logins are disabled because dashboards and datasources are provisioned from this repo. - **Docker-published ports bypass host firewalls such as `ufw`.** Only bind a port to a non-loopback address (e.g. `MONITORING_IP_GRAFANA`, `EL_IP_HTTP`) if a provider or network firewall restricts who can reach it. By default only the P2P ports are published on all interfaces. - **Avoid descriptive hostnames.** DNS names and TLS certificates naming the network or what the node runs (e.g. `mainnet`, `validator`) make a node easy to find with internet-wide scans. -- **If you ran a release before the Grafana hardening,** Grafana now starts from a fresh Docker volume and no longer uses `data/grafana`. After upgrading, run `./grafana/security-check.sh` (it may need `sudo`). It reports whether Grafana is published on a public address and lists any service accounts, tokens, users, datasources and plugins in the old `data/grafana` that this stack does not create. Report findings to security@obol.tech, then delete `data/grafana`. +- **If you ran a release before the Grafana hardening,** Grafana now starts from a fresh Docker volume and no longer uses `data/grafana`. After upgrading, run `./grafana/security-check.sh` (it may need `sudo`). It reports whether Grafana is published on a public address. It lists anything in the old `data/grafana` that this stack does not create, such as service accounts, tokens, users, datasources, plugins, organizations, dashboards, alert rules and contact points. It also lists changed or added files under `grafana/`, which older releases mounted writable. Report findings to security@obol.tech, then delete `data/grafana`. Grafana stores secrets with a default encryption key, so if your Grafana was reachable, rotate any credentials you stored in it, such as webhook URLs or tokens used by contact points. - **Upgrading leaves the old Grafana database behind.** Anything created in the Grafana UI, such as custom dashboards, alert rules, contact points or users, stays in the unused `data/grafana` and does not carry over to the fresh Grafana. Before upgrading, export any custom dashboards (Share > Export) and note your alert rules. Do not copy the old database across, as it may contain unauthorized changes. To set customizations up again, add dashboards as JSON files under `grafana/dashboards/`, or re-enable logins with your own admin password in a `docker-compose.override.yml`, adding `:docker-compose.override.yml` to the end of `COMPOSE_FILE` in `.env` so it is loaded: ```yaml @@ -219,11 +219,18 @@ docker compose up cl-grandine -d grafana: environment: GF_AUTH_DISABLE_LOGIN_FORM: "false" - GF_AUTH_BASIC_ENABLED: "true" + # Visitors must log in to see anything. + GF_AUTH_ANONYMOUS_ENABLED: "false" GF_SECURITY_ADMIN_PASSWORD: "" + # Generate with: openssl rand -hex 32 + GF_SECURITY_SECRET_KEY: "" ``` - `GF_SECURITY_ADMIN_PASSWORD` only applies when Grafana creates its database. If Grafana has already started, set the password with `docker compose exec grafana grafana cli admin reset-admin-password ''`. Keep Grafana bound to `127.0.0.1` either way. + **Set the admin password before you restart with this override.** `GF_SECURITY_ADMIN_PASSWORD` only applies when Grafana creates its database, and your Grafana has most likely already created it with the default `admin` / `admin`. Run `docker compose exec grafana grafana cli admin reset-admin-password ''` first, then `docker compose up -d`. + + `GF_SECURITY_SECRET_KEY` replaces Grafana's publicly known default key for encrypting stored secrets, such as the webhook URLs and tokens of contact points. Set it before adding any. Changing it later makes the secrets already stored unreadable, and you'll have to enter them again. + + Keep Grafana bound to `127.0.0.1` and reach it through an SSH tunnel. Logins go over plain HTTP, so only bind it elsewhere behind a provider or network firewall. # FAQs diff --git a/compose-cl.yml b/compose-cl.yml index b53e859f..785fd145 100644 --- a/compose-cl.yml +++ b/compose-cl.yml @@ -12,7 +12,7 @@ services: cl-grandine: profiles: [ cl-grandine ] - image: sifrai/grandine:${GRANDINE_VERSION:-3.0.0-rc.2} + image: sifrai/grandine:${CL_GRANDINE_VERSION:-${GRANDINE_VERSION:-3.0.0-rc.2}} restart: unless-stopped labels: - "alloy-monitored=${CL_GRANDINE_ALLOY_MONITORED:-true}" @@ -47,7 +47,7 @@ services: cl-lighthouse: profiles: [ cl-lighthouse ] - image: sigp/lighthouse:${LIGHTHOUSE_VERSION:-v8.3.2} + image: sigp/lighthouse:${CL_LIGHTHOUSE_VERSION:-${LIGHTHOUSE_VERSION:-v8.3.2}} restart: unless-stopped labels: - "alloy-monitored=${CL_LIGHTHOUSE_ALLOY_MONITORED:-true}" diff --git a/compose-el.yml b/compose-el.yml index bb841b27..36ba2b19 100644 --- a/compose-el.yml +++ b/compose-el.yml @@ -11,7 +11,7 @@ services: el-nethermind: profiles: [el-nethermind] - image: nethermind/nethermind:${EL_NETHERMIND_VERSION:-2.2.1} + image: nethermind/nethermind:${NETHERMIND_VERSION:-${EL_NETHERMIND_VERSION:-2.2.1}} restart: unless-stopped ports: - ${EL_PORT_P2P:-30303}:30303/tcp # P2P TCP @@ -30,6 +30,7 @@ services: --JsonRpc.EnginePort=8551 --JsonRpc.Host=0.0.0.0 --JsonRpc.Port=8545 + --JsonRpc.EnabledModules=[Eth,Subscribe,TxPool,Web3,Proof,Net,Health,Rpc] --Metrics.Enabled=true --Metrics.ExposePort=8008 --Sync.SnapSync=true @@ -48,7 +49,7 @@ services: el-reth: profiles: [el-reth] - image: ghcr.io/paradigmxyz/reth:${EL_RETH_VERSION:-v2.8.1} + image: ghcr.io/paradigmxyz/reth:${RETH_VERSION:-${EL_RETH_VERSION:-v2.8.1}} restart: unless-stopped ports: - ${EL_PORT_P2P:-30303}:30303/tcp # P2P TCP diff --git a/docker-compose.override.yml.sample b/docker-compose.override.yml.sample index 4aabb4cd..fef66f60 100644 --- a/docker-compose.override.yml.sample +++ b/docker-compose.override.yml.sample @@ -18,19 +18,19 @@ #nethermind: # Disable nethermind #profiles: [disable] - # Bind nethermind internal ports to host ports + # Bind nethermind internal ports to host ports. Keep 127.0.0.1: Docker-published ports bypass ufw and other host firewalls. #ports: - #- 8545:8545 # JSON-RPC - #- 8551:8551 # AUTH-RPC - #- 8008:8008 # Metrics + #- 127.0.0.1:8545:8545 # JSON-RPC + #- 127.0.0.1:8551:8551 # AUTH-RPC + #- 127.0.0.1:8008:8008 # Metrics #lighthouse: # Disable lighthouse #profiles: [disable] - # Bind lighthouse internal ports to host ports + # Bind lighthouse internal ports to host ports. Keep 127.0.0.1: Docker-published ports bypass ufw and other host firewalls. #ports: - #- 5052:5052 # HTTP - #- 5054:5054 # Metrics + #- 127.0.0.1:5052:5052 # HTTP + #- 127.0.0.1:5054:5054 # Metrics #charon: # Configure any additional env var flags in .env.charon.more @@ -38,28 +38,28 @@ # Uncomment the extra_hosts section if you are trying to communicate with a CL running in a different docker network on the same machine #extra_hosts: #- "host.docker.internal:host-gateway" - # Bind charon internal ports to host ports + # Bind charon internal ports to host ports. Keep 127.0.0.1: Docker-published ports bypass ufw and other host firewalls. #ports: - #- 3600:3600/tcp # Validator API - #- 3620:3620/tcp # Monitoring + #- 127.0.0.1:3600:3600/tcp # Validator API + #- 127.0.0.1:3620:3620/tcp # Monitoring #lodestar: # Disable lodestar #profiles: [disable] - # Bind lodestar internal ports to host ports + # Bind lodestar internal ports to host ports. Keep 127.0.0.1: Docker-published ports bypass ufw and other host firewalls. #ports: - #- 5064:5064 # Metrics + #- 127.0.0.1:5064:5064 # Metrics #prometheus: # Disable prometheus #profiles: [disable] - # Bind prometheus internal ports to host ports + # Bind prometheus internal ports to host ports. Keep 127.0.0.1: Docker-published ports bypass ufw and other host firewalls. #ports: - #- 9090:9090 # Metrics + #- 127.0.0.1:9090:9090 # Metrics #mev-boost: # Disable mev-boost #profiles: [disable] - # Bind mev-boost internal ports to host ports + # Bind mev-boost internal ports to host ports. Keep 127.0.0.1: Docker-published ports bypass ufw and other host firewalls. #ports: - #- 18550:18550 # Metrics + #- 127.0.0.1:18550:18550 # Metrics diff --git a/docker-compose.yml b/docker-compose.yml index 3ef5e6f9..01a6af28 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,6 +30,7 @@ services: --JsonRpc.EnginePort=8551 --JsonRpc.Host=0.0.0.0 --JsonRpc.Port=8545 + --JsonRpc.EnabledModules=[Eth,Subscribe,TxPool,Web3,Proof,Net,Health,Rpc] --Metrics.Enabled=true --Metrics.ExposePort=8008 --Sync.SnapSync=true @@ -195,10 +196,10 @@ services: security_opt: [no-new-privileges:true] cap_drop: [ALL] volumes: - - ./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml - - ./grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/datasource.yml + - ./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml:ro + - ./grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/datasource.yml:ro - ./grafana/grafana.ini:/etc/grafana/grafana.ini:ro - - ./grafana/dashboards:/etc/dashboards + - ./grafana/dashboards:/etc/dashboards:ro # A named volume lets Grafana run as its own non-root user. Nothing in it needs keeping: dashboards and datasources are provisioned. - grafana-data:/var/lib/grafana restart: unless-stopped diff --git a/grafana/security-check.sh b/grafana/security-check.sh index bf025fc8..78d94426 100755 --- a/grafana/security-check.sh +++ b/grafana/security-check.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Checks this node's Grafana for internet exposure and for signs of unauthorized changes. +# Checks this node's published ports and Grafana for internet exposure, and Grafana for signs of unauthorized changes. # Releases before the Grafana hardening granted anonymous users the Admin role and published # Grafana on all interfaces, so anyone who could reach port 3000 could change Grafana. # @@ -21,6 +21,7 @@ grafana-metricsdrilldown-app grafana-postgresql-datasource grafana-pyroscope-app influxdb jaeger loki mssql mysql opentsdb prometheus stackdriver tempo zipkin" EXPECTED_PLUGINS="${EXPECTED_PLUGINS//$'\n'/ }" findings=0 +exposed=0 warn() { echo " [!] $*" @@ -28,17 +29,44 @@ warn() { return 0 } +# exposure reports a port reachable from the internet. It is counted apart from signs of tampering. +exposure() { + echo " [!] $*" + exposed=$((exposed + 1)) + return 0 +} + echo "== Grafana port binding" if binding=$(docker compose port grafana 3000 2>/dev/null) && [[ -n "$binding" ]]; then echo " grafana:3000 is published on $binding" case "$binding" in 127.0.0.1:* | "[::1]:"*) echo " OK: only reachable from this host" ;; - *) warn "Grafana is published on a non-loopback address. Docker-published ports bypass ufw, so it is reachable unless a provider or network firewall blocks it. Set MONITORING_IP_GRAFANA=127.0.0.1 in .env and run: docker compose up -d grafana" ;; + *) exposure "Grafana is published on a non-loopback address. Docker-published ports bypass ufw, so it is reachable unless a provider or network firewall blocks it. Set MONITORING_IP_GRAFANA=127.0.0.1 in .env and run: docker compose up -d grafana" ;; esac else echo " grafana is not running or not published" fi +echo +echo "== Other published ports" +# Container ports that peers must reach: EL, CL (incl. Prysm's 13000/12000) and charon. +P2P_PORTS=" 30303 9000 12000 13000 3610 " +before=$exposed +while read -r service ports; do + IFS=',' read -ra mappings <<<"$ports" + for mapping in "${mappings[@]}"; do + mapping="${mapping# }" + [[ "$mapping" == *"->"* ]] || continue + host="${mapping%%->*}" + container_port="${mapping##*->}" + container_port="${container_port%%/*}" + case "$host" in 127.0.0.1:* | "[::1]:"* | "::1:"*) continue ;; esac + [[ "$P2P_PORTS" == *" $container_port "* || "$service" == grafana ]] && continue + exposure "$service publishes port $container_port on $host. Docker-published ports bypass ufw, so it is reachable unless a provider or network firewall blocks it. Bind it to 127.0.0.1 in .env or docker-compose.override.yml." + done +done < <(docker compose ps --format '{{.Service}} {{.Ports}}' 2>/dev/null) +[[ $exposed == "$before" ]] && echo " OK: only P2P ports are published on non-loopback addresses" + echo echo "== Grafana database ($DB)" if [[ ! -e "$DB" ]]; then @@ -52,6 +80,7 @@ elif ! command -v python3 >/dev/null; then else db_findings=$( python3 -I - "$DB" "$EXPECTED_DATASOURCES" <<'EOF' +import json import sqlite3 import sys @@ -75,12 +104,41 @@ for login, created in rows("SELECT login, created FROM user WHERE is_service_acc for name, kind, url, created in rows("SELECT name, type, url, created FROM data_source"): if name not in expected: print(f"datasource: {name} type={kind} url={url} (created {created})") +for name, created in rows("SELECT name, created FROM org WHERE id != 1"): + print(f"organization: {name} (created {created})") +for title, created in rows("SELECT title, created FROM dashboard WHERE id NOT IN (SELECT dashboard_id FROM dashboard_provisioning)"): + print(f"dashboard or folder not provisioned by this stack: {title} (created {created})") +for title, updated in rows("SELECT title, updated FROM alert_rule"): + print(f"alert rule: {title} (updated {updated})") +for name, kind, created in rows("SELECT name, type, created FROM alert_notification"): + print(f"notification channel: {name} type={kind} (created {created})") +contact_points = set() +for (config,) in rows("SELECT alertmanager_configuration FROM alert_configuration"): + try: + receivers = json.loads(config).get("alertmanager_config", {}).get("receivers") or [] + except (ValueError, AttributeError): + continue + for receiver in receivers: + for integration in receiver.get("grafana_managed_receiver_configs") or []: + # Grafana's default contact point emails a placeholder address. + if "example@email.com" not in json.dumps(integration.get("settings")): + contact_points.add((receiver.get("name"), integration.get("type"))) +for name, kind in sorted(contact_points, key=str): + print(f"contact point: {name} type={kind}") +# IDs keep counting up after rows are deleted, which reveals records that were created and later removed. +# Provisioning recreates a few datasources and dashboards on its own, so allow some of those. +ALLOWED_DELETIONS = {"user": 0, "api_key": 0, "org": 0, "alert_rule": 0, "data_source": 3, "dashboard": 10} +sequences = dict(rows("SELECT name, seq FROM sqlite_sequence")) +for table, allowed in ALLOWED_DELETIONS.items(): + count = rows(f"SELECT count(*) FROM {table}") + if table in sequences and count and sequences[table] - count[0][0] > allowed: + print(f"{sequences[table] - count[0][0]} {table} rows were created and later deleted") EOF ) if [[ -n "$db_findings" ]]; then while IFS= read -r line; do warn "$line"; done <<<"$db_findings" else - echo " OK: no service accounts, tokens, extra users or unexpected datasources" + echo " OK: no service accounts, tokens, users, datasources, organizations, dashboards or alerting changes" fi fi @@ -103,17 +161,40 @@ fi [[ "$unexpected_plugins" -eq 0 ]] && echo " OK: no unexpected plugins installed" echo +echo "== Repository files that Grafana mounted (grafana/)" +# Releases before the Grafana hardening mounted these writable, so Grafana could change its own provisioning. +if changed=$(git -c safe.directory="$PWD" status --porcelain --untracked-files=all -- grafana 2>/dev/null); then + if [[ -n "$changed" ]]; then + while IFS= read -r line; do warn "changed or added file: $line"; done <<<"$changed" + echo " Unless you made these changes, restore the files with: git checkout -- grafana && git clean -fd grafana" + echo " Also check 'git stash list', as 'git stash apply' during an upgrade would bring changes back." + else + echo " OK: grafana/ matches this release" + fi +else + echo " not a git checkout, cannot check" +fi + +echo +if [[ "$exposed" -gt 0 ]]; then + echo "Found $exposed exposed port(s). Fix the bindings listed above first." +fi if [[ "$findings" -eq 0 ]]; then - echo "No issues found." + echo "No signs of unauthorized changes found." else - echo "Found $findings issue(s). This stack provisions everything Grafana needs from files, so service accounts," - echo "tokens, extra users, extra datasources or plugins were most likely created by someone else." - echo "Please report it to security@obol.tech." + echo "Found $findings issue(s). This stack provisions everything Grafana needs from files, so anything listed" + echo "above that you did not create yourself was most likely created by someone else. Please report it to" + echo "security@obol.tech. Grafana encrypts stored secrets with a default key, so treat any credentials you" + echo "stored in it, such as webhook URLs or tokens of contact points, as exposed and rotate them." fi if [[ -d "$(dirname "$DB")" ]]; then echo + if [[ "$findings" -gt 0 ]]; then + echo "Before deleting $(dirname "$DB"), keep a copy outside this repository for the investigation:" + echo " sudo tar czf ~/grafana-evidence-\$(date +%F).tar.gz $(dirname "$DB") && sha256sum ~/grafana-evidence-*.tar.gz" + fi echo "Grafana no longer uses $(dirname "$DB"). Delete it with: sudo rm -rf $(dirname "$DB")" fi -[[ "$findings" -eq 0 ]] +[[ "$findings" -eq 0 && "$exposed" -eq 0 ]] From 7fa593f5572bd97b6f0c5b8884c31b9548bb39b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ois=C3=ADn=20Kyne?= Date: Wed, 28 Oct 2026 11:37:09 +0000 Subject: [PATCH 2/2] grafana: fix security-check.sh false positives and check Grafana 12+ dashboards Provisioning deletes and recreates datasources on every start, so stop checking deleted datasource IDs; the datasource names are still checked. Grafana 12 and later store dashboards and folders in the resource table, so also report ones not managed by file provisioning, and deleted ones from resource_history. List added files under grafana/ for review instead of counting them, since operators may add their own dashboards, and no longer suggest git clean. --- grafana/security-check.sh | 57 ++++++++++++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/grafana/security-check.sh b/grafana/security-check.sh index 78d94426..fc70c323 100755 --- a/grafana/security-check.sh +++ b/grafana/security-check.sh @@ -108,6 +108,32 @@ for name, created in rows("SELECT name, created FROM org WHERE id != 1"): print(f"organization: {name} (created {created})") for title, created in rows("SELECT title, created FROM dashboard WHERE id NOT IN (SELECT dashboard_id FROM dashboard_provisioning)"): print(f"dashboard or folder not provisioned by this stack: {title} (created {created})") + + +# Grafana 12 and later store dashboards and folders in the resource table, and every change in resource_history +# (action 3 is a delete). Dashboards provisioned from files are annotated as managed by file provisioning. +def unprovisioned(value): + """Returns the title and creation time of a stored dashboard or folder that file provisioning does not manage.""" + try: + obj = json.loads(value) + metadata = obj.get("metadata") or {} + except (TypeError, ValueError, AttributeError): + return None + if (metadata.get("annotations") or {}).get("grafana.app/managedBy") == "classic-file-provisioning": + return None + title = (obj.get("spec") or {}).get("title") or metadata.get("name", "?") + return title, metadata.get("creationTimestamp", "") + + +UNIFIED = "\"group\" IN ('dashboard.grafana.app', 'folder.grafana.app')" +for kind, value in rows(f"SELECT resource, value FROM resource WHERE {UNIFIED}"): + found = unprovisioned(value) + if found: + print(f"{kind[:-1]} not provisioned by this stack: {found[0]} (created {found[1]})") +for kind, value in rows(f"SELECT resource, value FROM resource_history WHERE action = 3 AND {UNIFIED}"): + found = unprovisioned(value) + if found: + print(f"{kind[:-1]} created and later deleted: {found[0]} (created {found[1]})") for title, updated in rows("SELECT title, updated FROM alert_rule"): print(f"alert rule: {title} (updated {updated})") for name, kind, created in rows("SELECT name, type, created FROM alert_notification"): @@ -126,8 +152,9 @@ for (config,) in rows("SELECT alertmanager_configuration FROM alert_configuratio for name, kind in sorted(contact_points, key=str): print(f"contact point: {name} type={kind}") # IDs keep counting up after rows are deleted, which reveals records that were created and later removed. -# Provisioning recreates a few datasources and dashboards on its own, so allow some of those. -ALLOWED_DELETIONS = {"user": 0, "api_key": 0, "org": 0, "alert_rule": 0, "data_source": 3, "dashboard": 10} +# Provisioning deletes and recreates datasources on every start, so their IDs are not checked; the +# datasource names are. Dashboards are only in this table before Grafana 12. +ALLOWED_DELETIONS = {"user": 0, "api_key": 0, "org": 0, "alert_rule": 0, "dashboard": 10} sequences = dict(rows("SELECT name, seq FROM sqlite_sequence")) for table, allowed in ALLOWED_DELETIONS.items(): count = rows(f"SELECT count(*) FROM {table}") @@ -163,13 +190,26 @@ fi echo echo "== Repository files that Grafana mounted (grafana/)" # Releases before the Grafana hardening mounted these writable, so Grafana could change its own provisioning. +added=0 if changed=$(git -c safe.directory="$PWD" status --porcelain --untracked-files=all -- grafana 2>/dev/null); then - if [[ -n "$changed" ]]; then - while IFS= read -r line; do warn "changed or added file: $line"; done <<<"$changed" - echo " Unless you made these changes, restore the files with: git checkout -- grafana && git clean -fd grafana" - echo " Also check 'git stash list', as 'git stash apply' during an upgrade would bring changes back." - else + while IFS= read -r line; do + [[ -n "$line" ]] || continue + if [[ "$line" == "??"* ]]; then + # Operators may add their own dashboards here, so added files are listed for review rather than counted. + echo " [?] added file, check that you added it yourself: ${line#?? }" + added=$((added + 1)) + else + warn "changed file: $line" + fi + done <<<"$changed" + if [[ "$added" -gt 0 ]]; then + echo " Delete any added file you did not add yourself, and report it." + fi + if [[ -z "$changed" ]]; then echo " OK: grafana/ matches this release" + elif [[ "$(grep -vc '^??' <<<"$changed")" -gt 0 ]]; then + echo " Unless you made these changes, restore the changed files with: git checkout -- grafana" + echo " Also check 'git stash list', as 'git stash apply' during an upgrade would bring changes back." fi else echo " not a git checkout, cannot check" @@ -179,6 +219,9 @@ echo if [[ "$exposed" -gt 0 ]]; then echo "Found $exposed exposed port(s). Fix the bindings listed above first." fi +if [[ "$added" -gt 0 ]]; then + echo "Found $added added file(s) under grafana/. Treat any you did not add yourself as a sign of unauthorized changes." +fi if [[ "$findings" -eq 0 ]]; then echo "No signs of unauthorized changes found." else