diff --git a/cmd/keybroker/main.go b/cmd/keybroker/main.go index 05fb126..eddd1d7 100644 --- a/cmd/keybroker/main.go +++ b/cmd/keybroker/main.go @@ -56,14 +56,16 @@ import ( type keyKind string const ( - // kindDaemon is a GPU-workload daemon key: SINGLE-USE + ephemeral + short TTL. - // Each workload gets its own throwaway credential (best tenant isolation) that + // kindDaemon is a GPU-workload daemon key: REUSABLE + ephemeral + bounded TTL. + // Each workload still gets its OWN freshly-minted key (best tenant isolation) that // headscale auto-reaps once the pod disconnects (no orphan node pile-up). It is - // single-use so a leaked key can register at most ONE node. On a disconnect the - // daemon rejoins by re-running `tailscale up`, which reactivates its EXISTING - // headscale node via the persisted node key (no authkey re-use) — so the reap - // window (headscale ephemeral_node_inactivity_timeout) must be long enough that a - // transient blip doesn't delete the node before the daemon reconnects. + // REUSABLE — not single-use — because on a flaky mesh a daemon's node gets reaped + // (headscale ephemeral_node_inactivity_timeout) during a transient blip, and after + // a reap the ONLY way back in is to re-register with the pre-auth key. A single-use + // key is already spent by then, so the daemon wedges forever, unable to rejoin. A + // reusable key lets every `tailscale up` retry re-register the node, which is what + // makes reconnection actually robust. The blast radius of a leaked reusable key is + // bounded by the ephemeral reap (nodes vanish on disconnect) and the TTL below. kindDaemon keyKind = "daemon" // kindController is a SandD controller key: REUSABLE + ephemeral + long TTL. // Reusable so it can re-register across restarts. Ephemeral — the SAME as a @@ -89,10 +91,12 @@ type keyPolicy struct { func policyFor(kind keyKind) (keyPolicy, bool) { switch kind { case kindDaemon: - // Single-use: reusable=false. Ephemeral so the node is reaped on disconnect. - // Short TTL: the key is consumed at boot, minutes after minting, so it only - // needs to outlive the gap between Provision and the daemon's first join. - return keyPolicy{reusable: false, ephemeral: true, expiration: "1h"}, true + // Reusable so a reaped daemon can re-register: on a flaky mesh the node is + // reaped during a blip, and re-auth via the pre-auth key is the only way back + // in — a single-use key would already be spent, wedging the daemon forever. + // Ephemeral so the node is still reaped on disconnect (no orphan pile-up). TTL + // bounds a leaked key; each workload gets its own freshly-minted one anyway. + return keyPolicy{reusable: true, ephemeral: true, expiration: "24h"}, true case kindController: // Reusable so it can re-register across restarts; ephemeral so the old node // is reaped on disconnect, freeing the stable MagicDNS name for the fresh pod diff --git a/cmd/keybroker/main_test.go b/cmd/keybroker/main_test.go index 81086e5..6f3432a 100644 --- a/cmd/keybroker/main_test.go +++ b/cmd/keybroker/main_test.go @@ -32,9 +32,9 @@ func TestPolicyFor(t *testing.T) { reusable bool ephemeral bool }{ - // Daemon: single-use (NOT reusable) + ephemeral — a throwaway per-workload - // credential that auto-reaps on disconnect. - {kindDaemon, true, false, true}, + // Daemon: reusable + ephemeral — a per-workload credential that auto-reaps on + // disconnect, but reusable so a reaped daemon can re-register after a blip. + {kindDaemon, true, true, true}, // Controller: reusable (survives restarts) + ephemeral (same as a daemon) so // the old node is reaped on disconnect, freeing its MagicDNS name to reclaim. {kindController, true, true, true}, @@ -92,15 +92,15 @@ func TestKeysHandler_Daemon(t *testing.T) { if resp.Key != "tskey-daemon-abc" { t.Errorf("key = %q, want tskey-daemon-abc", resp.Key) } - if resp.Reusable { - t.Errorf("daemon key must not be reusable") + if !resp.Reusable { + t.Errorf("daemon key must be reusable (so a reaped daemon can re-register)") } if !resp.Ephemeral { t.Errorf("daemon key must be ephemeral") } - // The runner must have been asked for the single-use ephemeral policy. - if seen.reusable || !seen.ephemeral { - t.Errorf("runner policy = %+v, want single-use ephemeral", *seen) + // The runner must have been asked for the reusable ephemeral policy. + if !seen.reusable || !seen.ephemeral { + t.Errorf("runner policy = %+v, want reusable ephemeral", *seen) } } diff --git a/config/sandd/README.md b/config/sandd/README.md index 9a65a76..98c48a4 100644 --- a/config/sandd/README.md +++ b/config/sandd/README.md @@ -24,7 +24,7 @@ it reaches headscale over a local unix socket. There is no static-key path. └──────────▲──────────────────▲──────────────────┘ │ POST /keys │ POST /keys │ ?kind=controller │ ?kind=daemon - │ (reusable) │ (single-use) + │ (reusable) │ (reusable) controller Nebula manager (at startup) (at Provision, per workload → bakes key into user-data) ``` @@ -113,11 +113,14 @@ default port even though it listens on 8080 in-pod (the Service maps 80 → 8080 The broker mints keys under one headscale user (`SANDD_KEYBROKER_USER`, default `nebula`) and **creates that user itself on startup** if it's missing — no manual -bootstrap. Key policies it owns (see `cmd/keybroker`): daemon keys are **single-use** -(each workload gets its own throwaway credential); the controller key is **reusable** -(re-registers across restarts). Both are **ephemeral**, so headscale auto-reaps a node -shortly after it disconnects — torn-down experiments don't pile up as OFFLINE nodes -squatting MagicDNS names, and the controller reclaims its stable name cleanly on +bootstrap. Key policies it owns (see `cmd/keybroker`): both daemon and controller keys +are **reusable** — the controller so it re-registers across restarts, the daemon so a +node reaped during a mesh blip can re-authenticate and rejoin (a single-use key would +be spent, wedging the daemon offline forever). Each workload still gets its own +freshly-minted daemon key, so a leak is scoped to that workload and bounded by the +key's TTL (daemon 24h, controller 720h). Both are **ephemeral**, so headscale auto-reaps +a node shortly after it disconnects — torn-down experiments don't pile up as OFFLINE +nodes squatting MagicDNS names, and the controller reclaims its stable name cleanly on restart (the reaped old node frees it). ### 2. Deploy the controller @@ -215,9 +218,11 @@ per instance, and a safe shared read-only mount (each container keeps its own wr `/var/lib/tailscale` + `/tmp`). `SANDD_TUNNEL_AUTHKEY` arrives as container env — the per-workload key the manager -minted (single-use + ephemeral), so even though the container can see it, it's a -throwaway good for one node and reaped on disconnect. The daemon is single-tenant, so a -container seeing its own key is fine. +minted (reusable + ephemeral, 24h TTL), so even though the container can see it, it's a +short-lived credential scoped to this one workload and its node is reaped on disconnect. +It is reusable (not single-use) so the daemon can re-authenticate if its node is reaped +during a mesh blip; the daemon is single-tenant, so a container seeing its own key is +fine. Both parts log to stderr with a `[sandd]` prefix, landing in the **EC2 console** (`aws ec2 get-console-output --instance-id --latest`) — the place to debug @@ -243,7 +248,7 @@ reconstructed — set an explicit `command`. This is a working DEV setup, not hardened: headscale uses SQLite (on a small PVC so its state survives pod restarts, but still a single-writer file DB) over plain HTTP. Each workload -already gets its own single-use ephemeral key, but for real use also: back headscale +already gets its own freshly-minted ephemeral key, but for real use also: back headscale with a PersistentVolume + real database + TLS, and add per-tenant **tags/ACLs** to minted keys so daemons are mesh-isolated, not just credential-distinct. See https://headscale.net. diff --git a/pkg/provider/aws/sandd_test.go b/pkg/provider/aws/sandd_test.go index c3210d9..a6c8496 100644 --- a/pkg/provider/aws/sandd_test.go +++ b/pkg/provider/aws/sandd_test.go @@ -78,7 +78,7 @@ func TestProvision_MintsPerDaemonKey(t *testing.T) { } // TestProvision_SeparateProvisionsMintSeparateKeys: two DISTINCT workloads => two -// mints, so each gets its OWN single-use credential (the isolation guarantee). +// mints, so each gets its OWN freshly-minted credential (the isolation guarantee). // Distinct claim names avoid the idempotency short-circuit (which would return the // existing instance without launching or minting again). func TestProvision_SeparateProvisionsMintSeparateKeys(t *testing.T) { diff --git a/pkg/sandd/keybroker.go b/pkg/sandd/keybroker.go index e7fad90..3a82d56 100644 --- a/pkg/sandd/keybroker.go +++ b/pkg/sandd/keybroker.go @@ -64,9 +64,11 @@ type keyResponse struct { Key string `json:"key"` } -// MintDaemonKey requests a fresh single-use, ephemeral daemon key from the broker. -// It satisfies provider.DaemonKeyMinter. The returned key is a secret; callers must -// not log it. +// MintDaemonKey requests a fresh reusable, ephemeral daemon key from the broker. +// Reusable so a reaped daemon can re-register after a mesh blip (a single-use key +// would be spent and the daemon could never rejoin); ephemeral so its node is still +// reaped on disconnect. It satisfies provider.DaemonKeyMinter. The returned key is a +// secret; callers must not log it. func (c *BrokerClient) MintDaemonKey(ctx context.Context) (string, error) { return c.mint(ctx, "daemon") } diff --git a/pkg/sandd/keybroker_test.go b/pkg/sandd/keybroker_test.go index ca75de9..e7f8ef5 100644 --- a/pkg/sandd/keybroker_test.go +++ b/pkg/sandd/keybroker_test.go @@ -55,7 +55,7 @@ func TestMintDaemonKey_Success(t *testing.T) { if gotMethod != http.MethodPost { t.Errorf("method = %s, want POST", gotMethod) } - // The daemon path must request the daemon kind (single-use + ephemeral). + // The daemon path must request the daemon kind (reusable + ephemeral). if gotKind != "daemon" { t.Errorf("kind = %q, want daemon", gotKind) }