use reusalbe key for daemon

cmd/keybroker/main.go

@@ -56,14 +56,16 @@ import (
 type keyKind string
 
 const (
-	// kindDaemon is a GPU-workload daemon key: SINGLE-USE + ephemeral + short TTL.
-	// Each workload gets its own throwaway credential (best tenant isolation) that
+	// kindDaemon is a GPU-workload daemon key: REUSABLE + ephemeral + bounded TTL.
+	// Each workload still gets its OWN freshly-minted key (best tenant isolation) that
 	// headscale auto-reaps once the pod disconnects (no orphan node pile-up). It is
-	// single-use so a leaked key can register at most ONE node. On a disconnect the
-	// daemon rejoins by re-running `tailscale up`, which reactivates its EXISTING
-	// headscale node via the persisted node key (no authkey re-use) — so the reap
-	// window (headscale ephemeral_node_inactivity_timeout) must be long enough that a
-	// transient blip doesn't delete the node before the daemon reconnects.
+	// REUSABLE — not single-use — because on a flaky mesh a daemon's node gets reaped
+	// (headscale ephemeral_node_inactivity_timeout) during a transient blip, and after
+	// a reap the ONLY way back in is to re-register with the pre-auth key. A single-use
+	// key is already spent by then, so the daemon wedges forever, unable to rejoin. A
+	// reusable key lets every `tailscale up` retry re-register the node, which is what
+	// makes reconnection actually robust. The blast radius of a leaked reusable key is
+	// bounded by the ephemeral reap (nodes vanish on disconnect) and the TTL below.
 	kindDaemon keyKind = "daemon"
 	// kindController is a SandD controller key: REUSABLE + ephemeral + long TTL.
 	// Reusable so it can re-register across restarts. Ephemeral — the SAME as a
@@ -89,10 +91,12 @@ type keyPolicy struct {
 func policyFor(kind keyKind) (keyPolicy, bool) {
 	switch kind {
 	case kindDaemon:
-		// Single-use: reusable=false. Ephemeral so the node is reaped on disconnect.
-		// Short TTL: the key is consumed at boot, minutes after minting, so it only
-		// needs to outlive the gap between Provision and the daemon's first join.
-		return keyPolicy{reusable: false, ephemeral: true, expiration: "1h"}, true
+		// Reusable so a reaped daemon can re-register: on a flaky mesh the node is
+		// reaped during a blip, and re-auth via the pre-auth key is the only way back
+		// in — a single-use key would already be spent, wedging the daemon forever.
+		// Ephemeral so the node is still reaped on disconnect (no orphan pile-up). TTL
+		// bounds a leaked key; each workload gets its own freshly-minted one anyway.
+		return keyPolicy{reusable: true, ephemeral: true, expiration: "24h"}, true
 	case kindController:
 		// Reusable so it can re-register across restarts; ephemeral so the old node
 		// is reaped on disconnect, freeing the stable MagicDNS name for the fresh pod

cmd/keybroker/main_test.go

@@ -32,9 +32,9 @@ func TestPolicyFor(t *testing.T) {
 		reusable  bool
 		ephemeral bool
 	}{
-		// Daemon: single-use (NOT reusable) + ephemeral — a throwaway per-workload
-		// credential that auto-reaps on disconnect.
-		{kindDaemon, true, false, true},
+		// Daemon: reusable + ephemeral — a per-workload credential that auto-reaps on
+		// disconnect, but reusable so a reaped daemon can re-register after a blip.
+		{kindDaemon, true, true, true},
 		// Controller: reusable (survives restarts) + ephemeral (same as a daemon) so
 		// the old node is reaped on disconnect, freeing its MagicDNS name to reclaim.
 		{kindController, true, true, true},
@@ -92,15 +92,15 @@ func TestKeysHandler_Daemon(t *testing.T) {
 	if resp.Key != "tskey-daemon-abc" {
 		t.Errorf("key = %q, want tskey-daemon-abc", resp.Key)
 	}
-	if resp.Reusable {
-		t.Errorf("daemon key must not be reusable")
+	if !resp.Reusable {
+		t.Errorf("daemon key must be reusable (so a reaped daemon can re-register)")
 	}
 	if !resp.Ephemeral {
 		t.Errorf("daemon key must be ephemeral")
 	}
-	// The runner must have been asked for the single-use ephemeral policy.
-	if seen.reusable || !seen.ephemeral {
-		t.Errorf("runner policy = %+v, want single-use ephemeral", *seen)
+	// The runner must have been asked for the reusable ephemeral policy.
+	if !seen.reusable || !seen.ephemeral {
+		t.Errorf("runner policy = %+v, want reusable ephemeral", *seen)
 	}
 }
 

config/sandd/README.md

@@ -24,7 +24,7 @@ it reaches headscale over a local unix socket. There is no static-key path.
    └──────────▲──────────────────▲──────────────────┘
               │ POST /keys        │ POST /keys
               │ ?kind=controller  │ ?kind=daemon
-              │ (reusable)        │ (single-use)
+              │ (reusable)        │ (reusable)
          controller          Nebula manager
          (at startup)        (at Provision, per workload → bakes key into user-data)
 ```
@@ -113,11 +113,14 @@ default port even though it listens on 8080 in-pod (the Service maps 80 → 8080
 
 The broker mints keys under one headscale user (`SANDD_KEYBROKER_USER`, default
 `nebula`) and **creates that user itself on startup** if it's missing — no manual
-bootstrap. Key policies it owns (see `cmd/keybroker`): daemon keys are **single-use**
-(each workload gets its own throwaway credential); the controller key is **reusable**
-(re-registers across restarts). Both are **ephemeral**, so headscale auto-reaps a node
-shortly after it disconnects — torn-down experiments don't pile up as OFFLINE nodes
-squatting MagicDNS names, and the controller reclaims its stable name cleanly on
+bootstrap. Key policies it owns (see `cmd/keybroker`): both daemon and controller keys
+are **reusable** — the controller so it re-registers across restarts, the daemon so a
+node reaped during a mesh blip can re-authenticate and rejoin (a single-use key would
+be spent, wedging the daemon offline forever). Each workload still gets its own
+freshly-minted daemon key, so a leak is scoped to that workload and bounded by the
+key's TTL (daemon 24h, controller 720h). Both are **ephemeral**, so headscale auto-reaps
+a node shortly after it disconnects — torn-down experiments don't pile up as OFFLINE
+nodes squatting MagicDNS names, and the controller reclaims its stable name cleanly on
 restart (the reaped old node frees it).
 
 ### 2. Deploy the controller
@@ -215,9 +218,11 @@ per instance, and a safe shared read-only mount (each container keeps its own wr
 `/var/lib/tailscale` + `/tmp`).
 
 `SANDD_TUNNEL_AUTHKEY` arrives as container env — the per-workload key the manager
-minted (single-use + ephemeral), so even though the container can see it, it's a
-throwaway good for one node and reaped on disconnect. The daemon is single-tenant, so a
-container seeing its own key is fine.
+minted (reusable + ephemeral, 24h TTL), so even though the container can see it, it's a
+short-lived credential scoped to this one workload and its node is reaped on disconnect.
+It is reusable (not single-use) so the daemon can re-authenticate if its node is reaped
+during a mesh blip; the daemon is single-tenant, so a container seeing its own key is
+fine.
 
 Both parts log to stderr with a `[sandd]` prefix, landing in the **EC2 console**
 (`aws ec2 get-console-output --instance-id <id> --latest`) — the place to debug
@@ -243,7 +248,7 @@ reconstructed — set an explicit `command`.
 
 This is a working DEV setup, not hardened: headscale uses SQLite (on a small PVC so its
 state survives pod restarts, but still a single-writer file DB) over plain HTTP. Each workload
-already gets its own single-use ephemeral key, but for real use also: back headscale
+already gets its own freshly-minted ephemeral key, but for real use also: back headscale
 with a PersistentVolume + real database + TLS, and add per-tenant **tags/ACLs** to
 minted keys so daemons are mesh-isolated, not just credential-distinct. See
 https://headscale.net.

pkg/provider/aws/sandd_test.go

@@ -78,7 +78,7 @@ func TestProvision_MintsPerDaemonKey(t *testing.T) {
 }
 
 // TestProvision_SeparateProvisionsMintSeparateKeys: two DISTINCT workloads => two
-// mints, so each gets its OWN single-use credential (the isolation guarantee).
+// mints, so each gets its OWN freshly-minted credential (the isolation guarantee).
 // Distinct claim names avoid the idempotency short-circuit (which would return the
 // existing instance without launching or minting again).
 func TestProvision_SeparateProvisionsMintSeparateKeys(t *testing.T) {

pkg/sandd/keybroker.go

@@ -64,9 +64,11 @@ type keyResponse struct {
 	Key string `json:"key"`
 }
 
-// MintDaemonKey requests a fresh single-use, ephemeral daemon key from the broker.
-// It satisfies provider.DaemonKeyMinter. The returned key is a secret; callers must
-// not log it.
+// MintDaemonKey requests a fresh reusable, ephemeral daemon key from the broker.
+// Reusable so a reaped daemon can re-register after a mesh blip (a single-use key
+// would be spent and the daemon could never rejoin); ephemeral so its node is still
+// reaped on disconnect. It satisfies provider.DaemonKeyMinter. The returned key is a
+// secret; callers must not log it.
 func (c *BrokerClient) MintDaemonKey(ctx context.Context) (string, error) {
 	return c.mint(ctx, "daemon")
 }

pkg/sandd/keybroker_test.go

@@ -55,7 +55,7 @@ func TestMintDaemonKey_Success(t *testing.T) {
 	if gotMethod != http.MethodPost {
 		t.Errorf("method = %s, want POST", gotMethod)
 	}
-	// The daemon path must request the daemon kind (single-use + ephemeral).
+	// The daemon path must request the daemon kind (reusable + ephemeral).
 	if gotKind != "daemon" {
 		t.Errorf("kind = %q, want daemon", gotKind)
 	}