diff --git a/.env.example b/.env.example index 51385d8..d1c9f69 100644 --- a/.env.example +++ b/.env.example @@ -1,11 +1,17 @@ -# Nebula provider credentials — SECRETS ONLY. +# Nebula provider credentials — SECRETS (plus one per-cluster endpoint). # # Copy to .env and fill in real values. .env is gitignored — never commit tokens. -# Consumed by hack/deploy.sh (make deploy-all), which turns these into one -# Kubernetes Secret PER PROVIDER. Non-secret config (image, namespace, Kind +# Consumed by hack/deploy.sh (make deploy-all), which turns the credentials into one +# Kubernetes Secret PER PROVIDER. Static non-secret config (image, namespace, Kind # cluster) is NOT here — pass it as make variables, e.g. # make deploy-all IMG=myrepo/nebula:v1 DEPLOY_KIND_CLUSTER=nebula-test-e2e # +# The lone non-secret that DOES belong here is SANDD_TUNNEL_SERVER (bottom): it is a +# per-cluster value discovered at deploy time (the headscale NLB hostname), so keeping +# it beside the deploy that consumes it — rather than a hand-passed make flag re-typed +# every run — is the whole point. It rides the SAME parsed-not-sourced path as the +# creds, so it can't leak into kubectl either. +# # cp .env.example .env # # edit .env # make deploy-all @@ -17,13 +23,43 @@ MODAL_TOKEN_ID= MODAL_TOKEN_SECRET= # --- AWS provider ---------------------------------------------------------- -# SECRETS ONLY. In production prefer IRSA / instance role and leave these blank -# (the SDK's default credential chain finds the role) — the AWS secret is then -# skipped, which is fine. Set them only for local/dev without a role. Both keys -# are required together; leave both blank to skip. +# SECRETS ONLY — the PROVISIONING identity (the AWS account that launches GPU +# instances). This is a SEPARATE identity from the one that talks to the cluster +# you deploy into (e.g. an EKS control plane in a different account). +# +# These use the standard AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY names — the +# same names `aws eks get-token` (kubectl's EKS auth plugin) reads — yet they do +# NOT collide with cluster auth: deploy.sh PARSES this file into a private array +# rather than sourcing/exporting it, so these values reach the provider Secret but +# never enter the environment of kubectl or any other child process. Your own +# ambient AWS credentials keep talking to the cluster. +# +# In production prefer IRSA / instance role and leave these blank (the SDK's +# default credential chain finds the role) — the AWS secret is then skipped, which +# is fine. Set them only for local/dev without a role. Both required together; +# leave both blank to skip. AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= # --- Additional providers (add as adapters land) --------------------------- # Each provider gets its OWN secret (see hack/deploy.sh PROVIDER_SECRETS), e.g.: # RUNPOD_API_KEY= + +# --- SandD mesh endpoint (NON-SECRET, per-cluster) ------------------------- +# The internet-facing headscale NLB hostname the SandD mesh dials, as a bare +# http:// (no port — the Service listens on 80; see config/sandd). It is only +# known AFTER config/samples/headscale-service.yaml provisions and changes on every +# recreate, so it can't live in the tracked manifests. deploy-all reads it from here +# and `make deploy` substitutes the __SANDD_TUNNEL_SERVER__ token into headscale's +# server_url and the nebula-sandd-config ConfigMap (byte-identical, as headscale +# requires) — no re-typing across files, no manager restart. +# +# Read it back after applying the Service (README config/sandd step 1): +# kubectl apply -f config/samples/headscale-service.yaml +# HS=$(kubectl -n nebula-system get svc nebula-headscale \ +# -o jsonpath='{.status.loadBalancer.ingress[0].hostname}') +# echo "SANDD_TUNNEL_SERVER=http://$HS" # paste the result below, then make deploy-all +# +# Leave blank to skip SandD wiring — the token is then left in place (an obviously +# broken render, surfaced by a deploy.sh warning), not a silent misconfig. +SANDD_TUNNEL_SERVER= diff --git a/Dockerfile.keybroker b/Dockerfile.keybroker new file mode 100644 index 0000000..64c202b --- /dev/null +++ b/Dockerfile.keybroker @@ -0,0 +1,37 @@ +# keybroker — the headscale pre-auth key broker (cmd/keybroker), packaged as a +# SIDECAR for the headscale pod. It shells out to the `headscale` CLI over the +# shared unix socket, so the final image must carry BOTH our Go binary AND the +# headscale CLI — hence the `FROM headscale/headscale` base rather than distroless. +# +# Build (from the repo root): +# docker build -f Dockerfile.keybroker -t inftyai/nebula-keybroker:latest . +# +# Keep the headscale tag here IN SYNC with config/sandd/headscale.yaml so the CLI +# in this sidecar speaks the same wire/flag version as the headscale it dials. + +# --- build the broker binary --------------------------------------------------- +FROM golang:1.24 AS builder +ARG TARGETOS +ARG TARGETARCH + +WORKDIR /workspace +COPY go.mod go.mod +COPY go.sum go.sum +RUN go mod download + +# Only the broker's own sources are needed (it is stdlib-only), but copy the +# module dirs it might import so the build is self-contained. +COPY cmd/keybroker/ cmd/keybroker/ + +# CGO off => a static binary that runs on the headscale base image regardless of +# its libc. +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ + go build -a -o keybroker ./cmd/keybroker + +# --- final image: headscale CLI + our broker ----------------------------------- +# FROM headscale so `headscale preauthkeys create` is on PATH. The broker calls it +# over the local unix socket shared with the real headscale container in the pod. +FROM headscale/headscale:0.23 +COPY --from=builder /workspace/keybroker /usr/local/bin/keybroker +# Override headscale's own entrypoint: this container runs the BROKER, not headscale. +ENTRYPOINT ["/usr/local/bin/keybroker"] diff --git a/Makefile b/Makefile index e9b163f..5f2ec36 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,12 @@ # Image URL to use all building/pushing image targets IMG ?= inftyai/nebula-controller:latest +# KEYBROKER_IMG is the SandD key-broker sidecar image (cmd/keybroker, +# Dockerfile.keybroker) that runs alongside headscale — see config/sandd. Built and +# pushed separately from the manager IMG because it is a distinct, optional image +# with its own base (FROM headscale, so it carries the headscale CLI). +KEYBROKER_IMG ?= inftyai/nebula-keybroker:latest + # NAMESPACE is where the manager runs (must match config/manager). Consumed by # hack/deploy.sh via the deploy-all target. NAMESPACE ?= nebula-system @@ -155,13 +161,21 @@ docker-build: ## Build docker image with the manager. docker-push: ## Push docker image with the manager. $(CONTAINER_TOOL) push ${IMG} +.PHONY: docker-build-keybroker +docker-build-keybroker: ## Build docker image for the SandD key-broker sidecar. + $(CONTAINER_TOOL) build -f Dockerfile.keybroker -t ${KEYBROKER_IMG} . + +.PHONY: docker-push-keybroker +docker-push-keybroker: ## Push the SandD key-broker sidecar image. + $(CONTAINER_TOOL) push ${KEYBROKER_IMG} + # PLATFORMS defines the target platforms for the manager image be built to provide support to multiple # architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to: # - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/ # - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/ # - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=> then the export will fail) # To adequately provide solutions that are compatible with multiple platforms, you should consider using this option. -PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le +PLATFORMS ?= linux/arm64,linux/amd64 .PHONY: docker-buildx docker-buildx: ## Build and push docker image for the manager for cross-platform support # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile @@ -195,7 +209,17 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified .PHONY: deploy deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build $(KUSTOMIZE_BUILD_FLAGS) config/default | $(KUBECTL) apply -f - + # SANDD_TUNNEL_SERVER is the internet-facing headscale NLB hostname, only known + # after that Service provisions and different on every recreate — so it can't be + # baked into the tracked manifests. The config/sandd overlay carries the literal + # token __SANDD_TUNNEL_SERVER__ in headscale's server_url and the nebula-sandd-config + # ConfigMap; substitute it here so both render correct on the FIRST apply (no + # manager/headscale restart). Driven from .env via deploy-all (make deploy-all), + # or pass SANDD_TUNNEL_SERVER=... directly. When unset the token is left as-is (the + # render is then obviously broken rather than silently pointing nowhere). + $(KUSTOMIZE) build $(KUSTOMIZE_BUILD_FLAGS) config/default | \ + $(if $(SANDD_TUNNEL_SERVER),sed 's|__SANDD_TUNNEL_SERVER__|$(SANDD_TUNNEL_SERVER)|g',cat) | \ + $(KUBECTL) apply -f - .PHONY: deploy-e2e deploy-e2e: manifests kustomize ## Deploy for e2e: config/default plus the fake-provider env var (baked in at deploy time, not via a post-deploy rollout). diff --git a/README.md b/README.md index 67668e8..0d0f1c7 100644 --- a/README.md +++ b/README.md @@ -96,3 +96,9 @@ placement controller owns those. - See [config/samples](config/samples) for example NodePools and a runnable workload. - See [docs/add-a-provider.md](docs/add-a-provider.md) to add a provider backend. - See [docs/architecture.md](docs/architecture.md) for design details. + +## License + +Apache-2.0 — see [LICENSE](LICENSE). Third-party components (e.g. Tailscale, used by +the optional SandD channel) are listed in +[THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md). diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md new file mode 100644 index 0000000..d12e230 --- /dev/null +++ b/THIRD_PARTY_NOTICES.md @@ -0,0 +1,15 @@ +# Third-party notices + +Nebula is licensed under Apache-2.0 (see [LICENSE](LICENSE)). It relies on the +following third-party software, which is licensed separately. + +## Tailscale + +The optional [SandD](https://github.com/InftyAI/SandD) access channel uses the +[Tailscale](https://github.com/tailscale/tailscale) client (`tailscale`/`tailscaled`), +© Tailscale Inc., licensed under +[BSD-3-Clause](https://github.com/tailscale/tailscale/blob/main/LICENSE). + +Nebula does not link or vendor it; the provider integration fetches the official +binary onto the GPU host at provision time. If you build and redistribute an image +with Tailscale baked in, retain its copyright notice per BSD-3-Clause. diff --git a/api/v1alpha1/nodepool_types.go b/api/v1alpha1/nodepool_types.go index 779de1b..a954007 100644 --- a/api/v1alpha1/nodepool_types.go +++ b/api/v1alpha1/nodepool_types.go @@ -136,9 +136,9 @@ const ( type FailoverPolicy struct { // BlocklistTTL is the BASE duration a failed placement is excluded before the // provider becomes a candidate for it again. The controller adds a random jitter - // (up to a minute) on top so Pods that failed for the same reason do not all - // retry the just-freed candidate in lockstep, so the effective exclusion is this - // value plus that jitter. + // (up to 30s) on top so Pods that failed for the same reason do not all retry the + // just-freed candidate in lockstep, so the effective exclusion is this value plus + // that jitter. // +kubebuilder:default="30s" BlocklistTTL metav1.Duration `json:"blocklistTTL,omitempty"` } diff --git a/cmd/keybroker/main.go b/cmd/keybroker/main.go new file mode 100644 index 0000000..05fb126 --- /dev/null +++ b/cmd/keybroker/main.go @@ -0,0 +1,347 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Command keybroker is a tiny HTTP service that mints headscale pre-auth keys on +// demand, so nothing else in the system needs headscale admin authority. +// +// WHY IT EXISTS: SandD daemons and the controller need a mesh pre-auth key to +// join, but neither can safely hold headscale's admin credentials — the daemon +// runs inside an untrusted tenant container, and the controller is user-facing +// sample code. Minting keys by hand also doesn't scale: every experiment spins up +// a fresh controller and every workload is a new daemon. This broker centralizes +// the privilege: it is the ONLY component that talks to headscale's admin surface, +// and it does so over headscale's LOCAL unix socket as a sidecar in the headscale +// pod — so there is no admin API key to store and no gRPC port to expose. Callers +// just POST /keys and get back a single, freshly-minted key string. +// +// SECURITY POSTURE: +// - It shells out to the `headscale` CLI over the shared unix socket +// (--config points at headscale's own config, which carries unix_socket). No +// API key, no network call to headscale. +// - It is reachable only in-cluster (a ClusterIP Service). It mints keys but +// cannot read or revoke them and exposes nothing else, so a compromised caller +// can at most cause it to mint extra ephemeral keys (which auto-reap). +// - The minted key is a secret: it is returned in the response body but NEVER +// logged. Only the key's metadata (kind, reusable, ephemeral) is logged. +package main + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + "os" + "os/exec" + "strconv" + "strings" + "time" +) + +// keyKind is the caller-declared purpose of the key, which selects its policy. The +// two consumers want opposite lifecycles (see policyFor), so the caller names its +// role rather than passing raw flags — the broker owns the policy, not the caller. +type keyKind string + +const ( + // kindDaemon is a GPU-workload daemon key: SINGLE-USE + ephemeral + short TTL. + // Each workload gets its own throwaway credential (best tenant isolation) that + // headscale auto-reaps once the pod disconnects (no orphan node pile-up). It is + // single-use so a leaked key can register at most ONE node. On a disconnect the + // daemon rejoins by re-running `tailscale up`, which reactivates its EXISTING + // headscale node via the persisted node key (no authkey re-use) — so the reap + // window (headscale ephemeral_node_inactivity_timeout) must be long enough that a + // transient blip doesn't delete the node before the daemon reconnects. + kindDaemon keyKind = "daemon" + // kindController is a SandD controller key: REUSABLE + ephemeral + long TTL. + // Reusable so it can re-register across restarts. Ephemeral — the SAME as a + // daemon — because the controller holds NO persistent state (no PVC): headscale + // reaps its node the instant the pod disconnects, which FREES the stable + // sandd-controller MagicDNS name for the next pod to reclaim (the name is pinned + // by the pod hostname, not by a persisted node key). This is what lets a restart + // reclaim the same name with no -suffix. It also sidesteps the port-443 dial + // wedge that a persisted /var/lib/tailscale caused. Long TTL (720h) just bounds a + // key that outlives a brief reap gap; ephemeral reaping is the real cleanup path. + kindController keyKind = "controller" +) + +// keyPolicy is the resolved (headscale-flag) shape of a key for a given kind. +type keyPolicy struct { + reusable bool + ephemeral bool + expiration string // headscale --expiration duration, e.g. "1h", "720h" +} + +// policyFor maps a kind onto its headscale flags. Unknown kinds are rejected by the +// caller before this is reached, so there is no default policy to fall through to. +func policyFor(kind keyKind) (keyPolicy, bool) { + switch kind { + case kindDaemon: + // Single-use: reusable=false. Ephemeral so the node is reaped on disconnect. + // Short TTL: the key is consumed at boot, minutes after minting, so it only + // needs to outlive the gap between Provision and the daemon's first join. + return keyPolicy{reusable: false, ephemeral: true, expiration: "1h"}, true + case kindController: + // Reusable so it can re-register across restarts; ephemeral so the old node + // is reaped on disconnect, freeing the stable MagicDNS name for the fresh pod + // to reclaim (the controller has no PVC, so nothing to preserve). Long TTL + // (720h) just bounds a key that outlives a brief reap gap. + return keyPolicy{reusable: true, ephemeral: true, expiration: "1h"}, true + default: + return keyPolicy{}, false + } +} + +// keyResponse is the JSON body returned to a caller. It carries the freshly-minted +// key plus the policy metadata, so a caller can log/inspect what it got WITHOUT the +// broker having to expose headscale's richer key object. +type keyResponse struct { + Key string `json:"key"` + Kind string `json:"kind"` + Reusable bool `json:"reusable"` + Ephemeral bool `json:"ephemeral"` + Expiration string `json:"expiration"` +} + +// headscalePreAuthKey is the subset of `headscale preauthkeys create -o json` we +// parse. headscale prints the created key object as JSON; we only need its `key`. +type headscalePreAuthKey struct { + Key string `json:"key"` +} + +// headscaleUser is the subset of `headscale users list -o json` we parse. We only +// need the name to decide whether the configured user already exists. +type headscaleUser struct { + Name string `json:"name"` +} + +// config is the broker's runtime configuration, all from env with sane defaults so +// the manifest can stay minimal. +type config struct { + // listenAddr is the HTTP bind address (SANDD_KEYBROKER_LISTEN, default :8090). + listenAddr string + // user is the headscale user keys are minted under (SANDD_KEYBROKER_USER, + // default "nebula"). The broker ensures it exists at startup (ensureHeadscaleUser), + // so no manual bootstrap is needed. + user string + // headscaleConfig is the path to headscale's config file, which carries the + // unix_socket the CLI dials (SANDD_KEYBROKER_HS_CONFIG, default the standard + // /etc/headscale/config.yaml the sidecar shares with headscale). + headscaleConfig string + // headscaleBin is the headscale CLI path (SANDD_KEYBROKER_HS_BIN, default + // "headscale" on PATH — the sidecar image is FROM headscale). + headscaleBin string +} + +func loadConfig() config { + return config{ + listenAddr: envOr("SANDD_KEYBROKER_LISTEN", ":8090"), + user: envOr("SANDD_KEYBROKER_USER", "nebula"), + headscaleConfig: envOr("SANDD_KEYBROKER_HS_CONFIG", "/etc/headscale/config.yaml"), + headscaleBin: envOr("SANDD_KEYBROKER_HS_BIN", "headscale"), + } +} + +func envOr(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} + +// commandRunner mints a key by invoking the headscale CLI. It is a seam so tests +// can inject a fake without a real headscale (the real one is runHeadscale). +type commandRunner func(cfg config, policy keyPolicy) (string, error) + +// userEnsurer creates cfg.user in headscale if it does not already exist. It is a +// seam so tests can inject a fake without a real headscale (the real one is +// ensureHeadscaleUser). +type userEnsurer func(cfg config) error + +// ensureUserRetries and ensureUserRetryDelay bound the startup wait for headscale's +// unix socket. ~10 attempts over 20s comfortably covers headscale opening its socket +// while the sidecars boot together, without hanging a genuinely broken deploy forever. +// Vars, not consts, so tests can shrink the delay to run without real sleeps. +var ( + ensureUserRetries = 10 + ensureUserRetryDelay = 2 * time.Second +) + +// ensureHeadscaleUserWithRetry retries ensure until it succeeds or the attempt budget +// is exhausted, absorbing the sidecar startup race where headscale's socket is not up +// yet. The ensure func is a parameter so tests drive it without a real headscale or +// real sleeps. Returns the last error if every attempt fails. +func ensureHeadscaleUserWithRetry(cfg config, ensure userEnsurer) error { + var err error + for attempt := 1; attempt <= ensureUserRetries; attempt++ { + if err = ensure(cfg); err == nil { + return nil + } + if attempt < ensureUserRetries { + log.Printf("ensure headscale user attempt %d/%d failed (retrying): %v", attempt, ensureUserRetries, err) + time.Sleep(ensureUserRetryDelay) + } + } + return err +} + +// ensureHeadscaleUser makes the configured user exist, idempotently: it lists users +// over the local unix socket, returns early if cfg.user is already present, and +// otherwise creates it. This replaces the manual `headscale users create` bootstrap +// step — the broker already has socket access, so it can self-provision the one user +// it mints under. Safe to run on every startup. +func ensureHeadscaleUser(cfg config) error { + listCmd := exec.Command(cfg.headscaleBin, "--config", cfg.headscaleConfig, "users", "list", "--output", "json") + var stdout, stderr strings.Builder + listCmd.Stdout = &stdout + listCmd.Stderr = &stderr + if err := listCmd.Run(); err != nil { + return fmt.Errorf("headscale users list failed: %w: %s", err, strings.TrimSpace(stderr.String())) + } + + var users []headscaleUser + if err := json.Unmarshal([]byte(stdout.String()), &users); err != nil { + return fmt.Errorf("parsing headscale users list: %w", err) + } + for _, u := range users { + if u.Name == cfg.user { + log.Printf("headscale user %q already exists", cfg.user) + return nil + } + } + + createCmd := exec.Command(cfg.headscaleBin, "--config", cfg.headscaleConfig, "users", "create", cfg.user) + var createErr strings.Builder + createCmd.Stderr = &createErr + if err := createCmd.Run(); err != nil { + return fmt.Errorf("headscale users create %q failed: %w: %s", cfg.user, err, strings.TrimSpace(createErr.String())) + } + log.Printf("created headscale user %q", cfg.user) + return nil +} + +// runHeadscale shells out to `headscale preauthkeys create -o json` over the local +// unix socket (via --config, which carries unix_socket) and returns the key string. +// It NEVER logs the key or the raw output (which contains the key). +func runHeadscale(cfg config, policy keyPolicy) (string, error) { + args := []string{ + "--config", cfg.headscaleConfig, + "preauthkeys", "create", + "--user", cfg.user, + "--expiration", policy.expiration, + "--output", "json", + } + if policy.reusable { + args = append(args, "--reusable") + } + if policy.ephemeral { + args = append(args, "--ephemeral") + } + + cmd := exec.Command(cfg.headscaleBin, args...) + // Capture stdout (the JSON key object) separately from stderr so a headscale + // error message can be surfaced without risking the key leaking into a log via + // combined output. + var stdout, stderr strings.Builder + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("headscale preauthkeys create failed: %w: %s", err, strings.TrimSpace(stderr.String())) + } + + var key headscalePreAuthKey + if err := json.Unmarshal([]byte(stdout.String()), &key); err != nil { + // Do NOT include stdout in the error: it is the key material. + return "", fmt.Errorf("parsing headscale key output: %w", err) + } + if key.Key == "" { + return "", fmt.Errorf("headscale returned an empty key") + } + return key.Key, nil +} + +// keysHandler handles POST /keys?kind=. It resolves the policy, +// mints one key, and returns it as JSON. GET/other methods and unknown kinds are +// rejected. The runner seam lets tests avoid a real headscale. +func keysHandler(cfg config, run commandRunner) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "only POST is supported", http.StatusMethodNotAllowed) + return + } + kind := keyKind(r.URL.Query().Get("kind")) + policy, ok := policyFor(kind) + if !ok { + http.Error(w, fmt.Sprintf("unknown or missing kind %q (want daemon|controller)", kind), http.StatusBadRequest) + return + } + + key, err := run(cfg, policy) + if err != nil { + // err carries headscale's stderr, never the key. Safe to log + return. + log.Printf("mint failed kind=%s: %v", kind, err) + http.Error(w, "failed to mint key", http.StatusBadGateway) + return + } + + // Log METADATA only — never the key itself. + log.Printf("minted key kind=%s reusable=%s ephemeral=%s exp=%s", + kind, strconv.FormatBool(policy.reusable), strconv.FormatBool(policy.ephemeral), policy.expiration) + + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(keyResponse{ + Key: key, + Kind: string(kind), + Reusable: policy.reusable, + Ephemeral: policy.ephemeral, + Expiration: policy.expiration, + }) + } +} + +func main() { + cfg := loadConfig() + + // Self-provision the user we mint under, so the mesh works without a manual + // `headscale users create`. The broker is a SIDECAR that starts alongside + // headscale, so its unix socket may not be up yet on the first attempt — retry + // with backoff rather than crash-looping on that startup race. Fatal only after + // the window elapses: a broker that can't guarantee its user exists would 502 on + // every mint, so fail loudly instead of serving broken. + if err := ensureHeadscaleUserWithRetry(cfg, ensureHeadscaleUser); err != nil { + log.Fatalf("ensuring headscale user %q: %v", cfg.user, err) + } + + mux := http.NewServeMux() + mux.HandleFunc("/keys", keysHandler(cfg, runHeadscale)) + // Liveness/readiness: the broker is up as soon as it can serve HTTP. We do NOT + // probe headscale here (that would need a real mint); mint errors surface as a + // 502 on /keys, which the caller retries. + mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) + }) + + srv := &http.Server{ + Addr: cfg.listenAddr, + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, + } + log.Printf("keybroker listening on %s (user=%s, headscale config=%s)", + cfg.listenAddr, cfg.user, cfg.headscaleConfig) + if err := srv.ListenAndServe(); err != nil { + log.Fatalf("keybroker server exited: %v", err) + } +} diff --git a/cmd/keybroker/main_test.go b/cmd/keybroker/main_test.go new file mode 100644 index 0000000..81086e5 --- /dev/null +++ b/cmd/keybroker/main_test.go @@ -0,0 +1,218 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestPolicyFor(t *testing.T) { + cases := []struct { + kind keyKind + ok bool + reusable bool + ephemeral bool + }{ + // Daemon: single-use (NOT reusable) + ephemeral — a throwaway per-workload + // credential that auto-reaps on disconnect. + {kindDaemon, true, false, true}, + // Controller: reusable (survives restarts) + ephemeral (same as a daemon) so + // the old node is reaped on disconnect, freeing its MagicDNS name to reclaim. + {kindController, true, true, true}, + {keyKind("bogus"), false, false, false}, + {keyKind(""), false, false, false}, + } + for _, tc := range cases { + t.Run(string(tc.kind), func(t *testing.T) { + p, ok := policyFor(tc.kind) + if ok != tc.ok { + t.Fatalf("policyFor(%q) ok = %v, want %v", tc.kind, ok, tc.ok) + } + if !ok { + return + } + if p.reusable != tc.reusable { + t.Errorf("reusable = %v, want %v", p.reusable, tc.reusable) + } + if p.ephemeral != tc.ephemeral { + t.Errorf("ephemeral = %v, want %v", p.ephemeral, tc.ephemeral) + } + if p.expiration == "" { + t.Errorf("expiration must not be empty") + } + }) + } +} + +// fakeRunner records the policy it was asked to mint and returns a canned key, so +// the handler can be tested without a real headscale. +func fakeRunner(key string, err error) (commandRunner, *keyPolicy) { + var seen keyPolicy + run := func(_ config, policy keyPolicy) (string, error) { + seen = policy + return key, err + } + return run, &seen +} + +func TestKeysHandler_Daemon(t *testing.T) { + run, seen := fakeRunner("tskey-daemon-abc", nil) + h := keysHandler(config{user: "nebula"}, run) + + req := httptest.NewRequest(http.MethodPost, "/keys?kind=daemon", nil) + rec := httptest.NewRecorder() + h(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + var resp keyResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decoding response: %v", err) + } + if resp.Key != "tskey-daemon-abc" { + t.Errorf("key = %q, want tskey-daemon-abc", resp.Key) + } + if resp.Reusable { + t.Errorf("daemon key must not be reusable") + } + if !resp.Ephemeral { + t.Errorf("daemon key must be ephemeral") + } + // The runner must have been asked for the single-use ephemeral policy. + if seen.reusable || !seen.ephemeral { + t.Errorf("runner policy = %+v, want single-use ephemeral", *seen) + } +} + +func TestKeysHandler_Controller(t *testing.T) { + run, seen := fakeRunner("tskey-ctrl-xyz", nil) + h := keysHandler(config{user: "nebula"}, run) + + req := httptest.NewRequest(http.MethodPost, "/keys?kind=controller", nil) + rec := httptest.NewRecorder() + h(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + if !seen.reusable || !seen.ephemeral { + t.Errorf("controller policy = %+v, want reusable + ephemeral", *seen) + } +} + +func TestKeysHandler_UnknownKind(t *testing.T) { + run, _ := fakeRunner("unused", nil) + h := keysHandler(config{user: "nebula"}, run) + + for _, q := range []string{"/keys?kind=bogus", "/keys"} { + req := httptest.NewRequest(http.MethodPost, q, nil) + rec := httptest.NewRecorder() + h(rec, req) + if rec.Code != http.StatusBadRequest { + t.Errorf("%s: status = %d, want 400", q, rec.Code) + } + } +} + +func TestKeysHandler_MethodNotAllowed(t *testing.T) { + run, _ := fakeRunner("unused", nil) + h := keysHandler(config{user: "nebula"}, run) + + req := httptest.NewRequest(http.MethodGet, "/keys?kind=daemon", nil) + rec := httptest.NewRecorder() + h(rec, req) + if rec.Code != http.StatusMethodNotAllowed { + t.Errorf("status = %d, want 405", rec.Code) + } +} + +func TestEnsureHeadscaleUserWithRetry_SucceedsFirstTry(t *testing.T) { + calls := 0 + err := ensureHeadscaleUserWithRetry(config{user: "nebula"}, func(config) error { + calls++ + return nil + }) + if err != nil { + t.Fatalf("err = %v, want nil", err) + } + if calls != 1 { + t.Errorf("ensure called %d times, want 1", calls) + } +} + +func TestEnsureHeadscaleUserWithRetry_RecoversAfterSocketRace(t *testing.T) { + // Simulate headscale's socket not being ready on the first few attempts, then + // coming up — the broker must keep trying rather than fatal on the first miss. + calls := 0 + err := ensureHeadscaleUserWithRetry(config{user: "nebula"}, func(config) error { + calls++ + if calls < 3 { + return fmt.Errorf("socket not ready") + } + return nil + }) + if err != nil { + t.Fatalf("err = %v, want nil after recovery", err) + } + if calls != 3 { + t.Errorf("ensure called %d times, want 3", calls) + } +} + +func TestEnsureHeadscaleUserWithRetry_ExhaustsAndReturnsLastError(t *testing.T) { + // Shrink the delay so exhausting every attempt doesn't wait the real ~18s. + orig := ensureUserRetryDelay + ensureUserRetryDelay = 0 + defer func() { ensureUserRetryDelay = orig }() + + calls := 0 + err := ensureHeadscaleUserWithRetry(config{user: "nebula"}, func(config) error { + calls++ + return fmt.Errorf("still down") + }) + if err == nil { + t.Fatal("err = nil, want the last failure") + } + if !strings.Contains(err.Error(), "still down") { + t.Errorf("err = %v, want it to carry the last failure", err) + } + if calls != ensureUserRetries { + t.Errorf("ensure called %d times, want %d", calls, ensureUserRetries) + } +} + +func TestKeysHandler_MintError(t *testing.T) { + run, _ := fakeRunner("", fmt.Errorf("headscale down")) + h := keysHandler(config{user: "nebula"}, run) + + req := httptest.NewRequest(http.MethodPost, "/keys?kind=daemon", nil) + rec := httptest.NewRecorder() + h(rec, req) + if rec.Code != http.StatusBadGateway { + t.Errorf("status = %d, want 502", rec.Code) + } + // The error body must NOT leak internal detail beyond a generic message. + if strings.Contains(rec.Body.String(), "headscale down") { + t.Errorf("response body leaked internal error: %s", rec.Body.String()) + } +} diff --git a/cmd/main.go b/cmd/main.go index d70e105..4033c3a 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -51,6 +51,7 @@ import ( awsprovider "github.com/InftyAI/Nebula/pkg/provider/aws" "github.com/InftyAI/Nebula/pkg/provider/fake" "github.com/InftyAI/Nebula/pkg/provider/modal" + "github.com/InftyAI/Nebula/pkg/sandd" "github.com/InftyAI/Nebula/pkg/vnode" // +kubebuilder:scaffold:imports ) @@ -351,7 +352,22 @@ func registerProviders(ctx context.Context, c client.Client) { // delivered via a Secret), and one account-global credential authorizes every // region. Registration only fails (and is a non-fatal skip) if the price catalog // cannot load — region config can no longer make it fail. - if p, err := awsprovider.NewSDKClient(ctx, awsRegionSource(c)); err != nil { + // SandD daemon (opt-in): SandD turns on when SANDD_KEYBROKER_URL points at the + // in-cluster key broker. When set, every AWS workload runs the SandD daemon inside + // its container in tunnel mode — commands and interactive shells run in the user's + // own environment over the mesh with no inbound access — and the adapter mints a + // FRESH single-use, ephemeral key per workload (tenant isolation + auto-reaped + // nodes). Unset => nil minter => the zero SanddConfig injects nothing, so this is + // off by default. Minted keys are secrets and are NEVER logged. + sanddCfg := provider.SanddConfig{ + ControlServer: os.Getenv("SANDD_TUNNEL_SERVER"), + ServerURL: os.Getenv("SANDD_SERVER_URL"), + } + if minter := sandd.NewBrokerClient(os.Getenv("SANDD_KEYBROKER_URL")); minter != nil { + sanddCfg.KeyMinter = minter + setupLog.Info("SandD per-daemon key minting enabled via key broker") + } + if p, err := awsprovider.NewSDKClient(ctx, awsRegionSource(c), sanddCfg); err != nil { setupLog.Info("skipping AWS provider registration", "reason", err.Error()) } else { provider.Register(p) diff --git a/config/crd/bases/nebula.inftyai.com_nodepools.yaml b/config/crd/bases/nebula.inftyai.com_nodepools.yaml index 61e03c6..ddfa71f 100644 --- a/config/crd/bases/nebula.inftyai.com_nodepools.yaml +++ b/config/crd/bases/nebula.inftyai.com_nodepools.yaml @@ -102,9 +102,9 @@ spec: description: |- BlocklistTTL is the BASE duration a failed placement is excluded before the provider becomes a candidate for it again. The controller adds a random jitter - (up to a minute) on top so Pods that failed for the same reason do not all - retry the just-freed candidate in lockstep, so the effective exclusion is this - value plus that jitter. + (up to 30s) on top so Pods that failed for the same reason do not all retry the + just-freed candidate in lockstep, so the effective exclusion is this value plus + that jitter. type: string type: object providers: diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index a0e5424..f429f20 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -34,6 +34,13 @@ resources: #- ../prometheus # [METRICS] Expose the controller manager metrics service. - metrics_service.yaml +# [SANDD] SandD access channel — deployed by default. Stands up headscale + the +# SandD controller so operators/agents can exec/shell into a workload container +# (kubectl exec does NOT work against a Nebula virtual node). This only stands up +# the in-cluster pieces; the manager starts injecting the daemon once the +# nebula-sandd-config Secret exists (still opt-in per cluster). To skip deploying +# these components, comment this line out. See config/sandd/README.md. +- ../sandd # [NETWORK POLICY] Protect the /metrics endpoint and Webhook Server with NetworkPolicy. # Only Pod(s) running a namespace labeled with 'metrics: enabled' will be able to gather the metrics. # Only CR(s) which requires webhooks and are applied on namespaces labeled with 'webhooks: enabled' will @@ -47,6 +54,11 @@ patches: - path: manager_metrics_patch.yaml target: kind: Deployment + # Scope to the manager ONLY. Without a name this matches EVERY Deployment in + # the build (e.g. headscale from ../sandd), injecting the manager-only + # --metrics-bind-address flag into `headscale serve`, which then errors: + # "unknown flag: --metrics-bind-address". Matches the pre-prefix name. + name: controller-manager # Uncomment the patches line if you enable Metrics and CertManager # [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. @@ -60,6 +72,9 @@ patches: - path: manager_webhook_patch.yaml target: kind: Deployment + # Scope to the manager ONLY (see the metrics patch above) — otherwise the + # webhook args/volumes get injected into headscale's Deployment too. + name: controller-manager # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index f8b02c0..50f71ce 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -97,6 +97,27 @@ spec: # - secretRef: # name: nebula-runpod-credentials # optional: true + # SandD sandbox daemon (cross-provider — see pkg/provider SanddConfig). Its + # SANDD_KEYBROKER_URL / SANDD_TUNNEL_SERVER / SANDD_SERVER_URL are projected + # as env and every supporting adapter runs the daemon inside the workload + # container, so an operator can exec/shell into the user's own environment + # over the mesh (kubectl exec does NOT work against a virtual node). + # + # optional=false: this ConfigMap is REQUIRED. If it is missing the manager + # will NOT start (it sits in CreateContainerConfigError until the ConfigMap + # exists) rather than booting with SandD silently off. The overlay applies + # the ConfigMap before this Deployment (kustomize orders ConfigMap ahead of + # Deployment), so a normal `make deploy-all` satisfies it; a manager without + # SandD config is treated as a misconfiguration, not a valid mode. + # + # It is a ConfigMap, not a Secret: it holds only URLs (public endpoints), no + # credentials — the actual mesh keys are minted on demand by the key broker + # and never stored here. See config/sandd for the full working deployment + # (headscale + controller). To turn injection OFF, blank SANDD_KEYBROKER_URL + # in the ConfigMap (nil minter => nothing injected) rather than deleting it. + - configMapRef: + name: nebula-sandd-config + optional: false ports: [] securityContext: readOnlyRootFilesystem: true diff --git a/config/samples/deployment.yaml b/config/samples/deployment.yaml index f45239b..f3e62cd 100644 --- a/config/samples/deployment.yaml +++ b/config/samples/deployment.yaml @@ -28,7 +28,7 @@ metadata: labels: app.kubernetes.io/managed-by: nebula spec: - replicas: 8 + replicas: 2 selector: matchLabels: app: gpu-workload-sample @@ -38,7 +38,7 @@ spec: app: gpu-workload-sample nebula.inftyai.com/enabled: "true" nebula.inftyai.com/nodepool: sample - nebula.inftyai.com/accelerator-type: a100-40gb + nebula.inftyai.com/accelerator-type: t4 spec: # Do NOT set nodeName or a provider nodeSelector yourself — the placement # controller fills the nodeSelector in when it ungates the Pod. Setting @@ -65,4 +65,4 @@ spec: # GPU count. Standard extended resource, so the scheduler's fit check # and provisioning read the same number. 8 => 8x the accelerator-type # above. - nvidia.com/gpu: "8" + nvidia.com/gpu: "1" diff --git a/config/samples/headscale-service.yaml b/config/samples/headscale-service.yaml new file mode 100644 index 0000000..d299cad --- /dev/null +++ b/config/samples/headscale-service.yaml @@ -0,0 +1,64 @@ +# headscale Service — hand-applied, NOT in the config/sandd overlay. +# +# It is kept out of the overlay because the overlay adds a `nebula-` name prefix, +# which would create a SECOND, differently-named Service and thus a second LB. So it +# lives here and sets its own namespace + `nebula-` name explicitly, matching the +# prefixed Deployment the overlay generates; its selector matches the pod labels, so +# it finds the Deployment's pods regardless. +# +# server_url (which MUST equal the address clients dial, or headscale rejects them) +# is this LB's AWS-assigned hostname. That name is only known AFTER the NLB +# provisions, so the flow is: apply this Service, read the hostname back, bake it into +# the three server_url/SANDD_TUNNEL_SERVER spots, then deploy headscale. See README +# step 1. +apiVersion: v1 +kind: Service +metadata: + name: nebula-headscale + namespace: nebula-system + labels: + app: headscale + # An NLB (L4 — headscale is raw HTTP/gRPC, so NOT an ALB). scheme=internet-facing: + # the mesh is CROSS-CLOUD — GPU boxes are provisioned across AWS regions and other + # providers (Modal, RunPod), none of which share a private network with the cluster + # VPC. A control plane every node must reach therefore has to be publicly routable, + # exactly like Tailscale's own control plane. An internal LB only worked for nodes + # in the same/peered VPC, which is not the case here. + # + # SECURITY: this endpoint is now public. It is defensible because the Tailscale + # control (Noise/ts2021) handshake is END-TO-END ENCRYPTED regardless of transport, + # and pre-auth keys are minted in-cluster by the broker and baked into instance + # user-data OUT OF BAND — they never traverse this endpoint. The gRPC admin API + # (:50443) that mints users/keys is NOT exposed here (in-pod only). Still, for real + # use put this behind TLS + the hardening in "Production notes" (README). + # + # The NLB's AWS-assigned hostname (k8s-...elb..amazonaws.com) is what you + # bake into server_url / SANDD_TUNNEL_SERVER; internet-facing => it resolves to + # PUBLIC IPs reachable from any cloud/region. The name is stable for the life of the + # Service (changes only on delete/recreate), which is why the read-back-then-deploy + # flow (README step 1) is safe. The LB listens on port 80 (see ports below) so + # server_url is a bare http://. + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: "external" + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip" + service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing" +spec: + # LoadBalancer so out-of-cluster GPU boxes — in any region or cloud — can reach the + # control server over the public internet. Swap for NodePort + DNS, or front with an + # Ingress, if you have no LB. + type: LoadBalancer + selector: + app: headscale + ports: + # Listen on the DEFAULT HTTP port 80 and forward to headscale's 8080. This is + # REQUIRED, not cosmetic: Tailscale's control (Noise/ts2021) handshake dialer + # only ever dials the standard ports — 443 for TLS, 80 for the plaintext + # fallback — and IGNORES any custom port in server_url. Exposing headscale on + # 8080 let the health check and key fetch through but left the Noise dial with + # nowhere to land, so registration timed out and no node ever joined the mesh. + # Serving on 80 gives that dial its plaintext target, so server_url is a bare + # http:// (no port). The gRPC admin API (:50443) is deliberately NOT + # exposed — it mints users/keys, so it stays in-pod (`kubectl exec`) only. + - name: http + port: 80 + targetPort: 8080 diff --git a/config/samples/nebula_v1alpha1_nodepool.yaml b/config/samples/nebula_v1alpha1_nodepool.yaml index 7b2c852..8295ac8 100644 --- a/config/samples/nebula_v1alpha1_nodepool.yaml +++ b/config/samples/nebula_v1alpha1_nodepool.yaml @@ -26,4 +26,4 @@ spec: # Inner axis: within the active capacity tier. strategy: Ordered failover: - blocklistTTL: 3m + blocklistTTL: 30s diff --git a/config/samples/sandd-controller.yaml b/config/samples/sandd-controller.yaml new file mode 100644 index 0000000..cc71bdb --- /dev/null +++ b/config/samples/sandd-controller.yaml @@ -0,0 +1,178 @@ +# SandD controller — a `Server()` in tunnel mode. This is the box YOU drive: +# `server.exec(daemon_id, ...)` / `server.new_session(...)` run commands and open +# PTYs on the GPU workloads. It joins the same headscale mesh, takes the first IP +# (100.64.0.1), and listens for daemons on :8765 OVER the mesh — no port is +# exposed to the cluster or the internet. +# +# SAMPLE, not part of the kustomize overlay: the controller is the USER-FACING +# piece — you replace the inline script below with your own controller logic — so +# it is applied by hand and adapted, not deployed as a fixed resource. It sets its +# own namespace and full name (nothing rewrites them here). Apply after the +# config/sandd overlay is up — see config/sandd/README.md. +# +# The image is SandD's tunnel-server build (Python `sandd` package + Tailscale +# client preinstalled). Build/push it from the SandD repo: +# docker build -f hack/docker/Dockerfile.server-tunnel -t inftyai/sandd-server-tunnel:latest. +# +# The controller mints its own mesh key at startup from the in-cluster key broker +# (SANDD_KEYBROKER_URL below), as REUSABLE + EPHEMERAL — the SAME policy the daemons +# use, and it holds NO persistent state (no PVC). This is deliberate: every start is +# a CLEAN start, exactly like a daemon. +# +# It still keeps its stable MagicDNS name (sandd-controller.nebula.mesh) across +# restarts, via two mechanisms that do NOT need a PVC: +# - hostname: sandd-controller (below) pins the OS hostname, from which Tailscale +# derives the node/MagicDNS name — so the name is requested identically each time. +# - the key is EPHEMERAL, so headscale reaps the old node the moment the pod +# disconnects, FREEING the name; the fresh pod then reclaims it cleanly (no +# -suffix). This is exactly why daemons never collide. strategy: Recreate ensures +# the old pod is fully gone (its node reaped) BEFORE the new one registers, so the +# two never race for the name. +# +# Why no PVC: a PVC on /var/lib/tailscale persisted stale dial state, which tripped +# Tailscale's control dialer into "forcing port 443" on restart — and headscale is +# HTTP-only on 80, so that dead-ended and the controller could never re-register +# ("first start works, restart doesn't"). Daemons never hit this because they have no +# persisted state. Dropping the PVC removes the wedge entirely. The headscale URL it +# joins (SANDD_TUNNEL_SERVER) is set inline in env below — see README.md, step 2. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: sandd-controller + namespace: nebula-system + labels: + app: sandd-controller +spec: + replicas: 1 + # Recreate (not the default RollingUpdate): the controller's mesh key is EPHEMERAL, + # so headscale reaps its node the moment the pod disconnects, which FREES the + # sandd-controller MagicDNS name for the next pod to reclaim. RollingUpdate starts + # the new pod BEFORE killing the old one, so the new one would try to register while + # the old node is still connected (not yet reaped) — a name collision that lands the + # new pod on sandd-controller-1 and strands daemons dialing the clean name. Recreate + # tears the old pod down first, so its node is reaped before the new one registers + # and the fresh pod reclaims the same name cleanly. + strategy: + type: Recreate + selector: + matchLabels: + app: sandd-controller + template: + metadata: + labels: + app: sandd-controller + spec: + # Pin the pod's OS hostname so it is DETERMINISTIC. Tailscale derives a + # node's name (and thus its MagicDNS name) from the OS hostname, so this is + # what makes the controller reliably reachable at + # sandd-controller. on the mesh — independent of join order or + # which 100.64.x.x IP it happens to get. headscale's base_domain is + # nebula.mesh (see config/sandd/headscale.yaml), so the daemons dial + # ws://sandd-controller.nebula.mesh:8765/ws (SANDD_SERVER_URL below). + hostname: sandd-controller + containers: + - name: controller + image: inftyai/sandd-server-tunnel:latest + # Start the Server in tunnel mode and keep it alive. It joins the mesh, then + # loops listing daemons and runs a few example commands (COMMANDS below) on + # each workload the first time it connects, printing the output to the pod log + # (`kubectl logs`) — just enough to show exec over the mesh works. Attach with + # `kubectl exec` to drive `server` yourself (see README, "Using it"), or edit + # COMMANDS / replace this with your own controller logic — that's the point. + command: ["python3", "-u", "-c"] + args: + - | + import os, time, json, urllib.request + from sandd import Server, TunnelConfig + + # Mint a reusable+ephemeral key for this controller from the in-cluster + # broker (crash-loops if unreachable — the broker is required, not optional). + # NEVER print the key itself, only that we got one. + broker = os.environ["SANDD_KEYBROKER_URL"].rstrip("/") + req = urllib.request.Request(f"{broker}/keys?kind=controller", method="POST") + with urllib.request.urlopen(req, timeout=10) as resp: + authkey = json.load(resp)["key"] + print("controller: minted key, joining mesh", flush=True) + + server = Server(connect="tunnel", tunnel_config=TunnelConfig( + authkey=authkey, server=os.environ["SANDD_TUNNEL_SERVER"])) + print("controller: ready, waiting for daemons", flush=True) + + # Commands run on each workload the first time it connects. Edit freely — + # each runs in the user's container over the mesh via server.exec. + COMMANDS = ["hostname", "nvidia-smi -L", "ls /"] + + # Run the commands on each daemon once. `seen` guards against re-running + # every poll. + seen = set() + while True: + try: + for d in server.list_daemons(): + if d.id in seen: + continue + seen.add(d.id) + for cmd in COMMANDS: + r = server.exec(d.id, cmd) + print(f"[{d.id}] $ {cmd} (exit={r.exit_code})\n{r.stdout.rstrip()}", flush=True) + except Exception as e: + print("controller: loop error:", e, flush=True) + time.sleep(5) + env: + # The headscale URL the controller joins. UNLIKE the daemons (which are + # out-of-cluster and MUST use the public NLB hostname) and headscale's own + # server_url, the controller runs IN-cluster, so it reaches headscale by its + # internal ClusterIP Service name — no per-cluster substitution, and it never + # touches the public NLB. That matters: routing an in-cluster client through + # the internet-facing NLB is a needless round-trip AND hairpins (blackholes) + # when the controller lands on the same node as the headscale pod, which is + # exactly what made /machine/register time out and fall back to a dead :443. + # The ClusterIP path is direct and can't hairpin. Port 80 (Service 80 -> 8080) + # satisfies Tailscale's Noise dialer (80/443 only). headscale accepts this + # even though its server_url is the public name: authkey registration doesn't + # require the client's login-server to byte-match server_url. Cross-namespace + # is fine — the .nebula-system suffix resolves from any namespace. The + # controller mints its OWN key from the broker, so no auth key here. + - name: SANDD_TUNNEL_SERVER + value: http://nebula-headscale.nebula-system + # In-cluster key broker: the controller mints its OWN reusable+ephemeral + # key at startup from this endpoint (see resolve_authkey above). This is + # the nebula-keybroker ClusterIP Service (config/sandd/headscale.yaml). + # Required — the controller crash-loops if the broker is unreachable. + - name: SANDD_KEYBROKER_URL + value: http://nebula-keybroker.nebula-system:8090 + ports: + # Informational only — the daemons reach this over the mesh, not via a + # Service. Not exposed cluster-wide. + - name: sandd-ws + containerPort: 8765 + # NO added capabilities. tailscaled runs in --tun=userspace-networking + # mode (userspace TCP/IP stack, no kernel TUN/route), so NET_ADMIN is not + # needed: inbound mesh TCP is forwarded to the local :8765 listener + # without touching the host's routing or firewall. NET_ADMIN would let the + # container rewrite routes/firewall rules and sniff/inject traffic, so we + # keep it off even on this trusted control-plane side. + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + # DON'T drop this if you rewrite the controller: on shutdown we log out of + # headscale so the ephemeral node is DEREGISTERED immediately, which frees + # the sandd-controller name for the next pod. Without it, headscale holds the + # old node as `offline` until its inactivity timeout (~30m) elapses; a restart + # inside that window finds the clean name still taken and registers as + # sandd-controller- — and every daemon dialing the clean MagicDNS name + # (sandd-controller.nebula.mesh) then hits a dead node ("Active daemons: 0"). + # Recreate guarantees this preStop finishes before the new pod starts, so the + # name is free in time. `|| true` keeps a failed logout (e.g. control plane + # already unreachable) from blocking teardown. + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "tailscale logout || true"] + resources: + requests: + cpu: "100m" + memory: "128Mi" + limits: + cpu: "500m" + memory: "512Mi" diff --git a/config/sandd/README.md b/config/sandd/README.md new file mode 100644 index 0000000..9a65a76 --- /dev/null +++ b/config/sandd/README.md @@ -0,0 +1,249 @@ +# SandD access channel + +Wires [SandD](https://github.com/InftyAI/SandD) into Nebula so an operator or agent +can run commands and open interactive shells **inside a Nebula-provisioned workload +container** — which `kubectl logs`/`exec` cannot do against a Nebula virtual node +(`pkg/vnode/handler.go` returns `NotFound`). + +The daemon runs **inside the workload container** (one per container, single-tenant), +so its shells see the user's own env, cwd, filesystem and code. It dials **out** to a +controller over a Tailscale/headscale mesh, so the GPU box needs no inbound access — +it works for instances behind NAT with no public IP, in any region or cloud (the mesh +is cross-cloud; only outbound reachability to the public headscale endpoint is needed). + +## Architecture + +**Minting keys** — the broker is the only component with headscale admin authority; +it reaches headscale over a local unix socket. There is no static-key path. + +``` + headscale pod + ┌──────────────────────────────────────────────┐ + │ keybroker sidecar ──unix socket──► headscale │ + │ (ClusterIP :8090) │ + └──────────▲──────────────────▲──────────────────┘ + │ POST /keys │ POST /keys + │ ?kind=controller │ ?kind=daemon + │ (reusable) │ (single-use) + controller Nebula manager + (at startup) (at Provision, per workload → bakes key into user-data) +``` + +**Using the mesh** — both sides join headscale, then the workload's daemon dials the +controller by its stable MagicDNS name (no inbound access to the box). + +``` + headscale (internet-facing LB :80 → 8080) + ▲ ▲ + joins mesh│ │joins mesh + ┌─────────────────┴────────┐ ┌───────┴──────────────────┐ + │ controller │◄──────│ GPU workload (EC2) │ + │ Server(tunnel) │ ws://…│ sandd --tunnel │ + │ sandd-controller. │ (mesh)│ dials controller by name │ + │ nebula.mesh:8765/ws │ │ DAEMON_ID = NodeClaim │ + └──────────────────────────┘ └───────────────────────────┘ +``` + +## What's in the overlay + +`config/sandd` is applied by `config/default` (the `- ../sandd` line), so `make deploy` +stands up everything except the hand-applied Service and controller: + +| File | Kind | Role | +|------|------|------| +| `headscale.yaml` | Deployment + ConfigMap + Service | Tailscale control server + the **`keybroker` sidecar** (mints pre-auth keys on demand over a local unix socket; in-cluster-only `nebula-keybroker` ClusterIP; the ONLY way keys are minted). | +| `manager-config.yaml` | ConfigMap `nebula-sandd-config` | **Required** — the manager mounts it as `envFrom` with `optional: false`, so it won't start without it. `SANDD_KEYBROKER_URL` is the injection switch: set, the manager mints a fresh per-daemon key from the broker; blank, injection is off (but keep the ConfigMap — blank the value, don't delete it). | + +Hand-applied (not in the overlay): + +| File | Kind | Role | +|------|------|------| +| `config/samples/headscale-service.yaml` | Service (internet-facing NLB, `:80` → `8080`) | `scheme=internet-facing` — publicly reachable so GPU boxes in **any region or cloud** can join the mesh (they share no private network with the cluster). Listens on **80** (Tailscale's Noise handshake only dials 80/443), so `server_url` is a bare `http://`. Its AWS-assigned ELB hostname is read back after it provisions (step 1). Kept out of the overlay so its `nebula-` prefix doesn't spawn a second LB. gRPC admin (`:50443`) is NOT exposed. The control handshake is end-to-end encrypted and keys never traverse it, but see "Production notes" for TLS/hardening. | +| `config/samples/sandd-controller.yaml` | Deployment `sandd-controller` | The box you drive (`server.exec` / `new_session`). A SandD `Server()` in tunnel mode; sets `SANDD_TUNNEL_SERVER` + `SANDD_KEYBROKER_URL` inline and mints its own key. Adapt the inline script to your own logic. | + +## Deploy + +**Prereqs:** a running Nebula control plane with the AWS provider configured +(`nebula-aws-credentials` + a NodePool referencing `aws`); GPU instances with outbound +internet access (they dial the public headscale endpoint — no inbound access or shared +VPC needed, which is why the mesh spans regions and clouds); and the AWS Load Balancer +Controller installed (it provisions the internet-facing NLB in step 1). + +### 0. Build the key-broker image + +```bash +# Override KEYBROKER_IMG for your own registry; then update the sidecar image: in +# config/sandd/headscale.yaml to match. +make docker-build-keybroker docker-push-keybroker +``` + +### 1. Apply the Service, read its hostname into `.env`, deploy + +headscale forces every client to dial the identical `server_url`, and the GPU boxes +live in other regions/clouds with no private path to the cluster — so the endpoint +must be publicly routable. That address is the internet-facing NLB's own AWS-assigned +hostname, which is only known once the LB provisions. So apply the Service first and +read it back: + +```bash +kubectl create namespace nebula-system # if not already present +kubectl apply -f config/samples/headscale-service.yaml + +# Poll until populated (NLB takes 1–3 min): +HS=$(kubectl -n nebula-system get svc nebula-headscale \ + -o jsonpath='{.status.loadBalancer.ingress[0].hostname}'); echo "$HS" + +echo "SANDD_TUNNEL_SERVER=http://$HS" >> .env # or edit .env by hand +make deploy-all # substitutes the hostname into every spot, then deploys +``` + +`make deploy-all` reads `SANDD_TUNNEL_SERVER` from `.env` and substitutes the +`__SANDD_TUNNEL_SERVER__` token wherever the endpoint must appear — `server_url` in +`config/sandd/headscale.yaml` and `SANDD_TUNNEL_SERVER` in +`config/sandd/manager-config.yaml` — so both render **byte-identical** (exactly what +headscale demands) on the first apply, with no manager or headscale restart. The URL +carries no port because the Service listens on **80**: Tailscale's control (Noise) +handshake only ever dials 80 or 443, never a custom port, so headscale must be on a +default port even though it listens on 8080 in-pod (the Service maps 80 → 8080). + +> The ELB hostname is stable for the life of the Service; if you delete and recreate +> the Service, just re-read it into `.env` and re-run `make deploy-all` — no manifest +> edits. (The controller in step 2 is hand-applied, so substitute the same value into +> it too — one `sed`, shown there.) + +The broker mints keys under one headscale user (`SANDD_KEYBROKER_USER`, default +`nebula`) and **creates that user itself on startup** if it's missing — no manual +bootstrap. Key policies it owns (see `cmd/keybroker`): daemon keys are **single-use** +(each workload gets its own throwaway credential); the controller key is **reusable** +(re-registers across restarts). Both are **ephemeral**, so headscale auto-reaps a node +shortly after it disconnects — torn-down experiments don't pile up as OFFLINE nodes +squatting MagicDNS names, and the controller reclaims its stable name cleanly on +restart (the reaped old node frees it). + +### 2. Deploy the controller + +Self-contained — it mints its own key at startup and reads its endpoints from inline +env. No substitution needed: unlike the daemons, the controller runs in-cluster, so it +dials headscale by its internal ClusterIP name (`SANDD_TUNNEL_SERVER: +http://nebula-headscale.nebula-system`), not the public NLB. So it's a plain apply: + +```bash +kubectl apply -f config/samples/sandd-controller.yaml + +# Confirm it joined (any 100.64.x.x — we address it by name, not IP): +CTRL=$(kubectl -n nebula-system get pod -l app=sandd-controller -o name | head -1) +kubectl -n nebula-system exec "$CTRL" -c controller -- tailscale ip -4 +``` + +Routing the in-cluster controller through the public NLB was a needless round-trip and +**hairpined** (AWS NLBs with `target-type: ip` blackhole traffic from a client that +lands on the *same node* as the target pod) — that made `/machine/register` time out and +fall back to a dead `:443`. The ClusterIP path is direct and can't hairpin. headscale +accepts it even though its `server_url` is the public name (authkey registration doesn't +require the client's login-server to byte-match `server_url`). + +It pins `hostname: sandd-controller` (the stable MagicDNS name) and holds **no +persistent state** — no PVC, an `emptyDir`-clean start every time, exactly like a +daemon. A restart still reclaims the *same* name (not a `-suffix` that would strand +daemons at `Active daemons: 0`) because the key is **ephemeral**: headscale reaps the +old node the instant the pod disconnects, freeing the name, and `strategy: Recreate` +guarantees the old pod is gone before the new one registers — so the fresh pod +reclaims the name with no collision. Deliberately **no PVC**: a persisted +`/var/lib/tailscale` reloaded stale dial state on restart and wedged Tailscale into +"forcing port 443" (headscale is HTTP-only on 80), so the controller could never +re-register — "first start works, restart doesn't". Daemons never hit this precisely +because they carry no state; the controller now matches them. No StorageClass needed. + +That's it — the manager read `nebula-sandd-config` at startup (step 1 applied it +before the manager pod started), so injection is already on. From now on **every AWS +workload** runs the daemon inside its container, addressed by `DAEMON_ID` = the +NodeClaim name, each with its own broker-minted key. (If you later *change* the config +on a running manager, restart it — `kubectl -n nebula-system rollout restart +deploy/nebula-controller-manager` — since it reads env only at startup.) + +## Using it + +Apply any Nebula workload — injection is a cluster decision, so the ordinary sample +works unchanged: + +```bash +# The workload places against a NodePool, so that must exist first. Apply both +# together (deployment.yaml targets the `sample` NodePool): +kubectl apply -f config/samples/nebula_v1alpha1_nodepool.yaml +kubectl apply -f config/samples/deployment.yaml # or your own workload +``` + +Then drive it from the controller: + +```bash +CTRL=$(kubectl -n nebula-system get pod -l app=sandd-controller -o name | head -1) +kubectl -n nebula-system exec -it "$CTRL" -c controller -- python3 - <<'PY' +from sandd import Server, TunnelConfig +import os, json, urllib.request +# Mint a key from the broker, same as the controller does at startup. +req = urllib.request.Request( + os.environ["SANDD_KEYBROKER_URL"].rstrip("/") + "/keys?kind=controller", method="POST") +with urllib.request.urlopen(req, timeout=10) as r: + authkey = json.load(r)["key"] +server = Server(connect="tunnel", tunnel_config=TunnelConfig( + authkey=authkey, server=os.environ["SANDD_TUNNEL_SERVER"])) +for d in server.list_daemons(): + print("daemon:", d.id) + print(server.exec(d.id, "nvidia-smi --query-gpu=name --format=csv,noheader").stdout) + print(server.exec(d.id, "ls /").stdout) # the USER's filesystem +PY +``` + +`d.id` is the NodeClaim name; `server.exec` runs in the workload container. For an +interactive shell use `server.new_session(d.id)` (PTY). + +## How injection works + +The AWS adapter injects SandD in two parts (`pkg/provider/aws/translate.go`), both +**fail-open** so they can never block the workload: + +1. **On the host, before `docker run`:** the user-data fetches the static (musl) + `sandd` binary + Tailscale bundle into `/opt/sandd`. The GPU AMI has `curl`+`tar`, + so the user's image needs nothing. +2. **`docker run`** bind-mounts `/opt/sandd` **read-only** and overrides the ENTRYPOINT + with a `/bin/sh` shim (`sanddShimScript`) that: puts `/opt/sandd` on `PATH`; starts + `sandd --tunnel` backgrounded (tailscaled in `--tun=userspace-networking` — no + `NET_ADMIN`, no `/dev/net/tun`); then `exec`s the user's command as PID 1. + +Fetching on the host means no fetcher/package manager in the user's image, one download +per instance, and a safe shared read-only mount (each container keeps its own writable +`/var/lib/tailscale` + `/tmp`). + +`SANDD_TUNNEL_AUTHKEY` arrives as container env — the per-workload key the manager +minted (single-use + ephemeral), so even though the container can see it, it's a +throwaway good for one node and reaped on disconnect. The daemon is single-tenant, so a +container seeing its own key is fine. + +Both parts log to stderr with a `[sandd]` prefix, landing in the **EC2 console** +(`aws ec2 get-console-output --instance-id --latest`) — the place to debug +bring-up. The daemon's own log is at `/tmp/sandd.log` inside the container. + +**Image requirements:** the shim needs only `/bin/sh` (binaries are mounted, not +fetched — even distroless-with-shell works); the **host** needs outbound network. A Pod +relying solely on its image's baked-in ENTRYPOINT (no `command`/`args`) can't be +reconstructed — set an explicit `command`. + +## Turning it off + +- **Stop injecting** (keep infra): blank `SANDD_KEYBROKER_URL` in `manager-config.yaml` + and restart the manager (empty broker URL => nil minter => nothing injected). Do NOT + delete the ConfigMap — it is a required mount (`optional: false`), so removing it + leaves the manager stuck in `CreateContainerConfigError`. +- **Remove the controller:** `kubectl delete -f config/samples/sandd-controller.yaml`. +- **Remove all infra:** comment out `- ../sandd` in + `config/default/kustomization.yaml` and re-apply — but keep `manager-config.yaml` + applied (with `SANDD_KEYBROKER_URL` blank) so the required mount is still satisfied. + +## Production notes + +This is a working DEV setup, not hardened: headscale uses SQLite (on a small PVC so its +state survives pod restarts, but still a single-writer file DB) over plain HTTP. Each workload +already gets its own single-use ephemeral key, but for real use also: back headscale +with a PersistentVolume + real database + TLS, and add per-tenant **tags/ACLs** to +minted keys so daemons are mesh-isolated, not just credential-distinct. See +https://headscale.net. diff --git a/config/sandd/headscale.yaml b/config/sandd/headscale.yaml new file mode 100644 index 0000000..6bbf287 --- /dev/null +++ b/config/sandd/headscale.yaml @@ -0,0 +1,233 @@ +# headscale — the Tailscale control server that coordinates the SandD mesh. +# +# ⚠️ DEVELOPMENT / EXAMPLE configuration. It uses SQLite on a small PVC (state +# survives Pod restarts, but it is still a single-writer file DB) and a plain +# HTTP listener. For production see https://headscale.net and back it with a +# real database + TLS. +# +# It MUST be reachable by out-of-cluster GPU boxes (they join the same mesh). The +# LoadBalancer Service that exposes it is hand-applied FIRST — it lives OUTSIDE this +# overlay (config/samples/headscale-service.yaml) precisely because everything in +# config/sandd/ is applied together by `make deploy`, whereas the Service must go up +# on its own so its ELB hostname is known before this Deployment starts. See that +# file and the README for why. This file (the ConfigMap + Deployment) is what the +# ../sandd kustomize overlay deploys. +# +# Namespace + the nebula- name prefix are added by config/default's kustomize, so +# they are omitted here (the ConfigMap ref below stays unprefixed too). +apiVersion: v1 +kind: ConfigMap +metadata: + name: headscale-config +data: + config.yaml: | + # server_url MUST equal the address clients actually dial. headscale rejects a + # client whose --login-server disagrees with this. It is the internet-facing NLB's + # AWS-assigned hostname (config/samples/headscale-service.yaml), only known AFTER + # the Service provisions. The literal token below is substituted by `make deploy` + # from SANDD_TUNNEL_SERVER (set in .env; deploy-all reads the hostname back — README + # step 1). This value is byte-identical to SANDD_TUNNEL_SERVER in nebula-sandd-config + # (same token, same substitution), which is exactly what headscale requires. The + # value is a bare http:// with NO port: the Service listens on 80 (Tailscale's + # Noise handshake only dials 80/443, never a custom port), and headscale itself still + # listens on 8080 in-pod (listen_addr below); the Service maps 80 -> 8080. + server_url: __SANDD_TUNNEL_SERVER__ + listen_addr: 0.0.0.0:8080 + metrics_listen_addr: 0.0.0.0:9090 + grpc_listen_addr: 0.0.0.0:50443 + grpc_allow_insecure: true + + # The keybroker sidecar (below) mints pre-auth keys via the headscale CLI over + # THIS local unix socket — no admin API key, no gRPC exposure. Both containers + # mount the same emptyDir at /var/run/headscale, so the CLI in the sidecar + # reaches headscale in-pod. Nothing outside the pod can touch it. + unix_socket: /var/run/headscale/headscale.sock + unix_socket_permission: "0770" + + # Mesh IP range (RFC 6598 CGNAT, as Tailscale/headscale recommend). Nodes are + # addressed by MagicDNS name (base_domain below), not raw IP, so which node + # gets which 100.64.x.x address does not matter. + prefixes: + v4: 100.64.0.0/10 + v6: fd7a:115c:a1e0::/48 + + private_key_path: /var/lib/headscale/private.key + noise: + private_key_path: /var/lib/headscale/noise_private.key + + database: + type: sqlite3 + sqlite: + path: /var/lib/headscale/db.sqlite + + log: + level: info + format: text + + dns: + magic_dns: true + base_domain: nebula.mesh + nameservers: + global: + - 1.1.1.1 + + derp: + server: + enabled: false + urls: + - https://controlplane.tailscale.com/derpmap/default + auto_update_enabled: true + update_frequency: 24h +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: headscale + labels: + app: headscale +spec: + replicas: 1 + # Recreate (not the default RollingUpdate): headscale's state is on a single + # ReadWriteOnce PVC (data volume below). RollingUpdate starts the new pod before + # killing the old one, so both would race for the same RWO volume -> the new pod + # hangs on a Multi-Attach error. Recreate tears the old pod down first. Fine at + # replicas: 1 (headscale is single-writer over SQLite anyway). + strategy: + type: Recreate + selector: + matchLabels: + app: headscale + template: + metadata: + labels: + app: headscale + spec: + containers: + - name: headscale + image: headscale/headscale:0.23 + args: ["serve"] + ports: + - name: http + containerPort: 8080 + # grpc (:50443) is intentionally NOT published on the LoadBalancer + # (see config/samples/headscale-service.yaml): the admin API mints + # users/keys, so it stays reachable only in-pod via `kubectl exec`. The + # process still listens on it (grpc_listen_addr) for that local CLI use. + volumeMounts: + - name: config + mountPath: /etc/headscale/config.yaml + subPath: config.yaml + readOnly: true + - name: data + mountPath: /var/lib/headscale + # Shared with the keybroker sidecar so its `headscale` CLI can dial this + # process's local admin socket (unix_socket above). + - name: socket + mountPath: /var/run/headscale + readinessProbe: + httpGet: + path: /health + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 10 + # keybroker — mints pre-auth keys on demand so nothing else needs headscale + # admin authority. Runs as a SIDECAR here (not a separate Deployment) so it + # can reach headscale over the shared LOCAL unix socket: no admin API key to + # store, no gRPC port exposed. It serves an in-cluster-only HTTP API + # (nebula-keybroker Service) that the manager (per-daemon keys) and the + # controller (its own key) call. Image built from Dockerfile.keybroker + # (FROM headscale, so the CLI is present); keep its headscale tag in sync + # with the headscale container above. + - name: keybroker + image: inftyai/nebula-keybroker:latest + env: + # The CLI reads unix_socket from headscale's config, so point the broker + # at the SAME config file the headscale container uses. + - name: SANDD_KEYBROKER_HS_CONFIG + value: /etc/headscale/config.yaml + # Keys are minted under this headscale user. The broker ensures it exists + # at startup (creates it if missing), so no manual bootstrap is needed. + - name: SANDD_KEYBROKER_USER + value: nebula + ports: + - name: broker + containerPort: 8090 + volumeMounts: + # Read the same config (for unix_socket) ... + - name: config + mountPath: /etc/headscale/config.yaml + subPath: config.yaml + readOnly: true + # ... and share the socket dir so the CLI reaches headscale in-pod. + - name: socket + mountPath: /var/run/headscale + readinessProbe: + httpGet: + path: /healthz + port: 8090 + initialDelaySeconds: 3 + periodSeconds: 10 + volumes: + - name: config + configMap: + # Unprefixed: config/default's namePrefix rewrites this reference to + # the generated nebula-headscale-config ConfigMap. + name: headscale-config + # headscale's SQLite DB + keys live here. Backed by a PVC (not emptyDir) so + # the DB SURVIVES pod recreation: with emptyDir, any reschedule (node + # scale-down, spot reclaim, rollout) wiped every registration, which + # orphaned already-joined nodes — in particular the PVC-backed controller, + # which then loops re-registering an identity headscale no longer knows. A + # PVC keeps that state, so a headscale restart no longer drops the mesh. + - name: data + persistentVolumeClaim: + claimName: headscale-data + # Ephemeral socket dir shared by headscale + keybroker so the sidecar's CLI + # reaches headscale's local admin socket. Never leaves the pod. + - name: socket + emptyDir: {} +--- +# Persists headscale's SQLite DB + private keys across pod recreation. Without it +# (emptyDir) a rescheduled headscale came up with an EMPTY database, invalidating +# every issued auth key and orphaning already-joined nodes — most visibly the +# controller, whose OWN identity is on a PVC, so it survived while headscale forgot +# it and looped re-registering. RWO is fine at replicas: 1 (headscale is a single +# SQLite writer); pair with strategy: Recreate on the Deployment so a rollout does +# not race two pods for this volume. gp3 to match the cluster default (same SC as +# the controller PVC); swap the storageClassName for your environment. This is still +# a DEV setup — for production also move off SQLite to a real DB + TLS (see README). +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: headscale-data + labels: + app: headscale +spec: + storageClassName: gp3 + accessModes: + - ReadWriteOnce + resources: + requests: + # SQLite DB + a couple of key files — tiny; 1Gi is the common minimum + # granularity for dynamically-provisioned block volumes. + storage: 1Gi +--- +# In-cluster API for the keybroker sidecar. ClusterIP ONLY — this mints mesh +# credentials, so it must never be exposed outside the cluster (unlike the +# headscale Service, which is a public LoadBalancer for out-of-cluster GPU boxes). +# The manager dials it to mint a per-daemon key at provision time; the SandD +# controller dials it at startup to mint its own key. +apiVersion: v1 +kind: Service +metadata: + name: keybroker + labels: + app: headscale +spec: + type: ClusterIP + selector: + app: headscale + ports: + - name: broker + port: 8090 + targetPort: broker diff --git a/config/sandd/kustomization.yaml b/config/sandd/kustomization.yaml new file mode 100644 index 0000000..feaafad --- /dev/null +++ b/config/sandd/kustomization.yaml @@ -0,0 +1,30 @@ +# [SANDD] SandD access channel — OPT-IN, disabled by default. +# +# SandD lets an operator/agent run commands and open interactive shells INSIDE a +# Nebula-provisioned workload container (kubectl logs/exec do NOT work against a +# Nebula virtual node — see pkg/vnode/handler.go). The daemon runs inside the +# workload, dials OUT to a controller over a Tailscale/headscale mesh, so it needs +# no inbound access to the GPU box. See pkg/provider.SanddConfig and the AWS +# entrypoint shim in pkg/provider/aws/translate.go. +# +# This overlay deploys only the shared INFRASTRUCTURE that backs the channel: +# - headscale.yaml : the Tailscale control server (assigns mesh IPs) +# +# headscale's LoadBalancer Service is deliberately NOT here: it must be applied on +# its own, FIRST, so its ELB hostname is known before headscale starts (that +# address is headscale's server_url). Everything in this folder is applied together +# by `make deploy`, so the Service lives in config/samples/headscale-service.yaml +# and is hand-applied — see README.md step 1. +# +# The CONTROLLER (a SandD Server() you drive with your own exec/session logic) is +# deliberately NOT here — it is the user-facing piece, so it ships as a hand-applied +# sample (config/samples/sandd-controller.yaml) you adapt, not a fixed resource. +# +# This overlay flips the manager ON: it ships nebula-sandd-config (manager-config.yaml), +# which the manager's optional envFrom reads to enable injection. So deploying this +# overlay + a controller is all it takes — but SANDD_TUNNEL_SERVER in that ConfigMap +# is a per-cluster placeholder (the headscale LB hostname), so injection only works +# once you fill it in (see README.md). Comment the resource out to keep SandD off. +resources: + - headscale.yaml + - manager-config.yaml diff --git a/config/sandd/manager-config.yaml b/config/sandd/manager-config.yaml new file mode 100644 index 0000000..1422e62 --- /dev/null +++ b/config/sandd/manager-config.yaml @@ -0,0 +1,35 @@ +# nebula-sandd-config — flips SandD injection ON for the manager. Deployed by this +# overlay (namePrefix in config/default turns "sandd-config" into "nebula-sandd-config"; +# the manager's envFrom references the pre-prefix name so kustomize rewrites both). +# +# A ConfigMap, not a Secret: it holds only URLs. The mesh keys are minted on demand +# by the key broker and never stored here. +# +# SANDD_TUNNEL_SERVER is headscale's server_url — the address the out-of-cluster GPU +# daemons dial to join the mesh. It is the internet-facing NLB's AWS-assigned hostname +# (see config/samples/headscale-service.yaml), resolving to PUBLIC IPs: the mesh spans +# clouds and AWS regions that share no private network with the cluster, so the control +# plane must be publicly routable (a ClusterIP or internal-LB name would be unreachable +# from a box in another region/cloud). Paste the hostname read back after the Service +# provisions (README step 1); keep it identical to server_url in headscale.yaml. +apiVersion: v1 +kind: ConfigMap +metadata: + name: sandd-config + namespace: nebula-system +data: + # In-cluster key broker (the switch that turns injection ON): the manager mints a + # fresh single-use, ephemeral key per workload from here. Stable internal Service + # DNS — no per-cluster edit needed. + SANDD_KEYBROKER_URL: "http://nebula-keybroker.nebula-system:8090" + # headscale control-server URL — the internet-facing NLB's AWS-assigned hostname. + # The literal token below is substituted by `make deploy` from SANDD_TUNNEL_SERVER + # (set it in .env; deploy-all reads it back). It stays a token in the tracked file + # because the hostname is per-cluster and changes on every Service recreate. Kept + # byte-identical to server_url in headscale.yaml (same token, same substitution). + SANDD_TUNNEL_SERVER: "__SANDD_TUNNEL_SERVER__" + # The controller's SandD WebSocket URL, reached OVER the mesh by MagicDNS name. + # Stable convention (headscale base_domain nebula.mesh + controller hostname), so + # no per-cluster edit needed. + # TODO: dynamically set this. + SANDD_SERVER_URL: "ws://sandd-controller.nebula.mesh:8765/ws" diff --git a/hack/deploy.sh b/hack/deploy.sh index 176ed81..1af9fe4 100755 --- a/hack/deploy.sh +++ b/hack/deploy.sh @@ -43,30 +43,56 @@ KUBECTL="${KUBECTL:-kubectl}" command -v "${KUBECTL}" >/dev/null 2>&1 || die "kubectl not found on PATH" # --- load .env (secrets only) ---------------------------------------------- +# We PARSE .env into a private associative array rather than `source`-ing it, so +# its values never enter this script's ENVIRONMENT. That is what lets .env use the +# standard AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY names safely: sourcing them +# (even without `set -a`, if they are already exported in the caller's shell) would +# hand the PROVISIONING identity to every child process — including `aws eks +# get-token`, kubectl's EKS auth plugin — and break cluster auth with a 401. Held +# in ENV_VARS[], the creds reach the Secret (via create_provider_secret) but not +# kubectl, so the deployer's own AWS identity keeps talking to the cluster. +declare -A ENV_VARS=() ENV_FILE="${ENV_FILE:-.env}" if [[ -f "${ENV_FILE}" ]]; then log "loading credentials from ${ENV_FILE}" - set -a # export everything defined while sourcing - # shellcheck disable=SC1090 - source "${ENV_FILE}" - set +a + while IFS= read -r line || [[ -n "${line}" ]]; do + line="${line%$'\r'}" # tolerate CRLF + [[ "${line}" =~ ^[[:space:]]*# ]] && continue # comment + [[ "${line}" =~ ^[[:space:]]*$ ]] && continue # blank + line="${line#export }" # allow `export KEY=val` + [[ "${line}" != *=* ]] && continue # not an assignment + local_key="${line%%=*}"; local_val="${line#*=}" + local_key="${local_key//[[:space:]]/}" # trim whitespace around the key + # strip one layer of matching surrounding quotes from the value + if [[ "${local_val}" == \"*\" || "${local_val}" == \'*\' ]]; then + local_val="${local_val:1:${#local_val}-2}" + fi + ENV_VARS["${local_key}"]="${local_val}" + done < "${ENV_FILE}" else warn "${ENV_FILE} not found; no provider credentials will be applied. Copy .env.example to .env." fi # --- per-provider secret table --------------------------------------------- # One entry per provider: "||". -# Keys are env var names read from .env; the Secret stores each verbatim (the -# SDKs read them by these exact names). To add a provider, append a row — -# nothing else in this script needs to change. +# Each key names both the variable read from .env (ENV_VARS[KEY]) and the field it +# is stored under in the Secret (the name the provider SDK reads inside the pod) — +# they are the same because .env is PARSED, not sourced, so a key can match the +# SDK's standard name without leaking into the deployer's environment (see the +# loader above). To add a provider, append a row — nothing else needs to change. PROVIDER_SECRETS=( # Only secrets belong here. "nebula-modal-credentials|MODAL_TOKEN_ID MODAL_TOKEN_SECRET|" - # AWS: creds are the only secret. The access key + secret are required together - # (a lone key is a misconfig), so a blank pair skips the Secret — the SDK then - # relies on IRSA / instance role, which is the preferred path. The region is - # NON-SECRET and lives on the manager Deployment, not in this Secret; the adapter - # self-configures the rest (GPU AMI + subnets). + # AWS: creds are the only secret. CRITICAL — these are the PROVISIONING identity + # (the account that launches GPU instances), which is DISTINCT from the identity + # that talks to the EKS control plane this deploy runs against. They use the + # standard AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY names — safe ONLY because the + # loader parses .env into ENV_VARS[] instead of exporting it, so these never reach + # `aws eks get-token` (kubectl's EKS auth plugin, which reads those exact env + # vars) and cannot hijack cluster auth. Both required together (a lone key is a + # misconfig) → a blank pair skips the Secret and the SDK falls back to IRSA / + # instance role (the preferred path). Region is NON-SECRET (on the manager + # Deployment); the adapter self-configures the rest (GPU AMI + subnets). "nebula-aws-credentials|AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY|" # "nebula-runpod-credentials|RUNPOD_API_KEY|" ) @@ -77,8 +103,9 @@ create_provider_secret() { local name="$1" required="$2" optional="$3" local args=() key val missing=0 + # Each key names both the .env variable (ENV_VARS[key]) and the Secret field. for key in ${required}; do - val="${!key:-}" + val="${ENV_VARS[${key}]:-}" if [[ -z "${val}" ]]; then missing=1 continue @@ -92,7 +119,7 @@ create_provider_secret() { fi for key in ${optional}; do - val="${!key:-}" + val="${ENV_VARS[${key}]:-}" [[ -n "${val}" ]] && args+=(--from-literal="${key}=${val}") done @@ -104,18 +131,24 @@ create_provider_secret() { --dry-run=client -o yaml | "${KUBECTL}" apply -f - } -# --- 1. build the image ---------------------------------------------------- -log "building image ${IMG}" -make docker-build IMG="${IMG}" - -# --- 2. make the image reachable by the cluster ---------------------------- +# --- 1. build the image, and make it reachable by the cluster -------------- +# Kind and a real registry differ in BOTH build arch and delivery: +# - Kind runs on the host's single architecture, so a plain single-arch +# `docker-build` + `kind load` is correct (and multi-arch buildx cannot +# `kind load` anyway — buildx only pushes). +# - A cloud registry may back nodes of a DIFFERENT arch than this machine +# (e.g. building on an arm64 Mac for amd64 EKS nodes), so we MUST build a +# multi-arch manifest — otherwise the node finds the image but "no match +# for platform in manifest". `docker-buildx` builds AND pushes in one step. if [[ -n "${KIND_CLUSTER}" ]]; then command -v kind >/dev/null 2>&1 || die "KIND_CLUSTER=${KIND_CLUSTER} set but 'kind' not found on PATH" + log "building image ${IMG}" + make docker-build IMG="${IMG}" log "loading ${IMG} into Kind cluster ${KIND_CLUSTER}" kind load docker-image "${IMG}" --name "${KIND_CLUSTER}" else - log "pushing ${IMG} (set KIND_CLUSTER to load into Kind instead)" - make docker-push IMG="${IMG}" + log "building + pushing multi-arch ${IMG} (set KIND_CLUSTER to load into Kind instead)" + make docker-buildx IMG="${IMG}" fi # --- 3. Secrets FIRST, so the manager mounts them on its very first boot ---- @@ -145,8 +178,20 @@ done # --- 4. install CRDs + deploy the manager ---------------------------------- # The pod mounts the cert Secret and reads provider creds at boot — both already # exist, so the manager comes up fully configured with no restart needed. +# +# SANDD_TUNNEL_SERVER is the ONE non-secret in .env (the internet-facing headscale +# NLB hostname). It is read from ENV_VARS[] (parsed, NOT sourced — same safety as the +# creds) and handed to `make deploy`, which substitutes the __SANDD_TUNNEL_SERVER__ +# token in headscale's server_url and the nebula-sandd-config ConfigMap so both render +# correct on first apply — no restart. Blank => `make deploy` leaves the token in place +# (an obviously-broken render, not a silent misconfig); the README's read-back step +# populates it. An env/flag SANDD_TUNNEL_SERVER overrides .env. +SANDD_TUNNEL_SERVER="${SANDD_TUNNEL_SERVER:-${ENV_VARS[SANDD_TUNNEL_SERVER]:-}}" +if [[ -z "${SANDD_TUNNEL_SERVER}" ]]; then + warn "SANDD_TUNNEL_SERVER unset (not in .env or env) — headscale server_url + SANDD_TUNNEL_SERVER will keep the __SANDD_TUNNEL_SERVER__ placeholder; set it (README step 1) and re-run." +fi log "installing CRDs and deploying the manager" -make deploy IMG="${IMG}" +make deploy IMG="${IMG}" SANDD_TUNNEL_SERVER="${SANDD_TUNNEL_SERVER}" # --- 5. inject the webhook CA bundle (server-side, no manager restart) ------ # This edits only the MutatingWebhookConfiguration, which just got created by diff --git a/keybroker b/keybroker new file mode 100755 index 0000000..92b495f Binary files /dev/null and b/keybroker differ diff --git a/pkg/provider/aws/aws.go b/pkg/provider/aws/aws.go index c061ec4..3c98537 100644 --- a/pkg/provider/aws/aws.go +++ b/pkg/provider/aws/aws.go @@ -142,6 +142,11 @@ type InstanceSpec struct { Region string // Tags carry Nebula identity; ClaimTagKey holds the NodeClaim name. Tags map[string]string + // Sandd, when Enabled, makes the workload's docker run start the SandD daemon + // INSIDE the container (via an entrypoint shim) so commands and interactive + // shells run in the user's own env/cwd/code over the tunnel. A zero value injects + // nothing. See buildUserData/writeSanddEntrypoint and provider.SanddConfig. + Sandd provider.SanddConfig } // EC2Instance is the adapter-level view of one EC2 instance as observed. @@ -211,6 +216,9 @@ type Provider struct { // regionSource reports the NodePool-declared regions to sweep in List/Offerings. // May be nil in tests, in which case sweepRegions uses only the cache keys. regionSource RegionSource + // sandd is the optional SandD daemon config stamped onto every InstanceSpec so + // buildUserData can bootstrap it. Zero value => disabled (no injection). + sandd provider.SanddConfig mu sync.Mutex clients map[string]Client // region -> Client, populated lazily by clientFor @@ -225,11 +233,16 @@ type Provider struct { // (admission requires each aws pool to list ≥1 region, and placement stamps it onto // the ProvisionRequest), and observed instances report their region from the // region-pinned client — so nothing needs a fallback, and no AWS_REGION env is read. -func New(newClient ClientFactory, cat catalog.Lookup, regionSource RegionSource) *Provider { +// +// sandd is the optional SandD daemon config baked into every instance's user-data; +// its zero value disables injection, so tests and the non-SandD path pass +// provider.SanddConfig{}. +func New(newClient ClientFactory, cat catalog.Lookup, regionSource RegionSource, sandd provider.SanddConfig) *Provider { return &Provider{ Base: catalog.Base{ProviderName: provider.ProviderAWS, Catalog: cat}, newClient: newClient, regionSource: regionSource, + sandd: sandd, clients: make(map[string]Client), } } @@ -244,6 +257,7 @@ func newSingleRegion(client Client, cat catalog.Lookup, region string) *Provider func(context.Context, string) (Client, error) { return client, nil }, cat, func() []string { return []string{region} }, + provider.SanddConfig{}, // single-region test convenience: no SandD injection ) // Pre-seed the cache so even a stray region lookup returns the fake rather than // invoking the (constant) factory. @@ -406,7 +420,7 @@ func (p *Provider) Provision(ctx context.Context, pod *corev1.Pod, req provider. return existing.ID, nil } - spec, err := p.instanceSpecFromPod(pod, req) + spec, err := p.instanceSpecFromPod(ctx, pod, req) if err != nil { return "", err } @@ -669,7 +683,9 @@ func splitID(instanceID string) (region, rawID string) { // instanceSpecFromPod reads the workload off the Pod (source of truth) and the // accelerator type (from the AcceleratorTypeLabel), maps it to an EC2 instance // type via the catalog, and stamps the claim tag, capacity tier, and region. -func (p *Provider) instanceSpecFromPod(pod *corev1.Pod, req provider.ProvisionRequest) (InstanceSpec, error) { +func (p *Provider) instanceSpecFromPod( + ctx context.Context, pod *corev1.Pod, req provider.ProvisionRequest, +) (InstanceSpec, error) { if len(pod.Spec.Containers) == 0 { return InstanceSpec{}, errors.New("aws: pod has no containers") } @@ -705,6 +721,15 @@ func (p *Provider) instanceSpecFromPod(pod *corev1.Pod, req provider.ProvisionRe return InstanceSpec{}, fmt.Errorf("aws: no EC2 instance type for %s x%d", canonical, count) } + // SandD daemon (opt-in): resolve THIS instance's mesh key by minting a fresh + // single-use, ephemeral key per workload (tenant isolation + auto-reaped nodes). + // This runs on the Provision path, so a mint failure aborts provisioning rather + // than launching a daemon that can never join the mesh. + sandd, err := p.resolveSanddConfig(ctx) + if err != nil { + return InstanceSpec{}, err + } + return InstanceSpec{ InstanceTypes: instanceTypes, Image: c.Image, @@ -714,9 +739,32 @@ func (p *Provider) instanceSpecFromPod(pod *corev1.Pod, req provider.ProvisionRe Spot: req.CapacityType == nebulav1alpha1.CapacitySpot, Region: req.Region, Tags: map[string]string{ClaimTagKey: req.ClaimName}, + // SandD daemon (opt-in): keyed by the claim name so a daemon that dials home + // correlates 1:1 with this instance's NodeClaim. Zero value => no injection. + Sandd: sandd, }, nil } +// resolveSanddConfig returns the per-instance SandD config. When SandD is disabled +// (zero config, no KeyMinter) it returns it unchanged. Otherwise it mints a FRESH +// per-daemon key and returns a copy with AuthKey set to it (and KeyMinter cleared, +// so the minted key — not the minter — is what buildUserData bakes in). A mint error +// is returned so Provision fails loudly rather than injecting a keyless (and thus +// useless) daemon. +func (p *Provider) resolveSanddConfig(ctx context.Context) (provider.SanddConfig, error) { + if p.sandd.KeyMinter == nil { + return p.sandd, nil + } + key, err := p.sandd.KeyMinter.MintDaemonKey(ctx) + if err != nil { + return provider.SanddConfig{}, fmt.Errorf("aws: minting SandD daemon key: %w", err) + } + perInstance := p.sandd + perInstance.AuthKey = key + perInstance.KeyMinter = nil // the resolved key travels on the spec, not the seam. + return perInstance, nil +} + // toInstance normalizes an observed EC2 instance into the provider-agnostic // Instance. func (p *Provider) toInstance(ec2 EC2Instance) provider.Instance { diff --git a/pkg/provider/aws/client.go b/pkg/provider/aws/client.go index 0ee0629..4b70865 100644 --- a/pkg/provider/aws/client.go +++ b/pkg/provider/aws/client.go @@ -30,6 +30,7 @@ import ( smithy "github.com/aws/smithy-go" logf "sigs.k8s.io/controller-runtime/pkg/log" + "github.com/InftyAI/Nebula/pkg/provider" "github.com/InftyAI/Nebula/pkg/provider/catalog" ) @@ -204,7 +205,14 @@ var _ Client = (*sdkClient)(nil) // // The catalog is loaded via catalog.Load() (embedded CSV / mounted ConfigMap), // identical to the other adapters. -func NewSDKClient(_ context.Context, regionSource RegionSource) (*Provider, error) { +// +// sandd is the OPTIONAL SandD daemon config: its zero value (empty AuthKey) leaves +// the bootstrap untouched, so passing provider.SanddConfig{} is a no-op. When set, +// every workload this provider launches runs the SandD daemon inside its container +// in tunnel mode (see buildUserData), the workload's command-execution/shell +// channel. Unlike credentials, the auth key IS accepted here — it is delivered to +// the controller as a secret and stamped into the (base64) user-data. +func NewSDKClient(_ context.Context, regionSource RegionSource, sandd provider.SanddConfig) (*Provider, error) { cat, err := catalog.Load() if err != nil { return nil, fmt.Errorf("aws: load price catalog: %w", err) @@ -213,7 +221,7 @@ func NewSDKClient(_ context.Context, regionSource RegionSource) (*Provider, erro factory := func(ctx context.Context, region string) (Client, error) { return newSDKClientForRegion(ctx, region) } - return New(factory, cat, regionSource), nil + return New(factory, cat, regionSource, sandd), nil } // newSDKClientForRegion builds one region-pinned sdkClient: it loads SDK config for diff --git a/pkg/provider/aws/client_test.go b/pkg/provider/aws/client_test.go index 544a161..88568de 100644 --- a/pkg/provider/aws/client_test.go +++ b/pkg/provider/aws/client_test.go @@ -347,6 +347,143 @@ func TestBuildUserData_QuotesHostileValues(t *testing.T) { } } +// TestBuildUserData_SanddDisabled: the zero SanddConfig is the default, so a spec +// that does not opt in must render EXACTLY the plain bootstrap — no sandd/tunnel +// tokens leak in, so existing clusters are unaffected. +func TestBuildUserData_SanddDisabled(t *testing.T) { + encoded, err := buildUserData(InstanceSpec{Image: "img"}) + if err != nil { + t.Fatalf("buildUserData: %v", err) + } + raw, _ := base64.StdEncoding.DecodeString(encoded) + script := string(raw) + if strings.Contains(script, "sandd") || strings.Contains(script, "tunnel") { + t.Fatalf("sandd disabled but bootstrap injected it:\n%s", script) + } +} + +// TestBuildUserData_SanddEnabled: when opted in, sandd runs INSIDE the container via +// an entrypoint shim so its shells see the user's env/cwd/code. The shim must +// override the image ENTRYPOINT with /bin/sh, carry daemon config as container env +// (incl. the claim-derived DAEMON_ID), start the daemon fail-open + backgrounded, +// and exec the user's effective command so the workload becomes PID 1. +func TestBuildUserData_SanddEnabled(t *testing.T) { + spec := InstanceSpec{ + Image: "img", + Command: []string{"python", "train.py"}, + Args: []string{"--epochs", "3"}, + Tags: map[string]string{ClaimTagKey: "claim-abc"}, + Sandd: provider.SanddConfig{ + AuthKey: "tskey-secret", + ControlServer: "http://headscale:8080", + ServerURL: "ws://100.64.0.1:8765/ws", + }, + } + encoded, err := buildUserData(spec) + if err != nil { + t.Fatalf("buildUserData: %v", err) + } + raw, _ := base64.StdEncoding.DecodeString(encoded) + script := string(raw) + + // The shim commandeers the container ENTRYPOINT as /bin/sh. + if !strings.Contains(script, "--entrypoint '/bin/sh'") { + t.Fatalf("sandd shim must override the image entrypoint with /bin/sh:\n%s", script) + } + + // Daemon config is delivered as container env (so a process INSIDE the container + // sees it), including the claim-derived DAEMON_ID and the tunnel key/server. The + // binary URLs are NOT container env anymore — the host fetches them (see below). + for _, want := range []string{ + "-e 'DAEMON_ID=claim-abc'", + "-e 'SERVER_URL=ws://100.64.0.1:8765/ws'", + "-e 'SANDD_TUNNEL_AUTHKEY=tskey-secret'", + "-e 'SANDD_TUNNEL_SERVER=http://headscale:8080'", + } { + if !strings.Contains(script, want) { + t.Fatalf("sandd shim missing container env %q:\n%s", want, script) + } + } + + // The binaries are fetched on the HOST (which has curl+tar) before docker run, so + // the user image needs no fetcher/package manager. Both URLs must appear in the + // host-side fetch, targeting sanddHostDir. + for _, want := range []string{sanddBinaryURL, tailscaleTarballURL, "mkdir -p " + sanddHostDir} { + if !strings.Contains(script, want) { + t.Fatalf("host fetch must download %q into %s:\n%s", want, sanddHostDir, script) + } + } + // The host fetch must precede docker run (binaries exist before the container mounts them). + if strings.Index(script, sanddBinaryURL) > strings.Index(script, "docker run") { + t.Fatalf("host fetch must run BEFORE docker run:\n%s", script) + } + // The binaries reach the container via a read-only bind mount, not an in-container fetch. + if !strings.Contains(script, "-v '"+sanddHostDir+":"+sanddHostDir+":ro'") { + t.Fatalf("sandd binaries must be bind-mounted read-only at %s:\n%s", sanddHostDir, script) + } + if !strings.Contains(script, `export PATH="`+sanddHostDir) { + t.Fatalf("shim must put the mounted binaries on PATH:\n%s", script) + } + // No in-container fetch/install: the image is not required to have curl/wget or a + // package manager. Guard against regressing to the old per-container-install model. + for _, absent := range []string{"apt-get", "apk add", "SANDD_BINARY_URL", "curl -fsSL \"$SANDD_BINARY_URL\""} { + if strings.Contains(script, absent) { + t.Fatalf("shim must NOT fetch/install inside the container (found %q):\n%s", absent, script) + } + } + + // Bring-up must be observable (not silent): steps log with a [sandd] prefix to + // stderr so they reach the EC2 instance console for debugging. + if !strings.Contains(script, "[sandd]") { + t.Fatalf("sandd bring-up must log steps with a [sandd] prefix:\n%s", script) + } + + // Fail-open + backgrounded: both the host fetch and the container bring-up are + // `( set +e ... ) || true` subshells; the daemon start ends in `&`. + if !strings.Contains(script, "set +e") || !strings.Contains(script, ") || true") { + t.Fatalf("sandd bring-up must be fail-open (set +e + || true):\n%s", script) + } + if !strings.Contains(script, "&\n") { + t.Fatalf("sandd daemon must be backgrounded:\n%s", script) + } + + // The shim execs the user's effective command so the workload is PID 1, and that + // command (image then argv) is rendered after the shim payload in kubelet order: + // Command[0]+Command[1:] then Args. + if !strings.Contains(script, `exec "$@"`) { + t.Fatalf("shim must exec the user command as PID 1:\n%s", script) + } + if !strings.Contains(script, "'img' '-c'") { + t.Fatalf("shim payload must follow the image on the run line:\n%s", script) + } + if !strings.Contains(script, "'sandd-shim' 'python' 'train.py' '--epochs' '3'\n") { + t.Fatalf("user effective command not rendered after the shim in order:\n%s", script) + } +} + +// TestBuildUserData_SanddQuotesHostileValues: sandd config values are shell-quoted +// like every other injected value, so a hostile auth key/URL cannot break out. +func TestBuildUserData_SanddQuotesHostileValues(t *testing.T) { + spec := InstanceSpec{ + Image: "img", + Tags: map[string]string{ClaimTagKey: "c"}, + Sandd: provider.SanddConfig{ + AuthKey: "k'; rm -rf /; echo '", + ControlServer: "http://h:8080", + ServerURL: "ws://s:8765/ws", + }, + } + encoded, err := buildUserData(spec) + if err != nil { + t.Fatalf("buildUserData: %v", err) + } + raw, _ := base64.StdEncoding.DecodeString(encoded) + script := string(raw) + if strings.Contains(script, "\nrm -rf /") { + t.Fatalf("hostile sandd value escaped quoting:\n%s", script) + } +} + func TestClassifyEC2Error(t *testing.T) { tests := []struct { name string diff --git a/pkg/provider/aws/sandd_test.go b/pkg/provider/aws/sandd_test.go new file mode 100644 index 0000000..c3210d9 --- /dev/null +++ b/pkg/provider/aws/sandd_test.go @@ -0,0 +1,146 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aws + +import ( + "context" + "errors" + "testing" + + "github.com/InftyAI/Nebula/pkg/provider" +) + +// fakeMinter is a provider.DaemonKeyMinter that returns a canned key (or error) and +// counts calls, so tests can assert a FRESH key is minted per provision. +type fakeMinter struct { + key string + err error + call int +} + +func (m *fakeMinter) MintDaemonKey(_ context.Context) (string, error) { + m.call++ + if m.err != nil { + return "", m.err + } + return m.key, nil +} + +// TestProvision_MintsPerDaemonKey: with a KeyMinter wired, each Provision mints a +// fresh key and that minted key (not the static one) is what lands on the spec. +func TestProvision_MintsPerDaemonKey(t *testing.T) { + f := &fakeClient{runID: "i-mint"} + p := newTestProvider(f) + minter := &fakeMinter{key: "tskey-fresh-123"} + p.sandd = provider.SanddConfig{ + AuthKey: "tskey-STATIC-should-not-be-used", + ControlServer: "http://headscale:8080", + ServerURL: "ws://ctrl.nebula.mesh:8765/ws", + KeyMinter: minter, + } + + if _, err := p.Provision(context.Background(), gpuPod("H100", 8), provider.ProvisionRequest{ + ClaimName: "claim-mint", + Region: testRegion, + }); err != nil { + t.Fatalf("Provision: %v", err) + } + + if minter.call != 1 { + t.Fatalf("minter called %d times, want 1", minter.call) + } + // The minted key, not the static AuthKey, must be baked onto the spec. + if got := f.lastSpec.Sandd.AuthKey; got != "tskey-fresh-123" { + t.Fatalf("spec AuthKey = %q, want the minted tskey-fresh-123", got) + } + // The minter seam must NOT ride onto the spec — only the resolved key travels. + if f.lastSpec.Sandd.KeyMinter != nil { + t.Fatalf("spec KeyMinter should be nil after resolution") + } + // The rest of the SandD config rides through unchanged. + if f.lastSpec.Sandd.ControlServer != "http://headscale:8080" { + t.Fatalf("ControlServer = %q", f.lastSpec.Sandd.ControlServer) + } +} + +// TestProvision_SeparateProvisionsMintSeparateKeys: two DISTINCT workloads => two +// mints, so each gets its OWN single-use credential (the isolation guarantee). +// Distinct claim names avoid the idempotency short-circuit (which would return the +// existing instance without launching or minting again). +func TestProvision_SeparateProvisionsMintSeparateKeys(t *testing.T) { + f := &fakeClient{runID: "i-x"} + p := newTestProvider(f) + minter := &fakeMinter{key: "tskey-k"} + p.sandd = provider.SanddConfig{ControlServer: "http://h:8080", KeyMinter: minter} + + for _, claim := range []string{"claim-1", "claim-2"} { + if _, err := p.Provision(context.Background(), gpuPod("H100", 8), provider.ProvisionRequest{ + ClaimName: claim, + Region: testRegion, + }); err != nil { + t.Fatalf("Provision %s: %v", claim, err) + } + } + if minter.call != 2 { + t.Fatalf("minter called %d times, want 2 (one fresh key per workload)", minter.call) + } +} + +// TestProvision_MintErrorAbortsProvision: a mint failure must abort provisioning +// (no instance launched), not silently inject a keyless daemon. +func TestProvision_MintErrorAbortsProvision(t *testing.T) { + f := &fakeClient{runID: "i-nope"} + p := newTestProvider(f) + p.sandd = provider.SanddConfig{ + ControlServer: "http://h:8080", + KeyMinter: &fakeMinter{err: errors.New("broker down")}, + } + + _, err := p.Provision(context.Background(), gpuPod("H100", 8), provider.ProvisionRequest{ + ClaimName: "claim-err", + Region: testRegion, + }) + if err == nil { + t.Fatalf("Provision succeeded, want mint error") + } + if f.runCnt != 0 { + t.Fatalf("RunInstance called %d times, want 0 (provision must abort before launch)", f.runCnt) + } +} + +// TestProvision_NoMinterInjectsNothing: with no KeyMinter, SandD is off — nothing is +// minted and no auth key lands on the spec (the daemon is not injected). +func TestProvision_NoMinterInjectsNothing(t *testing.T) { + f := &fakeClient{runID: "i-off"} + p := newTestProvider(f) + p.sandd = provider.SanddConfig{ + ControlServer: "http://h:8080", + } + + if _, err := p.Provision(context.Background(), gpuPod("H100", 8), provider.ProvisionRequest{ + ClaimName: "claim-off", + Region: testRegion, + }); err != nil { + t.Fatalf("Provision: %v", err) + } + if f.lastSpec.Sandd.Enabled() { + t.Fatalf("SandD should be disabled with no KeyMinter, got enabled spec %+v", f.lastSpec.Sandd) + } + if got := f.lastSpec.Sandd.AuthKey; got != "" { + t.Fatalf("spec AuthKey = %q, want empty (no key minted)", got) + } +} diff --git a/pkg/provider/aws/translate.go b/pkg/provider/aws/translate.go index ccb0444..8f54d53 100644 --- a/pkg/provider/aws/translate.go +++ b/pkg/provider/aws/translate.go @@ -52,9 +52,18 @@ func buildUserData(spec InstanceSpec) (string, error) { var b strings.Builder b.WriteString("#!/bin/bash\n") b.WriteString("set -euo pipefail\n") + // Pull first so an image error surfaces before we try to run. fmt.Fprintf(&b, "docker pull %s\n", shellQuote(spec.Image)) + // SandD (opt-in): fetch the daemon + Tailscale binaries on the HOST now, before + // the container starts. They are bind-mounted read-only into the container by + // writeSanddEntrypoint, so the user's image needs no fetcher. Fail-open: a failed + // download just leaves the mount empty and the shim skips the daemon. + if spec.Sandd.Enabled() { + fmt.Fprintf(&b, sanddHostFetchScript, sanddHostDir, sanddBinaryURL, tailscaleTarballURL) + } + // docker run --rm --gpus all, with env, then the image, then the workload's // command/args. Kubernetes container semantics are preserved by mapping the two // Pod fields the way the kubelet does, NOT by concatenating them: @@ -70,14 +79,27 @@ func buildUserData(spec InstanceSpec) (string, error) { for _, k := range sortedKeys(spec.Env) { fmt.Fprintf(&b, " -e %s", shellQuote(k+"="+spec.Env[k])) } - // runArgs are everything that follows the image: the entrypoint's own arguments - // (Command[1:]) then the CMD arguments (Args), in that order. + + // SandD daemon (opt-in): run it INSIDE the container so its shells/commands see + // the user's own env, cwd, filesystem and code (a host-side daemon would only see + // the host). We cannot assume the user's image has sandd, so a shell entrypoint + // shim fetches the static musl binary at boot, starts it backgrounded, then execs + // the user's real program — see writeSanddEntrypoint. This overrides the image's + // ENTRYPOINT with the shim, so the shim (not Docker) is responsible for launching + // the workload with the right Command/Args; runArgs is left empty in that case. var runArgs []string - if len(spec.Command) > 0 { - fmt.Fprintf(&b, " --entrypoint %s", shellQuote(spec.Command[0])) - runArgs = append(runArgs, spec.Command[1:]...) + if spec.Sandd.Enabled() { + runArgs = writeSanddEntrypoint(&b, spec) + } else { + // No shim: map Command/Args straight onto Docker's --entrypoint/CMD as before. + // runArgs are everything that follows the image: the entrypoint's own arguments + // (Command[1:]) then the CMD arguments (Args), in that order. + if len(spec.Command) > 0 { + fmt.Fprintf(&b, " --entrypoint %s", shellQuote(spec.Command[0])) + runArgs = append(runArgs, spec.Command[1:]...) + } + runArgs = append(runArgs, spec.Args...) } - runArgs = append(runArgs, spec.Args...) b.WriteString(" " + shellQuote(spec.Image)) for _, arg := range runArgs { @@ -88,6 +110,132 @@ func buildUserData(spec InstanceSpec) (string, error) { return base64.StdEncoding.EncodeToString([]byte(b.String())), nil } +// sanddBinaryURL is the statically-linked (musl) SandD daemon release asset. Being +// static, this one binary runs in any container image regardless of its libc, so the +// shim can fetch it into an arbitrary user image at boot. It is pinned to the same +// asset name install.sh resolves; amd64 matches the x86_64 GPU instance types. +const sanddBinaryURL = "https://github.com/InftyAI/SandD/releases/latest/download/sandd-linux-amd64" + +// tailscaleTarballURL is the static (no-libc) Tailscale bundle. sandd --tunnel does +// NOT install Tailscale — it requires `tailscale`/`tailscaled` already on PATH (see +// setup_tunnel in sandd) — so the shim must fetch them too. The static build runs in +// any image, and tailscaled's userspace-networking mode (below) needs no TUN device +// or NET_ADMIN, so this works in an unprivileged container. Pinned: unlike sandd, a +// tarball has no /latest/ redirect, and pinning the mesh client is prudent anyway. +const tailscaleTarballURL = "https://pkgs.tailscale.com/stable/tailscale_1.78.1_amd64.tgz" + +// sanddHostDir is where the host fetches the sandd + tailscale binaries and where +// they are bind-mounted (read-only) into the workload container. Fetching on the +// HOST — the AL2 GPU AMI, which has curl+tar — instead of inside the container means +// the user's image needs no fetcher or package manager (works on distroless too), +// and the download happens once per instance rather than per container start. +const sanddHostDir = "/opt/sandd" + +// sanddHostFetchScript downloads the static sandd + Tailscale binaries into +// sanddHostDir on the HOST, as a fragment of the user-data run BEFORE `docker run`. +// It is a fmt format string: %[1]s=dir, %[2]s=sandd URL, %[3]s=tailscale URL (all +// trusted package constants, so no shell-injection concern). +// +// FAIL-OPEN: wrapped in `( set +e ... ) || true` so a failed download never aborts +// the (set -euo pipefail) user-data — the container-side shim then simply finds no +// binary and skips the daemon, leaving the workload untouched. Every step logs to +// stderr with a `[sandd]` prefix, which reaches the instance CONSOLE +// (get-console-output) — the place to debug bring-up on a Nebula virtual node, since +// kubectl logs/exec do not work against it. +const sanddHostFetchScript = `( set +e + log() { echo "[sandd] $*" >&2; } + mkdir -p %[1]s + if curl -fsSL %[2]s -o %[1]s/sandd; then chmod +x %[1]s/sandd; log "fetched sandd binary" + else log "FAILED to fetch sandd binary from %[2]s — SandD disabled"; fi + if curl -fsSL %[3]s -o %[1]s/ts.tgz && tar -xzf %[1]s/ts.tgz -C %[1]s --strip-components=1; then + log "fetched tailscale bundle" + else log "FAILED to fetch/extract tailscale bundle from %[3]s — SandD disabled"; fi +) || true +` + +// sanddShimScript is the container ENTRYPOINT shim (run as `/bin/sh -c