From 785eb4f8d91a9b565895b7fbbca81e9d45bac59d Mon Sep 17 00:00:00 2001 From: kerthcet Date: Fri, 31 Jul 2026 17:08:39 +0100 Subject: [PATCH 01/10] change jitter Signed-off-by: kerthcet --- api/v1alpha1/nodepool_types.go | 6 +++--- config/crd/bases/nebula.inftyai.com_nodepools.yaml | 6 +++--- config/samples/nebula_v1alpha1_nodepool.yaml | 2 +- pkg/vnode/handler.go | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/api/v1alpha1/nodepool_types.go b/api/v1alpha1/nodepool_types.go index 779de1b..a954007 100644 --- a/api/v1alpha1/nodepool_types.go +++ b/api/v1alpha1/nodepool_types.go @@ -136,9 +136,9 @@ const ( type FailoverPolicy struct { // BlocklistTTL is the BASE duration a failed placement is excluded before the // provider becomes a candidate for it again. The controller adds a random jitter - // (up to a minute) on top so Pods that failed for the same reason do not all - // retry the just-freed candidate in lockstep, so the effective exclusion is this - // value plus that jitter. + // (up to 30s) on top so Pods that failed for the same reason do not all retry the + // just-freed candidate in lockstep, so the effective exclusion is this value plus + // that jitter. // +kubebuilder:default="30s" BlocklistTTL metav1.Duration `json:"blocklistTTL,omitempty"` } diff --git a/config/crd/bases/nebula.inftyai.com_nodepools.yaml b/config/crd/bases/nebula.inftyai.com_nodepools.yaml index 61e03c6..ddfa71f 100644 --- a/config/crd/bases/nebula.inftyai.com_nodepools.yaml +++ b/config/crd/bases/nebula.inftyai.com_nodepools.yaml @@ -102,9 +102,9 @@ spec: description: |- BlocklistTTL is the BASE duration a failed placement is excluded before the provider becomes a candidate for it again. The controller adds a random jitter - (up to a minute) on top so Pods that failed for the same reason do not all - retry the just-freed candidate in lockstep, so the effective exclusion is this - value plus that jitter. + (up to 30s) on top so Pods that failed for the same reason do not all retry the + just-freed candidate in lockstep, so the effective exclusion is this value plus + that jitter. type: string type: object providers: diff --git a/config/samples/nebula_v1alpha1_nodepool.yaml b/config/samples/nebula_v1alpha1_nodepool.yaml index 7b2c852..8295ac8 100644 --- a/config/samples/nebula_v1alpha1_nodepool.yaml +++ b/config/samples/nebula_v1alpha1_nodepool.yaml @@ -26,4 +26,4 @@ spec: # Inner axis: within the active capacity tier. strategy: Ordered failover: - blocklistTTL: 3m + blocklistTTL: 30s diff --git a/pkg/vnode/handler.go b/pkg/vnode/handler.go index 22fda2d..fab15e5 100644 --- a/pkg/vnode/handler.go +++ b/pkg/vnode/handler.go @@ -67,7 +67,7 @@ const defaultBlocklistTTL = 30 * time.Second // would otherwise carry an identical deadline for all of them. A per-record jitter // in [0, blocklistJitter) decorrelates those wake-ups so freed capacity is sampled // across a spread of moments instead of one thundering retry. -const blocklistJitter = 1 * time.Minute +const blocklistJitter = 30 * time.Second // Blocklister records a failed placement so the placement controller can fail over // to the next candidate instead of hot-looping against a provider that just From df0f929925e799fcf3c3b407b05b34b93f65a097 Mon Sep 17 00:00:00 2001 From: kerthcet Date: Fri, 31 Jul 2026 20:59:20 +0100 Subject: [PATCH 02/10] Integration with SandD Signed-off-by: kerthcet --- cmd/main.go | 12 +++- config/samples/deployment.yaml | 6 +- pkg/provider/aws/aws.go | 19 +++++- pkg/provider/aws/client.go | 12 +++- pkg/provider/aws/client_test.go | 106 ++++++++++++++++++++++++++++++++ pkg/provider/aws/translate.go | 57 +++++++++++++++++ pkg/provider/provider.go | 38 ++++++++++++ 7 files changed, 243 insertions(+), 7 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index d70e105..06f0d36 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -351,7 +351,17 @@ func registerProviders(ctx context.Context, c client.Client) { // delivered via a Secret), and one account-global credential authorizes every // region. Registration only fails (and is a non-fatal skip) if the price catalog // cannot load — region config can no longer make it fail. - if p, err := awsprovider.NewSDKClient(ctx, awsRegionSource(c)); err != nil { + // SandD daemon (opt-in): if SANDD_TUNNEL_AUTHKEY is set, every AWS instance also + // boots the SandD sandbox daemon in tunnel mode so commands and interactive + // shells can be run on the box over the mesh with no inbound access. Absent the + // authkey the zero SanddConfig injects nothing, so this is off by default. The + // authkey is a secret (delivered via a mounted Secret) and is NEVER logged. + sanddCfg := provider.SanddConfig{ + AuthKey: os.Getenv("SANDD_TUNNEL_AUTHKEY"), + ControlServer: os.Getenv("SANDD_TUNNEL_SERVER"), + ServerURL: os.Getenv("SANDD_SERVER_URL"), + } + if p, err := awsprovider.NewSDKClient(ctx, awsRegionSource(c), sanddCfg); err != nil { setupLog.Info("skipping AWS provider registration", "reason", err.Error()) } else { provider.Register(p) diff --git a/config/samples/deployment.yaml b/config/samples/deployment.yaml index f45239b..f3e62cd 100644 --- a/config/samples/deployment.yaml +++ b/config/samples/deployment.yaml @@ -28,7 +28,7 @@ metadata: labels: app.kubernetes.io/managed-by: nebula spec: - replicas: 8 + replicas: 2 selector: matchLabels: app: gpu-workload-sample @@ -38,7 +38,7 @@ spec: app: gpu-workload-sample nebula.inftyai.com/enabled: "true" nebula.inftyai.com/nodepool: sample - nebula.inftyai.com/accelerator-type: a100-40gb + nebula.inftyai.com/accelerator-type: t4 spec: # Do NOT set nodeName or a provider nodeSelector yourself — the placement # controller fills the nodeSelector in when it ungates the Pod. Setting @@ -65,4 +65,4 @@ spec: # GPU count. Standard extended resource, so the scheduler's fit check # and provisioning read the same number. 8 => 8x the accelerator-type # above. - nvidia.com/gpu: "8" + nvidia.com/gpu: "1" diff --git a/pkg/provider/aws/aws.go b/pkg/provider/aws/aws.go index c061ec4..682ab2f 100644 --- a/pkg/provider/aws/aws.go +++ b/pkg/provider/aws/aws.go @@ -142,6 +142,11 @@ type InstanceSpec struct { Region string // Tags carry Nebula identity; ClaimTagKey holds the NodeClaim name. Tags map[string]string + // Sandd, when Enabled, makes the cloud-init also install and start the SandD + // sandbox daemon on the host (backgrounded, before the workload container) so + // commands and interactive shells can be run on the box over the tunnel. A zero + // value injects nothing. See buildUserData and provider.SanddConfig. + Sandd provider.SanddConfig } // EC2Instance is the adapter-level view of one EC2 instance as observed. @@ -211,6 +216,9 @@ type Provider struct { // regionSource reports the NodePool-declared regions to sweep in List/Offerings. // May be nil in tests, in which case sweepRegions uses only the cache keys. regionSource RegionSource + // sandd is the optional SandD daemon config stamped onto every InstanceSpec so + // buildUserData can bootstrap it. Zero value => disabled (no injection). + sandd provider.SanddConfig mu sync.Mutex clients map[string]Client // region -> Client, populated lazily by clientFor @@ -225,11 +233,16 @@ type Provider struct { // (admission requires each aws pool to list ≥1 region, and placement stamps it onto // the ProvisionRequest), and observed instances report their region from the // region-pinned client — so nothing needs a fallback, and no AWS_REGION env is read. -func New(newClient ClientFactory, cat catalog.Lookup, regionSource RegionSource) *Provider { +// +// sandd is the optional SandD daemon config baked into every instance's user-data; +// its zero value disables injection, so tests and the non-SandD path pass +// provider.SanddConfig{}. +func New(newClient ClientFactory, cat catalog.Lookup, regionSource RegionSource, sandd provider.SanddConfig) *Provider { return &Provider{ Base: catalog.Base{ProviderName: provider.ProviderAWS, Catalog: cat}, newClient: newClient, regionSource: regionSource, + sandd: sandd, clients: make(map[string]Client), } } @@ -244,6 +257,7 @@ func newSingleRegion(client Client, cat catalog.Lookup, region string) *Provider func(context.Context, string) (Client, error) { return client, nil }, cat, func() []string { return []string{region} }, + provider.SanddConfig{}, // single-region test convenience: no SandD injection ) // Pre-seed the cache so even a stray region lookup returns the fake rather than // invoking the (constant) factory. @@ -714,6 +728,9 @@ func (p *Provider) instanceSpecFromPod(pod *corev1.Pod, req provider.ProvisionRe Spot: req.CapacityType == nebulav1alpha1.CapacitySpot, Region: req.Region, Tags: map[string]string{ClaimTagKey: req.ClaimName}, + // SandD daemon (opt-in): keyed by the claim name so a daemon that dials home + // correlates 1:1 with this instance's NodeClaim. Zero value => no injection. + Sandd: p.sandd, }, nil } diff --git a/pkg/provider/aws/client.go b/pkg/provider/aws/client.go index 0ee0629..3685e5c 100644 --- a/pkg/provider/aws/client.go +++ b/pkg/provider/aws/client.go @@ -30,6 +30,7 @@ import ( smithy "github.com/aws/smithy-go" logf "sigs.k8s.io/controller-runtime/pkg/log" + "github.com/InftyAI/Nebula/pkg/provider" "github.com/InftyAI/Nebula/pkg/provider/catalog" ) @@ -204,7 +205,14 @@ var _ Client = (*sdkClient)(nil) // // The catalog is loaded via catalog.Load() (embedded CSV / mounted ConfigMap), // identical to the other adapters. -func NewSDKClient(_ context.Context, regionSource RegionSource) (*Provider, error) { +// +// sandd is the OPTIONAL SandD daemon config: its zero value (empty AuthKey) leaves +// the bootstrap untouched, so passing provider.SanddConfig{} is a no-op. When set, +// every instance this provider launches also starts the SandD sandbox daemon in +// tunnel mode (see buildUserData), the box's command-execution/shell channel. Unlike +// credentials, the auth key IS accepted here — it is delivered to the controller as +// a secret and stamped into the (base64) user-data. +func NewSDKClient(_ context.Context, regionSource RegionSource, sandd provider.SanddConfig) (*Provider, error) { cat, err := catalog.Load() if err != nil { return nil, fmt.Errorf("aws: load price catalog: %w", err) @@ -213,7 +221,7 @@ func NewSDKClient(_ context.Context, regionSource RegionSource) (*Provider, erro factory := func(ctx context.Context, region string) (Client, error) { return newSDKClientForRegion(ctx, region) } - return New(factory, cat, regionSource), nil + return New(factory, cat, regionSource, sandd), nil } // newSDKClientForRegion builds one region-pinned sdkClient: it loads SDK config for diff --git a/pkg/provider/aws/client_test.go b/pkg/provider/aws/client_test.go index 544a161..dd8f2ee 100644 --- a/pkg/provider/aws/client_test.go +++ b/pkg/provider/aws/client_test.go @@ -347,6 +347,112 @@ func TestBuildUserData_QuotesHostileValues(t *testing.T) { } } +// TestBuildUserData_SanddDisabled: the zero SanddConfig is the default, so a spec +// that does not opt in must render EXACTLY the plain bootstrap — no sandd/tunnel +// tokens leak in, so existing clusters are unaffected. +func TestBuildUserData_SanddDisabled(t *testing.T) { + encoded, err := buildUserData(InstanceSpec{Image: "img"}) + if err != nil { + t.Fatalf("buildUserData: %v", err) + } + raw, _ := base64.StdEncoding.DecodeString(encoded) + script := string(raw) + if strings.Contains(script, "sandd") || strings.Contains(script, "tunnel") { + t.Fatalf("sandd disabled but bootstrap injected it:\n%s", script) + } +} + +// TestBuildUserData_SanddEnabled: when opted in, the daemon must be started BEFORE +// the workload `docker run`, be BACKGROUNDED (so cloud-init reaches the workload), +// and be FAIL-OPEN (its block cannot trip the workload's `set -euo pipefail`). It +// must carry the tunnel flags and the claim-derived daemon id. The daemon runs +// alongside the workload (it is the box's command/shell channel), so its bootstrap +// must not be able to abort the workload. +func TestBuildUserData_SanddEnabled(t *testing.T) { + spec := InstanceSpec{ + Image: "img", + Tags: map[string]string{ClaimTagKey: "claim-abc"}, + Sandd: provider.SanddConfig{ + AuthKey: "tskey-secret", + ControlServer: "http://headscale:8080", + ServerURL: "ws://100.64.0.1:8765/ws", + }, + } + encoded, err := buildUserData(spec) + if err != nil { + t.Fatalf("buildUserData: %v", err) + } + raw, _ := base64.StdEncoding.DecodeString(encoded) + script := string(raw) + + // Ordering: the sandd bootstrap precedes the workload pull/run — a daemon started + // after the foreground `docker run` would only come up once the workload exited. + sanddIdx := strings.Index(script, "setsid sandd") + runIdx := strings.Index(script, "docker run") + if sanddIdx < 0 { + t.Fatalf("sandd enabled but daemon not started:\n%s", script) + } + if runIdx < 0 || sanddIdx > runIdx { + t.Fatalf("sandd must start before the workload docker run (sandd=%d run=%d):\n%s", sanddIdx, runIdx, script) + } + + // Fail-open: the block is a `( set +e ... ) || true` subshell so a failed install + // cannot abort the workload under the script's pipefail. + if !strings.Contains(script, "set +e") || !strings.Contains(script, ") || true\n") { + t.Fatalf("sandd block must be fail-open (set +e + || true):\n%s", script) + } + + // Backgrounded: the daemon must be launched with a trailing `&` so cloud-init + // proceeds to the workload rather than blocking on the long-lived daemon. + if !strings.Contains(script, ">/var/log/sandd.log 2>&1 &\n") { + t.Fatalf("sandd daemon must be backgrounded:\n%s", script) + } + + // Tunnel flags + claim-derived id are present. + for _, want := range []string{ + "--tunnel", + "--server-url 'ws://100.64.0.1:8765/ws'", + "--tunnel-server 'http://headscale:8080'", + "--daemon-id 'claim-abc'", + } { + if !strings.Contains(script, want) { + t.Fatalf("sandd bootstrap missing %q:\n%s", want, script) + } + } + + // The auth key must NOT appear on the daemon command line (it would show in `ps`); + // it is stashed in a root-only file and read back via $(cat ...). + if strings.Contains(script, "--tunnel-authkey 'tskey-secret'") { + t.Fatalf("auth key must not be passed on the command line:\n%s", script) + } + if !strings.Contains(script, "/etc/sandd.authkey") { + t.Fatalf("auth key must be stashed in a root-only file:\n%s", script) + } +} + +// TestBuildUserData_SanddQuotesHostileValues: sandd config values are shell-quoted +// like every other injected value, so a hostile auth key/URL cannot break out. +func TestBuildUserData_SanddQuotesHostileValues(t *testing.T) { + spec := InstanceSpec{ + Image: "img", + Tags: map[string]string{ClaimTagKey: "c"}, + Sandd: provider.SanddConfig{ + AuthKey: "k'; rm -rf /; echo '", + ControlServer: "http://h:8080", + ServerURL: "ws://s:8765/ws", + }, + } + encoded, err := buildUserData(spec) + if err != nil { + t.Fatalf("buildUserData: %v", err) + } + raw, _ := base64.StdEncoding.DecodeString(encoded) + script := string(raw) + if strings.Contains(script, "\nrm -rf /") { + t.Fatalf("hostile sandd value escaped quoting:\n%s", script) + } +} + func TestClassifyEC2Error(t *testing.T) { tests := []struct { name string diff --git a/pkg/provider/aws/translate.go b/pkg/provider/aws/translate.go index ccb0444..b1b85ea 100644 --- a/pkg/provider/aws/translate.go +++ b/pkg/provider/aws/translate.go @@ -52,6 +52,17 @@ func buildUserData(spec InstanceSpec) (string, error) { var b strings.Builder b.WriteString("#!/bin/bash\n") b.WriteString("set -euo pipefail\n") + + // SandD daemon (opt-in): install and start it on the HOST, BEFORE the workload, + // so commands and interactive shells can be run on the box while the container + // runs. It goes first because the workload's `docker run` is a foreground process + // the script blocks on until it exits (that is by design — cloud-init "finishes" + // when the workload does); a daemon started after it would only come up once the + // workload was already gone. See writeSanddBootstrap for the fail-open guarantee. + if spec.Sandd.Enabled() { + writeSanddBootstrap(&b, spec.Sandd, spec.Tags[ClaimTagKey]) + } + // Pull first so an image error surfaces before we try to run. fmt.Fprintf(&b, "docker pull %s\n", shellQuote(spec.Image)) @@ -88,6 +99,52 @@ func buildUserData(spec InstanceSpec) (string, error) { return base64.StdEncoding.EncodeToString([]byte(b.String())), nil } +// sanddInstallURL is the SandD daemon install script. The bootstrap pipes it to +// bash with --tunnel so Tailscale is installed alongside the daemon (see the +// SandD install.sh --tunnel path). +const sanddInstallURL = "https://raw.githubusercontent.com/InftyAI/SandD/main/hack/scripts/install.sh" + +// writeSanddBootstrap appends the SandD sandbox-daemon bootstrap to the user-data +// script. daemonID is the NodeClaim name (from ClaimTagKey), so a daemon that dials +// home is identifiable as this exact instance. +// +// Two properties are load-bearing: +// - FAIL-OPEN: the workload runs under `set -euo pipefail`, and the daemon runs +// alongside it (it does not run the workload), so a failure to install or start +// the daemon must never abort the workload. The whole block runs inside a +// `( set +e ; ... ) || true` subshell so a failed install/curl/tailscale step +// cannot trip pipefail and kill the instance before `docker run`. +// - BACKGROUNDED & DETACHED: the daemon is a long-running process, so it is run +// with `setsid ... &` and its output redirected to a log file. If it blocked in +// the foreground, cloud-init would never reach `docker run` and the workload +// would never start. +// +// The auth key is written to a root-only file rather than passed on the command +// line so it does not linger in the process table (`ps`) for any workload the +// container can see. +func writeSanddBootstrap(b *strings.Builder, cfg provider.SanddConfig, daemonID string) { + b.WriteString("# --- SandD daemon (opt-in; fail-open, never aborts the workload) ---\n") + b.WriteString("(\n") + b.WriteString(" set +e\n") + // Install the daemon + Tailscale. Piped to bash; the subshell's `set +e` means a + // non-zero exit here is swallowed rather than tripping the outer pipefail. + fmt.Fprintf(b, " curl -fsSL %s | bash -s -- --tunnel\n", shellQuote(sanddInstallURL)) + // Keep the auth key off the command line: stash it in a root-only file the daemon + // reads via env, so it is not visible in `ps`. + b.WriteString(" install -m 600 /dev/null /etc/sandd.authkey\n") + fmt.Fprintf(b, " printf '%%s' %s > /etc/sandd.authkey\n", shellQuote(cfg.AuthKey)) + // Launch detached & backgrounded so cloud-init proceeds to the workload. Output + // goes to a log for post-hoc debugging via the daemon itself. + b.WriteString(" setsid sandd \\\n") + fmt.Fprintf(b, " --server-url %s \\\n", shellQuote(cfg.ServerURL)) + fmt.Fprintf(b, " --daemon-id %s \\\n", shellQuote(daemonID)) + b.WriteString(" --tunnel \\\n") + b.WriteString(" --tunnel-authkey \"$(cat /etc/sandd.authkey)\" \\\n") + fmt.Fprintf(b, " --tunnel-server %s \\\n", shellQuote(cfg.ControlServer)) + b.WriteString(" >/var/log/sandd.log 2>&1 &\n") + b.WriteString(") || true\n") +} + // sortedKeys returns m's keys in sorted order, for deterministic rendering. func sortedKeys(m map[string]string) []string { keys := make([]string, 0, len(m)) diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 093f3cb..2c00318 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -297,3 +297,41 @@ type BlockScope struct { // provider; it never spans providers. DenyAll bool } + +// SanddConfig configures the SandD sandbox daemon (see github.com/InftyAI/SandD) +// shared by every provider adapter: when set, a launched instance also runs SandD +// in tunnel mode, giving remote command execution and interactive shell/PTY +// sessions on the box with NO inbound access — the daemon dials OUT to the +// controller over the Tailscale/headscale mesh, which is what makes it work for +// instances in a private VPC with no public IP or open port. It is the access and +// control channel for the machine, not a troubleshooting-only add-on. +// +// It lives here, in the provider-agnostic seam, because it is not AWS-specific: it +// is a property of "let an agent run commands / shell into a Nebula-provisioned +// box" that any adapter can honour. Each adapter injects it in whatever way fits its +// launch model (AWS bakes it into cloud-init as a host daemon alongside the +// workload container; a serverless-sandbox provider would prepend it to the sandbox +// command). AWS is the first to wire it in. +// +// It is OPT-IN: a zero value (empty AuthKey) injects nothing, so the bootstrap is +// unchanged for clusters that do not enable it. AuthKey is a Tailscale/headscale +// auth key — a secret — delivered to the controller as one and never logged. The +// daemon runs ALONGSIDE the workload (it does not run it), so an adapter that +// honours it MUST launch the daemon such that its own failure can never abort the +// workload itself. +type SanddConfig struct { + // AuthKey is the Tailscale/headscale pre-auth key the daemon joins the mesh + // with (sandd --tunnel-authkey). Empty disables SandD injection entirely. + AuthKey string + // ControlServer is the headscale control-plane URL (sandd --tunnel-server), + // e.g. "http://headscale.internal:8080". Required when AuthKey is set. + ControlServer string + // ServerURL is the controller's SandD WebSocket URL reachable OVER the mesh + // (sandd --server-url), e.g. "ws://100.64.0.1:8765/ws". Required when AuthKey + // is set. + ServerURL string +} + +// Enabled reports whether the SandD daemon should be injected. A missing AuthKey +// means the operator did not opt in, so an adapter emits its plain bootstrap. +func (s SanddConfig) Enabled() bool { return s.AuthKey != "" } From 04d8702a662a5dffa5e7cf7f036c0e4a6dbc7ff1 Mon Sep 17 00:00:00 2001 From: kerthcet Date: Sat, 1 Aug 2026 11:51:13 +0100 Subject: [PATCH 03/10] add sandd Signed-off-by: kerthcet --- cmd/main.go | 11 +-- pkg/provider/aws/aws.go | 8 +- pkg/provider/aws/client.go | 8 +- pkg/provider/aws/client_test.go | 80 +++++++++--------- pkg/provider/aws/translate.go | 142 +++++++++++++++++++------------- pkg/provider/provider.go | 11 ++- 6 files changed, 145 insertions(+), 115 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 06f0d36..89e31e1 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -351,11 +351,12 @@ func registerProviders(ctx context.Context, c client.Client) { // delivered via a Secret), and one account-global credential authorizes every // region. Registration only fails (and is a non-fatal skip) if the price catalog // cannot load — region config can no longer make it fail. - // SandD daemon (opt-in): if SANDD_TUNNEL_AUTHKEY is set, every AWS instance also - // boots the SandD sandbox daemon in tunnel mode so commands and interactive - // shells can be run on the box over the mesh with no inbound access. Absent the - // authkey the zero SanddConfig injects nothing, so this is off by default. The - // authkey is a secret (delivered via a mounted Secret) and is NEVER logged. + // SandD daemon (opt-in): if SANDD_TUNNEL_AUTHKEY is set, every AWS workload runs + // the SandD daemon inside its container in tunnel mode, so commands and + // interactive shells can be run in the user's own environment over the mesh with + // no inbound access. Absent the authkey the zero SanddConfig injects nothing, so + // this is off by default. The authkey is a secret (delivered via a mounted + // Secret) and is NEVER logged. sanddCfg := provider.SanddConfig{ AuthKey: os.Getenv("SANDD_TUNNEL_AUTHKEY"), ControlServer: os.Getenv("SANDD_TUNNEL_SERVER"), diff --git a/pkg/provider/aws/aws.go b/pkg/provider/aws/aws.go index 682ab2f..0d46c58 100644 --- a/pkg/provider/aws/aws.go +++ b/pkg/provider/aws/aws.go @@ -142,10 +142,10 @@ type InstanceSpec struct { Region string // Tags carry Nebula identity; ClaimTagKey holds the NodeClaim name. Tags map[string]string - // Sandd, when Enabled, makes the cloud-init also install and start the SandD - // sandbox daemon on the host (backgrounded, before the workload container) so - // commands and interactive shells can be run on the box over the tunnel. A zero - // value injects nothing. See buildUserData and provider.SanddConfig. + // Sandd, when Enabled, makes the workload's docker run start the SandD daemon + // INSIDE the container (via an entrypoint shim) so commands and interactive + // shells run in the user's own env/cwd/code over the tunnel. A zero value injects + // nothing. See buildUserData/writeSanddEntrypoint and provider.SanddConfig. Sandd provider.SanddConfig } diff --git a/pkg/provider/aws/client.go b/pkg/provider/aws/client.go index 3685e5c..4b70865 100644 --- a/pkg/provider/aws/client.go +++ b/pkg/provider/aws/client.go @@ -208,10 +208,10 @@ var _ Client = (*sdkClient)(nil) // // sandd is the OPTIONAL SandD daemon config: its zero value (empty AuthKey) leaves // the bootstrap untouched, so passing provider.SanddConfig{} is a no-op. When set, -// every instance this provider launches also starts the SandD sandbox daemon in -// tunnel mode (see buildUserData), the box's command-execution/shell channel. Unlike -// credentials, the auth key IS accepted here — it is delivered to the controller as -// a secret and stamped into the (base64) user-data. +// every workload this provider launches runs the SandD daemon inside its container +// in tunnel mode (see buildUserData), the workload's command-execution/shell +// channel. Unlike credentials, the auth key IS accepted here — it is delivered to +// the controller as a secret and stamped into the (base64) user-data. func NewSDKClient(_ context.Context, regionSource RegionSource, sandd provider.SanddConfig) (*Provider, error) { cat, err := catalog.Load() if err != nil { diff --git a/pkg/provider/aws/client_test.go b/pkg/provider/aws/client_test.go index dd8f2ee..43c2cdf 100644 --- a/pkg/provider/aws/client_test.go +++ b/pkg/provider/aws/client_test.go @@ -362,16 +362,17 @@ func TestBuildUserData_SanddDisabled(t *testing.T) { } } -// TestBuildUserData_SanddEnabled: when opted in, the daemon must be started BEFORE -// the workload `docker run`, be BACKGROUNDED (so cloud-init reaches the workload), -// and be FAIL-OPEN (its block cannot trip the workload's `set -euo pipefail`). It -// must carry the tunnel flags and the claim-derived daemon id. The daemon runs -// alongside the workload (it is the box's command/shell channel), so its bootstrap -// must not be able to abort the workload. +// TestBuildUserData_SanddEnabled: when opted in, sandd runs INSIDE the container via +// an entrypoint shim so its shells see the user's env/cwd/code. The shim must +// override the image ENTRYPOINT with /bin/sh, carry daemon config as container env +// (incl. the claim-derived DAEMON_ID), start the daemon fail-open + backgrounded, +// and exec the user's effective command so the workload becomes PID 1. func TestBuildUserData_SanddEnabled(t *testing.T) { spec := InstanceSpec{ - Image: "img", - Tags: map[string]string{ClaimTagKey: "claim-abc"}, + Image: "img", + Command: []string{"python", "train.py"}, + Args: []string{"--epochs", "3"}, + Tags: map[string]string{ClaimTagKey: "claim-abc"}, Sandd: provider.SanddConfig{ AuthKey: "tskey-secret", ControlServer: "http://headscale:8080", @@ -385,48 +386,45 @@ func TestBuildUserData_SanddEnabled(t *testing.T) { raw, _ := base64.StdEncoding.DecodeString(encoded) script := string(raw) - // Ordering: the sandd bootstrap precedes the workload pull/run — a daemon started - // after the foreground `docker run` would only come up once the workload exited. - sanddIdx := strings.Index(script, "setsid sandd") - runIdx := strings.Index(script, "docker run") - if sanddIdx < 0 { - t.Fatalf("sandd enabled but daemon not started:\n%s", script) - } - if runIdx < 0 || sanddIdx > runIdx { - t.Fatalf("sandd must start before the workload docker run (sandd=%d run=%d):\n%s", sanddIdx, runIdx, script) - } - - // Fail-open: the block is a `( set +e ... ) || true` subshell so a failed install - // cannot abort the workload under the script's pipefail. - if !strings.Contains(script, "set +e") || !strings.Contains(script, ") || true\n") { - t.Fatalf("sandd block must be fail-open (set +e + || true):\n%s", script) - } - - // Backgrounded: the daemon must be launched with a trailing `&` so cloud-init - // proceeds to the workload rather than blocking on the long-lived daemon. - if !strings.Contains(script, ">/var/log/sandd.log 2>&1 &\n") { - t.Fatalf("sandd daemon must be backgrounded:\n%s", script) + // The shim commandeers the container ENTRYPOINT as /bin/sh. + if !strings.Contains(script, "--entrypoint '/bin/sh'") { + t.Fatalf("sandd shim must override the image entrypoint with /bin/sh:\n%s", script) } - // Tunnel flags + claim-derived id are present. + // Daemon config is delivered as container env (so a process INSIDE the container + // sees it), including the claim-derived DAEMON_ID and the tunnel key/server. for _, want := range []string{ - "--tunnel", - "--server-url 'ws://100.64.0.1:8765/ws'", - "--tunnel-server 'http://headscale:8080'", - "--daemon-id 'claim-abc'", + "-e 'DAEMON_ID=claim-abc'", + "-e 'SERVER_URL=ws://100.64.0.1:8765/ws'", + "-e 'SANDD_TUNNEL_AUTHKEY=tskey-secret'", + "-e 'SANDD_TUNNEL_SERVER=http://headscale:8080'", + "-e 'SANDD_BINARY_URL=" + sanddBinaryURL + "'", } { if !strings.Contains(script, want) { - t.Fatalf("sandd bootstrap missing %q:\n%s", want, script) + t.Fatalf("sandd shim missing container env %q:\n%s", want, script) } } - // The auth key must NOT appear on the daemon command line (it would show in `ps`); - // it is stashed in a root-only file and read back via $(cat ...). - if strings.Contains(script, "--tunnel-authkey 'tskey-secret'") { - t.Fatalf("auth key must not be passed on the command line:\n%s", script) + // Fail-open + backgrounded: the daemon bring-up is a `( set +e ... ) || true` + // subshell ending in `&`, so a failed fetch/start cannot stop the workload. + if !strings.Contains(script, "set +e") || !strings.Contains(script, ") || true") { + t.Fatalf("sandd bring-up must be fail-open (set +e + || true):\n%s", script) + } + if !strings.Contains(script, "&\n") { + t.Fatalf("sandd daemon must be backgrounded:\n%s", script) + } + + // The shim execs the user's effective command so the workload is PID 1, and that + // command (image then argv) is rendered after the shim payload in kubelet order: + // Command[0]+Command[1:] then Args. + if !strings.Contains(script, `exec "$@"`) { + t.Fatalf("shim must exec the user command as PID 1:\n%s", script) + } + if !strings.Contains(script, "'img' '-c'") { + t.Fatalf("shim payload must follow the image on the run line:\n%s", script) } - if !strings.Contains(script, "/etc/sandd.authkey") { - t.Fatalf("auth key must be stashed in a root-only file:\n%s", script) + if !strings.Contains(script, "'sandd-shim' 'python' 'train.py' '--epochs' '3'\n") { + t.Fatalf("user effective command not rendered after the shim in order:\n%s", script) } } diff --git a/pkg/provider/aws/translate.go b/pkg/provider/aws/translate.go index b1b85ea..b843fc7 100644 --- a/pkg/provider/aws/translate.go +++ b/pkg/provider/aws/translate.go @@ -53,16 +53,6 @@ func buildUserData(spec InstanceSpec) (string, error) { b.WriteString("#!/bin/bash\n") b.WriteString("set -euo pipefail\n") - // SandD daemon (opt-in): install and start it on the HOST, BEFORE the workload, - // so commands and interactive shells can be run on the box while the container - // runs. It goes first because the workload's `docker run` is a foreground process - // the script blocks on until it exits (that is by design — cloud-init "finishes" - // when the workload does); a daemon started after it would only come up once the - // workload was already gone. See writeSanddBootstrap for the fail-open guarantee. - if spec.Sandd.Enabled() { - writeSanddBootstrap(&b, spec.Sandd, spec.Tags[ClaimTagKey]) - } - // Pull first so an image error surfaces before we try to run. fmt.Fprintf(&b, "docker pull %s\n", shellQuote(spec.Image)) @@ -81,14 +71,27 @@ func buildUserData(spec InstanceSpec) (string, error) { for _, k := range sortedKeys(spec.Env) { fmt.Fprintf(&b, " -e %s", shellQuote(k+"="+spec.Env[k])) } - // runArgs are everything that follows the image: the entrypoint's own arguments - // (Command[1:]) then the CMD arguments (Args), in that order. + + // SandD daemon (opt-in): run it INSIDE the container so its shells/commands see + // the user's own env, cwd, filesystem and code (a host-side daemon would only see + // the host). We cannot assume the user's image has sandd, so a shell entrypoint + // shim fetches the static musl binary at boot, starts it backgrounded, then execs + // the user's real program — see writeSanddEntrypoint. This overrides the image's + // ENTRYPOINT with the shim, so the shim (not Docker) is responsible for launching + // the workload with the right Command/Args; runArgs is left empty in that case. var runArgs []string - if len(spec.Command) > 0 { - fmt.Fprintf(&b, " --entrypoint %s", shellQuote(spec.Command[0])) - runArgs = append(runArgs, spec.Command[1:]...) + if spec.Sandd.Enabled() { + runArgs = writeSanddEntrypoint(&b, spec) + } else { + // No shim: map Command/Args straight onto Docker's --entrypoint/CMD as before. + // runArgs are everything that follows the image: the entrypoint's own arguments + // (Command[1:]) then the CMD arguments (Args), in that order. + if len(spec.Command) > 0 { + fmt.Fprintf(&b, " --entrypoint %s", shellQuote(spec.Command[0])) + runArgs = append(runArgs, spec.Command[1:]...) + } + runArgs = append(runArgs, spec.Args...) } - runArgs = append(runArgs, spec.Args...) b.WriteString(" " + shellQuote(spec.Image)) for _, arg := range runArgs { @@ -99,50 +102,75 @@ func buildUserData(spec InstanceSpec) (string, error) { return base64.StdEncoding.EncodeToString([]byte(b.String())), nil } -// sanddInstallURL is the SandD daemon install script. The bootstrap pipes it to -// bash with --tunnel so Tailscale is installed alongside the daemon (see the -// SandD install.sh --tunnel path). -const sanddInstallURL = "https://raw.githubusercontent.com/InftyAI/SandD/main/hack/scripts/install.sh" +// sanddBinaryURL is the statically-linked (musl) SandD daemon release asset. Being +// static, this one binary runs in any container image regardless of its libc, so the +// shim can fetch it into an arbitrary user image at boot. It is pinned to the same +// asset name install.sh resolves; amd64 matches the x86_64 GPU instance types. +const sanddBinaryURL = "https://github.com/InftyAI/SandD/releases/latest/download/sandd-linux-amd64" -// writeSanddBootstrap appends the SandD sandbox-daemon bootstrap to the user-data -// script. daemonID is the NodeClaim name (from ClaimTagKey), so a daemon that dials -// home is identifiable as this exact instance. +// sanddShimScript is the container ENTRYPOINT shim (run as `/bin/sh -c