diff --git a/.env.example b/.env.example index d1c9f69..902d425 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,4 @@ -# Nebula provider credentials — SECRETS (plus one per-cluster endpoint). +# Nebula provider credentials — SECRETS. # # Copy to .env and fill in real values. .env is gitignored — never commit tokens. # Consumed by hack/deploy.sh (make deploy-all), which turns the credentials into one @@ -6,12 +6,6 @@ # cluster) is NOT here — pass it as make variables, e.g. # make deploy-all IMG=myrepo/nebula:v1 DEPLOY_KIND_CLUSTER=nebula-test-e2e # -# The lone non-secret that DOES belong here is SANDD_TUNNEL_SERVER (bottom): it is a -# per-cluster value discovered at deploy time (the headscale NLB hostname), so keeping -# it beside the deploy that consumes it — rather than a hand-passed make flag re-typed -# every run — is the whole point. It rides the SAME parsed-not-sourced path as the -# creds, so it can't leak into kubectl either. -# # cp .env.example .env # # edit .env # make deploy-all @@ -44,22 +38,3 @@ AWS_SECRET_ACCESS_KEY= # --- Additional providers (add as adapters land) --------------------------- # Each provider gets its OWN secret (see hack/deploy.sh PROVIDER_SECRETS), e.g.: # RUNPOD_API_KEY= - -# --- SandD mesh endpoint (NON-SECRET, per-cluster) ------------------------- -# The internet-facing headscale NLB hostname the SandD mesh dials, as a bare -# http:// (no port — the Service listens on 80; see config/sandd). It is only -# known AFTER config/samples/headscale-service.yaml provisions and changes on every -# recreate, so it can't live in the tracked manifests. deploy-all reads it from here -# and `make deploy` substitutes the __SANDD_TUNNEL_SERVER__ token into headscale's -# server_url and the nebula-sandd-config ConfigMap (byte-identical, as headscale -# requires) — no re-typing across files, no manager restart. -# -# Read it back after applying the Service (README config/sandd step 1): -# kubectl apply -f config/samples/headscale-service.yaml -# HS=$(kubectl -n nebula-system get svc nebula-headscale \ -# -o jsonpath='{.status.loadBalancer.ingress[0].hostname}') -# echo "SANDD_TUNNEL_SERVER=http://$HS" # paste the result below, then make deploy-all -# -# Leave blank to skip SandD wiring — the token is then left in place (an obviously -# broken render, surfaced by a deploy.sh warning), not a silent misconfig. -SANDD_TUNNEL_SERVER= diff --git a/Dockerfile.keybroker b/Dockerfile.keybroker deleted file mode 100644 index 64c202b..0000000 --- a/Dockerfile.keybroker +++ /dev/null @@ -1,37 +0,0 @@ -# keybroker — the headscale pre-auth key broker (cmd/keybroker), packaged as a -# SIDECAR for the headscale pod. It shells out to the `headscale` CLI over the -# shared unix socket, so the final image must carry BOTH our Go binary AND the -# headscale CLI — hence the `FROM headscale/headscale` base rather than distroless. -# -# Build (from the repo root): -# docker build -f Dockerfile.keybroker -t inftyai/nebula-keybroker:latest . -# -# Keep the headscale tag here IN SYNC with config/sandd/headscale.yaml so the CLI -# in this sidecar speaks the same wire/flag version as the headscale it dials. - -# --- build the broker binary --------------------------------------------------- -FROM golang:1.24 AS builder -ARG TARGETOS -ARG TARGETARCH - -WORKDIR /workspace -COPY go.mod go.mod -COPY go.sum go.sum -RUN go mod download - -# Only the broker's own sources are needed (it is stdlib-only), but copy the -# module dirs it might import so the build is self-contained. -COPY cmd/keybroker/ cmd/keybroker/ - -# CGO off => a static binary that runs on the headscale base image regardless of -# its libc. -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ - go build -a -o keybroker ./cmd/keybroker - -# --- final image: headscale CLI + our broker ----------------------------------- -# FROM headscale so `headscale preauthkeys create` is on PATH. The broker calls it -# over the local unix socket shared with the real headscale container in the pod. -FROM headscale/headscale:0.23 -COPY --from=builder /workspace/keybroker /usr/local/bin/keybroker -# Override headscale's own entrypoint: this container runs the BROKER, not headscale. -ENTRYPOINT ["/usr/local/bin/keybroker"] diff --git a/Makefile b/Makefile index 5f2ec36..a648480 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,6 @@ # Image URL to use all building/pushing image targets IMG ?= inftyai/nebula-controller:latest -# KEYBROKER_IMG is the SandD key-broker sidecar image (cmd/keybroker, -# Dockerfile.keybroker) that runs alongside headscale — see config/sandd. Built and -# pushed separately from the manager IMG because it is a distinct, optional image -# with its own base (FROM headscale, so it carries the headscale CLI). -KEYBROKER_IMG ?= inftyai/nebula-keybroker:latest - # NAMESPACE is where the manager runs (must match config/manager). Consumed by # hack/deploy.sh via the deploy-all target. NAMESPACE ?= nebula-system @@ -161,14 +155,6 @@ docker-build: ## Build docker image with the manager. docker-push: ## Push docker image with the manager. $(CONTAINER_TOOL) push ${IMG} -.PHONY: docker-build-keybroker -docker-build-keybroker: ## Build docker image for the SandD key-broker sidecar. - $(CONTAINER_TOOL) build -f Dockerfile.keybroker -t ${KEYBROKER_IMG} . - -.PHONY: docker-push-keybroker -docker-push-keybroker: ## Push the SandD key-broker sidecar image. - $(CONTAINER_TOOL) push ${KEYBROKER_IMG} - # PLATFORMS defines the target platforms for the manager image be built to provide support to multiple # architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to: # - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/ @@ -209,17 +195,7 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified .PHONY: deploy deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - # SANDD_TUNNEL_SERVER is the internet-facing headscale NLB hostname, only known - # after that Service provisions and different on every recreate — so it can't be - # baked into the tracked manifests. The config/sandd overlay carries the literal - # token __SANDD_TUNNEL_SERVER__ in headscale's server_url and the nebula-sandd-config - # ConfigMap; substitute it here so both render correct on the FIRST apply (no - # manager/headscale restart). Driven from .env via deploy-all (make deploy-all), - # or pass SANDD_TUNNEL_SERVER=... directly. When unset the token is left as-is (the - # render is then obviously broken rather than silently pointing nowhere). - $(KUSTOMIZE) build $(KUSTOMIZE_BUILD_FLAGS) config/default | \ - $(if $(SANDD_TUNNEL_SERVER),sed 's|__SANDD_TUNNEL_SERVER__|$(SANDD_TUNNEL_SERVER)|g',cat) | \ - $(KUBECTL) apply -f - + $(KUSTOMIZE) build $(KUSTOMIZE_BUILD_FLAGS) config/default | $(KUBECTL) apply -f - .PHONY: deploy-e2e deploy-e2e: manifests kustomize ## Deploy for e2e: config/default plus the fake-provider env var (baked in at deploy time, not via a post-deploy rollout). diff --git a/README.md b/README.md index 0d0f1c7..bccf94d 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,4 @@ placement controller owns those. ## License -Apache-2.0 — see [LICENSE](LICENSE). Third-party components (e.g. Tailscale, used by -the optional SandD channel) are listed in -[THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md). +Apache-2.0 — see [LICENSE](LICENSE). diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md deleted file mode 100644 index d12e230..0000000 --- a/THIRD_PARTY_NOTICES.md +++ /dev/null @@ -1,15 +0,0 @@ -# Third-party notices - -Nebula is licensed under Apache-2.0 (see [LICENSE](LICENSE)). It relies on the -following third-party software, which is licensed separately. - -## Tailscale - -The optional [SandD](https://github.com/InftyAI/SandD) access channel uses the -[Tailscale](https://github.com/tailscale/tailscale) client (`tailscale`/`tailscaled`), -© Tailscale Inc., licensed under -[BSD-3-Clause](https://github.com/tailscale/tailscale/blob/main/LICENSE). - -Nebula does not link or vendor it; the provider integration fetches the official -binary onto the GPU host at provision time. If you build and redistribute an image -with Tailscale baked in, retain its copyright notice per BSD-3-Clause. diff --git a/cmd/keybroker/main.go b/cmd/keybroker/main.go deleted file mode 100644 index eddd1d7..0000000 --- a/cmd/keybroker/main.go +++ /dev/null @@ -1,351 +0,0 @@ -/* -Copyright 2026 The InftyAI Team. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Command keybroker is a tiny HTTP service that mints headscale pre-auth keys on -// demand, so nothing else in the system needs headscale admin authority. -// -// WHY IT EXISTS: SandD daemons and the controller need a mesh pre-auth key to -// join, but neither can safely hold headscale's admin credentials — the daemon -// runs inside an untrusted tenant container, and the controller is user-facing -// sample code. Minting keys by hand also doesn't scale: every experiment spins up -// a fresh controller and every workload is a new daemon. This broker centralizes -// the privilege: it is the ONLY component that talks to headscale's admin surface, -// and it does so over headscale's LOCAL unix socket as a sidecar in the headscale -// pod — so there is no admin API key to store and no gRPC port to expose. Callers -// just POST /keys and get back a single, freshly-minted key string. -// -// SECURITY POSTURE: -// - It shells out to the `headscale` CLI over the shared unix socket -// (--config points at headscale's own config, which carries unix_socket). No -// API key, no network call to headscale. -// - It is reachable only in-cluster (a ClusterIP Service). It mints keys but -// cannot read or revoke them and exposes nothing else, so a compromised caller -// can at most cause it to mint extra ephemeral keys (which auto-reap). -// - The minted key is a secret: it is returned in the response body but NEVER -// logged. Only the key's metadata (kind, reusable, ephemeral) is logged. -package main - -import ( - "encoding/json" - "fmt" - "log" - "net/http" - "os" - "os/exec" - "strconv" - "strings" - "time" -) - -// keyKind is the caller-declared purpose of the key, which selects its policy. The -// two consumers want opposite lifecycles (see policyFor), so the caller names its -// role rather than passing raw flags — the broker owns the policy, not the caller. -type keyKind string - -const ( - // kindDaemon is a GPU-workload daemon key: REUSABLE + ephemeral + bounded TTL. - // Each workload still gets its OWN freshly-minted key (best tenant isolation) that - // headscale auto-reaps once the pod disconnects (no orphan node pile-up). It is - // REUSABLE — not single-use — because on a flaky mesh a daemon's node gets reaped - // (headscale ephemeral_node_inactivity_timeout) during a transient blip, and after - // a reap the ONLY way back in is to re-register with the pre-auth key. A single-use - // key is already spent by then, so the daemon wedges forever, unable to rejoin. A - // reusable key lets every `tailscale up` retry re-register the node, which is what - // makes reconnection actually robust. The blast radius of a leaked reusable key is - // bounded by the ephemeral reap (nodes vanish on disconnect) and the TTL below. - kindDaemon keyKind = "daemon" - // kindController is a SandD controller key: REUSABLE + ephemeral + long TTL. - // Reusable so it can re-register across restarts. Ephemeral — the SAME as a - // daemon — because the controller holds NO persistent state (no PVC): headscale - // reaps its node the instant the pod disconnects, which FREES the stable - // sandd-controller MagicDNS name for the next pod to reclaim (the name is pinned - // by the pod hostname, not by a persisted node key). This is what lets a restart - // reclaim the same name with no -suffix. It also sidesteps the port-443 dial - // wedge that a persisted /var/lib/tailscale caused. Long TTL (720h) just bounds a - // key that outlives a brief reap gap; ephemeral reaping is the real cleanup path. - kindController keyKind = "controller" -) - -// keyPolicy is the resolved (headscale-flag) shape of a key for a given kind. -type keyPolicy struct { - reusable bool - ephemeral bool - expiration string // headscale --expiration duration, e.g. "1h", "720h" -} - -// policyFor maps a kind onto its headscale flags. Unknown kinds are rejected by the -// caller before this is reached, so there is no default policy to fall through to. -func policyFor(kind keyKind) (keyPolicy, bool) { - switch kind { - case kindDaemon: - // Reusable so a reaped daemon can re-register: on a flaky mesh the node is - // reaped during a blip, and re-auth via the pre-auth key is the only way back - // in — a single-use key would already be spent, wedging the daemon forever. - // Ephemeral so the node is still reaped on disconnect (no orphan pile-up). TTL - // bounds a leaked key; each workload gets its own freshly-minted one anyway. - return keyPolicy{reusable: true, ephemeral: true, expiration: "24h"}, true - case kindController: - // Reusable so it can re-register across restarts; ephemeral so the old node - // is reaped on disconnect, freeing the stable MagicDNS name for the fresh pod - // to reclaim (the controller has no PVC, so nothing to preserve). Long TTL - // (720h) just bounds a key that outlives a brief reap gap. - return keyPolicy{reusable: true, ephemeral: true, expiration: "1h"}, true - default: - return keyPolicy{}, false - } -} - -// keyResponse is the JSON body returned to a caller. It carries the freshly-minted -// key plus the policy metadata, so a caller can log/inspect what it got WITHOUT the -// broker having to expose headscale's richer key object. -type keyResponse struct { - Key string `json:"key"` - Kind string `json:"kind"` - Reusable bool `json:"reusable"` - Ephemeral bool `json:"ephemeral"` - Expiration string `json:"expiration"` -} - -// headscalePreAuthKey is the subset of `headscale preauthkeys create -o json` we -// parse. headscale prints the created key object as JSON; we only need its `key`. -type headscalePreAuthKey struct { - Key string `json:"key"` -} - -// headscaleUser is the subset of `headscale users list -o json` we parse. We only -// need the name to decide whether the configured user already exists. -type headscaleUser struct { - Name string `json:"name"` -} - -// config is the broker's runtime configuration, all from env with sane defaults so -// the manifest can stay minimal. -type config struct { - // listenAddr is the HTTP bind address (SANDD_KEYBROKER_LISTEN, default :8090). - listenAddr string - // user is the headscale user keys are minted under (SANDD_KEYBROKER_USER, - // default "nebula"). The broker ensures it exists at startup (ensureHeadscaleUser), - // so no manual bootstrap is needed. - user string - // headscaleConfig is the path to headscale's config file, which carries the - // unix_socket the CLI dials (SANDD_KEYBROKER_HS_CONFIG, default the standard - // /etc/headscale/config.yaml the sidecar shares with headscale). - headscaleConfig string - // headscaleBin is the headscale CLI path (SANDD_KEYBROKER_HS_BIN, default - // "headscale" on PATH — the sidecar image is FROM headscale). - headscaleBin string -} - -func loadConfig() config { - return config{ - listenAddr: envOr("SANDD_KEYBROKER_LISTEN", ":8090"), - user: envOr("SANDD_KEYBROKER_USER", "nebula"), - headscaleConfig: envOr("SANDD_KEYBROKER_HS_CONFIG", "/etc/headscale/config.yaml"), - headscaleBin: envOr("SANDD_KEYBROKER_HS_BIN", "headscale"), - } -} - -func envOr(key, def string) string { - if v := os.Getenv(key); v != "" { - return v - } - return def -} - -// commandRunner mints a key by invoking the headscale CLI. It is a seam so tests -// can inject a fake without a real headscale (the real one is runHeadscale). -type commandRunner func(cfg config, policy keyPolicy) (string, error) - -// userEnsurer creates cfg.user in headscale if it does not already exist. It is a -// seam so tests can inject a fake without a real headscale (the real one is -// ensureHeadscaleUser). -type userEnsurer func(cfg config) error - -// ensureUserRetries and ensureUserRetryDelay bound the startup wait for headscale's -// unix socket. ~10 attempts over 20s comfortably covers headscale opening its socket -// while the sidecars boot together, without hanging a genuinely broken deploy forever. -// Vars, not consts, so tests can shrink the delay to run without real sleeps. -var ( - ensureUserRetries = 10 - ensureUserRetryDelay = 2 * time.Second -) - -// ensureHeadscaleUserWithRetry retries ensure until it succeeds or the attempt budget -// is exhausted, absorbing the sidecar startup race where headscale's socket is not up -// yet. The ensure func is a parameter so tests drive it without a real headscale or -// real sleeps. Returns the last error if every attempt fails. -func ensureHeadscaleUserWithRetry(cfg config, ensure userEnsurer) error { - var err error - for attempt := 1; attempt <= ensureUserRetries; attempt++ { - if err = ensure(cfg); err == nil { - return nil - } - if attempt < ensureUserRetries { - log.Printf("ensure headscale user attempt %d/%d failed (retrying): %v", attempt, ensureUserRetries, err) - time.Sleep(ensureUserRetryDelay) - } - } - return err -} - -// ensureHeadscaleUser makes the configured user exist, idempotently: it lists users -// over the local unix socket, returns early if cfg.user is already present, and -// otherwise creates it. This replaces the manual `headscale users create` bootstrap -// step — the broker already has socket access, so it can self-provision the one user -// it mints under. Safe to run on every startup. -func ensureHeadscaleUser(cfg config) error { - listCmd := exec.Command(cfg.headscaleBin, "--config", cfg.headscaleConfig, "users", "list", "--output", "json") - var stdout, stderr strings.Builder - listCmd.Stdout = &stdout - listCmd.Stderr = &stderr - if err := listCmd.Run(); err != nil { - return fmt.Errorf("headscale users list failed: %w: %s", err, strings.TrimSpace(stderr.String())) - } - - var users []headscaleUser - if err := json.Unmarshal([]byte(stdout.String()), &users); err != nil { - return fmt.Errorf("parsing headscale users list: %w", err) - } - for _, u := range users { - if u.Name == cfg.user { - log.Printf("headscale user %q already exists", cfg.user) - return nil - } - } - - createCmd := exec.Command(cfg.headscaleBin, "--config", cfg.headscaleConfig, "users", "create", cfg.user) - var createErr strings.Builder - createCmd.Stderr = &createErr - if err := createCmd.Run(); err != nil { - return fmt.Errorf("headscale users create %q failed: %w: %s", cfg.user, err, strings.TrimSpace(createErr.String())) - } - log.Printf("created headscale user %q", cfg.user) - return nil -} - -// runHeadscale shells out to `headscale preauthkeys create -o json` over the local -// unix socket (via --config, which carries unix_socket) and returns the key string. -// It NEVER logs the key or the raw output (which contains the key). -func runHeadscale(cfg config, policy keyPolicy) (string, error) { - args := []string{ - "--config", cfg.headscaleConfig, - "preauthkeys", "create", - "--user", cfg.user, - "--expiration", policy.expiration, - "--output", "json", - } - if policy.reusable { - args = append(args, "--reusable") - } - if policy.ephemeral { - args = append(args, "--ephemeral") - } - - cmd := exec.Command(cfg.headscaleBin, args...) - // Capture stdout (the JSON key object) separately from stderr so a headscale - // error message can be surfaced without risking the key leaking into a log via - // combined output. - var stdout, stderr strings.Builder - cmd.Stdout = &stdout - cmd.Stderr = &stderr - if err := cmd.Run(); err != nil { - return "", fmt.Errorf("headscale preauthkeys create failed: %w: %s", err, strings.TrimSpace(stderr.String())) - } - - var key headscalePreAuthKey - if err := json.Unmarshal([]byte(stdout.String()), &key); err != nil { - // Do NOT include stdout in the error: it is the key material. - return "", fmt.Errorf("parsing headscale key output: %w", err) - } - if key.Key == "" { - return "", fmt.Errorf("headscale returned an empty key") - } - return key.Key, nil -} - -// keysHandler handles POST /keys?kind=. It resolves the policy, -// mints one key, and returns it as JSON. GET/other methods and unknown kinds are -// rejected. The runner seam lets tests avoid a real headscale. -func keysHandler(cfg config, run commandRunner) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { - http.Error(w, "only POST is supported", http.StatusMethodNotAllowed) - return - } - kind := keyKind(r.URL.Query().Get("kind")) - policy, ok := policyFor(kind) - if !ok { - http.Error(w, fmt.Sprintf("unknown or missing kind %q (want daemon|controller)", kind), http.StatusBadRequest) - return - } - - key, err := run(cfg, policy) - if err != nil { - // err carries headscale's stderr, never the key. Safe to log + return. - log.Printf("mint failed kind=%s: %v", kind, err) - http.Error(w, "failed to mint key", http.StatusBadGateway) - return - } - - // Log METADATA only — never the key itself. - log.Printf("minted key kind=%s reusable=%s ephemeral=%s exp=%s", - kind, strconv.FormatBool(policy.reusable), strconv.FormatBool(policy.ephemeral), policy.expiration) - - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(keyResponse{ - Key: key, - Kind: string(kind), - Reusable: policy.reusable, - Ephemeral: policy.ephemeral, - Expiration: policy.expiration, - }) - } -} - -func main() { - cfg := loadConfig() - - // Self-provision the user we mint under, so the mesh works without a manual - // `headscale users create`. The broker is a SIDECAR that starts alongside - // headscale, so its unix socket may not be up yet on the first attempt — retry - // with backoff rather than crash-looping on that startup race. Fatal only after - // the window elapses: a broker that can't guarantee its user exists would 502 on - // every mint, so fail loudly instead of serving broken. - if err := ensureHeadscaleUserWithRetry(cfg, ensureHeadscaleUser); err != nil { - log.Fatalf("ensuring headscale user %q: %v", cfg.user, err) - } - - mux := http.NewServeMux() - mux.HandleFunc("/keys", keysHandler(cfg, runHeadscale)) - // Liveness/readiness: the broker is up as soon as it can serve HTTP. We do NOT - // probe headscale here (that would need a real mint); mint errors surface as a - // 502 on /keys, which the caller retries. - mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("ok")) - }) - - srv := &http.Server{ - Addr: cfg.listenAddr, - Handler: mux, - ReadHeaderTimeout: 5 * time.Second, - } - log.Printf("keybroker listening on %s (user=%s, headscale config=%s)", - cfg.listenAddr, cfg.user, cfg.headscaleConfig) - if err := srv.ListenAndServe(); err != nil { - log.Fatalf("keybroker server exited: %v", err) - } -} diff --git a/cmd/keybroker/main_test.go b/cmd/keybroker/main_test.go deleted file mode 100644 index 6f3432a..0000000 --- a/cmd/keybroker/main_test.go +++ /dev/null @@ -1,218 +0,0 @@ -/* -Copyright 2026 The InftyAI Team. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "encoding/json" - "fmt" - "net/http" - "net/http/httptest" - "strings" - "testing" -) - -func TestPolicyFor(t *testing.T) { - cases := []struct { - kind keyKind - ok bool - reusable bool - ephemeral bool - }{ - // Daemon: reusable + ephemeral — a per-workload credential that auto-reaps on - // disconnect, but reusable so a reaped daemon can re-register after a blip. - {kindDaemon, true, true, true}, - // Controller: reusable (survives restarts) + ephemeral (same as a daemon) so - // the old node is reaped on disconnect, freeing its MagicDNS name to reclaim. - {kindController, true, true, true}, - {keyKind("bogus"), false, false, false}, - {keyKind(""), false, false, false}, - } - for _, tc := range cases { - t.Run(string(tc.kind), func(t *testing.T) { - p, ok := policyFor(tc.kind) - if ok != tc.ok { - t.Fatalf("policyFor(%q) ok = %v, want %v", tc.kind, ok, tc.ok) - } - if !ok { - return - } - if p.reusable != tc.reusable { - t.Errorf("reusable = %v, want %v", p.reusable, tc.reusable) - } - if p.ephemeral != tc.ephemeral { - t.Errorf("ephemeral = %v, want %v", p.ephemeral, tc.ephemeral) - } - if p.expiration == "" { - t.Errorf("expiration must not be empty") - } - }) - } -} - -// fakeRunner records the policy it was asked to mint and returns a canned key, so -// the handler can be tested without a real headscale. -func fakeRunner(key string, err error) (commandRunner, *keyPolicy) { - var seen keyPolicy - run := func(_ config, policy keyPolicy) (string, error) { - seen = policy - return key, err - } - return run, &seen -} - -func TestKeysHandler_Daemon(t *testing.T) { - run, seen := fakeRunner("tskey-daemon-abc", nil) - h := keysHandler(config{user: "nebula"}, run) - - req := httptest.NewRequest(http.MethodPost, "/keys?kind=daemon", nil) - rec := httptest.NewRecorder() - h(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) - } - var resp keyResponse - if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { - t.Fatalf("decoding response: %v", err) - } - if resp.Key != "tskey-daemon-abc" { - t.Errorf("key = %q, want tskey-daemon-abc", resp.Key) - } - if !resp.Reusable { - t.Errorf("daemon key must be reusable (so a reaped daemon can re-register)") - } - if !resp.Ephemeral { - t.Errorf("daemon key must be ephemeral") - } - // The runner must have been asked for the reusable ephemeral policy. - if !seen.reusable || !seen.ephemeral { - t.Errorf("runner policy = %+v, want reusable ephemeral", *seen) - } -} - -func TestKeysHandler_Controller(t *testing.T) { - run, seen := fakeRunner("tskey-ctrl-xyz", nil) - h := keysHandler(config{user: "nebula"}, run) - - req := httptest.NewRequest(http.MethodPost, "/keys?kind=controller", nil) - rec := httptest.NewRecorder() - h(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("status = %d, want 200", rec.Code) - } - if !seen.reusable || !seen.ephemeral { - t.Errorf("controller policy = %+v, want reusable + ephemeral", *seen) - } -} - -func TestKeysHandler_UnknownKind(t *testing.T) { - run, _ := fakeRunner("unused", nil) - h := keysHandler(config{user: "nebula"}, run) - - for _, q := range []string{"/keys?kind=bogus", "/keys"} { - req := httptest.NewRequest(http.MethodPost, q, nil) - rec := httptest.NewRecorder() - h(rec, req) - if rec.Code != http.StatusBadRequest { - t.Errorf("%s: status = %d, want 400", q, rec.Code) - } - } -} - -func TestKeysHandler_MethodNotAllowed(t *testing.T) { - run, _ := fakeRunner("unused", nil) - h := keysHandler(config{user: "nebula"}, run) - - req := httptest.NewRequest(http.MethodGet, "/keys?kind=daemon", nil) - rec := httptest.NewRecorder() - h(rec, req) - if rec.Code != http.StatusMethodNotAllowed { - t.Errorf("status = %d, want 405", rec.Code) - } -} - -func TestEnsureHeadscaleUserWithRetry_SucceedsFirstTry(t *testing.T) { - calls := 0 - err := ensureHeadscaleUserWithRetry(config{user: "nebula"}, func(config) error { - calls++ - return nil - }) - if err != nil { - t.Fatalf("err = %v, want nil", err) - } - if calls != 1 { - t.Errorf("ensure called %d times, want 1", calls) - } -} - -func TestEnsureHeadscaleUserWithRetry_RecoversAfterSocketRace(t *testing.T) { - // Simulate headscale's socket not being ready on the first few attempts, then - // coming up — the broker must keep trying rather than fatal on the first miss. - calls := 0 - err := ensureHeadscaleUserWithRetry(config{user: "nebula"}, func(config) error { - calls++ - if calls < 3 { - return fmt.Errorf("socket not ready") - } - return nil - }) - if err != nil { - t.Fatalf("err = %v, want nil after recovery", err) - } - if calls != 3 { - t.Errorf("ensure called %d times, want 3", calls) - } -} - -func TestEnsureHeadscaleUserWithRetry_ExhaustsAndReturnsLastError(t *testing.T) { - // Shrink the delay so exhausting every attempt doesn't wait the real ~18s. - orig := ensureUserRetryDelay - ensureUserRetryDelay = 0 - defer func() { ensureUserRetryDelay = orig }() - - calls := 0 - err := ensureHeadscaleUserWithRetry(config{user: "nebula"}, func(config) error { - calls++ - return fmt.Errorf("still down") - }) - if err == nil { - t.Fatal("err = nil, want the last failure") - } - if !strings.Contains(err.Error(), "still down") { - t.Errorf("err = %v, want it to carry the last failure", err) - } - if calls != ensureUserRetries { - t.Errorf("ensure called %d times, want %d", calls, ensureUserRetries) - } -} - -func TestKeysHandler_MintError(t *testing.T) { - run, _ := fakeRunner("", fmt.Errorf("headscale down")) - h := keysHandler(config{user: "nebula"}, run) - - req := httptest.NewRequest(http.MethodPost, "/keys?kind=daemon", nil) - rec := httptest.NewRecorder() - h(rec, req) - if rec.Code != http.StatusBadGateway { - t.Errorf("status = %d, want 502", rec.Code) - } - // The error body must NOT leak internal detail beyond a generic message. - if strings.Contains(rec.Body.String(), "headscale down") { - t.Errorf("response body leaked internal error: %s", rec.Body.String()) - } -} diff --git a/cmd/main.go b/cmd/main.go index 4033c3a..d70e105 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -51,7 +51,6 @@ import ( awsprovider "github.com/InftyAI/Nebula/pkg/provider/aws" "github.com/InftyAI/Nebula/pkg/provider/fake" "github.com/InftyAI/Nebula/pkg/provider/modal" - "github.com/InftyAI/Nebula/pkg/sandd" "github.com/InftyAI/Nebula/pkg/vnode" // +kubebuilder:scaffold:imports ) @@ -352,22 +351,7 @@ func registerProviders(ctx context.Context, c client.Client) { // delivered via a Secret), and one account-global credential authorizes every // region. Registration only fails (and is a non-fatal skip) if the price catalog // cannot load — region config can no longer make it fail. - // SandD daemon (opt-in): SandD turns on when SANDD_KEYBROKER_URL points at the - // in-cluster key broker. When set, every AWS workload runs the SandD daemon inside - // its container in tunnel mode — commands and interactive shells run in the user's - // own environment over the mesh with no inbound access — and the adapter mints a - // FRESH single-use, ephemeral key per workload (tenant isolation + auto-reaped - // nodes). Unset => nil minter => the zero SanddConfig injects nothing, so this is - // off by default. Minted keys are secrets and are NEVER logged. - sanddCfg := provider.SanddConfig{ - ControlServer: os.Getenv("SANDD_TUNNEL_SERVER"), - ServerURL: os.Getenv("SANDD_SERVER_URL"), - } - if minter := sandd.NewBrokerClient(os.Getenv("SANDD_KEYBROKER_URL")); minter != nil { - sanddCfg.KeyMinter = minter - setupLog.Info("SandD per-daemon key minting enabled via key broker") - } - if p, err := awsprovider.NewSDKClient(ctx, awsRegionSource(c), sanddCfg); err != nil { + if p, err := awsprovider.NewSDKClient(ctx, awsRegionSource(c)); err != nil { setupLog.Info("skipping AWS provider registration", "reason", err.Error()) } else { provider.Register(p) diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index f429f20..6e42628 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -34,13 +34,6 @@ resources: #- ../prometheus # [METRICS] Expose the controller manager metrics service. - metrics_service.yaml -# [SANDD] SandD access channel — deployed by default. Stands up headscale + the -# SandD controller so operators/agents can exec/shell into a workload container -# (kubectl exec does NOT work against a Nebula virtual node). This only stands up -# the in-cluster pieces; the manager starts injecting the daemon once the -# nebula-sandd-config Secret exists (still opt-in per cluster). To skip deploying -# these components, comment this line out. See config/sandd/README.md. -- ../sandd # [NETWORK POLICY] Protect the /metrics endpoint and Webhook Server with NetworkPolicy. # Only Pod(s) running a namespace labeled with 'metrics: enabled' will be able to gather the metrics. # Only CR(s) which requires webhooks and are applied on namespaces labeled with 'webhooks: enabled' will @@ -54,10 +47,9 @@ patches: - path: manager_metrics_patch.yaml target: kind: Deployment - # Scope to the manager ONLY. Without a name this matches EVERY Deployment in - # the build (e.g. headscale from ../sandd), injecting the manager-only - # --metrics-bind-address flag into `headscale serve`, which then errors: - # "unknown flag: --metrics-bind-address". Matches the pre-prefix name. + # Scope to the manager ONLY (matches the pre-prefix name), so the + # manager-only --metrics-bind-address flag is not injected into any other + # Deployment in the build. name: controller-manager # Uncomment the patches line if you enable Metrics and CertManager @@ -72,8 +64,7 @@ patches: - path: manager_webhook_patch.yaml target: kind: Deployment - # Scope to the manager ONLY (see the metrics patch above) — otherwise the - # webhook args/volumes get injected into headscale's Deployment too. + # Scope to the manager ONLY (see the metrics patch above). name: controller-manager # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 50f71ce..f8b02c0 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -97,27 +97,6 @@ spec: # - secretRef: # name: nebula-runpod-credentials # optional: true - # SandD sandbox daemon (cross-provider — see pkg/provider SanddConfig). Its - # SANDD_KEYBROKER_URL / SANDD_TUNNEL_SERVER / SANDD_SERVER_URL are projected - # as env and every supporting adapter runs the daemon inside the workload - # container, so an operator can exec/shell into the user's own environment - # over the mesh (kubectl exec does NOT work against a virtual node). - # - # optional=false: this ConfigMap is REQUIRED. If it is missing the manager - # will NOT start (it sits in CreateContainerConfigError until the ConfigMap - # exists) rather than booting with SandD silently off. The overlay applies - # the ConfigMap before this Deployment (kustomize orders ConfigMap ahead of - # Deployment), so a normal `make deploy-all` satisfies it; a manager without - # SandD config is treated as a misconfiguration, not a valid mode. - # - # It is a ConfigMap, not a Secret: it holds only URLs (public endpoints), no - # credentials — the actual mesh keys are minted on demand by the key broker - # and never stored here. See config/sandd for the full working deployment - # (headscale + controller). To turn injection OFF, blank SANDD_KEYBROKER_URL - # in the ConfigMap (nil minter => nothing injected) rather than deleting it. - - configMapRef: - name: nebula-sandd-config - optional: false ports: [] securityContext: readOnlyRootFilesystem: true diff --git a/config/samples/headscale-service.yaml b/config/samples/headscale-service.yaml deleted file mode 100644 index d299cad..0000000 --- a/config/samples/headscale-service.yaml +++ /dev/null @@ -1,64 +0,0 @@ -# headscale Service — hand-applied, NOT in the config/sandd overlay. -# -# It is kept out of the overlay because the overlay adds a `nebula-` name prefix, -# which would create a SECOND, differently-named Service and thus a second LB. So it -# lives here and sets its own namespace + `nebula-` name explicitly, matching the -# prefixed Deployment the overlay generates; its selector matches the pod labels, so -# it finds the Deployment's pods regardless. -# -# server_url (which MUST equal the address clients dial, or headscale rejects them) -# is this LB's AWS-assigned hostname. That name is only known AFTER the NLB -# provisions, so the flow is: apply this Service, read the hostname back, bake it into -# the three server_url/SANDD_TUNNEL_SERVER spots, then deploy headscale. See README -# step 1. -apiVersion: v1 -kind: Service -metadata: - name: nebula-headscale - namespace: nebula-system - labels: - app: headscale - # An NLB (L4 — headscale is raw HTTP/gRPC, so NOT an ALB). scheme=internet-facing: - # the mesh is CROSS-CLOUD — GPU boxes are provisioned across AWS regions and other - # providers (Modal, RunPod), none of which share a private network with the cluster - # VPC. A control plane every node must reach therefore has to be publicly routable, - # exactly like Tailscale's own control plane. An internal LB only worked for nodes - # in the same/peered VPC, which is not the case here. - # - # SECURITY: this endpoint is now public. It is defensible because the Tailscale - # control (Noise/ts2021) handshake is END-TO-END ENCRYPTED regardless of transport, - # and pre-auth keys are minted in-cluster by the broker and baked into instance - # user-data OUT OF BAND — they never traverse this endpoint. The gRPC admin API - # (:50443) that mints users/keys is NOT exposed here (in-pod only). Still, for real - # use put this behind TLS + the hardening in "Production notes" (README). - # - # The NLB's AWS-assigned hostname (k8s-...elb..amazonaws.com) is what you - # bake into server_url / SANDD_TUNNEL_SERVER; internet-facing => it resolves to - # PUBLIC IPs reachable from any cloud/region. The name is stable for the life of the - # Service (changes only on delete/recreate), which is why the read-back-then-deploy - # flow (README step 1) is safe. The LB listens on port 80 (see ports below) so - # server_url is a bare http://. - annotations: - service.beta.kubernetes.io/aws-load-balancer-type: "external" - service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip" - service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing" -spec: - # LoadBalancer so out-of-cluster GPU boxes — in any region or cloud — can reach the - # control server over the public internet. Swap for NodePort + DNS, or front with an - # Ingress, if you have no LB. - type: LoadBalancer - selector: - app: headscale - ports: - # Listen on the DEFAULT HTTP port 80 and forward to headscale's 8080. This is - # REQUIRED, not cosmetic: Tailscale's control (Noise/ts2021) handshake dialer - # only ever dials the standard ports — 443 for TLS, 80 for the plaintext - # fallback — and IGNORES any custom port in server_url. Exposing headscale on - # 8080 let the health check and key fetch through but left the Noise dial with - # nowhere to land, so registration timed out and no node ever joined the mesh. - # Serving on 80 gives that dial its plaintext target, so server_url is a bare - # http:// (no port). The gRPC admin API (:50443) is deliberately NOT - # exposed — it mints users/keys, so it stays in-pod (`kubectl exec`) only. - - name: http - port: 80 - targetPort: 8080 diff --git a/config/samples/sandd-controller.yaml b/config/samples/sandd-controller.yaml deleted file mode 100644 index cc71bdb..0000000 --- a/config/samples/sandd-controller.yaml +++ /dev/null @@ -1,178 +0,0 @@ -# SandD controller — a `Server()` in tunnel mode. This is the box YOU drive: -# `server.exec(daemon_id, ...)` / `server.new_session(...)` run commands and open -# PTYs on the GPU workloads. It joins the same headscale mesh, takes the first IP -# (100.64.0.1), and listens for daemons on :8765 OVER the mesh — no port is -# exposed to the cluster or the internet. -# -# SAMPLE, not part of the kustomize overlay: the controller is the USER-FACING -# piece — you replace the inline script below with your own controller logic — so -# it is applied by hand and adapted, not deployed as a fixed resource. It sets its -# own namespace and full name (nothing rewrites them here). Apply after the -# config/sandd overlay is up — see config/sandd/README.md. -# -# The image is SandD's tunnel-server build (Python `sandd` package + Tailscale -# client preinstalled). Build/push it from the SandD repo: -# docker build -f hack/docker/Dockerfile.server-tunnel -t inftyai/sandd-server-tunnel:latest. -# -# The controller mints its own mesh key at startup from the in-cluster key broker -# (SANDD_KEYBROKER_URL below), as REUSABLE + EPHEMERAL — the SAME policy the daemons -# use, and it holds NO persistent state (no PVC). This is deliberate: every start is -# a CLEAN start, exactly like a daemon. -# -# It still keeps its stable MagicDNS name (sandd-controller.nebula.mesh) across -# restarts, via two mechanisms that do NOT need a PVC: -# - hostname: sandd-controller (below) pins the OS hostname, from which Tailscale -# derives the node/MagicDNS name — so the name is requested identically each time. -# - the key is EPHEMERAL, so headscale reaps the old node the moment the pod -# disconnects, FREEING the name; the fresh pod then reclaims it cleanly (no -# -suffix). This is exactly why daemons never collide. strategy: Recreate ensures -# the old pod is fully gone (its node reaped) BEFORE the new one registers, so the -# two never race for the name. -# -# Why no PVC: a PVC on /var/lib/tailscale persisted stale dial state, which tripped -# Tailscale's control dialer into "forcing port 443" on restart — and headscale is -# HTTP-only on 80, so that dead-ended and the controller could never re-register -# ("first start works, restart doesn't"). Daemons never hit this because they have no -# persisted state. Dropping the PVC removes the wedge entirely. The headscale URL it -# joins (SANDD_TUNNEL_SERVER) is set inline in env below — see README.md, step 2. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: sandd-controller - namespace: nebula-system - labels: - app: sandd-controller -spec: - replicas: 1 - # Recreate (not the default RollingUpdate): the controller's mesh key is EPHEMERAL, - # so headscale reaps its node the moment the pod disconnects, which FREES the - # sandd-controller MagicDNS name for the next pod to reclaim. RollingUpdate starts - # the new pod BEFORE killing the old one, so the new one would try to register while - # the old node is still connected (not yet reaped) — a name collision that lands the - # new pod on sandd-controller-1 and strands daemons dialing the clean name. Recreate - # tears the old pod down first, so its node is reaped before the new one registers - # and the fresh pod reclaims the same name cleanly. - strategy: - type: Recreate - selector: - matchLabels: - app: sandd-controller - template: - metadata: - labels: - app: sandd-controller - spec: - # Pin the pod's OS hostname so it is DETERMINISTIC. Tailscale derives a - # node's name (and thus its MagicDNS name) from the OS hostname, so this is - # what makes the controller reliably reachable at - # sandd-controller. on the mesh — independent of join order or - # which 100.64.x.x IP it happens to get. headscale's base_domain is - # nebula.mesh (see config/sandd/headscale.yaml), so the daemons dial - # ws://sandd-controller.nebula.mesh:8765/ws (SANDD_SERVER_URL below). - hostname: sandd-controller - containers: - - name: controller - image: inftyai/sandd-server-tunnel:latest - # Start the Server in tunnel mode and keep it alive. It joins the mesh, then - # loops listing daemons and runs a few example commands (COMMANDS below) on - # each workload the first time it connects, printing the output to the pod log - # (`kubectl logs`) — just enough to show exec over the mesh works. Attach with - # `kubectl exec` to drive `server` yourself (see README, "Using it"), or edit - # COMMANDS / replace this with your own controller logic — that's the point. - command: ["python3", "-u", "-c"] - args: - - | - import os, time, json, urllib.request - from sandd import Server, TunnelConfig - - # Mint a reusable+ephemeral key for this controller from the in-cluster - # broker (crash-loops if unreachable — the broker is required, not optional). - # NEVER print the key itself, only that we got one. - broker = os.environ["SANDD_KEYBROKER_URL"].rstrip("/") - req = urllib.request.Request(f"{broker}/keys?kind=controller", method="POST") - with urllib.request.urlopen(req, timeout=10) as resp: - authkey = json.load(resp)["key"] - print("controller: minted key, joining mesh", flush=True) - - server = Server(connect="tunnel", tunnel_config=TunnelConfig( - authkey=authkey, server=os.environ["SANDD_TUNNEL_SERVER"])) - print("controller: ready, waiting for daemons", flush=True) - - # Commands run on each workload the first time it connects. Edit freely — - # each runs in the user's container over the mesh via server.exec. - COMMANDS = ["hostname", "nvidia-smi -L", "ls /"] - - # Run the commands on each daemon once. `seen` guards against re-running - # every poll. - seen = set() - while True: - try: - for d in server.list_daemons(): - if d.id in seen: - continue - seen.add(d.id) - for cmd in COMMANDS: - r = server.exec(d.id, cmd) - print(f"[{d.id}] $ {cmd} (exit={r.exit_code})\n{r.stdout.rstrip()}", flush=True) - except Exception as e: - print("controller: loop error:", e, flush=True) - time.sleep(5) - env: - # The headscale URL the controller joins. UNLIKE the daemons (which are - # out-of-cluster and MUST use the public NLB hostname) and headscale's own - # server_url, the controller runs IN-cluster, so it reaches headscale by its - # internal ClusterIP Service name — no per-cluster substitution, and it never - # touches the public NLB. That matters: routing an in-cluster client through - # the internet-facing NLB is a needless round-trip AND hairpins (blackholes) - # when the controller lands on the same node as the headscale pod, which is - # exactly what made /machine/register time out and fall back to a dead :443. - # The ClusterIP path is direct and can't hairpin. Port 80 (Service 80 -> 8080) - # satisfies Tailscale's Noise dialer (80/443 only). headscale accepts this - # even though its server_url is the public name: authkey registration doesn't - # require the client's login-server to byte-match server_url. Cross-namespace - # is fine — the .nebula-system suffix resolves from any namespace. The - # controller mints its OWN key from the broker, so no auth key here. - - name: SANDD_TUNNEL_SERVER - value: http://nebula-headscale.nebula-system - # In-cluster key broker: the controller mints its OWN reusable+ephemeral - # key at startup from this endpoint (see resolve_authkey above). This is - # the nebula-keybroker ClusterIP Service (config/sandd/headscale.yaml). - # Required — the controller crash-loops if the broker is unreachable. - - name: SANDD_KEYBROKER_URL - value: http://nebula-keybroker.nebula-system:8090 - ports: - # Informational only — the daemons reach this over the mesh, not via a - # Service. Not exposed cluster-wide. - - name: sandd-ws - containerPort: 8765 - # NO added capabilities. tailscaled runs in --tun=userspace-networking - # mode (userspace TCP/IP stack, no kernel TUN/route), so NET_ADMIN is not - # needed: inbound mesh TCP is forwarded to the local :8765 listener - # without touching the host's routing or firewall. NET_ADMIN would let the - # container rewrite routes/firewall rules and sniff/inject traffic, so we - # keep it off even on this trusted control-plane side. - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: ["ALL"] - # DON'T drop this if you rewrite the controller: on shutdown we log out of - # headscale so the ephemeral node is DEREGISTERED immediately, which frees - # the sandd-controller name for the next pod. Without it, headscale holds the - # old node as `offline` until its inactivity timeout (~30m) elapses; a restart - # inside that window finds the clean name still taken and registers as - # sandd-controller- — and every daemon dialing the clean MagicDNS name - # (sandd-controller.nebula.mesh) then hits a dead node ("Active daemons: 0"). - # Recreate guarantees this preStop finishes before the new pod starts, so the - # name is free in time. `|| true` keeps a failed logout (e.g. control plane - # already unreachable) from blocking teardown. - lifecycle: - preStop: - exec: - command: ["/bin/sh", "-c", "tailscale logout || true"] - resources: - requests: - cpu: "100m" - memory: "128Mi" - limits: - cpu: "500m" - memory: "512Mi" diff --git a/config/sandd/README.md b/config/sandd/README.md deleted file mode 100644 index 98c48a4..0000000 --- a/config/sandd/README.md +++ /dev/null @@ -1,254 +0,0 @@ -# SandD access channel - -Wires [SandD](https://github.com/InftyAI/SandD) into Nebula so an operator or agent -can run commands and open interactive shells **inside a Nebula-provisioned workload -container** — which `kubectl logs`/`exec` cannot do against a Nebula virtual node -(`pkg/vnode/handler.go` returns `NotFound`). - -The daemon runs **inside the workload container** (one per container, single-tenant), -so its shells see the user's own env, cwd, filesystem and code. It dials **out** to a -controller over a Tailscale/headscale mesh, so the GPU box needs no inbound access — -it works for instances behind NAT with no public IP, in any region or cloud (the mesh -is cross-cloud; only outbound reachability to the public headscale endpoint is needed). - -## Architecture - -**Minting keys** — the broker is the only component with headscale admin authority; -it reaches headscale over a local unix socket. There is no static-key path. - -``` - headscale pod - ┌──────────────────────────────────────────────┐ - │ keybroker sidecar ──unix socket──► headscale │ - │ (ClusterIP :8090) │ - └──────────▲──────────────────▲──────────────────┘ - │ POST /keys │ POST /keys - │ ?kind=controller │ ?kind=daemon - │ (reusable) │ (reusable) - controller Nebula manager - (at startup) (at Provision, per workload → bakes key into user-data) -``` - -**Using the mesh** — both sides join headscale, then the workload's daemon dials the -controller by its stable MagicDNS name (no inbound access to the box). - -``` - headscale (internet-facing LB :80 → 8080) - ▲ ▲ - joins mesh│ │joins mesh - ┌─────────────────┴────────┐ ┌───────┴──────────────────┐ - │ controller │◄──────│ GPU workload (EC2) │ - │ Server(tunnel) │ ws://…│ sandd --tunnel │ - │ sandd-controller. │ (mesh)│ dials controller by name │ - │ nebula.mesh:8765/ws │ │ DAEMON_ID = NodeClaim │ - └──────────────────────────┘ └───────────────────────────┘ -``` - -## What's in the overlay - -`config/sandd` is applied by `config/default` (the `- ../sandd` line), so `make deploy` -stands up everything except the hand-applied Service and controller: - -| File | Kind | Role | -|------|------|------| -| `headscale.yaml` | Deployment + ConfigMap + Service | Tailscale control server + the **`keybroker` sidecar** (mints pre-auth keys on demand over a local unix socket; in-cluster-only `nebula-keybroker` ClusterIP; the ONLY way keys are minted). | -| `manager-config.yaml` | ConfigMap `nebula-sandd-config` | **Required** — the manager mounts it as `envFrom` with `optional: false`, so it won't start without it. `SANDD_KEYBROKER_URL` is the injection switch: set, the manager mints a fresh per-daemon key from the broker; blank, injection is off (but keep the ConfigMap — blank the value, don't delete it). | - -Hand-applied (not in the overlay): - -| File | Kind | Role | -|------|------|------| -| `config/samples/headscale-service.yaml` | Service (internet-facing NLB, `:80` → `8080`) | `scheme=internet-facing` — publicly reachable so GPU boxes in **any region or cloud** can join the mesh (they share no private network with the cluster). Listens on **80** (Tailscale's Noise handshake only dials 80/443), so `server_url` is a bare `http://`. Its AWS-assigned ELB hostname is read back after it provisions (step 1). Kept out of the overlay so its `nebula-` prefix doesn't spawn a second LB. gRPC admin (`:50443`) is NOT exposed. The control handshake is end-to-end encrypted and keys never traverse it, but see "Production notes" for TLS/hardening. | -| `config/samples/sandd-controller.yaml` | Deployment `sandd-controller` | The box you drive (`server.exec` / `new_session`). A SandD `Server()` in tunnel mode; sets `SANDD_TUNNEL_SERVER` + `SANDD_KEYBROKER_URL` inline and mints its own key. Adapt the inline script to your own logic. | - -## Deploy - -**Prereqs:** a running Nebula control plane with the AWS provider configured -(`nebula-aws-credentials` + a NodePool referencing `aws`); GPU instances with outbound -internet access (they dial the public headscale endpoint — no inbound access or shared -VPC needed, which is why the mesh spans regions and clouds); and the AWS Load Balancer -Controller installed (it provisions the internet-facing NLB in step 1). - -### 0. Build the key-broker image - -```bash -# Override KEYBROKER_IMG for your own registry; then update the sidecar image: in -# config/sandd/headscale.yaml to match. -make docker-build-keybroker docker-push-keybroker -``` - -### 1. Apply the Service, read its hostname into `.env`, deploy - -headscale forces every client to dial the identical `server_url`, and the GPU boxes -live in other regions/clouds with no private path to the cluster — so the endpoint -must be publicly routable. That address is the internet-facing NLB's own AWS-assigned -hostname, which is only known once the LB provisions. So apply the Service first and -read it back: - -```bash -kubectl create namespace nebula-system # if not already present -kubectl apply -f config/samples/headscale-service.yaml - -# Poll until populated (NLB takes 1–3 min): -HS=$(kubectl -n nebula-system get svc nebula-headscale \ - -o jsonpath='{.status.loadBalancer.ingress[0].hostname}'); echo "$HS" - -echo "SANDD_TUNNEL_SERVER=http://$HS" >> .env # or edit .env by hand -make deploy-all # substitutes the hostname into every spot, then deploys -``` - -`make deploy-all` reads `SANDD_TUNNEL_SERVER` from `.env` and substitutes the -`__SANDD_TUNNEL_SERVER__` token wherever the endpoint must appear — `server_url` in -`config/sandd/headscale.yaml` and `SANDD_TUNNEL_SERVER` in -`config/sandd/manager-config.yaml` — so both render **byte-identical** (exactly what -headscale demands) on the first apply, with no manager or headscale restart. The URL -carries no port because the Service listens on **80**: Tailscale's control (Noise) -handshake only ever dials 80 or 443, never a custom port, so headscale must be on a -default port even though it listens on 8080 in-pod (the Service maps 80 → 8080). - -> The ELB hostname is stable for the life of the Service; if you delete and recreate -> the Service, just re-read it into `.env` and re-run `make deploy-all` — no manifest -> edits. (The controller in step 2 is hand-applied, so substitute the same value into -> it too — one `sed`, shown there.) - -The broker mints keys under one headscale user (`SANDD_KEYBROKER_USER`, default -`nebula`) and **creates that user itself on startup** if it's missing — no manual -bootstrap. Key policies it owns (see `cmd/keybroker`): both daemon and controller keys -are **reusable** — the controller so it re-registers across restarts, the daemon so a -node reaped during a mesh blip can re-authenticate and rejoin (a single-use key would -be spent, wedging the daemon offline forever). Each workload still gets its own -freshly-minted daemon key, so a leak is scoped to that workload and bounded by the -key's TTL (daemon 24h, controller 720h). Both are **ephemeral**, so headscale auto-reaps -a node shortly after it disconnects — torn-down experiments don't pile up as OFFLINE -nodes squatting MagicDNS names, and the controller reclaims its stable name cleanly on -restart (the reaped old node frees it). - -### 2. Deploy the controller - -Self-contained — it mints its own key at startup and reads its endpoints from inline -env. No substitution needed: unlike the daemons, the controller runs in-cluster, so it -dials headscale by its internal ClusterIP name (`SANDD_TUNNEL_SERVER: -http://nebula-headscale.nebula-system`), not the public NLB. So it's a plain apply: - -```bash -kubectl apply -f config/samples/sandd-controller.yaml - -# Confirm it joined (any 100.64.x.x — we address it by name, not IP): -CTRL=$(kubectl -n nebula-system get pod -l app=sandd-controller -o name | head -1) -kubectl -n nebula-system exec "$CTRL" -c controller -- tailscale ip -4 -``` - -Routing the in-cluster controller through the public NLB was a needless round-trip and -**hairpined** (AWS NLBs with `target-type: ip` blackhole traffic from a client that -lands on the *same node* as the target pod) — that made `/machine/register` time out and -fall back to a dead `:443`. The ClusterIP path is direct and can't hairpin. headscale -accepts it even though its `server_url` is the public name (authkey registration doesn't -require the client's login-server to byte-match `server_url`). - -It pins `hostname: sandd-controller` (the stable MagicDNS name) and holds **no -persistent state** — no PVC, an `emptyDir`-clean start every time, exactly like a -daemon. A restart still reclaims the *same* name (not a `-suffix` that would strand -daemons at `Active daemons: 0`) because the key is **ephemeral**: headscale reaps the -old node the instant the pod disconnects, freeing the name, and `strategy: Recreate` -guarantees the old pod is gone before the new one registers — so the fresh pod -reclaims the name with no collision. Deliberately **no PVC**: a persisted -`/var/lib/tailscale` reloaded stale dial state on restart and wedged Tailscale into -"forcing port 443" (headscale is HTTP-only on 80), so the controller could never -re-register — "first start works, restart doesn't". Daemons never hit this precisely -because they carry no state; the controller now matches them. No StorageClass needed. - -That's it — the manager read `nebula-sandd-config` at startup (step 1 applied it -before the manager pod started), so injection is already on. From now on **every AWS -workload** runs the daemon inside its container, addressed by `DAEMON_ID` = the -NodeClaim name, each with its own broker-minted key. (If you later *change* the config -on a running manager, restart it — `kubectl -n nebula-system rollout restart -deploy/nebula-controller-manager` — since it reads env only at startup.) - -## Using it - -Apply any Nebula workload — injection is a cluster decision, so the ordinary sample -works unchanged: - -```bash -# The workload places against a NodePool, so that must exist first. Apply both -# together (deployment.yaml targets the `sample` NodePool): -kubectl apply -f config/samples/nebula_v1alpha1_nodepool.yaml -kubectl apply -f config/samples/deployment.yaml # or your own workload -``` - -Then drive it from the controller: - -```bash -CTRL=$(kubectl -n nebula-system get pod -l app=sandd-controller -o name | head -1) -kubectl -n nebula-system exec -it "$CTRL" -c controller -- python3 - <<'PY' -from sandd import Server, TunnelConfig -import os, json, urllib.request -# Mint a key from the broker, same as the controller does at startup. -req = urllib.request.Request( - os.environ["SANDD_KEYBROKER_URL"].rstrip("/") + "/keys?kind=controller", method="POST") -with urllib.request.urlopen(req, timeout=10) as r: - authkey = json.load(r)["key"] -server = Server(connect="tunnel", tunnel_config=TunnelConfig( - authkey=authkey, server=os.environ["SANDD_TUNNEL_SERVER"])) -for d in server.list_daemons(): - print("daemon:", d.id) - print(server.exec(d.id, "nvidia-smi --query-gpu=name --format=csv,noheader").stdout) - print(server.exec(d.id, "ls /").stdout) # the USER's filesystem -PY -``` - -`d.id` is the NodeClaim name; `server.exec` runs in the workload container. For an -interactive shell use `server.new_session(d.id)` (PTY). - -## How injection works - -The AWS adapter injects SandD in two parts (`pkg/provider/aws/translate.go`), both -**fail-open** so they can never block the workload: - -1. **On the host, before `docker run`:** the user-data fetches the static (musl) - `sandd` binary + Tailscale bundle into `/opt/sandd`. The GPU AMI has `curl`+`tar`, - so the user's image needs nothing. -2. **`docker run`** bind-mounts `/opt/sandd` **read-only** and overrides the ENTRYPOINT - with a `/bin/sh` shim (`sanddShimScript`) that: puts `/opt/sandd` on `PATH`; starts - `sandd --tunnel` backgrounded (tailscaled in `--tun=userspace-networking` — no - `NET_ADMIN`, no `/dev/net/tun`); then `exec`s the user's command as PID 1. - -Fetching on the host means no fetcher/package manager in the user's image, one download -per instance, and a safe shared read-only mount (each container keeps its own writable -`/var/lib/tailscale` + `/tmp`). - -`SANDD_TUNNEL_AUTHKEY` arrives as container env — the per-workload key the manager -minted (reusable + ephemeral, 24h TTL), so even though the container can see it, it's a -short-lived credential scoped to this one workload and its node is reaped on disconnect. -It is reusable (not single-use) so the daemon can re-authenticate if its node is reaped -during a mesh blip; the daemon is single-tenant, so a container seeing its own key is -fine. - -Both parts log to stderr with a `[sandd]` prefix, landing in the **EC2 console** -(`aws ec2 get-console-output --instance-id --latest`) — the place to debug -bring-up. The daemon's own log is at `/tmp/sandd.log` inside the container. - -**Image requirements:** the shim needs only `/bin/sh` (binaries are mounted, not -fetched — even distroless-with-shell works); the **host** needs outbound network. A Pod -relying solely on its image's baked-in ENTRYPOINT (no `command`/`args`) can't be -reconstructed — set an explicit `command`. - -## Turning it off - -- **Stop injecting** (keep infra): blank `SANDD_KEYBROKER_URL` in `manager-config.yaml` - and restart the manager (empty broker URL => nil minter => nothing injected). Do NOT - delete the ConfigMap — it is a required mount (`optional: false`), so removing it - leaves the manager stuck in `CreateContainerConfigError`. -- **Remove the controller:** `kubectl delete -f config/samples/sandd-controller.yaml`. -- **Remove all infra:** comment out `- ../sandd` in - `config/default/kustomization.yaml` and re-apply — but keep `manager-config.yaml` - applied (with `SANDD_KEYBROKER_URL` blank) so the required mount is still satisfied. - -## Production notes - -This is a working DEV setup, not hardened: headscale uses SQLite (on a small PVC so its -state survives pod restarts, but still a single-writer file DB) over plain HTTP. Each workload -already gets its own freshly-minted ephemeral key, but for real use also: back headscale -with a PersistentVolume + real database + TLS, and add per-tenant **tags/ACLs** to -minted keys so daemons are mesh-isolated, not just credential-distinct. See -https://headscale.net. diff --git a/config/sandd/headscale.yaml b/config/sandd/headscale.yaml deleted file mode 100644 index 6bbf287..0000000 --- a/config/sandd/headscale.yaml +++ /dev/null @@ -1,233 +0,0 @@ -# headscale — the Tailscale control server that coordinates the SandD mesh. -# -# ⚠️ DEVELOPMENT / EXAMPLE configuration. It uses SQLite on a small PVC (state -# survives Pod restarts, but it is still a single-writer file DB) and a plain -# HTTP listener. For production see https://headscale.net and back it with a -# real database + TLS. -# -# It MUST be reachable by out-of-cluster GPU boxes (they join the same mesh). The -# LoadBalancer Service that exposes it is hand-applied FIRST — it lives OUTSIDE this -# overlay (config/samples/headscale-service.yaml) precisely because everything in -# config/sandd/ is applied together by `make deploy`, whereas the Service must go up -# on its own so its ELB hostname is known before this Deployment starts. See that -# file and the README for why. This file (the ConfigMap + Deployment) is what the -# ../sandd kustomize overlay deploys. -# -# Namespace + the nebula- name prefix are added by config/default's kustomize, so -# they are omitted here (the ConfigMap ref below stays unprefixed too). -apiVersion: v1 -kind: ConfigMap -metadata: - name: headscale-config -data: - config.yaml: | - # server_url MUST equal the address clients actually dial. headscale rejects a - # client whose --login-server disagrees with this. It is the internet-facing NLB's - # AWS-assigned hostname (config/samples/headscale-service.yaml), only known AFTER - # the Service provisions. The literal token below is substituted by `make deploy` - # from SANDD_TUNNEL_SERVER (set in .env; deploy-all reads the hostname back — README - # step 1). This value is byte-identical to SANDD_TUNNEL_SERVER in nebula-sandd-config - # (same token, same substitution), which is exactly what headscale requires. The - # value is a bare http:// with NO port: the Service listens on 80 (Tailscale's - # Noise handshake only dials 80/443, never a custom port), and headscale itself still - # listens on 8080 in-pod (listen_addr below); the Service maps 80 -> 8080. - server_url: __SANDD_TUNNEL_SERVER__ - listen_addr: 0.0.0.0:8080 - metrics_listen_addr: 0.0.0.0:9090 - grpc_listen_addr: 0.0.0.0:50443 - grpc_allow_insecure: true - - # The keybroker sidecar (below) mints pre-auth keys via the headscale CLI over - # THIS local unix socket — no admin API key, no gRPC exposure. Both containers - # mount the same emptyDir at /var/run/headscale, so the CLI in the sidecar - # reaches headscale in-pod. Nothing outside the pod can touch it. - unix_socket: /var/run/headscale/headscale.sock - unix_socket_permission: "0770" - - # Mesh IP range (RFC 6598 CGNAT, as Tailscale/headscale recommend). Nodes are - # addressed by MagicDNS name (base_domain below), not raw IP, so which node - # gets which 100.64.x.x address does not matter. - prefixes: - v4: 100.64.0.0/10 - v6: fd7a:115c:a1e0::/48 - - private_key_path: /var/lib/headscale/private.key - noise: - private_key_path: /var/lib/headscale/noise_private.key - - database: - type: sqlite3 - sqlite: - path: /var/lib/headscale/db.sqlite - - log: - level: info - format: text - - dns: - magic_dns: true - base_domain: nebula.mesh - nameservers: - global: - - 1.1.1.1 - - derp: - server: - enabled: false - urls: - - https://controlplane.tailscale.com/derpmap/default - auto_update_enabled: true - update_frequency: 24h ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: headscale - labels: - app: headscale -spec: - replicas: 1 - # Recreate (not the default RollingUpdate): headscale's state is on a single - # ReadWriteOnce PVC (data volume below). RollingUpdate starts the new pod before - # killing the old one, so both would race for the same RWO volume -> the new pod - # hangs on a Multi-Attach error. Recreate tears the old pod down first. Fine at - # replicas: 1 (headscale is single-writer over SQLite anyway). - strategy: - type: Recreate - selector: - matchLabels: - app: headscale - template: - metadata: - labels: - app: headscale - spec: - containers: - - name: headscale - image: headscale/headscale:0.23 - args: ["serve"] - ports: - - name: http - containerPort: 8080 - # grpc (:50443) is intentionally NOT published on the LoadBalancer - # (see config/samples/headscale-service.yaml): the admin API mints - # users/keys, so it stays reachable only in-pod via `kubectl exec`. The - # process still listens on it (grpc_listen_addr) for that local CLI use. - volumeMounts: - - name: config - mountPath: /etc/headscale/config.yaml - subPath: config.yaml - readOnly: true - - name: data - mountPath: /var/lib/headscale - # Shared with the keybroker sidecar so its `headscale` CLI can dial this - # process's local admin socket (unix_socket above). - - name: socket - mountPath: /var/run/headscale - readinessProbe: - httpGet: - path: /health - port: 8080 - initialDelaySeconds: 5 - periodSeconds: 10 - # keybroker — mints pre-auth keys on demand so nothing else needs headscale - # admin authority. Runs as a SIDECAR here (not a separate Deployment) so it - # can reach headscale over the shared LOCAL unix socket: no admin API key to - # store, no gRPC port exposed. It serves an in-cluster-only HTTP API - # (nebula-keybroker Service) that the manager (per-daemon keys) and the - # controller (its own key) call. Image built from Dockerfile.keybroker - # (FROM headscale, so the CLI is present); keep its headscale tag in sync - # with the headscale container above. - - name: keybroker - image: inftyai/nebula-keybroker:latest - env: - # The CLI reads unix_socket from headscale's config, so point the broker - # at the SAME config file the headscale container uses. - - name: SANDD_KEYBROKER_HS_CONFIG - value: /etc/headscale/config.yaml - # Keys are minted under this headscale user. The broker ensures it exists - # at startup (creates it if missing), so no manual bootstrap is needed. - - name: SANDD_KEYBROKER_USER - value: nebula - ports: - - name: broker - containerPort: 8090 - volumeMounts: - # Read the same config (for unix_socket) ... - - name: config - mountPath: /etc/headscale/config.yaml - subPath: config.yaml - readOnly: true - # ... and share the socket dir so the CLI reaches headscale in-pod. - - name: socket - mountPath: /var/run/headscale - readinessProbe: - httpGet: - path: /healthz - port: 8090 - initialDelaySeconds: 3 - periodSeconds: 10 - volumes: - - name: config - configMap: - # Unprefixed: config/default's namePrefix rewrites this reference to - # the generated nebula-headscale-config ConfigMap. - name: headscale-config - # headscale's SQLite DB + keys live here. Backed by a PVC (not emptyDir) so - # the DB SURVIVES pod recreation: with emptyDir, any reschedule (node - # scale-down, spot reclaim, rollout) wiped every registration, which - # orphaned already-joined nodes — in particular the PVC-backed controller, - # which then loops re-registering an identity headscale no longer knows. A - # PVC keeps that state, so a headscale restart no longer drops the mesh. - - name: data - persistentVolumeClaim: - claimName: headscale-data - # Ephemeral socket dir shared by headscale + keybroker so the sidecar's CLI - # reaches headscale's local admin socket. Never leaves the pod. - - name: socket - emptyDir: {} ---- -# Persists headscale's SQLite DB + private keys across pod recreation. Without it -# (emptyDir) a rescheduled headscale came up with an EMPTY database, invalidating -# every issued auth key and orphaning already-joined nodes — most visibly the -# controller, whose OWN identity is on a PVC, so it survived while headscale forgot -# it and looped re-registering. RWO is fine at replicas: 1 (headscale is a single -# SQLite writer); pair with strategy: Recreate on the Deployment so a rollout does -# not race two pods for this volume. gp3 to match the cluster default (same SC as -# the controller PVC); swap the storageClassName for your environment. This is still -# a DEV setup — for production also move off SQLite to a real DB + TLS (see README). -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: headscale-data - labels: - app: headscale -spec: - storageClassName: gp3 - accessModes: - - ReadWriteOnce - resources: - requests: - # SQLite DB + a couple of key files — tiny; 1Gi is the common minimum - # granularity for dynamically-provisioned block volumes. - storage: 1Gi ---- -# In-cluster API for the keybroker sidecar. ClusterIP ONLY — this mints mesh -# credentials, so it must never be exposed outside the cluster (unlike the -# headscale Service, which is a public LoadBalancer for out-of-cluster GPU boxes). -# The manager dials it to mint a per-daemon key at provision time; the SandD -# controller dials it at startup to mint its own key. -apiVersion: v1 -kind: Service -metadata: - name: keybroker - labels: - app: headscale -spec: - type: ClusterIP - selector: - app: headscale - ports: - - name: broker - port: 8090 - targetPort: broker diff --git a/config/sandd/kustomization.yaml b/config/sandd/kustomization.yaml deleted file mode 100644 index feaafad..0000000 --- a/config/sandd/kustomization.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# [SANDD] SandD access channel — OPT-IN, disabled by default. -# -# SandD lets an operator/agent run commands and open interactive shells INSIDE a -# Nebula-provisioned workload container (kubectl logs/exec do NOT work against a -# Nebula virtual node — see pkg/vnode/handler.go). The daemon runs inside the -# workload, dials OUT to a controller over a Tailscale/headscale mesh, so it needs -# no inbound access to the GPU box. See pkg/provider.SanddConfig and the AWS -# entrypoint shim in pkg/provider/aws/translate.go. -# -# This overlay deploys only the shared INFRASTRUCTURE that backs the channel: -# - headscale.yaml : the Tailscale control server (assigns mesh IPs) -# -# headscale's LoadBalancer Service is deliberately NOT here: it must be applied on -# its own, FIRST, so its ELB hostname is known before headscale starts (that -# address is headscale's server_url). Everything in this folder is applied together -# by `make deploy`, so the Service lives in config/samples/headscale-service.yaml -# and is hand-applied — see README.md step 1. -# -# The CONTROLLER (a SandD Server() you drive with your own exec/session logic) is -# deliberately NOT here — it is the user-facing piece, so it ships as a hand-applied -# sample (config/samples/sandd-controller.yaml) you adapt, not a fixed resource. -# -# This overlay flips the manager ON: it ships nebula-sandd-config (manager-config.yaml), -# which the manager's optional envFrom reads to enable injection. So deploying this -# overlay + a controller is all it takes — but SANDD_TUNNEL_SERVER in that ConfigMap -# is a per-cluster placeholder (the headscale LB hostname), so injection only works -# once you fill it in (see README.md). Comment the resource out to keep SandD off. -resources: - - headscale.yaml - - manager-config.yaml diff --git a/config/sandd/manager-config.yaml b/config/sandd/manager-config.yaml deleted file mode 100644 index 1422e62..0000000 --- a/config/sandd/manager-config.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# nebula-sandd-config — flips SandD injection ON for the manager. Deployed by this -# overlay (namePrefix in config/default turns "sandd-config" into "nebula-sandd-config"; -# the manager's envFrom references the pre-prefix name so kustomize rewrites both). -# -# A ConfigMap, not a Secret: it holds only URLs. The mesh keys are minted on demand -# by the key broker and never stored here. -# -# SANDD_TUNNEL_SERVER is headscale's server_url — the address the out-of-cluster GPU -# daemons dial to join the mesh. It is the internet-facing NLB's AWS-assigned hostname -# (see config/samples/headscale-service.yaml), resolving to PUBLIC IPs: the mesh spans -# clouds and AWS regions that share no private network with the cluster, so the control -# plane must be publicly routable (a ClusterIP or internal-LB name would be unreachable -# from a box in another region/cloud). Paste the hostname read back after the Service -# provisions (README step 1); keep it identical to server_url in headscale.yaml. -apiVersion: v1 -kind: ConfigMap -metadata: - name: sandd-config - namespace: nebula-system -data: - # In-cluster key broker (the switch that turns injection ON): the manager mints a - # fresh single-use, ephemeral key per workload from here. Stable internal Service - # DNS — no per-cluster edit needed. - SANDD_KEYBROKER_URL: "http://nebula-keybroker.nebula-system:8090" - # headscale control-server URL — the internet-facing NLB's AWS-assigned hostname. - # The literal token below is substituted by `make deploy` from SANDD_TUNNEL_SERVER - # (set it in .env; deploy-all reads it back). It stays a token in the tracked file - # because the hostname is per-cluster and changes on every Service recreate. Kept - # byte-identical to server_url in headscale.yaml (same token, same substitution). - SANDD_TUNNEL_SERVER: "__SANDD_TUNNEL_SERVER__" - # The controller's SandD WebSocket URL, reached OVER the mesh by MagicDNS name. - # Stable convention (headscale base_domain nebula.mesh + controller hostname), so - # no per-cluster edit needed. - # TODO: dynamically set this. - SANDD_SERVER_URL: "ws://sandd-controller.nebula.mesh:8765/ws" diff --git a/hack/deploy.sh b/hack/deploy.sh index 1af9fe4..ec4a4a9 100755 --- a/hack/deploy.sh +++ b/hack/deploy.sh @@ -178,20 +178,8 @@ done # --- 4. install CRDs + deploy the manager ---------------------------------- # The pod mounts the cert Secret and reads provider creds at boot — both already # exist, so the manager comes up fully configured with no restart needed. -# -# SANDD_TUNNEL_SERVER is the ONE non-secret in .env (the internet-facing headscale -# NLB hostname). It is read from ENV_VARS[] (parsed, NOT sourced — same safety as the -# creds) and handed to `make deploy`, which substitutes the __SANDD_TUNNEL_SERVER__ -# token in headscale's server_url and the nebula-sandd-config ConfigMap so both render -# correct on first apply — no restart. Blank => `make deploy` leaves the token in place -# (an obviously-broken render, not a silent misconfig); the README's read-back step -# populates it. An env/flag SANDD_TUNNEL_SERVER overrides .env. -SANDD_TUNNEL_SERVER="${SANDD_TUNNEL_SERVER:-${ENV_VARS[SANDD_TUNNEL_SERVER]:-}}" -if [[ -z "${SANDD_TUNNEL_SERVER}" ]]; then - warn "SANDD_TUNNEL_SERVER unset (not in .env or env) — headscale server_url + SANDD_TUNNEL_SERVER will keep the __SANDD_TUNNEL_SERVER__ placeholder; set it (README step 1) and re-run." -fi log "installing CRDs and deploying the manager" -make deploy IMG="${IMG}" SANDD_TUNNEL_SERVER="${SANDD_TUNNEL_SERVER}" +make deploy IMG="${IMG}" # --- 5. inject the webhook CA bundle (server-side, no manager restart) ------ # This edits only the MutatingWebhookConfiguration, which just got created by diff --git a/keybroker b/keybroker deleted file mode 100755 index 92b495f..0000000 Binary files a/keybroker and /dev/null differ diff --git a/pkg/provider/aws/aws.go b/pkg/provider/aws/aws.go index 3c98537..d2f9a14 100644 --- a/pkg/provider/aws/aws.go +++ b/pkg/provider/aws/aws.go @@ -142,11 +142,6 @@ type InstanceSpec struct { Region string // Tags carry Nebula identity; ClaimTagKey holds the NodeClaim name. Tags map[string]string - // Sandd, when Enabled, makes the workload's docker run start the SandD daemon - // INSIDE the container (via an entrypoint shim) so commands and interactive - // shells run in the user's own env/cwd/code over the tunnel. A zero value injects - // nothing. See buildUserData/writeSanddEntrypoint and provider.SanddConfig. - Sandd provider.SanddConfig } // EC2Instance is the adapter-level view of one EC2 instance as observed. @@ -216,9 +211,6 @@ type Provider struct { // regionSource reports the NodePool-declared regions to sweep in List/Offerings. // May be nil in tests, in which case sweepRegions uses only the cache keys. regionSource RegionSource - // sandd is the optional SandD daemon config stamped onto every InstanceSpec so - // buildUserData can bootstrap it. Zero value => disabled (no injection). - sandd provider.SanddConfig mu sync.Mutex clients map[string]Client // region -> Client, populated lazily by clientFor @@ -233,16 +225,11 @@ type Provider struct { // (admission requires each aws pool to list ≥1 region, and placement stamps it onto // the ProvisionRequest), and observed instances report their region from the // region-pinned client — so nothing needs a fallback, and no AWS_REGION env is read. -// -// sandd is the optional SandD daemon config baked into every instance's user-data; -// its zero value disables injection, so tests and the non-SandD path pass -// provider.SanddConfig{}. -func New(newClient ClientFactory, cat catalog.Lookup, regionSource RegionSource, sandd provider.SanddConfig) *Provider { +func New(newClient ClientFactory, cat catalog.Lookup, regionSource RegionSource) *Provider { return &Provider{ Base: catalog.Base{ProviderName: provider.ProviderAWS, Catalog: cat}, newClient: newClient, regionSource: regionSource, - sandd: sandd, clients: make(map[string]Client), } } @@ -257,7 +244,6 @@ func newSingleRegion(client Client, cat catalog.Lookup, region string) *Provider func(context.Context, string) (Client, error) { return client, nil }, cat, func() []string { return []string{region} }, - provider.SanddConfig{}, // single-region test convenience: no SandD injection ) // Pre-seed the cache so even a stray region lookup returns the fake rather than // invoking the (constant) factory. @@ -420,7 +406,7 @@ func (p *Provider) Provision(ctx context.Context, pod *corev1.Pod, req provider. return existing.ID, nil } - spec, err := p.instanceSpecFromPod(ctx, pod, req) + spec, err := p.instanceSpecFromPod(pod, req) if err != nil { return "", err } @@ -684,7 +670,7 @@ func splitID(instanceID string) (region, rawID string) { // accelerator type (from the AcceleratorTypeLabel), maps it to an EC2 instance // type via the catalog, and stamps the claim tag, capacity tier, and region. func (p *Provider) instanceSpecFromPod( - ctx context.Context, pod *corev1.Pod, req provider.ProvisionRequest, + pod *corev1.Pod, req provider.ProvisionRequest, ) (InstanceSpec, error) { if len(pod.Spec.Containers) == 0 { return InstanceSpec{}, errors.New("aws: pod has no containers") @@ -721,15 +707,6 @@ func (p *Provider) instanceSpecFromPod( return InstanceSpec{}, fmt.Errorf("aws: no EC2 instance type for %s x%d", canonical, count) } - // SandD daemon (opt-in): resolve THIS instance's mesh key by minting a fresh - // single-use, ephemeral key per workload (tenant isolation + auto-reaped nodes). - // This runs on the Provision path, so a mint failure aborts provisioning rather - // than launching a daemon that can never join the mesh. - sandd, err := p.resolveSanddConfig(ctx) - if err != nil { - return InstanceSpec{}, err - } - return InstanceSpec{ InstanceTypes: instanceTypes, Image: c.Image, @@ -739,32 +716,9 @@ func (p *Provider) instanceSpecFromPod( Spot: req.CapacityType == nebulav1alpha1.CapacitySpot, Region: req.Region, Tags: map[string]string{ClaimTagKey: req.ClaimName}, - // SandD daemon (opt-in): keyed by the claim name so a daemon that dials home - // correlates 1:1 with this instance's NodeClaim. Zero value => no injection. - Sandd: sandd, }, nil } -// resolveSanddConfig returns the per-instance SandD config. When SandD is disabled -// (zero config, no KeyMinter) it returns it unchanged. Otherwise it mints a FRESH -// per-daemon key and returns a copy with AuthKey set to it (and KeyMinter cleared, -// so the minted key — not the minter — is what buildUserData bakes in). A mint error -// is returned so Provision fails loudly rather than injecting a keyless (and thus -// useless) daemon. -func (p *Provider) resolveSanddConfig(ctx context.Context) (provider.SanddConfig, error) { - if p.sandd.KeyMinter == nil { - return p.sandd, nil - } - key, err := p.sandd.KeyMinter.MintDaemonKey(ctx) - if err != nil { - return provider.SanddConfig{}, fmt.Errorf("aws: minting SandD daemon key: %w", err) - } - perInstance := p.sandd - perInstance.AuthKey = key - perInstance.KeyMinter = nil // the resolved key travels on the spec, not the seam. - return perInstance, nil -} - // toInstance normalizes an observed EC2 instance into the provider-agnostic // Instance. func (p *Provider) toInstance(ec2 EC2Instance) provider.Instance { diff --git a/pkg/provider/aws/client.go b/pkg/provider/aws/client.go index 4b70865..0ee0629 100644 --- a/pkg/provider/aws/client.go +++ b/pkg/provider/aws/client.go @@ -30,7 +30,6 @@ import ( smithy "github.com/aws/smithy-go" logf "sigs.k8s.io/controller-runtime/pkg/log" - "github.com/InftyAI/Nebula/pkg/provider" "github.com/InftyAI/Nebula/pkg/provider/catalog" ) @@ -205,14 +204,7 @@ var _ Client = (*sdkClient)(nil) // // The catalog is loaded via catalog.Load() (embedded CSV / mounted ConfigMap), // identical to the other adapters. -// -// sandd is the OPTIONAL SandD daemon config: its zero value (empty AuthKey) leaves -// the bootstrap untouched, so passing provider.SanddConfig{} is a no-op. When set, -// every workload this provider launches runs the SandD daemon inside its container -// in tunnel mode (see buildUserData), the workload's command-execution/shell -// channel. Unlike credentials, the auth key IS accepted here — it is delivered to -// the controller as a secret and stamped into the (base64) user-data. -func NewSDKClient(_ context.Context, regionSource RegionSource, sandd provider.SanddConfig) (*Provider, error) { +func NewSDKClient(_ context.Context, regionSource RegionSource) (*Provider, error) { cat, err := catalog.Load() if err != nil { return nil, fmt.Errorf("aws: load price catalog: %w", err) @@ -221,7 +213,7 @@ func NewSDKClient(_ context.Context, regionSource RegionSource, sandd provider.S factory := func(ctx context.Context, region string) (Client, error) { return newSDKClientForRegion(ctx, region) } - return New(factory, cat, regionSource, sandd), nil + return New(factory, cat, regionSource), nil } // newSDKClientForRegion builds one region-pinned sdkClient: it loads SDK config for diff --git a/pkg/provider/aws/client_test.go b/pkg/provider/aws/client_test.go index 88568de..206c840 100644 --- a/pkg/provider/aws/client_test.go +++ b/pkg/provider/aws/client_test.go @@ -347,10 +347,9 @@ func TestBuildUserData_QuotesHostileValues(t *testing.T) { } } -// TestBuildUserData_SanddDisabled: the zero SanddConfig is the default, so a spec -// that does not opt in must render EXACTLY the plain bootstrap — no sandd/tunnel -// tokens leak in, so existing clusters are unaffected. -func TestBuildUserData_SanddDisabled(t *testing.T) { +// TestBuildUserData_PlainBootstrap: the bootstrap renders only the plain +// docker-pull + docker-run flow — no sandd/tunnel tokens leak in. +func TestBuildUserData_PlainBootstrap(t *testing.T) { encoded, err := buildUserData(InstanceSpec{Image: "img"}) if err != nil { t.Fatalf("buildUserData: %v", err) @@ -358,129 +357,7 @@ func TestBuildUserData_SanddDisabled(t *testing.T) { raw, _ := base64.StdEncoding.DecodeString(encoded) script := string(raw) if strings.Contains(script, "sandd") || strings.Contains(script, "tunnel") { - t.Fatalf("sandd disabled but bootstrap injected it:\n%s", script) - } -} - -// TestBuildUserData_SanddEnabled: when opted in, sandd runs INSIDE the container via -// an entrypoint shim so its shells see the user's env/cwd/code. The shim must -// override the image ENTRYPOINT with /bin/sh, carry daemon config as container env -// (incl. the claim-derived DAEMON_ID), start the daemon fail-open + backgrounded, -// and exec the user's effective command so the workload becomes PID 1. -func TestBuildUserData_SanddEnabled(t *testing.T) { - spec := InstanceSpec{ - Image: "img", - Command: []string{"python", "train.py"}, - Args: []string{"--epochs", "3"}, - Tags: map[string]string{ClaimTagKey: "claim-abc"}, - Sandd: provider.SanddConfig{ - AuthKey: "tskey-secret", - ControlServer: "http://headscale:8080", - ServerURL: "ws://100.64.0.1:8765/ws", - }, - } - encoded, err := buildUserData(spec) - if err != nil { - t.Fatalf("buildUserData: %v", err) - } - raw, _ := base64.StdEncoding.DecodeString(encoded) - script := string(raw) - - // The shim commandeers the container ENTRYPOINT as /bin/sh. - if !strings.Contains(script, "--entrypoint '/bin/sh'") { - t.Fatalf("sandd shim must override the image entrypoint with /bin/sh:\n%s", script) - } - - // Daemon config is delivered as container env (so a process INSIDE the container - // sees it), including the claim-derived DAEMON_ID and the tunnel key/server. The - // binary URLs are NOT container env anymore — the host fetches them (see below). - for _, want := range []string{ - "-e 'DAEMON_ID=claim-abc'", - "-e 'SERVER_URL=ws://100.64.0.1:8765/ws'", - "-e 'SANDD_TUNNEL_AUTHKEY=tskey-secret'", - "-e 'SANDD_TUNNEL_SERVER=http://headscale:8080'", - } { - if !strings.Contains(script, want) { - t.Fatalf("sandd shim missing container env %q:\n%s", want, script) - } - } - - // The binaries are fetched on the HOST (which has curl+tar) before docker run, so - // the user image needs no fetcher/package manager. Both URLs must appear in the - // host-side fetch, targeting sanddHostDir. - for _, want := range []string{sanddBinaryURL, tailscaleTarballURL, "mkdir -p " + sanddHostDir} { - if !strings.Contains(script, want) { - t.Fatalf("host fetch must download %q into %s:\n%s", want, sanddHostDir, script) - } - } - // The host fetch must precede docker run (binaries exist before the container mounts them). - if strings.Index(script, sanddBinaryURL) > strings.Index(script, "docker run") { - t.Fatalf("host fetch must run BEFORE docker run:\n%s", script) - } - // The binaries reach the container via a read-only bind mount, not an in-container fetch. - if !strings.Contains(script, "-v '"+sanddHostDir+":"+sanddHostDir+":ro'") { - t.Fatalf("sandd binaries must be bind-mounted read-only at %s:\n%s", sanddHostDir, script) - } - if !strings.Contains(script, `export PATH="`+sanddHostDir) { - t.Fatalf("shim must put the mounted binaries on PATH:\n%s", script) - } - // No in-container fetch/install: the image is not required to have curl/wget or a - // package manager. Guard against regressing to the old per-container-install model. - for _, absent := range []string{"apt-get", "apk add", "SANDD_BINARY_URL", "curl -fsSL \"$SANDD_BINARY_URL\""} { - if strings.Contains(script, absent) { - t.Fatalf("shim must NOT fetch/install inside the container (found %q):\n%s", absent, script) - } - } - - // Bring-up must be observable (not silent): steps log with a [sandd] prefix to - // stderr so they reach the EC2 instance console for debugging. - if !strings.Contains(script, "[sandd]") { - t.Fatalf("sandd bring-up must log steps with a [sandd] prefix:\n%s", script) - } - - // Fail-open + backgrounded: both the host fetch and the container bring-up are - // `( set +e ... ) || true` subshells; the daemon start ends in `&`. - if !strings.Contains(script, "set +e") || !strings.Contains(script, ") || true") { - t.Fatalf("sandd bring-up must be fail-open (set +e + || true):\n%s", script) - } - if !strings.Contains(script, "&\n") { - t.Fatalf("sandd daemon must be backgrounded:\n%s", script) - } - - // The shim execs the user's effective command so the workload is PID 1, and that - // command (image then argv) is rendered after the shim payload in kubelet order: - // Command[0]+Command[1:] then Args. - if !strings.Contains(script, `exec "$@"`) { - t.Fatalf("shim must exec the user command as PID 1:\n%s", script) - } - if !strings.Contains(script, "'img' '-c'") { - t.Fatalf("shim payload must follow the image on the run line:\n%s", script) - } - if !strings.Contains(script, "'sandd-shim' 'python' 'train.py' '--epochs' '3'\n") { - t.Fatalf("user effective command not rendered after the shim in order:\n%s", script) - } -} - -// TestBuildUserData_SanddQuotesHostileValues: sandd config values are shell-quoted -// like every other injected value, so a hostile auth key/URL cannot break out. -func TestBuildUserData_SanddQuotesHostileValues(t *testing.T) { - spec := InstanceSpec{ - Image: "img", - Tags: map[string]string{ClaimTagKey: "c"}, - Sandd: provider.SanddConfig{ - AuthKey: "k'; rm -rf /; echo '", - ControlServer: "http://h:8080", - ServerURL: "ws://s:8765/ws", - }, - } - encoded, err := buildUserData(spec) - if err != nil { - t.Fatalf("buildUserData: %v", err) - } - raw, _ := base64.StdEncoding.DecodeString(encoded) - script := string(raw) - if strings.Contains(script, "\nrm -rf /") { - t.Fatalf("hostile sandd value escaped quoting:\n%s", script) + t.Fatalf("plain bootstrap must not inject sandd/tunnel:\n%s", script) } } diff --git a/pkg/provider/aws/sandd_test.go b/pkg/provider/aws/sandd_test.go deleted file mode 100644 index a6c8496..0000000 --- a/pkg/provider/aws/sandd_test.go +++ /dev/null @@ -1,146 +0,0 @@ -/* -Copyright 2026 The InftyAI Team. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package aws - -import ( - "context" - "errors" - "testing" - - "github.com/InftyAI/Nebula/pkg/provider" -) - -// fakeMinter is a provider.DaemonKeyMinter that returns a canned key (or error) and -// counts calls, so tests can assert a FRESH key is minted per provision. -type fakeMinter struct { - key string - err error - call int -} - -func (m *fakeMinter) MintDaemonKey(_ context.Context) (string, error) { - m.call++ - if m.err != nil { - return "", m.err - } - return m.key, nil -} - -// TestProvision_MintsPerDaemonKey: with a KeyMinter wired, each Provision mints a -// fresh key and that minted key (not the static one) is what lands on the spec. -func TestProvision_MintsPerDaemonKey(t *testing.T) { - f := &fakeClient{runID: "i-mint"} - p := newTestProvider(f) - minter := &fakeMinter{key: "tskey-fresh-123"} - p.sandd = provider.SanddConfig{ - AuthKey: "tskey-STATIC-should-not-be-used", - ControlServer: "http://headscale:8080", - ServerURL: "ws://ctrl.nebula.mesh:8765/ws", - KeyMinter: minter, - } - - if _, err := p.Provision(context.Background(), gpuPod("H100", 8), provider.ProvisionRequest{ - ClaimName: "claim-mint", - Region: testRegion, - }); err != nil { - t.Fatalf("Provision: %v", err) - } - - if minter.call != 1 { - t.Fatalf("minter called %d times, want 1", minter.call) - } - // The minted key, not the static AuthKey, must be baked onto the spec. - if got := f.lastSpec.Sandd.AuthKey; got != "tskey-fresh-123" { - t.Fatalf("spec AuthKey = %q, want the minted tskey-fresh-123", got) - } - // The minter seam must NOT ride onto the spec — only the resolved key travels. - if f.lastSpec.Sandd.KeyMinter != nil { - t.Fatalf("spec KeyMinter should be nil after resolution") - } - // The rest of the SandD config rides through unchanged. - if f.lastSpec.Sandd.ControlServer != "http://headscale:8080" { - t.Fatalf("ControlServer = %q", f.lastSpec.Sandd.ControlServer) - } -} - -// TestProvision_SeparateProvisionsMintSeparateKeys: two DISTINCT workloads => two -// mints, so each gets its OWN freshly-minted credential (the isolation guarantee). -// Distinct claim names avoid the idempotency short-circuit (which would return the -// existing instance without launching or minting again). -func TestProvision_SeparateProvisionsMintSeparateKeys(t *testing.T) { - f := &fakeClient{runID: "i-x"} - p := newTestProvider(f) - minter := &fakeMinter{key: "tskey-k"} - p.sandd = provider.SanddConfig{ControlServer: "http://h:8080", KeyMinter: minter} - - for _, claim := range []string{"claim-1", "claim-2"} { - if _, err := p.Provision(context.Background(), gpuPod("H100", 8), provider.ProvisionRequest{ - ClaimName: claim, - Region: testRegion, - }); err != nil { - t.Fatalf("Provision %s: %v", claim, err) - } - } - if minter.call != 2 { - t.Fatalf("minter called %d times, want 2 (one fresh key per workload)", minter.call) - } -} - -// TestProvision_MintErrorAbortsProvision: a mint failure must abort provisioning -// (no instance launched), not silently inject a keyless daemon. -func TestProvision_MintErrorAbortsProvision(t *testing.T) { - f := &fakeClient{runID: "i-nope"} - p := newTestProvider(f) - p.sandd = provider.SanddConfig{ - ControlServer: "http://h:8080", - KeyMinter: &fakeMinter{err: errors.New("broker down")}, - } - - _, err := p.Provision(context.Background(), gpuPod("H100", 8), provider.ProvisionRequest{ - ClaimName: "claim-err", - Region: testRegion, - }) - if err == nil { - t.Fatalf("Provision succeeded, want mint error") - } - if f.runCnt != 0 { - t.Fatalf("RunInstance called %d times, want 0 (provision must abort before launch)", f.runCnt) - } -} - -// TestProvision_NoMinterInjectsNothing: with no KeyMinter, SandD is off — nothing is -// minted and no auth key lands on the spec (the daemon is not injected). -func TestProvision_NoMinterInjectsNothing(t *testing.T) { - f := &fakeClient{runID: "i-off"} - p := newTestProvider(f) - p.sandd = provider.SanddConfig{ - ControlServer: "http://h:8080", - } - - if _, err := p.Provision(context.Background(), gpuPod("H100", 8), provider.ProvisionRequest{ - ClaimName: "claim-off", - Region: testRegion, - }); err != nil { - t.Fatalf("Provision: %v", err) - } - if f.lastSpec.Sandd.Enabled() { - t.Fatalf("SandD should be disabled with no KeyMinter, got enabled spec %+v", f.lastSpec.Sandd) - } - if got := f.lastSpec.Sandd.AuthKey; got != "" { - t.Fatalf("spec AuthKey = %q, want empty (no key minted)", got) - } -} diff --git a/pkg/provider/aws/translate.go b/pkg/provider/aws/translate.go index 8f54d53..48571d6 100644 --- a/pkg/provider/aws/translate.go +++ b/pkg/provider/aws/translate.go @@ -56,14 +56,6 @@ func buildUserData(spec InstanceSpec) (string, error) { // Pull first so an image error surfaces before we try to run. fmt.Fprintf(&b, "docker pull %s\n", shellQuote(spec.Image)) - // SandD (opt-in): fetch the daemon + Tailscale binaries on the HOST now, before - // the container starts. They are bind-mounted read-only into the container by - // writeSanddEntrypoint, so the user's image needs no fetcher. Fail-open: a failed - // download just leaves the mount empty and the shim skips the daemon. - if spec.Sandd.Enabled() { - fmt.Fprintf(&b, sanddHostFetchScript, sanddHostDir, sanddBinaryURL, tailscaleTarballURL) - } - // docker run --rm --gpus all, with env, then the image, then the workload's // command/args. Kubernetes container semantics are preserved by mapping the two // Pod fields the way the kubelet does, NOT by concatenating them: @@ -80,26 +72,15 @@ func buildUserData(spec InstanceSpec) (string, error) { fmt.Fprintf(&b, " -e %s", shellQuote(k+"="+spec.Env[k])) } - // SandD daemon (opt-in): run it INSIDE the container so its shells/commands see - // the user's own env, cwd, filesystem and code (a host-side daemon would only see - // the host). We cannot assume the user's image has sandd, so a shell entrypoint - // shim fetches the static musl binary at boot, starts it backgrounded, then execs - // the user's real program — see writeSanddEntrypoint. This overrides the image's - // ENTRYPOINT with the shim, so the shim (not Docker) is responsible for launching - // the workload with the right Command/Args; runArgs is left empty in that case. + // Map Command/Args straight onto Docker's --entrypoint/CMD. runArgs are everything + // that follows the image: the entrypoint's own arguments (Command[1:]) then the CMD + // arguments (Args), in that order. var runArgs []string - if spec.Sandd.Enabled() { - runArgs = writeSanddEntrypoint(&b, spec) - } else { - // No shim: map Command/Args straight onto Docker's --entrypoint/CMD as before. - // runArgs are everything that follows the image: the entrypoint's own arguments - // (Command[1:]) then the CMD arguments (Args), in that order. - if len(spec.Command) > 0 { - fmt.Fprintf(&b, " --entrypoint %s", shellQuote(spec.Command[0])) - runArgs = append(runArgs, spec.Command[1:]...) - } - runArgs = append(runArgs, spec.Args...) + if len(spec.Command) > 0 { + fmt.Fprintf(&b, " --entrypoint %s", shellQuote(spec.Command[0])) + runArgs = append(runArgs, spec.Command[1:]...) } + runArgs = append(runArgs, spec.Args...) b.WriteString(" " + shellQuote(spec.Image)) for _, arg := range runArgs { @@ -110,132 +91,6 @@ func buildUserData(spec InstanceSpec) (string, error) { return base64.StdEncoding.EncodeToString([]byte(b.String())), nil } -// sanddBinaryURL is the statically-linked (musl) SandD daemon release asset. Being -// static, this one binary runs in any container image regardless of its libc, so the -// shim can fetch it into an arbitrary user image at boot. It is pinned to the same -// asset name install.sh resolves; amd64 matches the x86_64 GPU instance types. -const sanddBinaryURL = "https://github.com/InftyAI/SandD/releases/latest/download/sandd-linux-amd64" - -// tailscaleTarballURL is the static (no-libc) Tailscale bundle. sandd --tunnel does -// NOT install Tailscale — it requires `tailscale`/`tailscaled` already on PATH (see -// setup_tunnel in sandd) — so the shim must fetch them too. The static build runs in -// any image, and tailscaled's userspace-networking mode (below) needs no TUN device -// or NET_ADMIN, so this works in an unprivileged container. Pinned: unlike sandd, a -// tarball has no /latest/ redirect, and pinning the mesh client is prudent anyway. -const tailscaleTarballURL = "https://pkgs.tailscale.com/stable/tailscale_1.78.1_amd64.tgz" - -// sanddHostDir is where the host fetches the sandd + tailscale binaries and where -// they are bind-mounted (read-only) into the workload container. Fetching on the -// HOST — the AL2 GPU AMI, which has curl+tar — instead of inside the container means -// the user's image needs no fetcher or package manager (works on distroless too), -// and the download happens once per instance rather than per container start. -const sanddHostDir = "/opt/sandd" - -// sanddHostFetchScript downloads the static sandd + Tailscale binaries into -// sanddHostDir on the HOST, as a fragment of the user-data run BEFORE `docker run`. -// It is a fmt format string: %[1]s=dir, %[2]s=sandd URL, %[3]s=tailscale URL (all -// trusted package constants, so no shell-injection concern). -// -// FAIL-OPEN: wrapped in `( set +e ... ) || true` so a failed download never aborts -// the (set -euo pipefail) user-data — the container-side shim then simply finds no -// binary and skips the daemon, leaving the workload untouched. Every step logs to -// stderr with a `[sandd]` prefix, which reaches the instance CONSOLE -// (get-console-output) — the place to debug bring-up on a Nebula virtual node, since -// kubectl logs/exec do not work against it. -const sanddHostFetchScript = `( set +e - log() { echo "[sandd] $*" >&2; } - mkdir -p %[1]s - if curl -fsSL %[2]s -o %[1]s/sandd; then chmod +x %[1]s/sandd; log "fetched sandd binary" - else log "FAILED to fetch sandd binary from %[2]s — SandD disabled"; fi - if curl -fsSL %[3]s -o %[1]s/ts.tgz && tar -xzf %[1]s/ts.tgz -C %[1]s --strip-components=1; then - log "fetched tailscale bundle" - else log "FAILED to fetch/extract tailscale bundle from %[3]s — SandD disabled"; fi -) || true -` - -// sanddShimScript is the container ENTRYPOINT shim (run as `/bin/sh -c